├── .coveragerc ├── .dockerignore ├── .editorconfig ├── .flake8 ├── .github ├── FUNDING.yml └── workflows │ └── build-bothub-engine-push-tag-dockerhub.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── .travis.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── Makefile ├── Makefile-help.txt ├── Procfile ├── README.md ├── app.json ├── bothub ├── __init__.py ├── api │ ├── __init__.py │ ├── grpc │ │ ├── __init__.py │ │ ├── connect_grpc_client.py │ │ ├── organization │ │ │ ├── __init__.py │ │ │ ├── handlers.py │ │ │ ├── serializers.py │ │ │ └── services.py │ │ ├── repository │ │ │ ├── __init__.py │ │ │ ├── handlers.py │ │ │ ├── serializers.py │ │ │ ├── services.py │ │ │ └── tests.py │ │ └── user │ │ │ ├── __init__.py │ │ │ ├── handlers.py │ │ │ ├── serializers.py │ │ │ ├── services.py │ │ │ └── tests.py │ └── v2 │ │ ├── __init__.py │ │ ├── account │ │ ├── __init__.py │ │ ├── serializers.py │ │ └── views.py │ │ ├── evaluate │ │ ├── __init__.py │ │ ├── filters.py │ │ ├── permissions.py │ │ ├── serializers.py │ │ ├── validators.py │ │ └── views.py │ │ ├── example │ │ ├── __init__.py │ │ └── serializers.py │ │ ├── examples │ │ ├── __init__.py │ │ ├── filters.py │ │ └── views.py │ │ ├── fields.py │ │ ├── groups │ │ ├── __init__.py │ │ ├── permissions.py │ │ ├── serializers.py │ │ └── views.py │ │ ├── internal │ │ ├── __init__.py │ │ ├── connect_rest_client.py │ │ ├── organization │ │ │ ├── __init__.py │ │ │ ├── permissions.py │ │ │ ├── serializers.py │ │ │ ├── tests.py │ │ │ └── views.py │ │ ├── permissions.py │ │ ├── repository │ │ │ ├── __init__.py │ │ │ ├── serializers.py │ │ │ ├── tests.py │ │ │ └── views.py │ │ └── user │ │ │ ├── __init__.py │ │ │ ├── serializers.py │ │ │ ├── tests.py │ │ │ └── views.py │ │ ├── knowledge_base │ │ ├── __init__.py │ │ ├── filters.py │ │ ├── permissions.py │ │ ├── serializers.py │ │ └── views.py │ │ ├── metadata.py │ │ ├── middleware.py │ │ ├── mixins.py │ │ ├── nlp │ │ ├── __init__.py │ │ ├── serializers.py │ │ └── views.py │ │ ├── organization │ │ ├── __init__.py │ │ ├── filters.py │ │ ├── permissions.py │ │ ├── serializers.py │ │ ├── validators.py │ │ └── views.py │ │ ├── repository │ │ ├── __init__.py │ │ ├── filters.py │ │ ├── permissions.py │ │ ├── serializers.py │ │ ├── validators.py │ │ └── views.py │ │ ├── routers.py │ │ ├── swagger.py │ │ ├── tests │ │ ├── __init__.py │ │ ├── test_account.py │ │ ├── test_evaluate.py │ │ ├── test_examples.py │ │ ├── test_groups.py │ │ ├── test_knowledge_base.py │ │ ├── test_logs.py │ │ ├── test_nlp.py │ │ ├── test_organization.py │ │ ├── test_qa_logs.py │ │ ├── test_repository.py │ │ ├── test_tasks.py │ │ ├── test_translation.py │ │ ├── test_versioning.py │ │ └── utils.py │ │ ├── translation │ │ ├── __init__.py │ │ ├── filters.py │ │ ├── permissions.py │ │ ├── serializers.py │ │ ├── validators.py │ │ └── views.py │ │ ├── translator │ │ ├── __init__.py │ │ ├── filters.py │ │ ├── permissions.py │ │ ├── serializers.py │ │ ├── validators.py │ │ └── views.py │ │ ├── urls.py │ │ ├── versionning │ │ ├── __init__.py │ │ ├── filters.py │ │ ├── permissions.py │ │ ├── serializers.py │ │ ├── validators.py │ │ └── views.py │ │ ├── views.py │ │ └── zeroshot │ │ ├── __init__.py │ │ ├── permissions.py │ │ ├── serializers.py │ │ ├── usecases │ │ ├── format_classification.py │ │ ├── format_prompt.py │ │ └── invoke_model.py │ │ └── views.py ├── asgi.py ├── authentication │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── authorization.py │ ├── keycloak_rest_client.py │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_auto_20180315_1343.py │ │ ├── 0003_auto_20180522_1705.py │ │ ├── 0004_auto_20180605_1357.py │ │ ├── 0005_auto_20180620_2059.py │ │ ├── 0006_auto_20200729_1220.py │ │ ├── 0007_auto_20210323_0035.py │ │ ├── 0008_user_language.py │ │ ├── 0009_auto_20210506_1453.py │ │ ├── 0010_alter_repositoryowner_name.py │ │ ├── 0011_alter_user_language.py │ │ └── __init__.py │ ├── models.py │ ├── static │ │ └── authentication │ │ │ └── emails │ │ │ └── welcome.png │ ├── templates │ │ └── authentication │ │ │ └── emails │ │ │ ├── reset_password.html │ │ │ ├── reset_password.txt │ │ │ ├── welcome.html │ │ │ └── welcome.txt │ └── tests.py ├── authorizations │ ├── __init__.py │ ├── consumers │ │ ├── __init__.py │ │ ├── authorizations_consumer.py │ │ └── handle.py │ └── usecases │ │ ├── __init__.py │ │ ├── dto.py │ │ ├── tests.py │ │ └── usecase.py ├── celery.py ├── common │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── documents │ │ ├── __init__.py │ │ ├── repositorybasicexample.py │ │ ├── repositorynlplog.py │ │ ├── repositoryqanlplog.py │ │ └── zeroshotlog.py │ ├── exceptions.py │ ├── helpers.py │ ├── languages.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ ├── enable_all_train.py │ │ │ ├── fill_db_using_fake_data.py │ │ │ ├── fix_repository_authorization.py │ │ │ ├── fix_repository_authorization_organization.py │ │ │ ├── fix_translations.py │ │ │ ├── grpc.py │ │ │ ├── search_index.py │ │ │ ├── start_all_repository_train.py │ │ │ └── transfer_train_aws.py │ ├── migrate_classifiers │ │ ├── __init__.py │ │ ├── classifiers.py │ │ └── wit │ │ │ ├── __init__.py │ │ │ └── type.py │ ├── migrations │ │ ├── 0040_initial.py │ │ ├── 0041_delete_examples_isdeleted.py │ │ ├── 0042_auto_20191212_2013.py │ │ ├── 0043_migrate_all_train_for_versioning.py │ │ ├── 0044_auto_20191217_1414.py │ │ ├── 0045_repositorynlplog_repositorynlplogintent.py │ │ ├── 0046_auto_20200306_1703.py │ │ ├── 0047_auto_20200310_1648.py │ │ ├── 0048_repositorynlplog_from_backend.py │ │ ├── 0049_repositoryexample_is_corrected.py │ │ ├── 0050_auto_20200320_2314.py │ │ ├── 0051_auto_20200403_1919.py │ │ ├── 0052_delete_evaluates_isdeleted.py │ │ ├── 0053_remove_repositoryevaluate_deleted_in.py │ │ ├── 0054_repositoryversion_is_deleted.py │ │ ├── 0055_auto_20200518_1810.py │ │ ├── 0056_auto_20200522_1120.py │ │ ├── 0057_migrate_trainers.py │ │ ├── 0058_auto_20200522_1522.py │ │ ├── 0059_auto_20200525_1122.py │ │ ├── 0060_migrate_labels.py │ │ ├── 0061_labels.py │ │ ├── 0062_fix_entities.py │ │ ├── 0063_repositoryqueuetask.py │ │ ├── 0064_auto_20200616_1642.py │ │ ├── 0065_auto_20200619_1243.py │ │ ├── 0066_repository_use_transformer_entities.py │ │ ├── 0067_repositoryversionlanguage_use_transformer_entities.py │ │ ├── 0068_fix_import_translations.py │ │ ├── 0069_auto_20200703_1437.py │ │ ├── 0070_auto_20200729_1220.py │ │ ├── 0071_auto_20200729_1632.py │ │ ├── 0072_auto_20200729_1704.py │ │ ├── 0073_fix_authorizations.py │ │ ├── 0074_auto_20200806_1501.py │ │ ├── 0075_auto_20200806_1805.py │ │ ├── 0076_auto_20200806_1830.py │ │ ├── 0077_auto_20200807_1410.py │ │ ├── 0078_auto_20200807_1425.py │ │ ├── 0079_auto_20200812_1513.py │ │ ├── 0080_repositoryintent.py │ │ ├── 0081_auto_20200821_1250.py │ │ ├── 0082_auto_20200821_1617.py │ │ ├── 0083_repositoryintent_created_at.py │ │ ├── 0084_repositoryreports.py │ │ ├── 0085_count_repositoryreports.py │ │ ├── 0086_fix_translations.py │ │ ├── 0087_repository_allow_search_examples.py │ │ ├── 0088_fix_translations.py │ │ ├── 0089_remove_repository_allow_search_examples.py │ │ ├── 0090_auto_20200921_1751.py │ │ ├── 0091_extensions.py │ │ ├── 0091_repository_count_authorizations.py │ │ ├── 0092_merge_20200930_1819.py │ │ ├── 0093_repositorytranslator.py │ │ ├── 0094_repositoryqueuetask_type_processing.py │ │ ├── 0095_auto_20201001_1944.py │ │ ├── 0096_auto_20201001_2005.py │ │ ├── 0097_repositorymigrate.py │ │ ├── 0097_repositoryscore.py │ │ ├── 0098_merge_20201008_1607.py │ │ ├── 0098_repositorymigrate_classifier.py │ │ ├── 0099_auto_20201008_1848.py │ │ ├── 0100_merge_20201009_1739.py │ │ ├── 0101_auto_20201020_1546.py │ │ ├── 0102_repositoryevaluateresult_cross_validation.py │ │ ├── 0103_auto_20210302_1431.py │ │ ├── 0104_qaknowledgebase.py │ │ ├── 0105_auto_20210309_1945.py │ │ ├── 0106_auto_20210311_1254.py │ │ ├── 0107_auto_20210311_1852.py │ │ ├── 0108_repository_repository_type.py │ │ ├── 0109_auto_20210601_0323.py │ │ ├── 0110_auto_20210601_1443.py │ │ ├── 0111_auto_20210908_1135.py │ │ ├── 0112_auto_20211201_1330.py │ │ ├── 0113_auto_20220216_1154.py │ │ ├── 0114_alter_repositoryversionlanguage_unique_together.py │ │ ├── 0115_auto_20220923_1543.py │ │ ├── 0116_auto_20230331_0118.py │ │ ├── 0117_alter_zeroshotoptions_option_uuid.py │ │ ├── 0118_alter_zeroshotoptions_option_uuid.py │ │ ├── 0119_repository_count_repository_train.py │ │ ├── 0120_auto_20231016_1255.py │ │ ├── 0121_zeroshotlogs_language.py │ │ ├── 0122_rename_categories_zeroshotlogs_options.py │ │ └── __init__.py │ ├── models.py │ ├── signals.py │ ├── static │ │ ├── bothub │ │ │ ├── emails │ │ │ │ ├── facebook.png │ │ │ │ ├── linkedin.png │ │ │ │ └── logo.png │ │ │ └── exporter │ │ │ │ ├── bothub.png │ │ │ │ └── example.xlsx │ │ └── common │ │ │ └── emails │ │ │ └── request.gif │ ├── tasks.py │ ├── templates │ │ ├── bothub │ │ │ └── emails │ │ │ │ └── base.html │ │ └── common │ │ │ └── emails │ │ │ ├── new_request.html │ │ │ ├── new_request.txt │ │ │ ├── new_role.html │ │ │ ├── new_role.txt │ │ │ ├── request_approved.html │ │ │ ├── request_approved.txt │ │ │ ├── request_rejected.html │ │ │ └── request_rejected.txt │ ├── tests.py │ ├── usecase │ │ ├── __init__.py │ │ └── repositorylog │ │ │ ├── __init__.py │ │ │ ├── dto.py │ │ │ ├── export.py │ │ │ ├── storage.py │ │ │ └── tests │ │ │ ├── __init__.py │ │ │ └── test_export.py │ └── views.py ├── event_driven │ ├── __init__.py │ ├── apps.py │ ├── connection │ │ ├── pymqp_connection.py │ │ └── rabbitmq_connection.py │ ├── consumer │ │ └── consumers.py │ ├── handle.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ └── edaconsume.py │ ├── migrations │ │ └── __init__.py │ ├── parsers │ │ ├── __init__.py │ │ ├── base_parser.py │ │ ├── exceptions.py │ │ └── json_parser.py │ ├── publisher │ │ ├── __init__.py │ │ └── rabbitmq_publisher.py │ └── signals.py ├── health │ ├── __init__.py │ ├── apps.py │ ├── checks.py │ └── views.py ├── locale │ └── pt_BR │ │ └── LC_MESSAGES │ │ ├── django.mo │ │ └── django.po ├── project │ ├── __init__.py │ ├── apps.py │ ├── consumers │ │ ├── project_consumer.py │ │ └── template_type_consumer.py │ ├── handle.py │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_projectintelligence.py │ │ ├── 0003_auto_20230830_1200.py │ │ ├── 0004_projectintelligence_integrated_by.py │ │ └── __init__.py │ ├── models.py │ ├── serializers.py │ ├── signals.py │ ├── usecases │ │ ├── exceptions.py │ │ ├── organization │ │ │ ├── __init__.py │ │ │ └── creation.py │ │ ├── project │ │ │ ├── __init__.py │ │ │ ├── creation.py │ │ │ └── projectdto.py │ │ └── template_type │ │ │ ├── creation.py │ │ │ ├── integration.py │ │ │ └── template_type_dto.py │ └── views.py ├── settings.py ├── translate.py ├── urls.py ├── utils.py └── wsgi.py ├── docker-compose.dev.yml ├── docker-compose.yml ├── docs └── quick-start.md ├── entrypoint.sh ├── gunicorn.conf.py ├── manage.py ├── poetry.lock ├── pyproject.toml ├── runtime.txt ├── setup.cfg └── setup.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = 3 | ./bothub/api 4 | ./bothub/authentication 5 | ./bothub/common 6 | omit = 7 | ./bothub/api/apps.py 8 | ./bothub/authentication/apps.py 9 | ./bothub/common/apps.py 10 | ./bothub/api/v2/swagger.py 11 | ./bothub/api/v2/tests/* 12 | ./bothub/api/v2/nlp/* 13 | ./bothub/authentication/tests.py 14 | ./bothub/common/tests.py 15 | ./bothub/common/management/* 16 | ./bothub/common/migrations/* 17 | 18 | [report] 19 | fail_under = 90 -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | env/ 3 | __pycache__/ 4 | staticfiles/ 5 | .DS_Store 6 | *.py[cod] 7 | settings.ini 8 | db.sqlite3 9 | .env -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 4 6 | end_of_line = lf 7 | charset = utf-8 8 | trim_trailing_whitespace = false 9 | insert_final_newline = false 10 | 11 | [*.yml] 12 | indent_size = 2 13 | 14 | [.flake8] 15 | indent_size = 2 16 | 17 | [*.{py,yml}] 18 | trim_trailing_whitespace = true 19 | insert_final_newline = true 20 | 21 | [Makefile] 22 | indent_style = tab 23 | 24 | [Makefile-help.txt] 25 | insert_final_newline = true 26 | 27 | [README.md] 28 | insert_final_newline = true -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 119 3 | ignore = E501,W503,E203 4 | exclude = 5 | ./env 6 | ./bothub/common/migrations 7 | ./manage.py 8 | ./bothub/common/languages.py 9 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | custom: ['https://www.ilhasoft.com.br/en/contact/'] 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Django stuff: 51 | *.log 52 | .static_storage/ 53 | .media/ 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # Jupyter Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | celerybeat-schedule.db 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # Environments 83 | .env 84 | .venv 85 | env/ 86 | venv/ 87 | ENV/ 88 | env.bak/ 89 | venv.bak/ 90 | 91 | # Spyder project settings 92 | .spyderproject 93 | .spyproject 94 | 95 | # Rope project settings 96 | .ropeproject 97 | 98 | # mkdocs documentation 99 | /site 100 | 101 | # mypy 102 | .mypy_cache/ 103 | 104 | # python decouple 105 | settings.ini 106 | 107 | # IDEs 108 | .vscode/ 109 | 110 | # db and staticfiles 111 | db.sqlite3 112 | staticfiles/ 113 | 114 | # OS 115 | .DS_Store 116 | 117 | #Pycharm 118 | .idea/ 119 | 120 | #ASDF 121 | .tool-versions 122 | 123 | #Local Docker Compose file 124 | dev-docker-compose.yml -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v2.3.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-yaml # checks yaml files for parseable syntax. 8 | - id: check-added-large-files 9 | - id: check-merge-conflict # checks for files that contain merge conflict strings. 10 | - id: check-toml # checks toml files for parseable syntax. 11 | 12 | - repo: https://github.com/pycqa/flake8 13 | rev: 4.0.1 14 | hooks: 15 | - id: flake8 16 | 17 | - repo: https://github.com/psf/black 18 | rev: 22.3.0 19 | hooks: 20 | - id: black 21 | exclude: ^migrations/ 22 | 23 | - repo: https://gitlab.com/smop/pre-commit-hooks 24 | rev: df034f88cf92b394e6f00a78fa97a2aa4e270e60 25 | hooks: 26 | - id: check-poetry # Validates the structure of the pyproject.toml file 27 | 28 | # todo: add tests 29 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # use container-based Ubuntu Trusty 2 | dist: bionic 3 | sudo: true 4 | 5 | language: python 6 | python: 7 | - "3.8.13" 8 | 9 | services: 10 | - docker 11 | - elasticsearch 12 | 13 | before_install: 14 | # install the chosen PG version 15 | - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install postgresql-11 postgresql-client-11 16 | - sudo -E sed -i -e '/local.*peer/s/postgres/all/' -e 's/peer\|md5/trust/g' /etc/postgresql/*/main/pg_hba.conf 17 | - sudo -E sed -i 's/port = 5433/port = 5432/' /etc/postgresql/*/main/postgresql.conf 18 | 19 | # give PG some time to finish setup 20 | - sleep 10 21 | 22 | # stop any running postgres versions 23 | - sudo -E service postgresql stop 10 24 | - sudo -E ps axuwww | grep -i postg 25 | 26 | # possibly a Travis bug but data directory sometimes not initialized 27 | - if [ ! -d /var/ramfs/postgresql/11/main ]; then sudo -u postgres /usr/lib/postgresql/11/bin/initdb -D /var/ramfs/postgresql/11/main; fi 28 | 29 | # start the chosen PG version 30 | - sudo -E systemctl -l restart postgresql@11-main 31 | - sudo -E systemctl -l status postgresql@11-main 32 | 33 | before_script: 34 | # setup test database 35 | - psql -U postgres -c "CREATE USER bothub WITH PASSWORD 'bothub';" 36 | - psql -U postgres -c "ALTER ROLE bothub WITH SUPERUSER;" 37 | - psql -U bothub postgres -c "CREATE DATABASE bothub;" 38 | 39 | install: 40 | - pip install --upgrade pip 41 | - pip install poetry==1.1.12 42 | - poetry config virtualenvs.create false --local 43 | - poetry install 44 | - pip install coveralls 45 | env: 46 | global: 47 | - SECRET_KEY=SK 48 | - SUPPORTED_LANGUAGES="en|pt" 49 | - DEFAULT_DATABASE="postgres://bothub:bothub@localhost:5432/bothub" 50 | - ELASTICSEARCH_DSL="http://localhost:9200" 51 | - ELASTICSEARCH_SIGNAL_PROCESSOR="realtime" 52 | - ELASTICSEARCH_NUMBER_OF_SHARDS=1 53 | - ELASTICSEARCH_NUMBER_OF_REPLICAS=0 54 | - USE_ELASTICSEARCH="false" 55 | script: 56 | - python manage.py migrate 57 | - python manage.py search_index --create -f 58 | - python manage.py search_index --populate -f 59 | - python manage.py collectstatic 60 | - flake8 61 | - travis_wait coverage run manage.py test 62 | after_success: 63 | - coveralls 64 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8.13-slim 2 | 3 | ENV WORKDIR /home/app 4 | WORKDIR $WORKDIR 5 | 6 | RUN apt-get update \ 7 | && apt-get install --no-install-recommends --no-install-suggests -y apt-utils \ 8 | && apt-get install --no-install-recommends --no-install-suggests -y gcc bzip2 git curl nginx libpq-dev gettext \ 9 | libgdal-dev python3-cffi python3-gdal vim 10 | 11 | RUN apt-get install make 12 | 13 | RUN pip install -U pip==21.2.2 setuptools==57.4.0 14 | RUN pip install poetry==1.2.2 15 | RUN pip install gunicorn==19.9.0 16 | RUN pip install gevent==21.12.0 17 | RUN pip install psycopg2-binary 18 | RUN apt-get install -y libjpeg-dev libgpgme-dev linux-libc-dev musl-dev libffi-dev libssl-dev 19 | ENV LIBRARY_PATH=/lib:/usr/lib 20 | 21 | COPY pyproject.toml pyproject.toml 22 | COPY poetry.lock poetry.lock 23 | 24 | RUN poetry config virtualenvs.create false \ 25 | && poetry install --no-interaction --no-ansi 26 | 27 | COPY . . 28 | 29 | RUN chmod +x ./entrypoint.sh 30 | ENTRYPOINT [ "./entrypoint.sh" ] 31 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | recursive-include data * -------------------------------------------------------------------------------- /Makefile-help.txt: -------------------------------------------------------------------------------- 1 | Make commands: 2 | make help - Show make commands help 3 | make check_environment - Check if all dependencies was installed 4 | make install_requirements - Install pip dependencies 5 | make lint - Show lint warnings and errors 6 | make test - Run unit tests and show coverage report 7 | make migrate - Update DB shema, apply migrations 8 | make start - Start development web server 9 | make migrations - Create DB migrations files 10 | make collectstatic - Collects the static files into STATIC_ROOT. 11 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: python manage.py migrate && python manage.py fill_db_using_fake_data && gunicorn bothub.wsgi -b 0.0.0.0:\$PORT --log-file - 2 | -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Bothub Engine", 3 | "image": "heroku/python", 4 | "repository": "https://github.com/Ilhasoft/bothub-engine/", 5 | "logo": "https://avatars0.githubusercontent.com/u/8379703?s=200&v=4", 6 | "keywords": ["bothub", "ia", "django", "python"], 7 | "website": "https://bothub.it", 8 | "addons": [ "heroku-postgresql" ], 9 | "env": { 10 | "DISABLE_COLLECTSTATIC": "1", 11 | "SECRET_KEY": "SK", 12 | "DEBUG": "1" 13 | }, 14 | "environments": { 15 | "test": { 16 | "scripts": { 17 | "test-setup": "python manage.py collectstatic --noinput", 18 | "test": "python manage.py test" 19 | } 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /bothub/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | from .celery import app as celery_app 3 | 4 | __all__ = ("celery_app",) 5 | -------------------------------------------------------------------------------- /bothub/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/__init__.py -------------------------------------------------------------------------------- /bothub/api/grpc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/grpc/__init__.py -------------------------------------------------------------------------------- /bothub/api/grpc/organization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/grpc/organization/__init__.py -------------------------------------------------------------------------------- /bothub/api/grpc/organization/handlers.py: -------------------------------------------------------------------------------- 1 | from bothub.api.grpc.organization.services import OrgService 2 | from weni.protobuf.intelligence import organization_pb2_grpc 3 | 4 | 5 | def grpc_handlers(server): 6 | organization_pb2_grpc.add_OrgControllerServicer_to_server( 7 | OrgService.as_servicer(), server 8 | ) 9 | -------------------------------------------------------------------------------- /bothub/api/grpc/repository/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/grpc/repository/__init__.py -------------------------------------------------------------------------------- /bothub/api/grpc/repository/handlers.py: -------------------------------------------------------------------------------- 1 | from .services import RepositoryService 2 | from weni.protobuf.intelligence import repository_pb2_grpc 3 | 4 | 5 | def grpc_handlers(server): 6 | repository_pb2_grpc.add_RepositoryControllerServicer_to_server( 7 | RepositoryService.as_servicer(), server 8 | ) 9 | -------------------------------------------------------------------------------- /bothub/api/grpc/repository/serializers.py: -------------------------------------------------------------------------------- 1 | from django_grpc_framework import proto_serializers 2 | from rest_framework import serializers 3 | 4 | from bothub.api.v2.repository.serializers import RepositoryCategorySerializer 5 | from bothub.common.models import Repository 6 | from weni.protobuf.intelligence import repository_pb2 7 | 8 | from bothub.utils import internal_serializer_fields 9 | 10 | 11 | class RepositoryProtoSerializer(proto_serializers.ModelProtoSerializer): 12 | owner__nickname = serializers.SerializerMethodField() 13 | intents = serializers.SerializerMethodField() 14 | available_languages = serializers.SerializerMethodField() 15 | categories_list = serializers.SerializerMethodField() 16 | 17 | class Meta: 18 | model = Repository 19 | proto_class = repository_pb2.Repository 20 | fields = internal_serializer_fields 21 | 22 | def get_owner__nickname(self, repository: Repository): 23 | return repository.owner.nickname 24 | 25 | def get_intents(self, repository: Repository): 26 | return repository.get_formatted_intents() 27 | 28 | def get_available_languages(self, repository: Repository): 29 | return repository.available_languages() 30 | 31 | def get_categories_list(self, repository: Repository): 32 | return RepositoryCategorySerializer(repository.categories, many=True).data 33 | -------------------------------------------------------------------------------- /bothub/api/grpc/repository/services.py: -------------------------------------------------------------------------------- 1 | from django_grpc_framework import generics 2 | from django_grpc_framework import mixins 3 | 4 | from bothub.common.models import Repository 5 | from .serializers import RepositoryProtoSerializer 6 | 7 | 8 | class RepositoryService(mixins.ListModelMixin, generics.GenericService): 9 | queryset = Repository.objects.all() 10 | serializer_class = RepositoryProtoSerializer 11 | 12 | def filter_queryset(self, queryset): 13 | org_id = self.request.org_id 14 | 15 | queryset = queryset.filter(name__icontains=self.request.name) 16 | 17 | if org_id: 18 | queryset = queryset.filter(authorizations__user__pk=org_id) 19 | 20 | return queryset[:20] 21 | 22 | def RetrieveAuthorization(self, request, context): 23 | repository = Repository.objects.get( 24 | authorizations__uuid=self.request.repository_authorization 25 | ) 26 | 27 | return RepositoryProtoSerializer(repository).message 28 | -------------------------------------------------------------------------------- /bothub/api/grpc/user/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/grpc/user/__init__.py -------------------------------------------------------------------------------- /bothub/api/grpc/user/handlers.py: -------------------------------------------------------------------------------- 1 | from bothub.api.grpc.user.services import ( 2 | UserPermissionService, 3 | UserService, 4 | UserLanguageService, 5 | ) 6 | from weni.protobuf.intelligence import authentication_pb2_grpc 7 | 8 | 9 | def grpc_handlers(server): 10 | authentication_pb2_grpc.add_UserControllerServicer_to_server( 11 | UserService.as_servicer(), server 12 | ) 13 | authentication_pb2_grpc.add_UserPermissionControllerServicer_to_server( 14 | UserPermissionService.as_servicer(), server 15 | ) 16 | authentication_pb2_grpc.add_UserLanguageControllerServicer_to_server( 17 | UserLanguageService.as_servicer(), server 18 | ) 19 | -------------------------------------------------------------------------------- /bothub/api/grpc/user/serializers.py: -------------------------------------------------------------------------------- 1 | from django_grpc_framework import proto_serializers 2 | from rest_framework import serializers 3 | 4 | from bothub.authentication.models import User 5 | from weni.protobuf.intelligence import authentication_pb2 6 | 7 | 8 | class UserProtoSerializer(proto_serializers.ModelProtoSerializer): 9 | class Meta: 10 | model = User 11 | proto_class = authentication_pb2.User 12 | fields = [ 13 | "id", 14 | "email", 15 | "nickname", 16 | "name", 17 | "language", 18 | "joined_at", 19 | "is_active", 20 | "is_superuser", 21 | ] 22 | 23 | 24 | class UserPermissionProtoSerializer(proto_serializers.ProtoSerializer): 25 | role = serializers.IntegerField() 26 | 27 | class Meta: 28 | proto_class = authentication_pb2.Permission 29 | 30 | 31 | class UserLanguageProtoSerializer(proto_serializers.ModelProtoSerializer): 32 | class Meta: 33 | model = User 34 | proto_class = authentication_pb2.User 35 | fields = ["language"] 36 | read_only = [ 37 | "id", 38 | "email", 39 | "nickname", 40 | "name", 41 | "joined_at", 42 | "is_active", 43 | "is_superuser", 44 | ] 45 | -------------------------------------------------------------------------------- /bothub/api/v2/__init__.py: -------------------------------------------------------------------------------- 1 | from rest_framework import permissions 2 | 3 | 4 | READ_METHODS = permissions.SAFE_METHODS 5 | WRITE_METHODS = ["POST", "PUT", "PATCH"] 6 | ADMIN_METHODS = ["DELETE"] 7 | -------------------------------------------------------------------------------- /bothub/api/v2/account/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/account/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/evaluate/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/evaluate/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/evaluate/permissions.py: -------------------------------------------------------------------------------- 1 | from rest_framework import permissions 2 | 3 | from bothub.common.models import Repository 4 | from .. import READ_METHODS 5 | from .. import WRITE_METHODS 6 | 7 | 8 | class RepositoryEvaluatePermission(permissions.BasePermission): 9 | def has_permission(self, request, view): 10 | try: 11 | repository_uuid = ( 12 | request.data.get("repository") 13 | if request.method in WRITE_METHODS 14 | else request.GET.get("repository_uuid") 15 | ) 16 | repository = Repository.objects.get(uuid=repository_uuid) 17 | authorization = repository.get_user_authorization(request.user) 18 | 19 | if request.method in READ_METHODS and not request.user.is_authenticated: 20 | return authorization.can_read 21 | 22 | if request.user.is_authenticated: 23 | if request.method in READ_METHODS: 24 | return authorization.can_read 25 | if request.method in WRITE_METHODS: 26 | return authorization.can_write 27 | return authorization.is_admin or authorization.is_owner 28 | return False 29 | except Repository.DoesNotExist: 30 | return False 31 | 32 | 33 | class RepositoryEvaluateResultPermission(permissions.BasePermission): 34 | def has_permission(self, request, view): 35 | try: 36 | repository = Repository.objects.get(uuid=request.GET.get("repository_uuid")) 37 | authorization = repository.get_user_authorization(request.user) 38 | 39 | if request.method in READ_METHODS: 40 | return authorization.can_read 41 | return authorization.can_contribute 42 | except Repository.DoesNotExist: 43 | return False 44 | -------------------------------------------------------------------------------- /bothub/api/v2/example/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/example/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/examples/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/fields.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | 3 | from bothub.common.models import RepositoryEntity 4 | from bothub.common.models import RepositoryVersionLanguage 5 | 6 | 7 | class ModelMultipleChoiceField(serializers.ManyRelatedField): 8 | pass 9 | 10 | 11 | class TextField(serializers.CharField): 12 | pass 13 | 14 | 15 | class PasswordField(serializers.CharField): 16 | def __init__(self, *args, **kwargs): 17 | kwargs.pop("trim_whitespace", None) 18 | super().__init__(trim_whitespace=False, **kwargs) 19 | 20 | 21 | class EntityText(serializers.CharField): 22 | pass 23 | 24 | 25 | class EntityValueField(serializers.CharField): 26 | def __init__(self, *args, validators=[], **kwargs): 27 | kwargs.pop("max_length", 0) 28 | kwargs.pop("help_text", "") 29 | 30 | value_field = RepositoryEntity._meta.get_field("value") 31 | 32 | super().__init__( 33 | *args, 34 | max_length=value_field.max_length, 35 | validators=(validators + value_field.validators), 36 | **kwargs 37 | ) 38 | 39 | def to_representation(self, obj): 40 | return obj.value # pragma: no cover 41 | 42 | 43 | class RepositoryVersionRelatedField(serializers.PrimaryKeyRelatedField): 44 | def to_representation(self, value): 45 | version = RepositoryVersionLanguage.objects.get( 46 | pk=int(value.pk) 47 | ).repository_version.pk 48 | return version 49 | -------------------------------------------------------------------------------- /bothub/api/v2/groups/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/groups/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/groups/permissions.py: -------------------------------------------------------------------------------- 1 | from rest_framework import permissions 2 | 3 | from .. import READ_METHODS, WRITE_METHODS 4 | 5 | 6 | class RepositoryEntityGroupHasPermission(permissions.BasePermission): 7 | def has_object_permission(self, request, view, obj): # pragma: no cover 8 | authorization = obj.repository_version.repository.get_user_authorization( 9 | request.user 10 | ) 11 | if request.method in READ_METHODS: 12 | return authorization.can_read 13 | if request.user.is_authenticated: 14 | if request.method in WRITE_METHODS: 15 | return authorization.can_write 16 | return authorization.is_admin 17 | return False 18 | -------------------------------------------------------------------------------- /bothub/api/v2/groups/serializers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | 3 | from bothub.api.v2.repository.validators import CanContributeInRepositoryValidator 4 | from bothub.common.models import RepositoryEntityGroup, Repository 5 | 6 | 7 | class RepositoryEntityGroupSeralizer(serializers.ModelSerializer): 8 | class Meta: 9 | model = RepositoryEntityGroup 10 | fields = ["id", "value", "created_at", "repository", "repository_version"] 11 | ref_name = None 12 | 13 | read_only = ["id", "created_at"] 14 | 15 | repository = serializers.PrimaryKeyRelatedField( 16 | queryset=Repository.objects, 17 | write_only=True, 18 | required=True, 19 | validators=[CanContributeInRepositoryValidator()], 20 | ) 21 | 22 | def create(self, validated_data): 23 | validated_data.pop("repository") 24 | return super().create(validated_data) 25 | -------------------------------------------------------------------------------- /bothub/api/v2/groups/views.py: -------------------------------------------------------------------------------- 1 | from rest_framework import mixins 2 | from rest_framework.permissions import IsAuthenticated 3 | from rest_framework.viewsets import GenericViewSet 4 | 5 | from bothub.api.v2.groups.permissions import RepositoryEntityGroupHasPermission 6 | from bothub.api.v2.groups.serializers import RepositoryEntityGroupSeralizer 7 | from bothub.api.v2.metadata import Metadata 8 | from bothub.common.models import RepositoryEntityGroup 9 | 10 | 11 | class RepositoryEntityGroupViewSet( 12 | mixins.CreateModelMixin, 13 | mixins.UpdateModelMixin, 14 | mixins.DestroyModelMixin, 15 | GenericViewSet, 16 | ): 17 | queryset = RepositoryEntityGroup.objects 18 | serializer_class = RepositoryEntityGroupSeralizer 19 | permission_classes = [IsAuthenticated, RepositoryEntityGroupHasPermission] 20 | metadata_class = Metadata 21 | -------------------------------------------------------------------------------- /bothub/api/v2/internal/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/internal/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/internal/organization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/internal/organization/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/internal/organization/permissions.py: -------------------------------------------------------------------------------- 1 | from rest_framework import permissions 2 | 3 | from bothub.utils import get_user 4 | 5 | 6 | class InternalOrganizationAdminHasPermission(permissions.BasePermission): 7 | def has_object_permission(self, request, view, obj): 8 | authorization = obj.organization.get_organization_authorization( 9 | get_user(request.query_params.get("user_email", None)) 10 | ) 11 | return authorization.is_admin 12 | -------------------------------------------------------------------------------- /bothub/api/v2/internal/permissions.py: -------------------------------------------------------------------------------- 1 | from rest_framework import permissions 2 | 3 | 4 | class ModuleHasPermission(permissions.BasePermission): 5 | def has_permission(self, request, view): # pragma: no cover 6 | return request.user.has_perm("authentication.can_communicate_internally") 7 | 8 | def has_object_permission(self, request, view, obj): 9 | return self.has_permission(request, view) 10 | -------------------------------------------------------------------------------- /bothub/api/v2/internal/repository/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/internal/repository/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/internal/repository/serializers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | 3 | from bothub.api.v2.repository.serializers import RepositoryCategorySerializer 4 | from bothub.common.models import Repository 5 | 6 | from bothub.utils import internal_serializer_fields 7 | 8 | 9 | class InternalRepositorySerializer(serializers.ModelSerializer): 10 | owner__nickname = serializers.SerializerMethodField() 11 | intents = serializers.SerializerMethodField() 12 | available_languages = serializers.SerializerMethodField() 13 | categories_list = serializers.SerializerMethodField() 14 | 15 | class Meta: 16 | model = Repository 17 | fields = internal_serializer_fields 18 | 19 | def get_owner__nickname(self, repository: Repository): 20 | return repository.owner.nickname 21 | 22 | def get_intents(self, repository: Repository): 23 | return repository.get_formatted_intents() 24 | 25 | def get_available_languages(self, repository: Repository): 26 | return repository.available_languages() 27 | 28 | def get_categories_list(self, repository: Repository): 29 | return RepositoryCategorySerializer(repository.categories, many=True).data 30 | -------------------------------------------------------------------------------- /bothub/api/v2/internal/repository/views.py: -------------------------------------------------------------------------------- 1 | from rest_framework import mixins, status 2 | from rest_framework.decorators import action 3 | from rest_framework.filters import SearchFilter 4 | from rest_framework.response import Response 5 | from rest_framework.viewsets import GenericViewSet 6 | 7 | from bothub.common.models import Repository 8 | 9 | from bothub.api.v2.internal.repository.serializers import InternalRepositorySerializer 10 | from bothub.api.v2.internal.permissions import ModuleHasPermission 11 | 12 | 13 | class InternalRepositoriesViewSet(mixins.ListModelMixin, GenericViewSet): 14 | serializer_class = InternalRepositorySerializer 15 | queryset = Repository.objects 16 | permission_classes = [ModuleHasPermission] 17 | filter_backends = [SearchFilter] 18 | search_fields = ["$name", "^name", "=name"] 19 | 20 | def get_queryset(self, *args, **kwargs): 21 | queryset = self.queryset.all() 22 | name = self.request.query_params.get("name", None) 23 | if name: 24 | queryset = self.queryset.filter(name__icontains=name) 25 | org_id = self.request.query_params.get("org_id", None) 26 | if org_id: 27 | queryset = queryset.filter(authorizations__user__pk=org_id) 28 | 29 | return queryset[:20] 30 | 31 | @action(detail=True, methods=["GET"], url_name="retrieve-authorization") 32 | def retrieve_authorization(self, request, **kwargs): 33 | auth = self.request.query_params.get("repository_authorization", None) 34 | repository = Repository.objects.none() 35 | if auth: 36 | try: 37 | repository = Repository.objects.get(authorizations__uuid=auth) 38 | except Repository.DoesNotExist: 39 | return Response({}, status=status.HTTP_404_NOT_FOUND) 40 | serialized_data = InternalRepositorySerializer(repository) 41 | 42 | return Response(serialized_data.data) 43 | -------------------------------------------------------------------------------- /bothub/api/v2/internal/user/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/internal/user/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/internal/user/serializers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | 3 | from bothub.authentication.models import User 4 | 5 | 6 | class UserSerializer(serializers.ModelSerializer): 7 | class Meta: 8 | model = User 9 | fields = [ 10 | "id", 11 | "email", 12 | "nickname", 13 | "name", 14 | "language", 15 | "joined_at", 16 | "is_active", 17 | "is_superuser", 18 | ] 19 | 20 | 21 | class UserPermissionSerializer(serializers.Serializer): 22 | role = serializers.IntegerField() 23 | 24 | 25 | class UserLanguageSerializer(serializers.ModelSerializer): 26 | class Meta: 27 | model = User 28 | fields = ["language"] 29 | read_only = [ 30 | "id", 31 | "email", 32 | "nickname", 33 | "name", 34 | "joined_at", 35 | "is_active", 36 | "is_superuser", 37 | ] 38 | -------------------------------------------------------------------------------- /bothub/api/v2/knowledge_base/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/knowledge_base/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/knowledge_base/permissions.py: -------------------------------------------------------------------------------- 1 | from rest_framework import permissions 2 | 3 | from bothub.api.v2 import READ_METHODS 4 | 5 | 6 | class QABasePermission(permissions.BasePermission): 7 | def has_object_permission(self, request, view, obj): 8 | authorization = obj.get_user_authorization(request.user) 9 | if request.method in READ_METHODS: 10 | return authorization.can_read 11 | return authorization.can_contribute 12 | 13 | 14 | class QAKnowledgeBasePermission(QABasePermission): 15 | pass 16 | 17 | 18 | class QAtextPermission(QABasePermission): 19 | pass 20 | -------------------------------------------------------------------------------- /bothub/api/v2/middleware.py: -------------------------------------------------------------------------------- 1 | from django.utils import translation 2 | 3 | 4 | class UserLanguageMiddleware: 5 | def __init__(self, get_response): 6 | self.get_response = get_response 7 | 8 | def __call__(self, request): 9 | user = getattr(request, "user", None) 10 | if not user or user.is_anonymous: 11 | return self.get_response(request) 12 | 13 | user_language = getattr(user, "language", None) 14 | if not user_language: 15 | return self.get_response(request) 16 | 17 | translation.activate(user_language) 18 | 19 | response = self.get_response(request) 20 | 21 | return response 22 | -------------------------------------------------------------------------------- /bothub/api/v2/mixins.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import get_object_or_404 2 | 3 | 4 | class MultipleFieldLookupMixin(object): 5 | """ 6 | Apply this mixin to any view or viewset to get multiple field filtering 7 | based on a `lookup_fields` attribute, instead of the default single field 8 | filtering. 9 | """ 10 | 11 | def get_object(self): # pragma: no cover 12 | queryset = self.get_queryset() 13 | queryset = self.filter_queryset(queryset) 14 | filter = {} 15 | for field in self.lookup_fields: 16 | if self.kwargs.get(field): 17 | filter[field] = self.kwargs[field] 18 | obj = get_object_or_404(queryset, **filter) 19 | self.check_object_permissions(self.request, obj) 20 | return obj 21 | -------------------------------------------------------------------------------- /bothub/api/v2/nlp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/nlp/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/organization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/organization/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/organization/filters.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import ValidationError as DjangoValidationError 2 | from django.utils.translation import ugettext_lazy as _ 3 | from django_filters import rest_framework as filters 4 | from rest_framework.exceptions import NotFound 5 | from rest_framework.exceptions import PermissionDenied 6 | 7 | from bothub.common.models import OrganizationAuthorization, Organization 8 | 9 | 10 | class OrganizationAuthorizationFilter(filters.FilterSet): 11 | class Meta: 12 | model = OrganizationAuthorization 13 | fields = ["organization"] 14 | 15 | org_nickname = filters.CharFilter( 16 | field_name="organization", 17 | method="filter_organization_nickname", 18 | required=True, 19 | help_text=_("Organization Nickname"), 20 | ) 21 | 22 | def filter_organization_nickname(self, queryset, name, value): 23 | request = self.request 24 | try: 25 | org = Organization.objects.get(nickname=value) 26 | authorization = org.get_organization_authorization(request.user) 27 | if not authorization.is_admin: 28 | raise PermissionDenied() 29 | return queryset.filter(organization=org) 30 | except Organization.DoesNotExist: 31 | raise NotFound(_("Organization {} does not exist").format(value)) 32 | except DjangoValidationError: 33 | raise NotFound(_("Invalid organization UUID")) 34 | -------------------------------------------------------------------------------- /bothub/api/v2/organization/permissions.py: -------------------------------------------------------------------------------- 1 | from rest_framework import permissions 2 | 3 | from bothub.api.v2 import READ_METHODS, WRITE_METHODS 4 | 5 | 6 | class OrganizationAdminManagerAuthorization(permissions.BasePermission): 7 | def has_object_permission(self, request, view, obj): 8 | authorization = obj.organization.get_organization_authorization(request.user) 9 | return authorization.is_admin 10 | 11 | 12 | class OrganizationHasPermission(permissions.BasePermission): 13 | def has_object_permission(self, request, view, obj): 14 | authorization = obj.get_organization_authorization(request.user) 15 | if request.method in READ_METHODS and not request.user.is_authenticated: 16 | return authorization.can_read 17 | 18 | if request.user.is_authenticated: 19 | if request.method in READ_METHODS: 20 | return authorization.can_read 21 | if request.method in WRITE_METHODS: 22 | return authorization.can_write 23 | return authorization.is_admin 24 | return False 25 | -------------------------------------------------------------------------------- /bothub/api/v2/organization/validators.py: -------------------------------------------------------------------------------- 1 | from django.utils.translation import ugettext_lazy as _ 2 | from rest_framework.exceptions import ValidationError 3 | 4 | from bothub.common.models import Organization 5 | 6 | 7 | class OrganizationNotExistValidator(object): 8 | def __call__(self, attrs): 9 | nickname = attrs.get("nickname") 10 | 11 | if Organization.objects.filter(nickname=nickname).count() > 0: 12 | raise ValidationError(_("This organization nickname already exists")) 13 | -------------------------------------------------------------------------------- /bothub/api/v2/repository/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/repository/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/swagger.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from drf_yasg2.generators import OpenAPISchemaGenerator, EndpointEnumerator 3 | 4 | 5 | class CustomEndpointEnumerator(EndpointEnumerator): 6 | """ 7 | Add custom setting to exclude views 8 | """ 9 | 10 | def should_include_endpoint( 11 | self, path, callback, app_name="", namespace="", url_name=None 12 | ): 13 | view = "{}.{}".format(callback.__module__, callback.__qualname__) 14 | if view in settings.DRF_YASG_EXCLUDE_VIEWS: 15 | return False 16 | return super().should_include_endpoint( 17 | path, callback, app_name, namespace, url_name 18 | ) 19 | 20 | 21 | class CustomOpenAPISchemaGenerator(OpenAPISchemaGenerator): 22 | """ 23 | We want change default endpoint enumerator class 24 | """ 25 | 26 | endpoint_enumerator_class = CustomEndpointEnumerator 27 | -------------------------------------------------------------------------------- /bothub/api/v2/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/tests/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/tests/utils.py: -------------------------------------------------------------------------------- 1 | from bothub.utils import check_module_permission 2 | from rest_framework.authtoken.models import Token 3 | 4 | from bothub.authentication.models import User 5 | from bothub.common import languages 6 | from bothub.common.models import Repository 7 | 8 | 9 | def create_user_and_token(nickname="fake", module=False): 10 | user = User.objects.create_user("{}@user.com".format(nickname), nickname) 11 | if module is True: 12 | check_module_permission({"can_communicate_internally": module}, user) 13 | user = User.objects.get(email=user.email) 14 | token, create = Token.objects.get_or_create(user=user) 15 | return (user, token) 16 | 17 | 18 | def get_valid_mockups(categories): 19 | return [ 20 | { 21 | "name": "Repository 1", 22 | "slug": "repository-1", 23 | "description": "", 24 | "language": languages.LANGUAGE_EN, 25 | "categories": [category.pk for category in categories], 26 | }, 27 | { 28 | "name": "Repository 2", 29 | "description": "", 30 | "language": languages.LANGUAGE_PT, 31 | "categories": [category.pk for category in categories], 32 | }, 33 | ] 34 | 35 | 36 | def get_invalid_mockups(categories): 37 | return [ 38 | { 39 | "name": "", 40 | "slug": "repository-1", 41 | "language": languages.LANGUAGE_EN, 42 | "categories": [category.pk for category in categories], 43 | }, 44 | { 45 | "name": "Repository 3", 46 | "language": "out", 47 | "categories": [category.pk for category in categories], 48 | "is_private": False, 49 | }, 50 | ] 51 | 52 | 53 | def create_repository_from_mockup(owner, categories, **mockup): 54 | r = Repository.objects.create(owner_id=owner.id, **mockup) 55 | r.current_version() 56 | for category in categories: 57 | r.categories.add(category) 58 | return r 59 | -------------------------------------------------------------------------------- /bothub/api/v2/translation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/translation/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/translation/permissions.py: -------------------------------------------------------------------------------- 1 | from rest_framework import permissions 2 | 3 | from bothub.api.v2 import READ_METHODS 4 | 5 | 6 | class RepositoryTranslatedExamplePermission(permissions.BasePermission): 7 | def has_object_permission(self, request, view, obj): 8 | repository = ( 9 | obj.original_example.repository_version_language.repository_version.repository 10 | ) 11 | authorization = repository.get_user_authorization(request.user) 12 | if request.method in READ_METHODS: 13 | return authorization.can_read 14 | return authorization.can_translate 15 | 16 | 17 | class RepositoryTranslatedExampleExporterPermission(permissions.BasePermission): 18 | def has_object_permission(self, request, view, obj): 19 | authorization = obj.repository.get_user_authorization(request.user) 20 | return authorization.can_translate 21 | -------------------------------------------------------------------------------- /bothub/api/v2/translator/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/translator/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/translator/permissions.py: -------------------------------------------------------------------------------- 1 | from rest_framework import permissions 2 | 3 | from bothub.common.models import Repository 4 | from .. import READ_METHODS, WRITE_METHODS 5 | 6 | 7 | class RepositoryExampleTranslatorPermission(permissions.BasePermission): 8 | def has_permission(self, request, view): 9 | try: 10 | repository = Repository.objects.get( 11 | pk=request.auth.repository_version_language.repository_version.repository.pk 12 | ) 13 | authorization = repository.get_user_authorization(request.user) 14 | 15 | if request.method in READ_METHODS: 16 | return authorization.can_read 17 | return authorization.can_contribute 18 | except Repository.DoesNotExist: 19 | return False 20 | except AttributeError: 21 | return False 22 | 23 | 24 | class RepositoryTranslatorPermission(permissions.BasePermission): 25 | def has_object_permission(self, request, view, obj): 26 | authorization = obj.repository_version_language.repository_version.repository.get_user_authorization( 27 | request.user 28 | ) 29 | if request.method in READ_METHODS and not request.user.is_authenticated: 30 | return authorization.can_read 31 | 32 | if request.user.is_authenticated: 33 | if request.method in READ_METHODS: 34 | return authorization.can_read 35 | if request.method in WRITE_METHODS: 36 | return authorization.can_write 37 | return authorization.is_admin 38 | return False 39 | -------------------------------------------------------------------------------- /bothub/api/v2/translator/validators.py: -------------------------------------------------------------------------------- 1 | from django.utils.translation import ugettext_lazy as _ 2 | from rest_framework.exceptions import PermissionDenied 3 | 4 | 5 | class CanContributeInRepositoryExampleTranslatorValidator(object): 6 | def __call__(self, value): 7 | repository = value.repository_version_language.repository_version.repository 8 | user_authorization = repository.get_user_authorization(self.request.user) 9 | 10 | if not user_authorization.can_translate: 11 | raise PermissionDenied(_("You can't contribute in this repository")) 12 | 13 | if ( 14 | not value.repository_version_language.repository_version 15 | == self.request.auth.repository_version_language.repository_version 16 | ): 17 | raise PermissionDenied(_("You can't contribute in this repository")) 18 | 19 | def set_context(self, serializer): 20 | self.request = serializer.context.get("request") 21 | -------------------------------------------------------------------------------- /bothub/api/v2/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path, include 2 | 3 | from . import views 4 | from .routers import router 5 | 6 | from .zeroshot.views import ZeroShotRepositoryAPIView, ZeroShotOptionsTextAPIView, ZeroShotFastPredictAPIView 7 | from bothub.project.views import ProjectAPIView 8 | 9 | 10 | urlpatterns = [ 11 | path( 12 | "repository-shortcut///", 13 | views.repository_shortcut, 14 | name="repository-shortcut", 15 | ), 16 | path( 17 | "check-user-legacy//", 18 | views.check_user_legacy, 19 | name="check-user-legacy", 20 | ), 21 | path("", include(router.urls)), 22 | path( 23 | "repository/nlp/zeroshot", ZeroShotRepositoryAPIView.as_view(), name="zeroshot" 24 | ), 25 | path( 26 | "repository/nlp/zeroshot/options", 27 | ZeroShotOptionsTextAPIView.as_view(), 28 | name="zeroshot-options", 29 | ), 30 | path("project", ProjectAPIView.as_view(), name="project"), 31 | path( 32 | "repository/nlp/zeroshot/zeroshot-fast-predict", 33 | ZeroShotFastPredictAPIView.as_view(), 34 | name="zeroshot-fast-prediction" 35 | ) 36 | ] 37 | -------------------------------------------------------------------------------- /bothub/api/v2/versionning/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/versionning/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/versionning/filters.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import ValidationError as DjangoValidationError 2 | from django.utils.translation import ugettext_lazy as _ 3 | from django_filters import rest_framework as filters 4 | from rest_framework.exceptions import NotFound, PermissionDenied 5 | 6 | from bothub.common.models import Repository, RepositoryVersion 7 | 8 | 9 | class VersioningFilter(filters.FilterSet): 10 | class Meta: 11 | model = RepositoryVersion 12 | fields = ["repository"] 13 | 14 | repository = filters.CharFilter( 15 | field_name="repository", 16 | method="filter_repository_uuid", 17 | required=True, 18 | help_text=_("Repository's UUID"), 19 | ) 20 | 21 | def filter_repository_uuid(self, queryset, name, value): 22 | request = self.request 23 | try: 24 | repository = Repository.objects.get(uuid=value) 25 | authorization = repository.get_user_authorization(request.user) 26 | if not authorization.can_read: 27 | raise PermissionDenied() # pragma: no cover 28 | return RepositoryVersion.objects.filter(repository=repository).filter( 29 | is_deleted=False 30 | ) 31 | except Repository.DoesNotExist: 32 | raise NotFound(_("Repository {} does not exist").format(value)) 33 | except DjangoValidationError: # pragma: no cover 34 | raise NotFound(_("Invalid repository_uuid")) # pragma: no cover 35 | -------------------------------------------------------------------------------- /bothub/api/v2/versionning/permissions.py: -------------------------------------------------------------------------------- 1 | from rest_framework import permissions 2 | 3 | from .. import READ_METHODS 4 | from .. import WRITE_METHODS 5 | 6 | 7 | class RepositoryVersionHasPermission(permissions.BasePermission): 8 | def has_object_permission(self, request, view, obj): 9 | authorization = obj.repository.get_user_authorization(request.user) 10 | if request.method in READ_METHODS: 11 | return authorization.can_read # pragma: no cover 12 | if request.user.is_authenticated: 13 | if request.method in WRITE_METHODS: 14 | return authorization.can_write 15 | return authorization.is_admin # pragma: no cover 16 | return False # pragma: no cover 17 | -------------------------------------------------------------------------------- /bothub/api/v2/versionning/validators.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from django.utils.translation import ugettext_lazy as _ 4 | from rest_framework.exceptions import ValidationError 5 | 6 | from bothub.common.models import RepositoryVersion 7 | 8 | 9 | class VersionNameNotExistValidator(object): 10 | def __call__(self, attrs): 11 | repository = attrs.get("repository") 12 | name = attrs.get("name") 13 | 14 | if name and re.search("[^A-Za-z0-9]+", name): 15 | raise ValidationError( 16 | _("Only letters and numbers allowed") 17 | ) # pragma: no cover 18 | 19 | if ( 20 | RepositoryVersion.objects.filter(repository=repository, name=name).count() 21 | > 0 22 | ): 23 | raise ValidationError(_("This Repository Version already exists")) 24 | 25 | 26 | class CanUseNameVersionValidator(object): 27 | def __call__(self, value): 28 | if re.search("[^A-Za-z0-9]+", value): 29 | raise ValidationError(_("Only letters and numbers allowed")) 30 | -------------------------------------------------------------------------------- /bothub/api/v2/versionning/views.py: -------------------------------------------------------------------------------- 1 | from django_filters.rest_framework import DjangoFilterBackend 2 | from rest_framework import mixins 3 | from rest_framework.filters import SearchFilter 4 | from rest_framework.filters import OrderingFilter 5 | from rest_framework.permissions import IsAuthenticated 6 | from rest_framework.viewsets import GenericViewSet 7 | 8 | from bothub.api.v2.metadata import Metadata 9 | from bothub.api.v2.versionning.permissions import RepositoryVersionHasPermission 10 | from bothub.api.v2.versionning.serializers import RepositoryVersionSeralizer 11 | from bothub.common.models import RepositoryVersion 12 | from .filters import VersioningFilter 13 | 14 | 15 | class RepositoryVersionViewSet( 16 | mixins.ListModelMixin, 17 | mixins.CreateModelMixin, 18 | mixins.UpdateModelMixin, 19 | mixins.DestroyModelMixin, 20 | GenericViewSet, 21 | ): 22 | queryset = RepositoryVersion.objects.order_by("-is_default") 23 | serializer_class = RepositoryVersionSeralizer 24 | permission_classes = [IsAuthenticated, RepositoryVersionHasPermission] 25 | filter_backends = [DjangoFilterBackend, OrderingFilter, SearchFilter] 26 | search_fields = ["$name", "^name", "=name"] 27 | metadata_class = Metadata 28 | 29 | def list(self, request, *args, **kwargs): 30 | self.filter_class = VersioningFilter 31 | 32 | return super().list(request, *args, **kwargs) 33 | 34 | def perform_destroy(self, instance): # pragma: no cover 35 | instance.is_deleted = True 36 | instance.save(update_fields=["is_deleted"]) 37 | -------------------------------------------------------------------------------- /bothub/api/v2/views.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.conf import settings 4 | from django.http import JsonResponse, HttpResponse 5 | from django.shortcuts import get_object_or_404 6 | from django.shortcuts import redirect 7 | from django.views.decorators.csrf import csrf_exempt 8 | 9 | from bothub.authentication.models import User 10 | from bothub.common.models import Repository 11 | 12 | 13 | def repository_shortcut(self, **kwargs): # pragma: no cover 14 | repository = get_object_or_404(Repository, **kwargs) 15 | if "repository_version" in self.GET: 16 | version = self.GET.get("repository_version") 17 | else: 18 | version = repository.current_version().repository_version.pk 19 | response = redirect(f"/v2/repository/info/{repository.uuid}/{version}/") 20 | return response 21 | 22 | 23 | @csrf_exempt 24 | def check_user_legacy(request, email: str): # pragma: no cover 25 | try: 26 | if settings.SECRET_KEY_CHECK_LEGACY_USER: 27 | prefix, token = request.headers.get("Authorization").split() 28 | if ( 29 | prefix.lower() != "bearer" 30 | or token != settings.SECRET_KEY_CHECK_LEGACY_USER 31 | ): 32 | return HttpResponse(status=404) 33 | except AttributeError: 34 | return HttpResponse(status=404) 35 | 36 | if request.method == "GET": 37 | obj = get_object_or_404(User, email__iexact=email) 38 | return JsonResponse( 39 | { 40 | "username": obj.nickname.lower(), 41 | "email": obj.email.lower(), 42 | "firstName": obj.name, 43 | "lastName": "", 44 | "enabled": obj.is_active, 45 | "emailVerified": False, 46 | "attributes": {}, 47 | "roles": [], 48 | "groups": [], 49 | } 50 | ) 51 | elif request.method == "POST": 52 | obj = get_object_or_404(User, nickname__iexact=email) 53 | body_unicode = request.body.decode("utf-8") 54 | body = json.loads(body_unicode) 55 | check_password = obj.check_password(raw_password=body.get("password")) 56 | return JsonResponse({}) if check_password else HttpResponse(status=404) 57 | return HttpResponse(status=404) 58 | -------------------------------------------------------------------------------- /bothub/api/v2/zeroshot/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/zeroshot/__init__.py -------------------------------------------------------------------------------- /bothub/api/v2/zeroshot/permissions.py: -------------------------------------------------------------------------------- 1 | from rest_framework import permissions 2 | 3 | from django.conf import settings 4 | 5 | class ZeroshotTokenPermission(permissions.BasePermission): 6 | def has_permission(self, request, view): 7 | token = request.META.get('HTTP_AUTHORIZATION') 8 | 9 | if token: 10 | return token == f'Bearer {settings.FLOWS_TOKEN_ZEROSHOT}' 11 | else: 12 | return False 13 | -------------------------------------------------------------------------------- /bothub/api/v2/zeroshot/serializers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/api/v2/zeroshot/serializers.py -------------------------------------------------------------------------------- /bothub/api/v2/zeroshot/usecases/format_prompt.py: -------------------------------------------------------------------------------- 1 | BASE_PROMPT = """<|begin_of_text|><|start_header_id|>system<|end_header_id|> 2 | 3 | {system_prompt}<|eot_id|><|start_header_id|>user<|end_header_id|> 4 | 5 | {input}<|eot_id|><|start_header_id|>assistant<|end_header_id|> 6 | 7 | """ 8 | 9 | class FormatPrompt: 10 | const_prompt_data = { 11 | "system_prompt": "Task: Classify the 'User' message within a chatbot about: {context}. Carefully consider the context and respond with ONLY ONE tag of the class that best represents the intent of the 'User' message with the below categories.\n\n\n{classes_formatted}\n", 12 | "question": "{input}", 13 | } 14 | 15 | def generate_prompt(self, language: str, zeroshot_data: dict): 16 | context = zeroshot_data.get("context") 17 | input = zeroshot_data.get("text") 18 | all_classes = self.setup_ids_on_classes(zeroshot_data.get("options")) 19 | classes_formatted = self.format_classes(all_classes) 20 | 21 | system_prompt = self.const_prompt_data["system_prompt"].format(context=context, classes_formatted=classes_formatted) 22 | question = self.const_prompt_data["question"].format(input=input) 23 | 24 | prompt = BASE_PROMPT.format(system_prompt=system_prompt, input=question) 25 | return prompt 26 | 27 | def setup_ids_on_classes(self, all_classes): 28 | for index, class_obj in enumerate(all_classes): 29 | id = index + 1 30 | class_obj["id"] = id 31 | return all_classes 32 | 33 | def format_classes(self, all_classes): 34 | classes_formatted = '\n'.join([f"A{mclass['id']}: {mclass['class']} - {mclass['context']}" for index, mclass in enumerate(all_classes)]) 35 | classes_formatted += f"\nA{len(all_classes)+1}: none - if there is insufficient information or if the User message doesn't fit any class" 36 | return classes_formatted 37 | 38 | def get_default_language(self): 39 | return "por" -------------------------------------------------------------------------------- /bothub/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for weni project. 3 | It exposes the ASGI callable as a module-level variable named ``application``. 4 | For more information on this file, see 5 | https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/ 6 | """ 7 | 8 | import os 9 | 10 | from django.core.asgi import get_asgi_application 11 | 12 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "weni.settings") 13 | 14 | application = get_asgi_application() 15 | -------------------------------------------------------------------------------- /bothub/authentication/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/authentication/__init__.py -------------------------------------------------------------------------------- /bothub/authentication/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from .models import User 4 | 5 | 6 | @admin.register(User) 7 | class UserAdmin(admin.ModelAdmin): 8 | pass 9 | -------------------------------------------------------------------------------- /bothub/authentication/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class AuthenticationConfig(AppConfig): 5 | name = "bothub.authentication" 6 | -------------------------------------------------------------------------------- /bothub/authentication/keycloak_rest_client.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | from django.conf import settings 4 | 5 | class KeycloakRESTClient: 6 | 7 | def get_user_token(self, email, password): 8 | response = requests.post( 9 | url=settings.OIDC_OP_TOKEN_ENDPOINT, 10 | headers={'Content-Type': 'application/x-www-form-urlencoded'}, 11 | data={ 12 | "grant_type": "password", 13 | "username": email, 14 | "password": password, 15 | "client_id": settings.OIDC_RP_CLIENT_ID, 16 | "client_secret": settings.OIDC_RP_CLIENT_SECRET 17 | } 18 | ) 19 | token = None 20 | if response.status_code == 200: 21 | token = response.json()["access_token"] 22 | return dict(status_code=response.status_code, token=f"Bearer {token}") 23 | 24 | def get_user_info(self, email, password): 25 | user_token_response = self.get_user_token(email=email, password=password) 26 | if user_token_response.get("status_code") == 200: 27 | formatted_header = { 28 | "Content-Type": "application/json; charset: utf-8", 29 | "Authorization": user_token_response.get("token") 30 | } 31 | response = requests.get( 32 | url=settings.OIDC_OP_USER_ENDPOINT, 33 | headers=formatted_header 34 | ) 35 | user_response = dict() 36 | if response.status_code == 200: 37 | response_json = response.json() 38 | user_response = dict( 39 | status_code=response.status_code, 40 | email=response_json.get("email") 41 | ) 42 | else: 43 | user_response = dict(status_code=response.status_code, message="cannot get that user") 44 | else: 45 | user_response = dict(status_code=user_token_response.get("status_code"), message="cannot get that user") 46 | return user_response -------------------------------------------------------------------------------- /bothub/authentication/migrations/0002_auto_20180315_1343.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.2 on 2018-03-15 16:43 2 | 3 | import django.core.validators 4 | from django.db import migrations, models 5 | import re 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("authentication", "0001_initial")] 11 | 12 | operations = [ 13 | migrations.RenameField(model_name="user", old_name="nick", new_name="nickname"), 14 | migrations.AlterField( 15 | model_name="user", 16 | name="nickname", 17 | field=models.CharField( 18 | help_text="User's nickname, using letters, numbers, underscores and hyphens without spaces.", 19 | max_length=16, 20 | validators=[ 21 | django.core.validators.RegexValidator( 22 | re.compile("^[-a-zA-Z0-9_]+\\Z"), 23 | "Enter a valid 'nickname' consisting of letters, numbers, underscores or hyphens.", 24 | "invalid", 25 | ) 26 | ], 27 | verbose_name="nickname", 28 | ), 29 | ), 30 | migrations.AlterField( 31 | model_name="user", 32 | name="email", 33 | field=models.EmailField( 34 | help_text="User's email.", 35 | max_length=254, 36 | unique=True, 37 | verbose_name="email", 38 | ), 39 | ), 40 | migrations.AlterField( 41 | model_name="user", 42 | name="locale", 43 | field=models.CharField( 44 | help_text="User's locale.", max_length=48, verbose_name="locale" 45 | ), 46 | ), 47 | migrations.AlterField( 48 | model_name="user", 49 | name="name", 50 | field=models.CharField( 51 | help_text="User's name.", max_length=32, verbose_name="name" 52 | ), 53 | ), 54 | ] 55 | -------------------------------------------------------------------------------- /bothub/authentication/migrations/0003_auto_20180522_1705.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.2 on 2018-05-22 17:05 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("authentication", "0002_auto_20180315_1343")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="user", 13 | name="locale", 14 | field=models.CharField( 15 | blank=True, 16 | help_text="User's locale.", 17 | max_length=48, 18 | verbose_name="locale", 19 | ), 20 | ) 21 | ] 22 | -------------------------------------------------------------------------------- /bothub/authentication/migrations/0004_auto_20180605_1357.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.2 on 2018-06-05 13:57 2 | 3 | import django.core.validators 4 | from django.db import migrations, models 5 | import re 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [("authentication", "0003_auto_20180522_1705")] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="user", 15 | name="nickname", 16 | field=models.CharField( 17 | help_text="User's nickname, using letters, numbers, underscores and hyphens without spaces.", 18 | max_length=16, 19 | unique=True, 20 | validators=[ 21 | django.core.validators.RegexValidator( 22 | re.compile("^[-a-zA-Z0-9_]+\\Z"), 23 | "Enter a valid 'nickname' consisting of letters, numbers, underscores or hyphens.", 24 | "invalid", 25 | ) 26 | ], 27 | verbose_name="nickname", 28 | ), 29 | ) 30 | ] 31 | -------------------------------------------------------------------------------- /bothub/authentication/migrations/0005_auto_20180620_2059.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.6 on 2018-06-20 20:59 2 | 3 | import bothub.authentication.models 4 | import django.core.validators 5 | from django.db import migrations, models 6 | import re 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("authentication", "0004_auto_20180605_1357")] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="user", 16 | name="nickname", 17 | field=models.CharField( 18 | help_text="User's nickname, using letters, numbers, underscores and hyphens without spaces.", 19 | max_length=16, 20 | unique=True, 21 | validators=[ 22 | django.core.validators.RegexValidator( 23 | re.compile("^[-a-zA-Z0-9_]+\\Z"), 24 | "Enter a valid 'nickname' consisting of letters, numbers, underscores or hyphens.", 25 | "invalid", 26 | ), 27 | bothub.authentication.models.validate_user_nickname_value, 28 | ], 29 | verbose_name="nickname", 30 | ), 31 | ) 32 | ] 33 | -------------------------------------------------------------------------------- /bothub/authentication/migrations/0007_auto_20210323_0035.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.19 on 2021-03-23 00:35 2 | 3 | import bothub.authentication.models 4 | import django.core.validators 5 | from django.db import migrations, models 6 | import re 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [("authentication", "0006_auto_20200729_1220")] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="repositoryowner", 16 | name="nickname", 17 | field=models.CharField( 18 | help_text="User's or Organization nickname, using letters, numbers, underscores and hyphens without spaces.", 19 | max_length=150, 20 | unique=True, 21 | validators=[ 22 | django.core.validators.RegexValidator( 23 | re.compile("^[-a-zA-Z0-9_]+\\Z"), 24 | "Enter a valid 'nickname' consisting of letters, numbers, underscores or hyphens.", 25 | "invalid", 26 | ), 27 | bothub.authentication.models.validate_user_nickname_value, 28 | ], 29 | verbose_name="nickname", 30 | ), 31 | ) 32 | ] 33 | -------------------------------------------------------------------------------- /bothub/authentication/migrations/0008_user_language.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.19 on 2021-03-31 14:14 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("authentication", "0007_auto_20210323_0035")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="user", 13 | name="language", 14 | field=models.CharField( 15 | blank=True, 16 | choices=[("en", "English"), ("pt-br", "Brazilian Portuguese")], 17 | max_length=5, 18 | null=True, 19 | ), 20 | ) 21 | ] 22 | -------------------------------------------------------------------------------- /bothub/authentication/migrations/0009_auto_20210506_1453.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.22 on 2021-05-06 14:53 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("authentication", "0008_user_language")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="user", 13 | name="language", 14 | field=models.CharField( 15 | blank=True, 16 | choices=[("en-us", "English"), ("pt-br", "Brazilian Portuguese")], 17 | max_length=5, 18 | null=True, 19 | ), 20 | ) 21 | ] 22 | -------------------------------------------------------------------------------- /bothub/authentication/migrations/0010_alter_repositoryowner_name.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.8 on 2021-10-28 21:08 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("authentication", "0009_auto_20210506_1453"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="repositoryowner", 15 | name="name", 16 | field=models.CharField( 17 | help_text="User's name.", max_length=150, verbose_name="name" 18 | ), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /bothub/authentication/migrations/0011_alter_user_language.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.8 on 2022-03-03 13:51 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('authentication', '0010_alter_repositoryowner_name'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='user', 15 | name='language', 16 | field=models.CharField(blank=True, choices=[('en-us', 'English'), ('pt-br', 'Brazilian Portuguese'), ('es', 'Spanish')], max_length=5, null=True), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /bothub/authentication/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/authentication/migrations/__init__.py -------------------------------------------------------------------------------- /bothub/authentication/static/authentication/emails/welcome.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/authentication/static/authentication/emails/welcome.png -------------------------------------------------------------------------------- /bothub/authentication/templates/authentication/emails/reset_password.html: -------------------------------------------------------------------------------- 1 |

Reset your account password,

2 |

Click here and follow the instructions to set a new password.

3 |

Or visit the link bellow:
{{ reset_url }}

4 |

 

5 |

If you did not request a password change, disregard this email.

-------------------------------------------------------------------------------- /bothub/authentication/templates/authentication/emails/reset_password.txt: -------------------------------------------------------------------------------- 1 | Reset your account password, 2 | 3 | Visit the link below and follow the instructions to set a new password: 4 | {{ reset_url }} 5 | 6 | If you did not request a password change, disregard this email. -------------------------------------------------------------------------------- /bothub/authentication/templates/authentication/emails/welcome.html: -------------------------------------------------------------------------------- 1 | {% extends 'bothub/emails/base.html' %} 2 | 3 | {% load static i18n %} 4 | 5 | {% block before-content %} 6 |
7 | {% trans 'Welcome to Bothub' %} 8 |

{% trans 'Welcome to Bothub' %}

9 |
10 | {% endblock %} 11 | 12 | {% block main-content %} 13 |

{% blocktrans %}Hello, {{ name }}!{% endblocktrans %}

14 |

{% trans 'You\'ve successfully signed up for Bothub and now you can start creating, improving and translating bots on the platform.' %}

15 |

{% trans 'We are happy to have you with us and hope you enjoy Bothub.' %}

16 | {% endblock %} 17 | -------------------------------------------------------------------------------- /bothub/authentication/templates/authentication/emails/welcome.txt: -------------------------------------------------------------------------------- 1 | {% load i18n %}{% trans 'Welcome to Bothub' %} 2 | 3 | {% blocktrans %}Hello, {{ name }}!{% endblocktrans %} 4 | {% trans 'You\'ve successfully signed up for Bothub and now you can start creating, improving and translating bots on the platform.' %} 5 | {% trans 'We are happy to have you with us and hope you enjoy Bothub.' %} -------------------------------------------------------------------------------- /bothub/authentication/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from django.db import IntegrityError 3 | 4 | from .models import User 5 | 6 | 7 | class AuthenticationTestCase(TestCase): 8 | def test_new_user(self): 9 | User.objects.create_user("fake@user.com", "fake") 10 | 11 | def test_new_superuser(self): 12 | User.objects.create_superuser("fake@user.com", "fake") 13 | 14 | def test_new_user_fail_without_email(self): 15 | with self.assertRaises(ValueError): 16 | User.objects._create_user("", "fake") 17 | 18 | def test_new_user_fail_without_nickname(self): 19 | with self.assertRaises(ValueError): 20 | User.objects._create_user("fake@user.com", "") 21 | 22 | def test_new_superuser_fail_issuperuser_false(self): 23 | with self.assertRaises(ValueError): 24 | User.objects.create_superuser("fake@user.com", "fake", is_superuser=False) 25 | 26 | def test_user_unique_nickname(self): 27 | User.objects.create_user("user1@user.com", "fake") 28 | with self.assertRaises(IntegrityError): 29 | User.objects.create_user("user2@user.com", "fake") 30 | -------------------------------------------------------------------------------- /bothub/authorizations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/authorizations/__init__.py -------------------------------------------------------------------------------- /bothub/authorizations/consumers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/authorizations/consumers/__init__.py -------------------------------------------------------------------------------- /bothub/authorizations/consumers/authorizations_consumer.py: -------------------------------------------------------------------------------- 1 | import amqp 2 | from sentry_sdk import capture_exception 3 | 4 | from bothub.event_driven.parsers import JSONParser 5 | from bothub.event_driven.consumer.consumers import EDAConsumer 6 | from bothub.authorizations.usecases import AuthorizationsUsecase 7 | from bothub.authorizations.usecases.dto import OrgAuthDTO 8 | 9 | 10 | class OrgAuthConsumer(EDAConsumer): # pragma: no cover 11 | def consume(self, message: amqp.Message): 12 | print(f"[OrgAuthConsumer] - Consuming a message. Body: {message.body}") 13 | 14 | try: 15 | body = JSONParser.parse(message.body) 16 | 17 | action = body.get("action") 18 | 19 | org_auth_dto = OrgAuthDTO( 20 | user=body.get("user_email"), 21 | org_id=body.get("org_intelligence"), 22 | role=body.get("role") 23 | ) 24 | 25 | usecase = AuthorizationsUsecase() 26 | usecase.dispatch(action)(org_auth_dto) 27 | 28 | message.channel.basic_ack(message.delivery_tag) 29 | 30 | except Exception as exception: 31 | capture_exception(exception) 32 | message.channel.basic_reject(message.delivery_tag, requeue=False) 33 | print(f"[OrgAuthConsumer] - Message rejected by: {exception}") 34 | -------------------------------------------------------------------------------- /bothub/authorizations/consumers/handle.py: -------------------------------------------------------------------------------- 1 | from amqp.channel import Channel 2 | 3 | from bothub.authorizations.consumers.authorizations_consumer import OrgAuthConsumer 4 | 5 | 6 | 7 | def handle_consumers(channel: Channel) -> None: 8 | channel.basic_consume("artificial-intelligence.authorizations", callback=OrgAuthConsumer().handle) 9 | -------------------------------------------------------------------------------- /bothub/authorizations/usecases/__init__.py: -------------------------------------------------------------------------------- 1 | from bothub.authorizations.usecases.usecase import AuthorizationsUsecase 2 | -------------------------------------------------------------------------------- /bothub/authorizations/usecases/dto.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | 4 | @dataclass 5 | class OrgAuthDTO: 6 | user: str 7 | role: int 8 | org_id: int 9 | -------------------------------------------------------------------------------- /bothub/celery.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | import os 3 | from celery import Celery, schedules 4 | 5 | from bothub import settings 6 | 7 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "bothub.settings") 8 | 9 | app = Celery("bothub") 10 | app.config_from_object("django.conf:settings", namespace="CELERY") 11 | 12 | app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) 13 | 14 | app.conf.beat_schedule = { 15 | "check-training-status": { 16 | "task": "bothub.common.tasks.trainings_check_task", 17 | "schedule": 5.0, 18 | }, 19 | "delete-nlp-logs": { 20 | "task": "bothub.common.tasks.delete_nlp_logs", 21 | "schedule": schedules.crontab(hour="22", minute=0), 22 | }, 23 | "repositories-count-authorizations": { 24 | "task": "bothub.common.tasks.repositories_count_authorizations", 25 | "schedule": schedules.crontab(hour="8", minute=0), 26 | }, 27 | "repository-score": { 28 | "task": "bothub.common.tasks.repository_score", 29 | "schedule": schedules.crontab(minute="*/5"), 30 | }, 31 | } 32 | 33 | 34 | @app.task(bind=True) 35 | def debug_task(self): 36 | print("Request: {0!r}".format(self.request)) 37 | -------------------------------------------------------------------------------- /bothub/common/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | from bothub.celery import app as celery_app 3 | 4 | __all__ = ("celery_app",) 5 | -------------------------------------------------------------------------------- /bothub/common/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.utils.html import format_html 3 | from django.shortcuts import reverse 4 | 5 | from bothub.common.models import Repository, Organization 6 | from bothub.common.models import RepositoryVersion 7 | from bothub.common.models import RepositoryCategory 8 | 9 | 10 | class RepositoryVersionInline(admin.TabularInline): 11 | model = RepositoryVersion 12 | extra = 0 13 | can_delete = False 14 | 15 | fields = [ 16 | "name", 17 | "last_update", 18 | "repository", 19 | "created_by", 20 | "created_at", 21 | "is_deleted", 22 | "download_bot_data", 23 | ] 24 | readonly_fields = fields 25 | 26 | def download_bot_data(self, obj): # pragma: no cover 27 | trainers = [] 28 | if obj: 29 | for version in obj.version_languages: 30 | if ( 31 | version.get_bot_data.bot_data != "" 32 | and version.get_bot_data.bot_data is not None 33 | ): 34 | trainers.append( 35 | f"{version.language.upper()}" 36 | ) 37 | if not trainers: 38 | return "-" 39 | return format_html(f"Download Bot Data {' - '.join(trainers)}") 40 | 41 | 42 | @admin.register(Repository) 43 | class RepositoryAdmin(admin.ModelAdmin): 44 | list_display = ["__str__", "uuid", "language", "is_private", "created_at"] 45 | search_fields = ["name", "uuid", "language", "owner__nickname", "slug"] 46 | list_filter = ["is_private", "language", "categories"] 47 | inlines = [RepositoryVersionInline] 48 | 49 | 50 | @admin.register(RepositoryCategory) 51 | class RepositoryCategoryAdmin(admin.ModelAdmin): 52 | list_display = ["__str__", "icon"] 53 | 54 | 55 | @admin.register(Organization) 56 | class OrganizationAdmin(admin.ModelAdmin): 57 | list_display = ["__str__", "pk", "nickname", "created_at"] 58 | search_fields = ["name", "pk", "nickname"] 59 | list_filter = ["verificated"] 60 | -------------------------------------------------------------------------------- /bothub/common/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class CommonConfig(AppConfig): 5 | name = "bothub.common" 6 | -------------------------------------------------------------------------------- /bothub/common/documents/__init__.py: -------------------------------------------------------------------------------- 1 | from bothub.common.documents.repositorynlplog import RepositoryNLPLogDocument 2 | from bothub.common.documents.repositoryqanlplog import RepositoryQANLPLogDocument 3 | from bothub.common.documents.repositorybasicexample import RepositoryExampleDocument 4 | 5 | __all__ = ( 6 | "RepositoryNLPLogDocument", 7 | "RepositoryQANLPLogDocument", 8 | "RepositoryExampleDocument", 9 | ) 10 | -------------------------------------------------------------------------------- /bothub/common/documents/repositorybasicexample.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django_elasticsearch_dsl import Document, Index, fields 3 | 4 | from bothub.common.models import ( 5 | RepositoryExample, 6 | RepositoryExampleEntity, 7 | RepositoryIntent, 8 | RepositoryVersionLanguage, 9 | ) 10 | 11 | REPOSITORYBASICEXAMPLE_INDEX = Index(settings.ELASTICSEARCH_INDEX_NAMES[__name__]) 12 | 13 | 14 | @REPOSITORYBASICEXAMPLE_INDEX.doc_type 15 | class RepositoryExampleDocument(Document): 16 | time_based = False 17 | 18 | repository_version_language = fields.ObjectField( 19 | properties={ 20 | "pk": fields.IntegerField(), 21 | "language": fields.TextField(fields={"raw": fields.KeywordField()}), 22 | } 23 | ) 24 | intent = fields.ObjectField( 25 | properties={"text": fields.TextField(fields={"raw": fields.KeywordField()})} 26 | ) 27 | entities = fields.NestedField( 28 | properties={ 29 | "entity": fields.ObjectField( 30 | properties={ 31 | "value": fields.TextField(fields={"raw": fields.KeywordField()}) 32 | } 33 | ) 34 | } 35 | ) 36 | pk = fields.IntegerField() 37 | 38 | class Django: 39 | model = RepositoryExample 40 | fields = ["id", "text"] 41 | related_models = [ 42 | RepositoryVersionLanguage, 43 | RepositoryIntent, 44 | RepositoryExampleEntity, 45 | ] 46 | 47 | def get_queryset(self): 48 | return ( 49 | super(RepositoryExampleDocument, self) 50 | .get_queryset() 51 | .select_related("repository_version_language", "intent") 52 | .prefetch_related("entities", "translations") 53 | ) 54 | 55 | def get_instances_from_related(self, related_instance): 56 | if isinstance(related_instance, RepositoryVersionLanguage): 57 | return related_instance.added.all() 58 | elif isinstance(related_instance, RepositoryIntent): 59 | return related_instance.repositoryexample_set.all() 60 | elif isinstance(related_instance, RepositoryExampleEntity): 61 | return related_instance.repository_example 62 | -------------------------------------------------------------------------------- /bothub/common/documents/repositoryqanlplog.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.conf import settings 4 | from django_elasticsearch_dsl import Index, fields 5 | from bothub.utils import TimeBasedDocument 6 | 7 | from bothub.common.models import QALogs 8 | 9 | REPOSITORYQANLPLOG_INDEX_NAME = settings.ELASTICSEARCH_INDEX_NAMES[__name__] 10 | REPOSITORYQANLPLOG_INDEX = Index(REPOSITORYQANLPLOG_INDEX_NAME) 11 | 12 | 13 | @REPOSITORYQANLPLOG_INDEX.doc_type 14 | class RepositoryQANLPLogDocument(TimeBasedDocument): 15 | time_based = True 16 | 17 | user = fields.IntegerField(attr="user.id") 18 | knowledge_base = fields.IntegerField(attr="knowledge_base.id") 19 | nlp_log = fields.NestedField( 20 | properties={ 21 | "answers": fields.NestedField( 22 | properties={ 23 | "text": fields.TextField(fields={"raw": fields.KeywordField()}), 24 | "confidence": fields.FloatField(), 25 | } 26 | ), 27 | "id": fields.IntegerField(), 28 | } 29 | ) 30 | text = fields.IntegerField() 31 | repository_uuid = fields.TextField(fields={"raw": fields.KeywordField()}) 32 | 33 | pk = fields.IntegerField() 34 | 35 | class Django: 36 | model = QALogs 37 | fields = [ 38 | "id", 39 | "answer", 40 | "language", 41 | "confidence", 42 | "question", 43 | "from_backend", 44 | "user_agent", 45 | "created_at", 46 | ] 47 | 48 | def prepare_text(self, obj): 49 | try: 50 | return obj.knowledge_base.texts.filter(language=obj.language).first().id 51 | except AttributeError: 52 | return None 53 | 54 | def prepare_nlp_log(self, obj): 55 | return json.loads(obj.nlp_log) 56 | 57 | def prepare_repository_uuid(self, obj): 58 | return obj.knowledge_base.repository.uuid 59 | -------------------------------------------------------------------------------- /bothub/common/documents/zeroshotlog.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.conf import settings 4 | from django_elasticsearch_dsl import Index, fields 5 | from bothub.utils import TimeBasedDocument 6 | 7 | from bothub.common.models import ZeroshotLogs 8 | 9 | REPOSITORYZEROSHOTLOG_INDEX_NAME = settings.ELASTICSEARCH_INDEX_NAMES[__name__] 10 | REPOSITORYZEROSHOTLOG_INDEX = Index(REPOSITORYZEROSHOTLOG_INDEX_NAME) 11 | 12 | 13 | @REPOSITORYZEROSHOTLOG_INDEX.doc_type 14 | class ZeroshotLogDocument(TimeBasedDocument): 15 | time_based = True 16 | 17 | text = fields.TextField( 18 | analyzer='english', 19 | fields={ 20 | 'raw': fields.KeywordField() 21 | } 22 | ) 23 | classification = fields.TextField( 24 | analyzer='english' 25 | ) 26 | other = fields.BooleanField() 27 | categories = fields.ObjectField(properties={ 28 | 'name': fields.TextField(), 29 | 'value': fields.FloatField() 30 | }) 31 | nlp_log = fields.TextField( 32 | analyzer='english' 33 | ) 34 | created_at = fields.DateField() 35 | 36 | class Django: 37 | model = ZeroshotLogs 38 | fields = [ 39 | "id", 40 | "text", 41 | "options", 42 | "classification", 43 | "other", 44 | "created_at", 45 | ] 46 | 47 | -------------------------------------------------------------------------------- /bothub/common/exceptions.py: -------------------------------------------------------------------------------- 1 | class BotHubException(Exception): 2 | pass 3 | 4 | 5 | class RepositoryUpdateAlreadyStartedTraining(BotHubException): 6 | pass 7 | 8 | 9 | class RepositoryUpdateAlreadyTrained(BotHubException): 10 | pass 11 | 12 | 13 | class TrainingNotAllowed(BotHubException): 14 | pass 15 | 16 | 17 | class DoesNotHaveTranslation(BotHubException): 18 | pass 19 | -------------------------------------------------------------------------------- /bothub/common/helpers.py: -------------------------------------------------------------------------------- 1 | from transformers import GPT2TokenizerFast 2 | import re 3 | 4 | from django.conf import settings 5 | from typing import Tuple, List 6 | 7 | 8 | class ChatGPTTokenText: 9 | max_tokens: int = settings.GPT_MAX_TOKENS 10 | 11 | def count_tokens(self, text: str) -> Tuple[int, List[str]]: 12 | tokenizer = GPT2TokenizerFast.from_pretrained("gpt2") 13 | 14 | sentence_boundary_pattern = r'(?<=[.!?])\s+(?=[^\d])' 15 | sentence_boundaries = [(m.start(), m.end()) for m in re.finditer(sentence_boundary_pattern, text)] 16 | 17 | chunks = [] 18 | current_chunk = [] 19 | current_token_count = 0 20 | current_position = 0 21 | total_token_count = 0 22 | 23 | for boundary_start, boundary_end in sentence_boundaries: 24 | sentence = text[current_position : boundary_start + 1] 25 | current_position = boundary_end 26 | 27 | token_count = len(tokenizer(sentence)["input_ids"]) 28 | total_token_count += token_count 29 | 30 | if current_token_count + token_count <= self.max_tokens: 31 | current_chunk.append(sentence) 32 | current_token_count += token_count 33 | else: 34 | chunks.append(''.join(current_chunk)) 35 | current_chunk = [sentence] 36 | current_token_count = token_count 37 | 38 | return (total_token_count, chunks) 39 | -------------------------------------------------------------------------------- /bothub/common/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/common/management/__init__.py -------------------------------------------------------------------------------- /bothub/common/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/common/management/commands/__init__.py -------------------------------------------------------------------------------- /bothub/common/management/commands/enable_all_train.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | from django.utils import timezone 3 | 4 | from bothub.common.models import RepositoryVersionLanguage 5 | 6 | 7 | class Command(BaseCommand): 8 | def handle(self, *args, **kwargs): 9 | RepositoryVersionLanguage.objects.all().update(last_update=timezone.now()) 10 | print("END") 11 | -------------------------------------------------------------------------------- /bothub/common/management/commands/fix_repository_authorization.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.management.base import BaseCommand 3 | 4 | from bothub.common.models import RepositoryAuthorization, RequestRepositoryAuthorization 5 | 6 | 7 | class Command(BaseCommand): 8 | def handle(self, *args, **kwargs): 9 | print("Updating...") 10 | old_settings = settings.SEND_EMAILS 11 | settings.SEND_EMAILS = False 12 | for auth in RepositoryAuthorization.objects.all(): 13 | if auth.user == auth.repository.owner: 14 | continue 15 | if ( 16 | RequestRepositoryAuthorization.objects.filter( 17 | repository=auth.repository, user=auth.user 18 | ).count() 19 | == 0 20 | ): 21 | RequestRepositoryAuthorization.objects.create( 22 | user=auth.user, 23 | repository=auth.repository, 24 | approved_by=auth.repository.owner, 25 | ) 26 | settings.SEND_EMAILS = old_settings 27 | -------------------------------------------------------------------------------- /bothub/common/management/commands/fix_repository_authorization_organization.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.management.base import BaseCommand 3 | 4 | from bothub.common.models import RepositoryAuthorization, RequestRepositoryAuthorization 5 | 6 | 7 | class Command(BaseCommand): 8 | def handle(self, *args, **kwargs): 9 | print("Updating...") 10 | old_settings = settings.SEND_EMAILS 11 | settings.SEND_EMAILS = False 12 | for auth in RepositoryAuthorization.objects.all(): 13 | if auth.user.is_organization: 14 | RequestRepositoryAuthorization.objects.create( 15 | user=auth.user, repository=auth.repository, approved_by=auth.user 16 | ) 17 | settings.SEND_EMAILS = old_settings 18 | -------------------------------------------------------------------------------- /bothub/common/management/commands/fix_translations.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | from django.db import transaction 3 | from django.db.models import F, Q 4 | 5 | from bothub.common.models import RepositoryTranslatedExample 6 | 7 | BATCH_SIZE = 5000 8 | 9 | 10 | class Command(BaseCommand): 11 | def handle(self, *args, **kwargs): 12 | translations = RepositoryTranslatedExample.objects.exclude( 13 | Q(repository_version_language__language=F("language")) 14 | ) 15 | 16 | num_updated = 0 17 | max_id = -1 18 | while True: 19 | batch = list(translations.filter(id__gt=max_id).order_by("id")[:BATCH_SIZE]) 20 | if not batch: 21 | break 22 | 23 | with transaction.atomic(): 24 | for translation in batch: 25 | repository_version_language = translation.original_example.repository_version_language.repository_version.get_version_language( 26 | translation.language 27 | ) 28 | translation.repository_version_language = ( 29 | repository_version_language 30 | ) 31 | translation.save(update_fields=["repository_version_language"]) 32 | 33 | num_updated += len(batch) 34 | print(f" > Updated {num_updated} translations") 35 | 36 | max_id = batch[-1].id 37 | -------------------------------------------------------------------------------- /bothub/common/management/commands/grpc.py: -------------------------------------------------------------------------------- 1 | import grpc 2 | from concurrent import futures 3 | from django_grpc_framework.management.commands.grpcrunserver import ( 4 | Command as BaseCommand, 5 | ) 6 | from django_grpc_framework.settings import grpc_settings 7 | 8 | 9 | class Command(BaseCommand): 10 | def add_arguments(self, parser): 11 | super().add_arguments(parser) 12 | parser.add_argument( 13 | "--server-key", dest="server_key", help="Server Key Certificate path" 14 | ) 15 | parser.add_argument( 16 | "--server-crt", dest="server_crt", help="Server CTR Certificate path" 17 | ) 18 | 19 | def handle(self, *args, **options): 20 | self.server_key = options["server_key"] 21 | self.server_crt = options["server_crt"] 22 | super().handle(*args, **options) 23 | 24 | def _serve(self): 25 | server = grpc.server( 26 | futures.ThreadPoolExecutor(max_workers=self.max_workers), 27 | interceptors=grpc_settings.SERVER_INTERCEPTORS, 28 | ) 29 | grpc_settings.ROOT_HANDLERS_HOOK(server) 30 | 31 | if self.server_crt and self.server_key: 32 | # read in key and certificate 33 | private_key = open(self.server_key, "rb").read() 34 | certificate_chain = open(self.server_crt, "rb").read() 35 | 36 | # create server credentials 37 | server_credentials = grpc.ssl_server_credentials( 38 | ((private_key, certificate_chain),) 39 | ) 40 | 41 | # add secure port using crendentials 42 | server.add_secure_port(self.address, server_credentials) 43 | else: 44 | server.add_insecure_port(self.address) 45 | server.start() 46 | server.wait_for_termination() 47 | -------------------------------------------------------------------------------- /bothub/common/management/commands/start_all_repository_train.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from django.core.management.base import BaseCommand 3 | 4 | from bothub.common.models import RepositoryVersion 5 | 6 | 7 | class Command(BaseCommand): 8 | def handle(self, *args, **kwargs): 9 | repository_version_language = RepositoryVersion.objects.all() 10 | 11 | for version in repository_version_language: 12 | user_authorization = version.repository.get_user_authorization( 13 | user=version.repository.owner 14 | ) 15 | 16 | t = threading.Thread( 17 | target=version.repository.request_nlp_train, 18 | args=(user_authorization, {"repository_version": version.pk}), 19 | ) 20 | t.start() 21 | -------------------------------------------------------------------------------- /bothub/common/management/commands/transfer_train_aws.py: -------------------------------------------------------------------------------- 1 | import base64 2 | 3 | from django.conf import settings 4 | from django.core.management.base import BaseCommand 5 | 6 | from bothub.common.models import RepositoryNLPTrain 7 | from bothub.utils import send_bot_data_file_aws 8 | 9 | 10 | class Command(BaseCommand): 11 | def handle(self, *args, **kwargs): 12 | if settings.AWS_SEND: 13 | repo = RepositoryNLPTrain.objects.all().exclude(bot_data__exact="") 14 | for update in repo: 15 | try: 16 | repository_update = RepositoryNLPTrain.objects.get(pk=update.pk) 17 | bot_data = send_bot_data_file_aws( 18 | update.pk, base64.b64decode(update.bot_data) 19 | ) 20 | repository_update.bot_data = bot_data 21 | repository_update.save(update_fields=["bot_data"]) 22 | print( 23 | "Updating bot_data repository_update {}".format(str(update.pk)) 24 | ) 25 | except Exception as e: 26 | print("Error " + str(update.pk)) 27 | print(str(e)) 28 | else: 29 | print("You need to configure the environment variables for AWS.") 30 | -------------------------------------------------------------------------------- /bothub/common/migrate_classifiers/__init__.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | 3 | from django.conf import settings 4 | from django.utils.module_loading import import_string 5 | 6 | TYPES = OrderedDict({}) 7 | 8 | 9 | def register_classifier_type(type_class): 10 | """ 11 | Registers a classifier type 12 | """ 13 | global TYPES 14 | 15 | if not type_class.slug: 16 | type_class.slug = type_class.__module__.split(".")[-2] 17 | 18 | if type_class.slug in TYPES: # pragma: no cover 19 | raise ValueError( 20 | "More than one classifier type with slug: %s" % type_class.slug 21 | ) 22 | TYPES[type_class.slug] = type_class() 23 | 24 | 25 | def reload_classifier_types(): 26 | """ 27 | Re-loads the dynamic classifier types 28 | """ 29 | global TYPES 30 | 31 | TYPES = OrderedDict({}) 32 | for class_name in settings.BASE_MIGRATIONS_TYPES: 33 | register_classifier_type(import_string(class_name)) 34 | 35 | 36 | reload_classifier_types() 37 | -------------------------------------------------------------------------------- /bothub/common/migrate_classifiers/classifiers.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from abc import ABCMeta 3 | 4 | logger = logging.getLogger(__name__) 5 | 6 | 7 | class ClassifierType(metaclass=ABCMeta): 8 | """ 9 | ClassifierType is our abstract base type for custom NLU providers. Each provider will 10 | provides a way to export phrases, intentions and entities, 11 | """ 12 | 13 | # the verbose name for this classifier type 14 | name = None 15 | 16 | # the short code for this classifier type (< 16 chars, lowercase) 17 | slug = None 18 | 19 | repository_version = None 20 | auth_token = None 21 | language = None 22 | 23 | def migrate(self): 24 | """ 25 | Must implement every classifier rule to receive phrases, intentions and consequently entities 26 | """ 27 | raise NotImplementedError("classifier types must implement migrate") 28 | -------------------------------------------------------------------------------- /bothub/common/migrate_classifiers/wit/__init__.py: -------------------------------------------------------------------------------- 1 | from .type import WitType # noqa 2 | -------------------------------------------------------------------------------- /bothub/common/migrations/0041_delete_examples_isdeleted.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | 3 | 4 | def noop(apps, schema_editor): # pragma: no cover 5 | pass 6 | 7 | 8 | def delete_examples_already_deleted(apps, schema_editor): # pragma: no cover 9 | RepositoryExample = apps.get_model("common", "RepositoryExample") 10 | RepositoryEvaluate = apps.get_model("common", "RepositoryEvaluate") 11 | RepositoryExample.objects.filter(deleted_in__isnull=False).delete() 12 | RepositoryEvaluate.objects.filter(deleted_in__isnull=False).delete() 13 | 14 | 15 | class Migration(migrations.Migration): 16 | 17 | dependencies = [("common", "0040_initial")] 18 | 19 | operations = [migrations.RunPython(delete_examples_already_deleted, noop)] 20 | -------------------------------------------------------------------------------- /bothub/common/migrations/0044_auto_20191217_1414.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.11 on 2019-12-17 14:14 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [("common", "0043_migrate_all_train_for_versioning")] 10 | 11 | operations = [ 12 | migrations.RemoveField(model_name="repositoryupdate", name="by"), 13 | migrations.RemoveField(model_name="repositoryupdate", name="repository"), 14 | migrations.RemoveField(model_name="repository", name="total_updates"), 15 | migrations.RemoveField( 16 | model_name="repositoryevaluate", name="repository_update" 17 | ), 18 | migrations.RemoveField( 19 | model_name="repositoryevaluateresult", name="repository_update" 20 | ), 21 | migrations.RemoveField(model_name="repositoryexample", name="deleted_in"), 22 | migrations.RemoveField( 23 | model_name="repositoryexample", name="repository_update" 24 | ), 25 | migrations.RemoveField( 26 | model_name="repositorytranslatedexample", name="repository_update" 27 | ), 28 | migrations.AlterField( 29 | model_name="repositoryevaluate", 30 | name="deleted_in", 31 | field=models.ForeignKey( 32 | blank=True, 33 | null=True, 34 | on_delete=django.db.models.deletion.CASCADE, 35 | related_name="deleted_evaluate", 36 | to="common.RepositoryVersionLanguage", 37 | ), 38 | ), 39 | migrations.DeleteModel(name="RepositoryUpdate"), 40 | ] 41 | -------------------------------------------------------------------------------- /bothub/common/migrations/0046_auto_20200306_1703.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.11 on 2020-03-06 17:03 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0045_repositorynlplog_repositorynlplogintent")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="repository", 13 | name="algorithm", 14 | field=models.CharField( 15 | choices=[ 16 | ("statistical_model", "Statistical Model"), 17 | ( 18 | "neural_network_internal", 19 | "Neural Network with internal vocabulary", 20 | ), 21 | ( 22 | "neural_network_external", 23 | "Neural Network with external vocabulary (BETA)", 24 | ), 25 | ], 26 | default="neural_network_internal", 27 | max_length=24, 28 | verbose_name="algorithm", 29 | ), 30 | ) 31 | ] 32 | -------------------------------------------------------------------------------- /bothub/common/migrations/0047_auto_20200310_1648.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.11 on 2020-03-10 16:48 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0046_auto_20200306_1703")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="repository", 13 | name="algorithm", 14 | field=models.CharField( 15 | choices=[ 16 | ( 17 | "neural_network_internal", 18 | "Neural Network with internal vocabulary", 19 | ), 20 | ( 21 | "neural_network_external", 22 | "Neural Network with external vocabulary (BETA)", 23 | ), 24 | ], 25 | default="neural_network_internal", 26 | max_length=24, 27 | verbose_name="algorithm", 28 | ), 29 | ), 30 | migrations.AlterField( 31 | model_name="repositoryversionlanguage", 32 | name="algorithm", 33 | field=models.CharField( 34 | choices=[ 35 | ( 36 | "neural_network_internal", 37 | "Neural Network with internal vocabulary", 38 | ), 39 | ( 40 | "neural_network_external", 41 | "Neural Network with external vocabulary (BETA)", 42 | ), 43 | ], 44 | default="neural_network_internal", 45 | max_length=24, 46 | verbose_name="algorithm", 47 | ), 48 | ), 49 | ] 50 | -------------------------------------------------------------------------------- /bothub/common/migrations/0048_repositorynlplog_from_backend.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.11 on 2020-03-11 12:45 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0047_auto_20200310_1648")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="repositorynlplog", 13 | name="from_backend", 14 | field=models.BooleanField(default=True), 15 | preserve_default=False, 16 | ) 17 | ] 18 | -------------------------------------------------------------------------------- /bothub/common/migrations/0049_repositoryexample_is_corrected.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.11 on 2020-03-12 13:46 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0048_repositorynlplog_from_backend")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="repositoryexample", 13 | name="is_corrected", 14 | field=models.BooleanField(default=False), 15 | ) 16 | ] 17 | -------------------------------------------------------------------------------- /bothub/common/migrations/0050_auto_20200320_2314.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.11 on 2020-03-20 23:14 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0049_repositoryexample_is_corrected")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="repositoryevaluateresultscore", 13 | name="support", 14 | field=models.FloatField(null=True), 15 | ) 16 | ] 17 | -------------------------------------------------------------------------------- /bothub/common/migrations/0051_auto_20200403_1919.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.11 on 2020-04-03 22:19 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0050_auto_20200320_2314")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="repositoryauthorization", 13 | name="role", 14 | field=models.PositiveIntegerField( 15 | choices=[ 16 | (0, "not set"), 17 | (1, "user"), 18 | (2, "contributor"), 19 | (3, "admin"), 20 | (4, "translate"), 21 | ], 22 | default=0, 23 | verbose_name="role", 24 | ), 25 | ) 26 | ] 27 | -------------------------------------------------------------------------------- /bothub/common/migrations/0052_delete_evaluates_isdeleted.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | 3 | 4 | def noop(apps, schema_editor): # pragma: no cover 5 | pass 6 | 7 | 8 | def delete_evaluates_already_deleted(apps, schema_editor): # pragma: no cover 9 | RepositoryEvaluate = apps.get_model("common", "RepositoryEvaluate") 10 | RepositoryEvaluate.objects.filter(deleted_in__isnull=False).delete() 11 | 12 | 13 | class Migration(migrations.Migration): 14 | 15 | dependencies = [("common", "0051_auto_20200403_1919")] 16 | 17 | operations = [migrations.RunPython(delete_evaluates_already_deleted, noop)] 18 | -------------------------------------------------------------------------------- /bothub/common/migrations/0053_remove_repositoryevaluate_deleted_in.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.11 on 2020-04-09 13:46 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0052_delete_evaluates_isdeleted")] 9 | 10 | operations = [ 11 | migrations.RemoveField(model_name="repositoryevaluate", name="deleted_in") 12 | ] 13 | -------------------------------------------------------------------------------- /bothub/common/migrations/0054_repositoryversion_is_deleted.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.11 on 2020-04-22 19:05 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0053_remove_repositoryevaluate_deleted_in")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="repositoryversion", 13 | name="is_deleted", 14 | field=models.BooleanField(default=False, verbose_name="is deleted"), 15 | ) 16 | ] 17 | -------------------------------------------------------------------------------- /bothub/common/migrations/0056_auto_20200522_1120.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.11 on 2020-05-22 14:20 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [("common", "0055_auto_20200518_1810")] 10 | 11 | operations = [ 12 | migrations.CreateModel( 13 | name="RepositoryNLPTrain", 14 | fields=[ 15 | ( 16 | "id", 17 | models.AutoField( 18 | auto_created=True, 19 | primary_key=True, 20 | serialize=False, 21 | verbose_name="ID", 22 | ), 23 | ), 24 | ("bot_data", models.TextField(blank=True, verbose_name="bot data")), 25 | ( 26 | "rasa_version", 27 | models.CharField(max_length=20, verbose_name="Rasa Version Code"), 28 | ), 29 | ( 30 | "created_at", 31 | models.DateTimeField(auto_now_add=True, verbose_name="created at"), 32 | ), 33 | ], 34 | options={"verbose_name": "repository nlp train"}, 35 | ), 36 | migrations.AddField( 37 | model_name="repositorynlptrain", 38 | name="repositoryversionlanguage", 39 | field=models.ForeignKey( 40 | on_delete=django.db.models.deletion.CASCADE, 41 | related_name="trainers", 42 | to="common.RepositoryVersionLanguage", 43 | ), 44 | ), 45 | migrations.AlterUniqueTogether( 46 | name="repositorynlptrain", 47 | unique_together={("repositoryversionlanguage", "rasa_version")}, 48 | ), 49 | ] 50 | -------------------------------------------------------------------------------- /bothub/common/migrations/0057_migrate_trainers.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | from django.conf import settings 3 | 4 | 5 | def noop(apps, schema_editor): # pragma: no cover 6 | pass 7 | 8 | 9 | def migrate(apps, schema_editor): # pragma: no cover 10 | RepositoryVersionLanguage = apps.get_model("common", "RepositoryVersionLanguage") 11 | RepositoryNLPTrain = apps.get_model("common", "RepositoryNLPTrain") 12 | 13 | for version in RepositoryVersionLanguage.objects.all(): 14 | if version.bot_data: 15 | RepositoryNLPTrain.objects.create( 16 | bot_data=version.bot_data, 17 | repositoryversionlanguage=version, 18 | rasa_version=settings.BOTHUB_NLP_RASA_VERSION, 19 | ) 20 | 21 | 22 | class Migration(migrations.Migration): 23 | 24 | dependencies = [("common", "0056_auto_20200522_1120")] 25 | 26 | operations = [ 27 | migrations.RunPython(migrate, noop), 28 | migrations.RemoveField(model_name="repositoryversionlanguage", name="bot_data"), 29 | ] 30 | -------------------------------------------------------------------------------- /bothub/common/migrations/0058_auto_20200522_1522.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.11 on 2020-05-22 18:22 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [("common", "0057_migrate_trainers")] 10 | 11 | operations = [ 12 | migrations.AlterField( 13 | model_name="repositorynlplogintent", 14 | name="repository_nlp_log", 15 | field=models.ForeignKey( 16 | editable=False, 17 | null=True, 18 | on_delete=django.db.models.deletion.CASCADE, 19 | related_name="repository_nlp_log", 20 | to="common.RepositoryNLPLog", 21 | ), 22 | ) 23 | ] 24 | -------------------------------------------------------------------------------- /bothub/common/migrations/0061_labels.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.11 on 2020-05-20 15:20 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0060_migrate_labels")] 9 | 10 | operations = [ 11 | migrations.RemoveField(model_name="repositoryentitylabel", name="repository"), 12 | migrations.RemoveField(model_name="repositoryentity", name="label"), 13 | migrations.RemoveField(model_name="repositoryentity", name="repository"), 14 | migrations.DeleteModel(name="RepositoryEntityLabel"), 15 | ] 16 | -------------------------------------------------------------------------------- /bothub/common/migrations/0062_fix_entities.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | 3 | 4 | def noop(apps, schema_editor): # pragma: no cover 5 | pass 6 | 7 | 8 | def migrate(apps, schema_editor): # pragma: no cover 9 | RepositoryVersion = apps.get_model("common", "RepositoryVersion") 10 | RepositoryEntity = apps.get_model("common", "RepositoryEntity") 11 | RepositoryExampleEntity = apps.get_model("common", "RepositoryExampleEntity") 12 | 13 | for repository_version in RepositoryVersion.objects.all(): 14 | RepositoryEntity.objects.exclude( 15 | pk__in=RepositoryExampleEntity.objects.filter( 16 | repository_example__repository_version_language__repository_version=repository_version 17 | ).values("entity") 18 | ).filter(repository_version=repository_version).delete() 19 | 20 | 21 | class Migration(migrations.Migration): 22 | 23 | dependencies = [("common", "0061_labels")] 24 | 25 | operations = [migrations.RunPython(migrate, noop)] 26 | -------------------------------------------------------------------------------- /bothub/common/migrations/0064_auto_20200616_1642.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-06-16 19:42 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0063_repositoryqueuetask")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="repositoryqueuetask", 13 | name="status", 14 | field=models.PositiveIntegerField( 15 | choices=[ 16 | (0, "Pending"), 17 | (2, "Success"), 18 | (1, "Training"), 19 | (3, "Failed"), 20 | ], 21 | default=0, 22 | verbose_name="Status Queue NLP", 23 | ), 24 | ) 25 | ] 26 | -------------------------------------------------------------------------------- /bothub/common/migrations/0065_auto_20200619_1243.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-06-19 15:43 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0064_auto_20200616_1642")] 9 | 10 | operations = [ 11 | migrations.AlterUniqueTogether( 12 | name="repositoryqueuetask", unique_together=set() 13 | ) 14 | ] 15 | -------------------------------------------------------------------------------- /bothub/common/migrations/0066_repository_use_transformer_entities.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-06-29 15:56 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0065_auto_20200619_1243")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="repository", 13 | name="use_transformer_entities", 14 | field=models.BooleanField( 15 | default=False, 16 | help_text="When selected, the entities will be trained on a better entities recognition model but the training time will be significantly increased", 17 | verbose_name="Use Transformer Entities", 18 | ), 19 | ) 20 | ] 21 | -------------------------------------------------------------------------------- /bothub/common/migrations/0067_repositoryversionlanguage_use_transformer_entities.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-06-29 15:59 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0066_repository_use_transformer_entities")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="repositoryversionlanguage", 13 | name="use_transformer_entities", 14 | field=models.BooleanField(default=False), 15 | ) 16 | ] 17 | -------------------------------------------------------------------------------- /bothub/common/migrations/0068_fix_import_translations.py: -------------------------------------------------------------------------------- 1 | from django.db import migrations 2 | 3 | 4 | def noop(apps, schema_editor): # pragma: no cover 5 | pass 6 | 7 | 8 | def migrate(apps, schema_editor): # pragma: no cover 9 | RepositoryTranslatedExample = apps.get_model( 10 | "common", "RepositoryTranslatedExample" 11 | ) 12 | RepositoryVersionLanguage = apps.get_model("common", "RepositoryVersionLanguage") 13 | 14 | for translated in list(RepositoryTranslatedExample.objects.all().iterator()): 15 | if translated.repository_version_language is None: 16 | ( 17 | repository_version_language, 18 | created, 19 | ) = RepositoryVersionLanguage.objects.get_or_create( 20 | repository_version=translated.original_example.repository_version_language.repository_version, 21 | language=translated.language, 22 | ) 23 | translated.repository_version_language = repository_version_language 24 | translated.save(update_fields=["repository_version_language"]) 25 | continue 26 | if translated.repository_version_language.language != translated.language: 27 | ( 28 | repository_version_language, 29 | created, 30 | ) = RepositoryVersionLanguage.objects.get_or_create( 31 | repository_version=translated.repository_version_language.repository_version, 32 | language=translated.language, 33 | ) 34 | translated.repository_version_language = repository_version_language 35 | translated.save(update_fields=["repository_version_language"]) 36 | 37 | 38 | class Migration(migrations.Migration): 39 | 40 | dependencies = [ 41 | ("common", "0067_repositoryversionlanguage_use_transformer_entities") 42 | ] 43 | 44 | operations = [migrations.RunPython(migrate, noop)] 45 | -------------------------------------------------------------------------------- /bothub/common/migrations/0071_auto_20200729_1632.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-07-29 19:32 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0070_auto_20200729_1220")] 9 | 10 | operations = [ 11 | migrations.AddIndex( 12 | model_name="repositorynlplog", 13 | index=models.Index( 14 | fields=["repository_version_language", "user"], 15 | name="common_repo_nlp_log_idx", 16 | ), 17 | ), 18 | migrations.AddIndex( 19 | model_name="repositoryversion", 20 | index=models.Index( 21 | fields=["created_by", "repository"], 22 | name="common_repository_version_idx", 23 | ), 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /bothub/common/migrations/0072_auto_20200729_1704.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-07-29 20:04 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0071_auto_20200729_1632")] 9 | 10 | operations = [ 11 | migrations.RemoveIndex( 12 | model_name="repositorynlplog", name="common_repo_nlp_log_idx" 13 | ), 14 | migrations.AddIndex( 15 | model_name="repositorynlplog", 16 | index=models.Index( 17 | condition=models.Q(from_backend=False), 18 | fields=["repository_version_language", "user"], 19 | name="common_repo_nlp_log_idx", 20 | ), 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /bothub/common/migrations/0073_fix_authorizations.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-07-31 11:35 2 | 3 | from django.db import migrations 4 | 5 | 6 | def noop(apps, schema_editor): # pragma: no cover 7 | pass 8 | 9 | 10 | def update_authorization(apps, schema_editor): # pragma: no cover 11 | RepositoryAuthorization = apps.get_model("common", "RepositoryAuthorization") 12 | User = apps.get_model("authentication", "User") 13 | ROLE_ADMIN = 3 14 | 15 | def is_owner(auth): 16 | try: 17 | user = auth.user 18 | except User.DoesNotExist: # pragma: no cover 19 | return False # pragma: no cover 20 | return auth.repository.owner == user 21 | 22 | for auth in RepositoryAuthorization.objects.all(): 23 | if is_owner(auth): 24 | auth.role = ROLE_ADMIN 25 | auth.save(update_fields=["role"]) 26 | 27 | 28 | class Migration(migrations.Migration): 29 | 30 | dependencies = [ 31 | ("authentication", "0006_auto_20200729_1220"), 32 | ("common", "0072_auto_20200729_1704"), 33 | ] 34 | 35 | operations = [migrations.RunPython(update_authorization, noop)] 36 | -------------------------------------------------------------------------------- /bothub/common/migrations/0078_auto_20200807_1425.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-08-07 17:25 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0077_auto_20200807_1410")] 9 | 10 | operations = [ 11 | migrations.AlterModelOptions( 12 | name="repositorynlplog", 13 | options={ 14 | "ordering": ["-created_at"], 15 | "verbose_name": "repository nlp logs", 16 | }, 17 | ) 18 | ] 19 | -------------------------------------------------------------------------------- /bothub/common/migrations/0079_auto_20200812_1513.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-08-12 18:13 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [("common", "0078_auto_20200807_1425")] 10 | 11 | operations = [ 12 | migrations.AlterField( 13 | model_name="requestrepositoryauthorization", 14 | name="approved_by", 15 | field=models.ForeignKey( 16 | blank=True, 17 | null=True, 18 | on_delete=django.db.models.deletion.CASCADE, 19 | to="authentication.RepositoryOwner", 20 | ), 21 | ), 22 | migrations.AlterField( 23 | model_name="requestrepositoryauthorization", 24 | name="user", 25 | field=models.ForeignKey( 26 | on_delete=django.db.models.deletion.CASCADE, 27 | related_name="requests", 28 | to="authentication.RepositoryOwner", 29 | ), 30 | ), 31 | ] 32 | -------------------------------------------------------------------------------- /bothub/common/migrations/0081_auto_20200821_1250.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-08-21 15:50 2 | 3 | from django.db import migrations 4 | from django.db.models import Q 5 | 6 | 7 | def noop(apps, schema_editor): # pragma: no cover 8 | pass 9 | 10 | 11 | def migration(apps, schema_editor): # pragma: no cover 12 | RepositoryExample = apps.get_model("common", "RepositoryExample") 13 | RepositoryIntent = apps.get_model("common", "RepositoryIntent") 14 | 15 | chunk = [] 16 | 17 | # Delete all rows null that does not belong to any intelligence 18 | RepositoryExample.objects.filter( 19 | Q(old_intent__isnull=True) | Q(repository_version_language__isnull=True) 20 | ).delete() 21 | 22 | for i, example in enumerate( 23 | RepositoryExample.objects.only( 24 | "repository_version_language", "old_intent", "intent" 25 | ).iterator(chunk_size=10000) 26 | ): 27 | if ( 28 | example.repository_version_language is not None 29 | and example.old_intent is not None 30 | ): 31 | intent, created = RepositoryIntent.objects.get_or_create( 32 | repository_version=example.repository_version_language.repository_version, 33 | text=example.old_intent, 34 | ) 35 | example.intent = intent 36 | chunk.append(example) 37 | if i % 10000 == 0 and chunk: 38 | RepositoryExample.objects.bulk_update(chunk, ["intent"]) 39 | chunk = [] 40 | if chunk: 41 | RepositoryExample.objects.bulk_update(chunk, ["intent"]) 42 | 43 | 44 | class Migration(migrations.Migration): 45 | 46 | dependencies = [("common", "0080_repositoryintent")] 47 | 48 | operations = [migrations.RunPython(migration, noop)] 49 | -------------------------------------------------------------------------------- /bothub/common/migrations/0082_auto_20200821_1617.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-08-21 19:17 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [("common", "0081_auto_20200821_1250")] 10 | 11 | operations = [ 12 | migrations.RemoveField(model_name="repositoryexample", name="old_intent"), 13 | migrations.AlterField( 14 | model_name="repositoryexample", 15 | name="intent", 16 | field=models.ForeignKey( 17 | default=None, 18 | on_delete=django.db.models.deletion.CASCADE, 19 | to="common.RepositoryIntent", 20 | ), 21 | preserve_default=False, 22 | ), 23 | migrations.AlterField( 24 | model_name="repositoryexample", 25 | name="repository_version_language", 26 | field=models.ForeignKey( 27 | default=None, 28 | editable=False, 29 | on_delete=django.db.models.deletion.CASCADE, 30 | related_name="added", 31 | to="common.RepositoryVersionLanguage", 32 | ), 33 | preserve_default=False, 34 | ), 35 | migrations.AlterUniqueTogether( 36 | name="repositoryintent", unique_together={("repository_version", "text")} 37 | ), 38 | ] 39 | -------------------------------------------------------------------------------- /bothub/common/migrations/0083_repositoryintent_created_at.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-08-21 20:52 2 | 3 | from django.db import migrations, models 4 | import django.utils.timezone 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [("common", "0082_auto_20200821_1617")] 10 | 11 | operations = [ 12 | migrations.AddField( 13 | model_name="repositoryintent", 14 | name="created_at", 15 | field=models.DateTimeField( 16 | auto_now_add=True, 17 | default=django.utils.timezone.now, 18 | verbose_name="created at", 19 | ), 20 | preserve_default=False, 21 | ) 22 | ] 23 | -------------------------------------------------------------------------------- /bothub/common/migrations/0084_repositoryreports.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-08-31 18:24 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ("authentication", "0006_auto_20200729_1220"), 11 | ("common", "0083_repositoryintent_created_at"), 12 | ] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name="RepositoryReports", 17 | fields=[ 18 | ( 19 | "id", 20 | models.AutoField( 21 | auto_created=True, 22 | primary_key=True, 23 | serialize=False, 24 | verbose_name="ID", 25 | ), 26 | ), 27 | ("count_reports", models.IntegerField(default=0)), 28 | ("report_date", models.DateField(verbose_name="report date")), 29 | ( 30 | "repository_version_language", 31 | models.ForeignKey( 32 | editable=False, 33 | on_delete=django.db.models.deletion.CASCADE, 34 | related_name="repository_reports", 35 | to="common.RepositoryVersionLanguage", 36 | ), 37 | ), 38 | ( 39 | "user", 40 | models.ForeignKey( 41 | on_delete=django.db.models.deletion.CASCADE, 42 | to="authentication.RepositoryOwner", 43 | ), 44 | ), 45 | ], 46 | options={ 47 | "verbose_name": "repository report", 48 | "verbose_name_plural": "repository reports", 49 | }, 50 | ) 51 | ] 52 | -------------------------------------------------------------------------------- /bothub/common/migrations/0085_count_repositoryreports.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-08-31 18:24 2 | 3 | from django.db import migrations 4 | 5 | 6 | def noop(apps, schema_editor): # pragma: no cover 7 | pass 8 | 9 | 10 | def migration(apps, schema_editor): # pragma: no cover 11 | RepositoryNLPLog = apps.get_model("common", "RepositoryNLPLog") 12 | RepositoryReports = apps.get_model("common", "RepositoryReports") 13 | 14 | for log in ( 15 | RepositoryNLPLog.objects.filter(created_at__gte="2020-09-01") 16 | .only("repository_version_language", "user", "created_at") 17 | .iterator() 18 | ): 19 | report, created = RepositoryReports.objects.get_or_create( 20 | repository_version_language=log.repository_version_language, 21 | user=log.user, 22 | report_date=log.created_at.date(), 23 | ) 24 | report.count_reports += 1 25 | report.save(update_fields=["count_reports"]) 26 | 27 | 28 | class Migration(migrations.Migration): 29 | 30 | dependencies = [("common", "0084_repositoryreports")] 31 | 32 | operations = [migrations.RunPython(migration, noop)] 33 | -------------------------------------------------------------------------------- /bothub/common/migrations/0086_fix_translations.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-08-31 18:24 2 | 3 | from django.db import migrations 4 | 5 | 6 | def noop(apps, schema_editor): # pragma: no cover 7 | pass 8 | 9 | 10 | def migration(apps, schema_editor): # pragma: no cover 11 | RepositoryVersionLanguage = apps.get_model("common", "RepositoryVersionLanguage") 12 | RepositoryTranslatedExample = apps.get_model( 13 | "common", "RepositoryTranslatedExample" 14 | ) 15 | 16 | def get_version_language(repository_version, language): 17 | version_language, created = RepositoryVersionLanguage.objects.get_or_create( 18 | repository_version=repository_version, language=language 19 | ) 20 | return version_language 21 | 22 | chunk = [] 23 | 24 | for i, translated in enumerate( 25 | RepositoryTranslatedExample.objects.only( 26 | "repository_version_language", "original_example" 27 | ).iterator(chunk_size=10000) 28 | ): 29 | if translated.repository_version_language.language != translated.language: 30 | repository_version_language = get_version_language( 31 | translated.original_example.repository_version_language.repository_version, 32 | translated.language, 33 | ) 34 | translated.repository_version_language = repository_version_language 35 | chunk.append(translated) 36 | if i % 10000 == 0 and chunk: 37 | RepositoryTranslatedExample.objects.bulk_update( 38 | chunk, ["repository_version_language"] 39 | ) 40 | chunk = [] 41 | if chunk: 42 | RepositoryTranslatedExample.objects.bulk_update( 43 | chunk, ["repository_version_language"] 44 | ) 45 | 46 | 47 | class Migration(migrations.Migration): 48 | 49 | dependencies = [("common", "0085_count_repositoryreports")] 50 | 51 | operations = [migrations.RunPython(migration, noop)] 52 | -------------------------------------------------------------------------------- /bothub/common/migrations/0087_repository_allow_search_examples.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-09-08 17:28 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0086_fix_translations")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="repository", 13 | name="allow_search_examples", 14 | field=models.BooleanField( 15 | default=False, verbose_name="Authorizes third-party phrase search" 16 | ), 17 | ) 18 | ] 19 | -------------------------------------------------------------------------------- /bothub/common/migrations/0088_fix_translations.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.12 on 2020-08-31 18:24 2 | 3 | from django.db import migrations, transaction 4 | from django.db.models import Q, F 5 | 6 | BATCH_SIZE = 5000 7 | 8 | 9 | def reverse(apps, schema_editor): # pragma: no cover 10 | pass 11 | 12 | 13 | def fix_translations_rv_language_fields(apps, schema_editor): # pragma: no cover 14 | RepositoryVersionLanguage = apps.get_model("common", "RepositoryVersionLanguage") 15 | RepositoryTranslatedExample = apps.get_model( 16 | "common", "RepositoryTranslatedExample" 17 | ) 18 | 19 | def get_version_language(repository_version, language): 20 | version_language, created = RepositoryVersionLanguage.objects.get_or_create( 21 | repository_version=repository_version, language=language 22 | ) 23 | return version_language 24 | 25 | translations = RepositoryTranslatedExample.objects.exclude( 26 | Q(repository_version_language__language=F("language")) 27 | ) 28 | 29 | num_updated = 0 30 | max_id = -1 31 | while True: 32 | batch = list(translations.filter(id__gt=max_id).order_by("id")[:BATCH_SIZE]) 33 | if not batch: 34 | break 35 | 36 | with transaction.atomic(): 37 | for translation in batch: 38 | repository_version_language = get_version_language( 39 | translation.original_example.repository_version_language.repository_version, 40 | translation.language, 41 | ) 42 | translation.repository_version_language = repository_version_language 43 | translation.save(update_fields=["repository_version_language"]) 44 | 45 | num_updated += len(batch) 46 | print(f" > Updated {num_updated} translations") 47 | 48 | max_id = batch[-1].id 49 | 50 | 51 | def apply_manual(): # pragma: no cover 52 | from django.apps import apps 53 | 54 | fix_translations_rv_language_fields(apps, None) 55 | 56 | 57 | class Migration(migrations.Migration): 58 | 59 | dependencies = [("common", "0087_repository_allow_search_examples")] 60 | 61 | operations = [migrations.RunPython(fix_translations_rv_language_fields, reverse)] 62 | -------------------------------------------------------------------------------- /bothub/common/migrations/0089_remove_repository_allow_search_examples.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-09-18 15:17 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0088_fix_translations")] 9 | 10 | operations = [ 11 | migrations.RemoveField(model_name="repository", name="allow_search_examples") 12 | ] 13 | -------------------------------------------------------------------------------- /bothub/common/migrations/0090_auto_20200921_1751.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-09-21 17:51 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [("common", "0089_remove_repository_allow_search_examples")] 10 | 11 | operations = [ 12 | migrations.AlterField( 13 | model_name="repositoryintent", 14 | name="repository_version", 15 | field=models.ForeignKey( 16 | on_delete=django.db.models.deletion.CASCADE, 17 | related_name="version_intents", 18 | to="common.RepositoryVersion", 19 | ), 20 | ) 21 | ] 22 | -------------------------------------------------------------------------------- /bothub/common/migrations/0091_extensions.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-09-30 11:00 2 | from django.contrib.postgres.operations import UnaccentExtension, TrigramExtension 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0090_auto_20200921_1751")] 9 | 10 | operations = [UnaccentExtension(), TrigramExtension()] 11 | -------------------------------------------------------------------------------- /bothub/common/migrations/0091_repository_count_authorizations.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-09-24 14:18 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0090_auto_20200921_1751")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="repository", 13 | name="count_authorizations", 14 | field=models.IntegerField( 15 | default=0, verbose_name="Authorization count calculated by celery" 16 | ), 17 | ) 18 | ] 19 | -------------------------------------------------------------------------------- /bothub/common/migrations/0092_merge_20200930_1819.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.15 on 2020-09-30 18:19 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("common", "0091_extensions"), 10 | ("common", "0091_repository_count_authorizations"), 11 | ] 12 | 13 | operations = [] 14 | -------------------------------------------------------------------------------- /bothub/common/migrations/0094_repositoryqueuetask_type_processing.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.15 on 2020-10-01 19:05 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0093_repositorytranslator")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="repositoryqueuetask", 13 | name="type_processing", 14 | field=models.PositiveIntegerField( 15 | choices=[(0, "NLP Tranining"), (1, "Repository Auto Translation")], 16 | default=0, 17 | verbose_name="Type Processing", 18 | ), 19 | preserve_default=False, 20 | ) 21 | ] 22 | -------------------------------------------------------------------------------- /bothub/common/migrations/0095_auto_20201001_1944.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.15 on 2020-10-01 19:44 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0094_repositoryqueuetask_type_processing")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="repositoryqueuetask", 13 | name="from_queue", 14 | field=models.PositiveIntegerField( 15 | choices=[(0, "Ai Platform"), (1, "Celery")], 16 | default=1, 17 | verbose_name="From Queue NLP", 18 | ), 19 | ) 20 | ] 21 | -------------------------------------------------------------------------------- /bothub/common/migrations/0096_auto_20201001_2005.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.15 on 2020-10-01 20:05 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0095_auto_20201001_1944")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="repositoryqueuetask", 13 | name="status", 14 | field=models.PositiveIntegerField( 15 | choices=[ 16 | (0, "Pending"), 17 | (2, "Success"), 18 | (1, "Processing"), 19 | (3, "Failed"), 20 | ], 21 | default=0, 22 | verbose_name="Status Queue NLP", 23 | ), 24 | ) 25 | ] 26 | -------------------------------------------------------------------------------- /bothub/common/migrations/0097_repositorymigrate.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-10-07 16:11 2 | 3 | import bothub.common.languages 4 | from django.db import migrations, models 5 | import django.db.models.deletion 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ("authentication", "0006_auto_20200729_1220"), 12 | ("common", "0096_auto_20201001_2005"), 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name="RepositoryMigrate", 18 | fields=[ 19 | ( 20 | "id", 21 | models.AutoField( 22 | auto_created=True, 23 | primary_key=True, 24 | serialize=False, 25 | verbose_name="ID", 26 | ), 27 | ), 28 | ( 29 | "language", 30 | models.CharField( 31 | max_length=5, 32 | validators=[bothub.common.languages.validate_language], 33 | verbose_name="language", 34 | ), 35 | ), 36 | ("auth_token", models.TextField()), 37 | ("created", models.DateTimeField(auto_now_add=True)), 38 | ( 39 | "repository_version", 40 | models.ForeignKey( 41 | on_delete=django.db.models.deletion.DO_NOTHING, 42 | related_name="repository_migrate", 43 | to="common.RepositoryVersion", 44 | ), 45 | ), 46 | ( 47 | "user", 48 | models.ForeignKey( 49 | on_delete=django.db.models.deletion.CASCADE, 50 | to="authentication.RepositoryOwner", 51 | ), 52 | ), 53 | ], 54 | options={ 55 | "verbose_name": "repository migrate", 56 | "verbose_name_plural": "repository migrates", 57 | }, 58 | ) 59 | ] 60 | -------------------------------------------------------------------------------- /bothub/common/migrations/0097_repositoryscore.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-10-07 21:04 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [("common", "0096_auto_20201001_2005")] 10 | 11 | operations = [ 12 | migrations.CreateModel( 13 | name="RepositoryScore", 14 | fields=[ 15 | ( 16 | "id", 17 | models.AutoField( 18 | auto_created=True, 19 | primary_key=True, 20 | serialize=False, 21 | verbose_name="ID", 22 | ), 23 | ), 24 | ("intents_balance_score", models.FloatField(default=0.0)), 25 | ("intents_balance_recommended", models.TextField(null=True)), 26 | ("intents_size_score", models.FloatField(default=0.0)), 27 | ("intents_size_recommended", models.TextField(null=True)), 28 | ("evaluate_size_score", models.FloatField(default=0.0)), 29 | ("evaluate_size_recommended", models.TextField(null=True)), 30 | ( 31 | "repository", 32 | models.ForeignKey( 33 | editable=False, 34 | on_delete=django.db.models.deletion.CASCADE, 35 | related_name="repository_score", 36 | to="common.Repository", 37 | ), 38 | ), 39 | ], 40 | options={ 41 | "verbose_name": "repository score", 42 | "verbose_name_plural": "repository scores", 43 | "unique_together": {("repository",)}, 44 | }, 45 | ) 46 | ] 47 | -------------------------------------------------------------------------------- /bothub/common/migrations/0098_merge_20201008_1607.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-10-08 16:07 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("common", "0097_repositoryscore"), 10 | ("common", "0097_repositorymigrate"), 11 | ] 12 | 13 | operations = [] 14 | -------------------------------------------------------------------------------- /bothub/common/migrations/0098_repositorymigrate_classifier.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-10-08 18:34 2 | 3 | import bothub.utils 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [("common", "0097_repositorymigrate")] 10 | 11 | operations = [ 12 | migrations.AddField( 13 | model_name="repositorymigrate", 14 | name="classifier", 15 | field=models.CharField( 16 | default="wit", 17 | editable=False, 18 | max_length=30, 19 | validators=[bothub.utils.validate_classifier], 20 | verbose_name="classifier", 21 | ), 22 | preserve_default=False, 23 | ) 24 | ] 25 | -------------------------------------------------------------------------------- /bothub/common/migrations/0099_auto_20201008_1848.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-10-08 18:48 2 | 3 | import bothub.utils 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [("common", "0098_repositorymigrate_classifier")] 10 | 11 | operations = [ 12 | migrations.AlterField( 13 | model_name="repositorymigrate", 14 | name="classifier", 15 | field=models.CharField( 16 | editable=False, 17 | max_length=16, 18 | validators=[bothub.utils.validate_classifier], 19 | verbose_name="classifier", 20 | ), 21 | ) 22 | ] 23 | -------------------------------------------------------------------------------- /bothub/common/migrations/0100_merge_20201009_1739.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-10-09 17:39 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("common", "0099_auto_20201008_1848"), 10 | ("common", "0098_merge_20201008_1607"), 11 | ] 12 | 13 | operations = [] 14 | -------------------------------------------------------------------------------- /bothub/common/migrations/0101_auto_20201020_1546.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-10-20 15:46 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("authentication", "0006_auto_20200729_1220"), 10 | ("common", "0100_merge_20201009_1739"), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterUniqueTogether( 15 | name="repositoryreports", 16 | unique_together={("repository_version_language", "user", "report_date")}, 17 | ) 18 | ] 19 | -------------------------------------------------------------------------------- /bothub/common/migrations/0102_repositoryevaluateresult_cross_validation.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.17 on 2020-12-03 11:36 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0101_auto_20201020_1546")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="repositoryevaluateresult", 13 | name="cross_validation", 14 | field=models.BooleanField(default=False, verbose_name="cross validation"), 15 | ) 16 | ] 17 | -------------------------------------------------------------------------------- /bothub/common/migrations/0103_auto_20210302_1431.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.17 on 2021-03-02 14:31 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0102_repositoryevaluateresult_cross_validation")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="repositoryqueuetask", 13 | name="type_processing", 14 | field=models.PositiveIntegerField( 15 | choices=[ 16 | (0, "NLP Tranining"), 17 | (1, "Repository Auto Translation"), 18 | (2, "Evaluate Cross Validation"), 19 | ], 20 | verbose_name="Type Processing", 21 | ), 22 | ) 23 | ] 24 | -------------------------------------------------------------------------------- /bothub/common/migrations/0105_auto_20210309_1945.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.17 on 2021-03-09 19:45 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0104_qaknowledgebase")] 9 | 10 | operations = [ 11 | migrations.AlterField( 12 | model_name="qaknowledgebase", 13 | name="last_update", 14 | field=models.DateTimeField(auto_now=True, verbose_name="last update"), 15 | ) 16 | ] 17 | -------------------------------------------------------------------------------- /bothub/common/migrations/0107_auto_20210311_1852.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.17 on 2021-03-11 18:52 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0106_auto_20210311_1254")] 9 | 10 | operations = [ 11 | migrations.AlterUniqueTogether( 12 | name="qacontext", unique_together={("knowledge_base", "language")} 13 | ) 14 | ] 15 | -------------------------------------------------------------------------------- /bothub/common/migrations/0108_repository_repository_type.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.19 on 2021-03-29 14:23 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0107_auto_20210311_1852")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="repository", 13 | name="repository_type", 14 | field=models.CharField( 15 | choices=[("classifier", "Classifier"), ("content", "Content")], 16 | default="classifier", 17 | max_length=10, 18 | verbose_name="repository type", 19 | ), 20 | ) 21 | ] 22 | -------------------------------------------------------------------------------- /bothub/common/migrations/0109_auto_20210601_0323.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.22 on 2021-06-01 03:23 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0108_repository_repository_type")] 9 | 10 | operations = [ 11 | migrations.AlterModelOptions( 12 | name="repositorynlplog", 13 | options={"ordering": ["-id"], "verbose_name": "repository nlp logs"}, 14 | ) 15 | ] 16 | -------------------------------------------------------------------------------- /bothub/common/migrations/0110_auto_20210601_1443.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.22 on 2021-06-01 14:43 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [("common", "0109_auto_20210601_0323")] 9 | 10 | operations = [ 11 | migrations.AddIndex( 12 | model_name="repositoryexample", 13 | index=models.Index( 14 | fields=["repository_version_language"], 15 | name="common_repository_example_idx", 16 | ), 17 | ), 18 | migrations.AddIndex( 19 | model_name="repositorytranslatedexample", 20 | index=models.Index( 21 | fields=["repository_version_language", "language"], 22 | name="repository_version_idx", 23 | ), 24 | ), 25 | migrations.AddIndex( 26 | model_name="repositoryversionlanguage", 27 | index=models.Index( 28 | fields=["repository_version"], name="vs_repository_version_idx" 29 | ), 30 | ), 31 | ] 32 | -------------------------------------------------------------------------------- /bothub/common/migrations/0112_auto_20211201_1330.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.8 on 2021-12-01 13:30 2 | 3 | import django.core.validators 4 | from django.db import migrations, models 5 | import re 6 | 7 | 8 | VALIDATE_REGEX = ".[-a-zA-Z_]" 9 | VALIDATE_TEXT = "Enter a valid value that have letters in it" 10 | 11 | 12 | class Migration(migrations.Migration): 13 | 14 | dependencies = [ 15 | ("common", "0111_auto_20210908_1135"), 16 | ] 17 | 18 | operations = [ 19 | migrations.AlterField( 20 | model_name="qatext", 21 | name="text", 22 | field=models.TextField( 23 | help_text="QA context text", 24 | max_length=25000, 25 | validators=[ 26 | django.core.validators.RegexValidator( 27 | re.compile(VALIDATE_REGEX), 28 | VALIDATE_TEXT, 29 | "invalid", 30 | ) 31 | ], 32 | verbose_name="text", 33 | ), 34 | ), 35 | migrations.AlterField( 36 | model_name="repositoryexample", 37 | name="text", 38 | field=models.TextField( 39 | help_text="Example text", 40 | validators=[ 41 | django.core.validators.RegexValidator( 42 | re.compile(VALIDATE_REGEX), 43 | VALIDATE_TEXT, 44 | "invalid", 45 | ) 46 | ], 47 | verbose_name="text", 48 | ), 49 | ), 50 | migrations.AlterField( 51 | model_name="repositorytranslatedexample", 52 | name="text", 53 | field=models.TextField( 54 | help_text="Translation text", 55 | validators=[ 56 | django.core.validators.RegexValidator( 57 | re.compile(VALIDATE_REGEX), 58 | VALIDATE_TEXT, 59 | "invalid", 60 | ) 61 | ], 62 | verbose_name="text", 63 | ), 64 | ), 65 | ] 66 | -------------------------------------------------------------------------------- /bothub/common/migrations/0113_auto_20220216_1154.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.8 on 2022-02-16 11:54 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("common", "0112_auto_20211201_1330"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="qatext", 15 | name="text", 16 | field=models.TextField( 17 | help_text="QA context text", max_length=25000, verbose_name="text" 18 | ), 19 | ), 20 | migrations.AlterField( 21 | model_name="repositoryexample", 22 | name="text", 23 | field=models.TextField(help_text="Example text", verbose_name="text"), 24 | ), 25 | migrations.AlterField( 26 | model_name="repositorytranslatedexample", 27 | name="text", 28 | field=models.TextField(help_text="Translation text", verbose_name="text"), 29 | ), 30 | ] 31 | -------------------------------------------------------------------------------- /bothub/common/migrations/0114_alter_repositoryversionlanguage_unique_together.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.12 on 2022-03-18 14:08 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('common', '0113_auto_20220216_1154'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterUniqueTogether( 14 | name='repositoryversionlanguage', 15 | unique_together={('language', 'repository_version')}, 16 | ), 17 | ] 18 | -------------------------------------------------------------------------------- /bothub/common/migrations/0115_auto_20220923_1543.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.15 on 2022-09-23 15:43 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('common', '0114_alter_repositoryversionlanguage_unique_together'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='organizationauthorization', 15 | name='role', 16 | field=models.PositiveIntegerField(choices=[(0, 'not set'), (1, 'user'), (2, 'contributor'), (3, 'admin')], default=0, verbose_name='role'), 17 | ), 18 | migrations.AlterField( 19 | model_name='repositoryauthorization', 20 | name='role', 21 | field=models.PositiveIntegerField(choices=[(0, 'not set'), (1, 'user'), (2, 'contributor'), (3, 'admin')], default=0, verbose_name='role'), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /bothub/common/migrations/0117_alter_zeroshotoptions_option_uuid.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.15 on 2023-03-31 03:14 2 | 3 | from django.db import migrations, models 4 | import uuid 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('common', '0116_auto_20230331_0118'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='zeroshotoptions', 16 | name='option_uuid', 17 | field=models.UUIDField(default=uuid.UUID('e740f18a-14bd-42e4-b744-cfa75e867188')), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /bothub/common/migrations/0118_alter_zeroshotoptions_option_uuid.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.15 on 2023-06-17 03:02 2 | 3 | from django.db import migrations, models 4 | import uuid 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('common', '0117_alter_zeroshotoptions_option_uuid'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='zeroshotoptions', 16 | name='option_uuid', 17 | field=models.UUIDField(default=uuid.uuid4), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /bothub/common/migrations/0119_repository_count_repository_train.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.20 on 2023-07-13 20:18 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('common', '0118_alter_zeroshotoptions_option_uuid'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='repository', 15 | name='count_repository_train', 16 | field=models.IntegerField(default=0, verbose_name='Train count'), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /bothub/common/migrations/0120_auto_20231016_1255.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.21 on 2023-10-16 12:55 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('common', '0119_repository_count_repository_train'), 10 | ] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name='ZeroshotLogs', 15 | fields=[ 16 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 17 | ('text', models.TextField(help_text='Text to analyze')), 18 | ('classification', models.TextField()), 19 | ('other', models.BooleanField()), 20 | ('categories', models.JSONField()), 21 | ('nlp_log', models.TextField(blank=True)), 22 | ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created at')), 23 | ], 24 | options={ 25 | 'verbose_name': 'zeroshot nlp logs', 26 | }, 27 | ), 28 | migrations.AddIndex( 29 | model_name='zeroshotlogs', 30 | index=models.Index(fields=['nlp_log'], name='common_zeroshot_log_idx'), 31 | ), 32 | ] 33 | -------------------------------------------------------------------------------- /bothub/common/migrations/0121_zeroshotlogs_language.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.21 on 2023-10-30 19:23 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('common', '0120_auto_20231016_1255'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='zeroshotlogs', 15 | name='language', 16 | field=models.CharField(blank=True, max_length=64, null=True, verbose_name='Language'), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /bothub/common/migrations/0122_rename_categories_zeroshotlogs_options.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.21 on 2023-12-07 16:40 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('common', '0121_zeroshotlogs_language'), 10 | ] 11 | 12 | operations = [ 13 | migrations.RenameField( 14 | model_name='zeroshotlogs', 15 | old_name='categories', 16 | new_name='options', 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /bothub/common/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/common/migrations/__init__.py -------------------------------------------------------------------------------- /bothub/common/signals.py: -------------------------------------------------------------------------------- 1 | from django.db import transaction 2 | from django.conf import settings 3 | from bothub.common.tasks import handle_save 4 | from django_elasticsearch_dsl.registries import registry 5 | from django_elasticsearch_dsl.signals import RealTimeSignalProcessor 6 | 7 | 8 | class CelerySignalProcessor(RealTimeSignalProcessor): 9 | def handle_save(self, sender, instance, **kwargs): 10 | app_label = instance._meta.app_label 11 | model_name = instance._meta.model_name 12 | model = instance._meta.concrete_model 13 | 14 | if settings.USE_ELASTICSEARCH and ( 15 | model in registry._models or model in registry._related_models 16 | ): 17 | transaction.on_commit( 18 | lambda: handle_save.apply_async( 19 | args=[instance.pk, app_label, model_name], 20 | queue=settings.ELASTICSEARCH_CUSTOM_QUEUE, 21 | ) 22 | ) 23 | 24 | def handle_pre_delete(self, sender, instance, **kwargs): 25 | """ 26 | Logs deletions are now handled in delete_nlp_logs task 27 | """ 28 | pass 29 | 30 | def handle_delete(self, sender, instance, **kwargs): 31 | """ 32 | Logs deletions are now handled in delete_nlp_logs task 33 | """ 34 | pass 35 | -------------------------------------------------------------------------------- /bothub/common/static/bothub/emails/facebook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/common/static/bothub/emails/facebook.png -------------------------------------------------------------------------------- /bothub/common/static/bothub/emails/linkedin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/common/static/bothub/emails/linkedin.png -------------------------------------------------------------------------------- /bothub/common/static/bothub/emails/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/common/static/bothub/emails/logo.png -------------------------------------------------------------------------------- /bothub/common/static/bothub/exporter/bothub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/common/static/bothub/exporter/bothub.png -------------------------------------------------------------------------------- /bothub/common/static/bothub/exporter/example.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/common/static/bothub/exporter/example.xlsx -------------------------------------------------------------------------------- /bothub/common/static/common/emails/request.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/common/static/common/emails/request.gif -------------------------------------------------------------------------------- /bothub/common/templates/common/emails/new_request.html: -------------------------------------------------------------------------------- 1 | {% extends 'bothub/emails/base.html' %} 2 | 3 | {% load i18n static %} 4 | 5 | {% block before-content %} 6 |
7 | {% trans 'New Authorization Request' %} 8 |

{% trans 'New Authorization Request' %}

9 |
10 | {% endblock %} 11 | 12 | {% block main-content %} 13 |

{% blocktrans %}You receive new authorization request from {{ user_name }} to repository {{ repository_name }}.{% endblocktrans %}

14 |
{{ text }}
15 |
{{ user_name }}
16 | {% endblock %} 17 | 18 | {% block after-content %} 19 | {% if repository_url %} 20 | 23 | {% endif %} 24 | {% endblock %} 25 | -------------------------------------------------------------------------------- /bothub/common/templates/common/emails/new_request.txt: -------------------------------------------------------------------------------- 1 | {% load i18n %}{% trans 'New Authorization Request' %} 2 | 3 | {% blocktrans %}You receive new authorization request from {{ user_name }} to repository {{ repository_name }}.{% endblocktrans %} 4 | 5 | "{{ text }}" 6 | - {{ user_name }}{% if repository_url %} 7 | 8 | Access {{ repository_name }}: 9 | {{ repository_url }}{% endif %} -------------------------------------------------------------------------------- /bothub/common/templates/common/emails/new_role.html: -------------------------------------------------------------------------------- 1 | {% extends 'bothub/emails/base.html' %} 2 | 3 | {% load i18n %} 4 | 5 | {% block main-content %} 6 |

{% blocktrans %}Hi, {{ user_name }}{% endblocktrans %}

7 | {% with new_role_lower=new_role|lower %} 8 |

{% blocktrans %}{{ responsible_name }} added you as {{ new_role_lower }} in the repository {{ repository_name }}.{% endblocktrans %}

9 | {% endwith %} 10 | {% endblock %} 11 | 12 | {% block after-content %} 13 | {% if repository_url %} 14 | 17 | {% endif %} 18 | {% endblock %} 19 | -------------------------------------------------------------------------------- /bothub/common/templates/common/emails/new_role.txt: -------------------------------------------------------------------------------- 1 | {% load i18n %}{% blocktrans %}Hi, {{ user_name }}{% endblocktrans %} 2 | 3 | {% with new_role_lower=new_role|lower %}{% blocktrans %}{{ responsible_name }} added you as {{ new_role_lower }} in the repository {{ repository_name }}.{% endblocktrans %}{% endwith %}{% if repository_url %} 4 | 5 | Access {{ repository_name }}: 6 | {{ repository_url }}{% endif %} 7 | -------------------------------------------------------------------------------- /bothub/common/templates/common/emails/request_approved.html: -------------------------------------------------------------------------------- 1 | {% extends 'bothub/emails/base.html' %} 2 | 3 | {% load i18n static %} 4 | 5 | {% block before-content %} 6 |
7 | {% trans 'Authorization Request Approved' %} 8 |

{% trans 'Authorization Request Approved' %}

9 |
10 | {% endblock %} 11 | 12 | {% block main-content %} 13 |

{% blocktrans %}Your authorization request to repository {{ repository_name }} was approved by {{ admin_name }}.{% endblocktrans %}

14 | {% endblock %} 15 | 16 | {% block after-content %} 17 | {% if repository_url %} 18 | 21 | {% endif %} 22 | {% endblock %} 23 | -------------------------------------------------------------------------------- /bothub/common/templates/common/emails/request_approved.txt: -------------------------------------------------------------------------------- 1 | {% load i18n %}{% trans 'Authorization Request Approved' %} 2 | 3 | {% blocktrans %}Your authorization request to repository {{ repository_name }} was approved by {{ admin_name }}.{% endblocktrans %}{% if repository_url %} 4 | 5 | Access {{ repository_name }}: 6 | {{ repository_url }}{% endif %} 7 | -------------------------------------------------------------------------------- /bothub/common/templates/common/emails/request_rejected.html: -------------------------------------------------------------------------------- 1 | {% extends 'bothub/emails/base.html' %} 2 | 3 | {% load i18n %} 4 | 5 | {% block main-content %} 6 |

{% blocktrans %}We’re sorry to say that, but your request to access the repository {{ repository_name }} was denied.{% endblocktrans %}

7 |

{% trans 'Keep up contributing and creating bots with Bothub.' %}

8 | {% endblock %} 9 | -------------------------------------------------------------------------------- /bothub/common/templates/common/emails/request_rejected.txt: -------------------------------------------------------------------------------- 1 | {% load i18n %}{% blocktrans %}We’re sorry to say that, but your request to access the repository {{ repository_name }} was denied.{% endblocktrans %} 2 | {% trans 'Keep up contributing and creating bots with Bothub.' %} -------------------------------------------------------------------------------- /bothub/common/usecase/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/common/usecase/__init__.py -------------------------------------------------------------------------------- /bothub/common/usecase/repositorylog/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/common/usecase/repositorylog/__init__.py -------------------------------------------------------------------------------- /bothub/common/usecase/repositorylog/dto.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from dataclasses import dataclass 3 | 4 | 5 | @dataclass 6 | class FileResponseDTO: 7 | status: int = None 8 | file_url: str = None 9 | err: str = None 10 | file_name: str = None 11 | 12 | 13 | class FileDataBase(ABC): 14 | 15 | @abstractmethod 16 | def add_file(file) -> FileResponseDTO: 17 | ... 18 | -------------------------------------------------------------------------------- /bothub/common/usecase/repositorylog/storage.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | import boto3 3 | from os.path import basename 4 | from django.conf import settings 5 | 6 | from .dto import FileDataBase, FileResponseDTO 7 | 8 | 9 | class s3FileDatabase(FileDataBase): 10 | def __init__(self) -> None: 11 | self.s3_client = boto3.client( 12 | 's3', 13 | aws_access_key_id=settings.AWS_ACCESS_KEY_ID, 14 | aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, 15 | region_name=settings.AWS_REGION_NAME 16 | ) 17 | 18 | def add_file(self, file) -> FileResponseDTO: 19 | file_name = basename("export_logs.xlsx") 20 | name, extension = file_name.split(".") 21 | file_name = f"{name}-{uuid.uuid4()}.{extension}" 22 | response = FileResponseDTO() 23 | try: 24 | self.s3_client.upload_fileobj(file, settings.AWS_BUCKET_NAME, file_name) 25 | response.status = 0 26 | response.file_url = f"https://{settings.AWS_BUCKET_NAME}.s3.{settings.AWS_REGION_NAME}.amazonaws.com/{file_name}" 27 | response.file_name = file_name 28 | except Exception as exception: 29 | response.status = 1 30 | response.err = str(exception) 31 | return response 32 | 33 | def create_presigned_url( 34 | self, 35 | file_name: str, 36 | expiration: int = 3600 37 | ) -> str: 38 | 39 | return self.s3_client.generate_presigned_url( 40 | 'get_object', 41 | Params={ 42 | 'Bucket': settings.AWS_BUCKET_NAME, 43 | 'Key': file_name 44 | }, 45 | ExpiresIn=expiration 46 | ) 47 | -------------------------------------------------------------------------------- /bothub/common/usecase/repositorylog/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/common/usecase/repositorylog/tests/__init__.py -------------------------------------------------------------------------------- /bothub/common/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import get_object_or_404 2 | from django.http import HttpResponseRedirect 3 | from django.core.exceptions import ValidationError 4 | from django.contrib.admin.views.decorators import staff_member_required 5 | 6 | from bothub.common.models import RepositoryNLPTrain 7 | 8 | 9 | @staff_member_required 10 | def download_bot_data(self, update_id): # pragma: no cover 11 | update = get_object_or_404(RepositoryNLPTrain, pk=update_id) 12 | if update.bot_data is None or update.bot_data == "": 13 | raise ValidationError(f"Update #{update.pk} not trained at.") 14 | response = HttpResponseRedirect( 15 | update.repositoryversionlanguage.get_bot_data.bot_data 16 | ) 17 | return response 18 | -------------------------------------------------------------------------------- /bothub/event_driven/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/event_driven/__init__.py -------------------------------------------------------------------------------- /bothub/event_driven/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class EventDrivenConfig(AppConfig): 5 | default_auto_field = 'django.db.models.BigAutoField' 6 | name = 'bothub.event_driven' 7 | -------------------------------------------------------------------------------- /bothub/event_driven/connection/pymqp_connection.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import amqp 4 | 5 | from bothub.event_driven.connection.rabbitmq_connection import RabbitMQConnection 6 | 7 | 8 | class PyAMQPConnectionBackend: 9 | _start_message = "[+] Connection established. Waiting for events" 10 | 11 | def __init__(self, handle_consumers: callable): 12 | self._handle_consumers = handle_consumers 13 | self.rabbitmq_instance = RabbitMQConnection() 14 | 15 | def _drain_events(self, connection: amqp.connection.Connection): 16 | while True: 17 | connection.drain_events() 18 | 19 | def start_consuming(self): 20 | while True: 21 | try: 22 | channel = self.rabbitmq_instance.channel 23 | 24 | self._handle_consumers(channel) 25 | 26 | print(self._start_message) 27 | 28 | self._drain_events(self.rabbitmq_instance.connection) 29 | 30 | except (amqp.exceptions.AMQPError, ConnectionRefusedError, OSError) as error: 31 | print(f"[-] Connection error: {error}") 32 | print(" [+] Reconnecting in 5 seconds...") 33 | time.sleep(5) 34 | self.rabbitmq_instance._establish_connection() 35 | except Exception as error: 36 | # TODO: Handle exceptions with RabbitMQ 37 | print("error on drain_events:", type(error), error) 38 | time.sleep(5) 39 | self.rabbitmq_instance._establish_connection() 40 | -------------------------------------------------------------------------------- /bothub/event_driven/connection/rabbitmq_connection.py: -------------------------------------------------------------------------------- 1 | import amqp 2 | import time 3 | 4 | from django.conf import settings 5 | 6 | class RabbitMQConnection: 7 | _instance = None 8 | 9 | def __new__(cls): 10 | if cls._instance is None: 11 | cls._instance = super(RabbitMQConnection, cls).__new__(cls) 12 | cls._instance.connect() 13 | return cls._instance 14 | 15 | def _establish_connection(self): 16 | self.connection = amqp.Connection( 17 | host=settings.EDA_BROKER_HOST, 18 | virtual_host=settings.EDA_VIRTUAL_HOST, 19 | userid=settings.EDA_BROKER_USER, 20 | password=settings.EDA_BROKER_PASSWORD, 21 | port=settings.EDA_BROKER_PORT 22 | ) 23 | self.channel = self.connection.channel() 24 | 25 | def connect(self): 26 | try: 27 | if not hasattr(self, "connection"): 28 | self._establish_connection() 29 | except Exception as e: 30 | print("Error while connecting to RabbitMQ:", str(e)) 31 | time.sleep(5) # Espera antes de tentar reconectar 32 | self._establish_connection() 33 | 34 | def make_connection(self): 35 | if self.connection.is_closing: 36 | self._establish_connection() 37 | return self.connection.is_alive -------------------------------------------------------------------------------- /bothub/event_driven/consumer/consumers.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | import amqp 4 | 5 | from bothub.event_driven.signals import message_started, message_finished 6 | 7 | 8 | class EDAConsumer(ABC): # pragma: no cover 9 | def handle(self, message: amqp.Message): 10 | message_started.send(sender=self) 11 | try: 12 | self.consume(message) 13 | finally: 14 | message_finished.send(sender=self) 15 | 16 | @abstractmethod 17 | def consume(self, message: amqp.Message): 18 | pass 19 | -------------------------------------------------------------------------------- /bothub/event_driven/handle.py: -------------------------------------------------------------------------------- 1 | from amqp.channel import Channel 2 | 3 | from bothub.project.handle import handle_consumers as project_handle_consumers 4 | from bothub.authorizations.consumers.handle import handle_consumers as authorizations_consumers 5 | 6 | def handle_consumers(channel: Channel) -> None: 7 | project_handle_consumers(channel) 8 | authorizations_consumers(channel) 9 | -------------------------------------------------------------------------------- /bothub/event_driven/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/event_driven/management/__init__.py -------------------------------------------------------------------------------- /bothub/event_driven/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/event_driven/management/commands/__init__.py -------------------------------------------------------------------------------- /bothub/event_driven/management/commands/edaconsume.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.management.base import BaseCommand 3 | from django.utils.module_loading import import_string 4 | 5 | 6 | handle_consumers_function = import_string(settings.EDA_CONSUMERS_HANDLE) 7 | connection_backend = import_string(settings.EDA_CONNECTION_BACKEND)(handle_consumers_function) 8 | 9 | 10 | class Command(BaseCommand): 11 | def handle(self, *args, **options): 12 | connection_backend.start_consuming() 13 | -------------------------------------------------------------------------------- /bothub/event_driven/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/event_driven/migrations/__init__.py -------------------------------------------------------------------------------- /bothub/event_driven/parsers/__init__.py: -------------------------------------------------------------------------------- 1 | from .json_parser import JSONParser # noqa: F401 2 | -------------------------------------------------------------------------------- /bothub/event_driven/parsers/base_parser.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractstaticmethod 2 | 3 | 4 | class BaseParser(ABC): # pragma: no cover 5 | @abstractstaticmethod 6 | def parse(stream, encoding=None): 7 | pass 8 | -------------------------------------------------------------------------------- /bothub/event_driven/parsers/exceptions.py: -------------------------------------------------------------------------------- 1 | class ParseError(Exception): 2 | pass 3 | -------------------------------------------------------------------------------- /bothub/event_driven/parsers/json_parser.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from .base_parser import BaseParser 4 | from .exceptions import ParseError 5 | 6 | 7 | class JSONParser(BaseParser): 8 | @staticmethod 9 | def parse(stream, encoding="utf-8"): 10 | """ 11 | Parses the incoming bytestream as JSON and returns the resulting data. 12 | """ 13 | 14 | if not stream: 15 | raise ParseError("JSON parse error - stream cannot be empty") 16 | 17 | try: 18 | decoded_stream = stream.decode(encoding) 19 | return json.loads(decoded_stream) 20 | except ValueError as exc: 21 | raise ParseError("JSON parse error - %s" % str(exc)) 22 | -------------------------------------------------------------------------------- /bothub/event_driven/publisher/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/event_driven/publisher/__init__.py -------------------------------------------------------------------------------- /bothub/event_driven/publisher/rabbitmq_publisher.py: -------------------------------------------------------------------------------- 1 | import amqp 2 | import json 3 | from time import sleep 4 | 5 | from typing import Dict 6 | 7 | from django.conf import settings 8 | 9 | from bothub.event_driven.connection.rabbitmq_connection import RabbitMQConnection 10 | 11 | class RabbitMQPublisher: 12 | 13 | def __init__(self) -> None: 14 | self.rabbitmq_connection = RabbitMQConnection() 15 | 16 | def send_message(self, body: Dict, exchange: str, routing_key: str): 17 | sended = False 18 | while not sended: 19 | try: 20 | self.rabbitmq_connection.channel.basic_publish( 21 | exchange=exchange, 22 | routing_key=routing_key, 23 | msg=amqp.Message( 24 | body=json.dumps(body).encode(), 25 | properties={"delivery_mode": 2}, 26 | content_type="application/octet-stream", 27 | ) 28 | ) 29 | sended = True 30 | except Exception as err: 31 | print(f"error: {err}") 32 | self.rabbitmq_connection._establish_connection() 33 | sleep(settings.EDA_WAIT_TIME_RETRY) -------------------------------------------------------------------------------- /bothub/event_driven/signals.py: -------------------------------------------------------------------------------- 1 | from django.dispatch import Signal 2 | from django.db import reset_queries, close_old_connections 3 | 4 | 5 | message_started = Signal() 6 | message_finished = Signal() 7 | 8 | # db connection state managed similarly to the wsgi handler 9 | message_started.connect(reset_queries) 10 | message_started.connect(close_old_connections) 11 | message_finished.connect(close_old_connections) 12 | -------------------------------------------------------------------------------- /bothub/health/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/health/__init__.py -------------------------------------------------------------------------------- /bothub/health/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class HealthConfig(AppConfig): 5 | name = "health" 6 | -------------------------------------------------------------------------------- /bothub/health/checks.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from ..settings import env 4 | from rest_framework import status 5 | 6 | 7 | logger = logging.getLogger("bothub.health.checks") 8 | 9 | CHECK_ACCESSIBLE_API_URL = env.str("CHECK_ACCESSIBLE_API_URL") 10 | 11 | 12 | def check_database_connection(**kwargs): 13 | from django.db import connections 14 | from django.db.utils import OperationalError 15 | 16 | if len(connections.all()) == 0: 17 | return False 18 | logger.info("found {} database connection".format(len(connections.all()))) 19 | for i, conn in enumerate(connections.all(), 1): 20 | try: 21 | conn.cursor() 22 | logger.info("#{} db connection OKAY".format(i)) 23 | except OperationalError: 24 | logger.warning("#{} db connection ERROR".format(i)) 25 | return False 26 | return True 27 | 28 | 29 | def check_accessible_api(request, **kwargs): 30 | import requests 31 | 32 | if CHECK_ACCESSIBLE_API_URL: 33 | logger.info("requesting {}".format(CHECK_ACCESSIBLE_API_URL)) 34 | response = requests.get(CHECK_ACCESSIBLE_API_URL) 35 | else: 36 | url = "http://{}/200/".format(request.META.get("HTTP_HOST")) 37 | logger.info("requesting to {}".format(url)) 38 | response = requests.get(url) 39 | logger.info( 40 | "{} response status code {}".format( 41 | CHECK_ACCESSIBLE_API_URL, response.status_code 42 | ) 43 | ) 44 | if response.status_code is status.HTTP_200_OK: 45 | return True 46 | return False 47 | -------------------------------------------------------------------------------- /bothub/health/views.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | from functools import reduce 3 | 4 | from django.http import HttpResponse 5 | from rest_framework import status 6 | 7 | from .checks import check_database_connection 8 | from .checks import check_accessible_api 9 | 10 | CHECKS = [check_database_connection, check_accessible_api] 11 | 12 | 13 | def ping(request): 14 | checks_status = OrderedDict( 15 | map(lambda check: (check.__name__, check(request=request)), CHECKS) 16 | ) 17 | healthy = reduce( 18 | lambda current, status: current and status, checks_status.values(), True 19 | ) 20 | content = "{}\n{}".format( 21 | "OK" if healthy else "something wrong happened", 22 | "\n".join(map(lambda x: "{}: {}".format(*x), checks_status.items())), 23 | ) 24 | status_code = status.HTTP_200_OK if healthy else status.HTTP_503_SERVICE_UNAVAILABLE 25 | return HttpResponse(content=content, content_type="text/plain", status=status_code) 26 | 27 | 28 | def r200(request): 29 | return HttpResponse() 30 | -------------------------------------------------------------------------------- /bothub/locale/pt_BR/LC_MESSAGES/django.mo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/locale/pt_BR/LC_MESSAGES/django.mo -------------------------------------------------------------------------------- /bothub/project/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/project/__init__.py -------------------------------------------------------------------------------- /bothub/project/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | class ProjectConfig(AppConfig): 4 | name = 'bothub.project' 5 | 6 | def ready(self): 7 | from bothub.project.signals import integrate_ai # noqa: F401 8 | -------------------------------------------------------------------------------- /bothub/project/consumers/project_consumer.py: -------------------------------------------------------------------------------- 1 | import amqp 2 | from sentry_sdk import capture_exception 3 | 4 | from ..usecases.project.projectdto import ProjectCreationDTO 5 | from ..usecases.project.creation import ProjectCreationUseCase 6 | from ..usecases.organization.creation import OrganizationAuthorizationCreateUsecase 7 | 8 | from bothub.event_driven.parsers import JSONParser 9 | from bothub.event_driven.consumer.consumers import EDAConsumer 10 | 11 | 12 | class ProjectConsumer(EDAConsumer): # pragma: no cover 13 | def consume(self, message: amqp.Message): 14 | print(f"[ProjectConsumer] - Consuming a message. Body: {message.body}") 15 | 16 | try: 17 | body = JSONParser.parse(message.body) 18 | 19 | project_dto = ProjectCreationDTO( 20 | uuid=body.get("uuid"), 21 | name=body.get("name"), 22 | is_template=body.get("is_template"), 23 | date_format=body.get("date_format"), 24 | template_type_uuid=body.get("template_type_uuid"), 25 | timezone=body.get("timezone"), 26 | organization_id=body.get("organization_id") 27 | ) 28 | 29 | project_creation = ProjectCreationUseCase() 30 | project_creation.create_project(project_dto, body.get("user_email")) 31 | auth_creation = OrganizationAuthorizationCreateUsecase() 32 | auth_creation.eda_consume_organization_authorization( 33 | authorizations=body.get("authorizations"), 34 | organization_id=body.get("organization_id") 35 | ) 36 | 37 | message.channel.basic_ack(message.delivery_tag) 38 | 39 | except Exception as exception: 40 | capture_exception(exception) 41 | message.channel.basic_reject(message.delivery_tag, requeue=False) 42 | print(f"[ProjectConsumer] - Message rejected by: {exception}") 43 | -------------------------------------------------------------------------------- /bothub/project/consumers/template_type_consumer.py: -------------------------------------------------------------------------------- 1 | import amqp 2 | from sentry_sdk import capture_exception 3 | 4 | from bothub.event_driven.parsers import JSONParser 5 | from bothub.event_driven.consumer.consumers import EDAConsumer 6 | from ..usecases.template_type.creation import TemplateTypeCreationUseCase 7 | from ..usecases.template_type.template_type_dto import TemplateTypeDTO 8 | 9 | 10 | class TemplateTypeConsumer(EDAConsumer): # pragma: no cover 11 | def consume(self, message: amqp.Message): 12 | print(f"[TemplateTypeConsumer] - Consuming a message. Body: {message.body}") 13 | try: 14 | body = JSONParser.parse(message.body) 15 | 16 | template_type_dto = TemplateTypeDTO( 17 | uuid=body.get("uuid"), 18 | name=body.get("name"), 19 | project_uuid=body.get("project_uuid") 20 | ) 21 | 22 | template_type_creation = TemplateTypeCreationUseCase() 23 | template_type_creation.create_template_type(template_type_dto) 24 | 25 | message.channel.basic_ack(message.delivery_tag) 26 | 27 | except Exception as exception: 28 | capture_exception(exception) 29 | message.channel.basic_reject(message.delivery_tag, requeue=False) 30 | print(f"[TemplateTypeConsumer] - Message rejected by: {exception}") 31 | -------------------------------------------------------------------------------- /bothub/project/handle.py: -------------------------------------------------------------------------------- 1 | from amqp.channel import Channel 2 | 3 | from .consumers.project_consumer import ProjectConsumer 4 | from .consumers.template_type_consumer import TemplateTypeConsumer 5 | 6 | 7 | 8 | def handle_consumers(channel: Channel) -> None: 9 | channel.basic_consume("artificial-intelligence.projects", callback=ProjectConsumer().handle) 10 | channel.basic_consume("artificial-intelligence.template-types", callback=TemplateTypeConsumer().handle) 11 | -------------------------------------------------------------------------------- /bothub/project/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.15 on 2023-06-17 03:02 2 | 3 | from django.conf import settings 4 | from django.db import migrations, models 5 | import django.db.models.deletion 6 | import timezone_field.fields 7 | import uuid 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | initial = True 13 | 14 | dependencies = [ 15 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 16 | ('common', '0118_alter_zeroshotoptions_option_uuid'), 17 | ] 18 | 19 | operations = [ 20 | migrations.CreateModel( 21 | name='Project', 22 | fields=[ 23 | ('created_at', models.DateTimeField(auto_now_add=True, verbose_name='created at')), 24 | ('date_format', models.CharField(max_length=1, verbose_name='Date Format')), 25 | ('is_template', models.BooleanField(default=False)), 26 | ('is_active', models.BooleanField(default=True)), 27 | ('name', models.TextField(verbose_name='name')), 28 | ('timezone', timezone_field.fields.TimeZoneField(verbose_name='Timezone')), 29 | ('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='UUID')), 30 | ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='projects', to=settings.AUTH_USER_MODEL)), 31 | ('organization', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='projects', to='common.organization')), 32 | ], 33 | ), 34 | ] 35 | -------------------------------------------------------------------------------- /bothub/project/migrations/0002_projectintelligence.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.20 on 2023-08-29 16:57 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | import uuid 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('common', '0119_repository_count_repository_train'), 12 | ('project', '0001_initial'), 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='ProjectIntelligence', 18 | fields=[ 19 | ('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='UUID')), 20 | ('access_token', models.CharField(blank=True, max_length=255, null=True, verbose_name='Access token')), 21 | ('name', models.TextField(verbose_name='name')), 22 | ('integrated_at', models.DateTimeField(auto_now_add=True, verbose_name='created at')), 23 | ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='intelligences', to='project.project')), 24 | ('repositories', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='project_intelligence', to='common.repository')), 25 | ], 26 | ), 27 | ] 28 | -------------------------------------------------------------------------------- /bothub/project/migrations/0003_auto_20230830_1200.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.20 on 2023-08-30 12:00 2 | 3 | from django.db import migrations, models 4 | import django.db.models.deletion 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('common', '0119_repository_count_repository_train'), 11 | ('project', '0002_projectintelligence'), 12 | ] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name='TemplateType', 17 | fields=[ 18 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 19 | ('uuid', models.UUIDField(blank=True, null=True, verbose_name='UUID')), 20 | ('name', models.CharField(max_length=255)), 21 | ('setup', models.JSONField(default=dict)), 22 | ], 23 | ), 24 | migrations.RemoveField( 25 | model_name='projectintelligence', 26 | name='repositories', 27 | ), 28 | migrations.AddField( 29 | model_name='projectintelligence', 30 | name='repository', 31 | field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='project_intelligences', to='common.repository'), 32 | ), 33 | migrations.AddField( 34 | model_name='project', 35 | name='template_type', 36 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='template_type', to='project.templatetype'), 37 | ), 38 | ] 39 | -------------------------------------------------------------------------------- /bothub/project/migrations/0004_projectintelligence_integrated_by.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.20 on 2023-09-06 19:22 2 | 3 | from django.conf import settings 4 | from django.db import migrations, models 5 | import django.db.models.deletion 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 12 | ('project', '0003_auto_20230830_1200'), 13 | ] 14 | 15 | operations = [ 16 | migrations.AddField( 17 | model_name='projectintelligence', 18 | name='integrated_by', 19 | field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='project_intelligences', to=settings.AUTH_USER_MODEL), 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /bothub/project/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/project/migrations/__init__.py -------------------------------------------------------------------------------- /bothub/project/serializers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | 3 | from bothub.project.models import Project 4 | 5 | 6 | class ProjectSerializer(serializers.ModelSerializer): 7 | class Meta: 8 | model = Project 9 | fields = [ 10 | "is_template", 11 | "name", 12 | "uuid", 13 | "timezone", 14 | ] 15 | ref_name = None 16 | 17 | timezone = serializers.CharField() 18 | -------------------------------------------------------------------------------- /bothub/project/signals.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.db.models.signals import post_save, post_delete 3 | from django.dispatch import receiver 4 | 5 | from bothub.project.models import ProjectIntelligence 6 | from bothub.event_driven.publisher.rabbitmq_publisher import RabbitMQPublisher 7 | 8 | @receiver(post_save, sender=ProjectIntelligence) 9 | def integrate_ai(sender, instance, created, **kwargs): 10 | 11 | if created: 12 | rabbitmq_publisher = RabbitMQPublisher() 13 | body = { 14 | "uuid": str(instance.uuid), 15 | "access_token": str(instance.access_token), 16 | "name": instance.name, 17 | "repository": str(instance.repository.uuid), 18 | "project_uuid": str(instance.project.uuid), 19 | "user_email": instance.integrated_by.email, 20 | } 21 | rabbitmq_publisher.send_message(body=body, exchange="intelligences.topic", routing_key="") 22 | print(f"[ intelligencePublisher ] message sent - {body} to intelligences.topic") 23 | -------------------------------------------------------------------------------- /bothub/project/usecases/exceptions.py: -------------------------------------------------------------------------------- 1 | class InvalidProjectData(Exception): 2 | pass 3 | 4 | 5 | class InvalidTemplateTypeData(Exception): 6 | pass 7 | 8 | 9 | class InvalidOrganizationData(Exception): 10 | pass 11 | -------------------------------------------------------------------------------- /bothub/project/usecases/organization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/project/usecases/organization/__init__.py -------------------------------------------------------------------------------- /bothub/project/usecases/organization/creation.py: -------------------------------------------------------------------------------- 1 | from bothub.common.models import Organization, OrganizationAuthorization 2 | from bothub.authentication.models import User 3 | from bothub.project.usecases.exceptions import InvalidOrganizationData 4 | 5 | 6 | class OrganizationAuthorizationCreateUsecase: 7 | 8 | def get_or_create_user_by_email(self, email: str) -> tuple: 9 | return User.objects.get_or_create(email=email) 10 | 11 | def get_or_create_organization_authorization( 12 | self, 13 | organization: Organization, 14 | user: User, 15 | role: int 16 | ) -> OrganizationAuthorization: 17 | 18 | return OrganizationAuthorization.objects.get_or_create( 19 | organization=organization, 20 | user=user, 21 | role=role 22 | ) 23 | 24 | def get_organization_by_id(self, organization_id) -> Organization: 25 | try: 26 | return Organization.objects.get(pk=organization_id) 27 | except Organization.DoesNotExist: 28 | raise InvalidOrganizationData(f"Organization {organization_id} does not exists!") 29 | 30 | def eda_consume_organization_authorization( 31 | self, 32 | authorizations: list, 33 | organization_id: int, 34 | ): 35 | 36 | organization = self.get_organization_by_id(organization_id) 37 | 38 | for authorization in authorizations: 39 | user, _ = self.get_or_create_user_by_email(authorization.get("user_email")) 40 | self.get_or_create_organization_authorization( 41 | organization=organization, 42 | user=user, 43 | role=authorization.get("role") 44 | ) 45 | -------------------------------------------------------------------------------- /bothub/project/usecases/project/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weni-ai/bothub-engine/00f223dba93eb6b0877abe48963b2d7041cbcc05/bothub/project/usecases/project/__init__.py -------------------------------------------------------------------------------- /bothub/project/usecases/project/projectdto.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | 4 | @dataclass 5 | class ProjectCreationDTO: 6 | uuid: str 7 | name: str 8 | is_template: bool 9 | date_format: str 10 | template_type_uuid: str 11 | timezone: str 12 | organization_id: int 13 | -------------------------------------------------------------------------------- /bothub/project/usecases/template_type/creation.py: -------------------------------------------------------------------------------- 1 | from .template_type_dto import TemplateTypeDTO 2 | from bothub.project.models import ProjectIntelligence, TemplateType, Project 3 | from bothub.project.usecases.exceptions import InvalidTemplateTypeData 4 | 5 | 6 | class TemplateTypeCreationUseCase: 7 | 8 | def get_repository_info_by_project(self, project): 9 | 10 | info = {"repositories": []} 11 | p_intelligence_queryset = ProjectIntelligence.objects.filter(project=project) 12 | for project_intelligence in p_intelligence_queryset: 13 | info["repositories"].append( 14 | { 15 | "uuid": str(project_intelligence.repository.uuid) 16 | } 17 | ) 18 | return info 19 | 20 | 21 | def create_template_type(self, template_type_dto: TemplateTypeDTO): 22 | try: 23 | project = Project.objects.get(uuid=template_type_dto.project_uuid) 24 | except Project.DoesNotExist: 25 | raise InvalidTemplateTypeData(f"Project `{template_type_dto.project_uuid}` does not exists!") 26 | setup = self.get_repository_info_by_project(project) 27 | template_type, created = TemplateType.objects.get_or_create(uuid=template_type_dto.uuid, defaults=dict(name=template_type_dto.name, setup=setup)) 28 | if not created: 29 | template_type.setup = setup 30 | template_type.name = template_type_dto.name 31 | template_type.save() 32 | return template_type 33 | -------------------------------------------------------------------------------- /bothub/project/usecases/template_type/template_type_dto.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | 4 | @dataclass 5 | class TemplateTypeDTO: 6 | uuid: str 7 | name: str 8 | project_uuid: str 9 | -------------------------------------------------------------------------------- /bothub/translate.py: -------------------------------------------------------------------------------- 1 | import time 2 | import requests 3 | from django.conf import settings 4 | 5 | # enforce quotas (https://cloud.google.com/translate/quotas) (very naive implementation) 6 | 7 | quota_char = 0 8 | quota_limit = 100000 9 | quota_wait = 100 10 | 11 | 12 | def translate(text, source_lang, target_language): 13 | global quota_char 14 | # dont translate source language 15 | if target_language == source_lang: 16 | return text 17 | 18 | data = { 19 | "q": text, 20 | "target": target_language, 21 | "format": "text", 22 | "source": source_lang, 23 | "model": "nmt", 24 | } 25 | 26 | headers = {"Content-Type": "application/json; charset: utf-8"} 27 | 28 | if not settings.GOOGLE_API_TRANSLATION_KEY: 29 | raise Exception("GOOGLE_API_TRANSLATION_KEY credential has not been set") 30 | 31 | URL = f"https://translation.googleapis.com/language/translate/v2?key={settings.GOOGLE_API_TRANSLATION_KEY}" 32 | 33 | quota_char += len(str(data)) 34 | if quota_char >= quota_limit: 35 | print("Would hit rate limit - waiting %s seconds" % (quota_wait + 5)) 36 | time.sleep(quota_wait + 5) 37 | print("Resuming after rate limit") 38 | quota_char = 0 39 | 40 | req = requests.post(URL, headers=headers, json=data) 41 | response = req.json() 42 | 43 | if ( 44 | "error" in response 45 | and "code" in response["error"] 46 | and response["error"]["code"] == 403 47 | ): 48 | print("Rate limit hit - waiting %s seconds" % (quota_wait + 5)) 49 | time.sleep(quota_wait + 5) 50 | quota_char = 0 51 | print("Resuming after rate limit") 52 | return translate(text, source_lang, target_language) 53 | 54 | return response.get("data").get("translations")[0].get("translatedText") 55 | -------------------------------------------------------------------------------- /bothub/wsgi.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from django.core.wsgi import get_wsgi_application 4 | 5 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "bothub.settings") 6 | 7 | application = get_wsgi_application() 8 | -------------------------------------------------------------------------------- /docker-compose.dev.yml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | 3 | services: 4 | bothub: 5 | volumes: 6 | - ./bothub:/home/app/bothub -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | cd $WORKDIR 3 | python manage.py collectstatic --noinput 4 | 5 | if [ "$RUN_AS_DEVELOPMENT_MODE" = true -o "$RUN_AS_DEVELOPMENT_MODE" = "True" ] 6 | then 7 | python3 manage.py runserver 0.0.0.0:80 8 | else 9 | gunicorn bothub.wsgi --timeout 999999 -c gunicorn.conf.py 10 | fi -------------------------------------------------------------------------------- /gunicorn.conf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import multiprocessing 3 | 4 | bind = "0.0.0.0:80" 5 | workers = os.environ.get("GUNICORN_WORKERS", multiprocessing.cpu_count() * 2 + 1) 6 | worker_class = "gevent" 7 | raw_env = ["DJANGO_SETTINGS_MODULE=bothub.settings"] 8 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "bothub.settings") 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError as exc: 10 | raise ImportError( 11 | "Couldn't import Django. Are you sure it's installed and " 12 | "available on your PYTHONPATH environment variable? Did you " 13 | "forget to activate a virtual environment?" 14 | ) from exc 15 | execute_from_command_line(sys.argv) 16 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | exclude = ''' 3 | /( 4 | | migrations 5 | )/ 6 | ''' 7 | 8 | [tool.poetry] 9 | name = "weni-ai" 10 | version = "2.1.29" 11 | description = "weni-ai: weni-ai-engine" 12 | authors = ["weni.ai"] 13 | license = "MPL-2.0" 14 | 15 | [tool.poetry.dependencies] 16 | python = "~=3.8.13" 17 | django = "~=3.2.6" 18 | djangorestframework = "~=3.12.4" 19 | django-filter = "~=2.4.0" 20 | django-cors-headers = "~=3.7.0" 21 | requests = "~=2.23.0" 22 | coreapi = "~=2.3.3" 23 | whitenoise = "~=5.3.0" 24 | pytz = "~=2020.1" 25 | drf-yasg2 = "~=1.19.4" 26 | gunicorn = "~=19.9.0" 27 | gevent = "22.10.2" 28 | packaging = "~=20.4" 29 | boto3 = "~=1.35.8" 30 | sentry-sdk = "~=1.3.1" 31 | redis = "*" 32 | celery = "~=5.1.2" 33 | django-celery-beat = "~=2.2.1" 34 | django-celery-results = "~=2.2.0" 35 | python-slugify = "~=4.0.0" 36 | openpyxl = "~=3.0.3" 37 | kombu = "~=5.1.0" 38 | matplot = "~=0.1.9" 39 | django-redis = "~=4.12.1" 40 | elastic-apm = "~=6.3.3" 41 | mozilla-django-oidc = "~=2.0.0" 42 | djangorestframework-recaptcha = "~=0.2.0" 43 | djangogrpcframework = "~=0.2.1" 44 | grpcio = "~=1.39.0" 45 | grpcio-tools = "~=1.39.0" 46 | django-elasticsearch-dsl = "~=7.2.0" 47 | django-elasticsearch-dsl-drf = "~=0.22.1" 48 | elasticsearch = "<7.14.0" 49 | elasticsearch-dsl = "~=7.4.0" 50 | psycopg2-binary = "~=2.9.1" 51 | weni-protobuffers = "~=1.2.1" 52 | black = "21.7b0" 53 | Pillow = "~=8.4.0" 54 | django-csp = "^3.7" 55 | django-environ = "^0.9.0" 56 | transformers = "^4.33.2" 57 | 58 | [tool.poetry.dev-dependencies] 59 | flake8 = "~=4.0.0" 60 | requests-mock = "~=1.8.0" 61 | coverage = "~=5.5" 62 | ipython = "*" 63 | black = "==21.7b0" 64 | autopep8 = "^1.6.0" 65 | pre-commit = "^2.20.0" 66 | 67 | [build-system] 68 | requires = ["poetry-core>=1.0.0"] 69 | build-backend = "poetry.core.masonry.api" 70 | -------------------------------------------------------------------------------- /runtime.txt: -------------------------------------------------------------------------------- 1 | python-3.8.13 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal=1 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | 4 | setup( 5 | name="bothub-engine", 6 | version="1.19.3", 7 | description="Bothub Engine", 8 | packages=find_packages(), 9 | install_requires=[ 10 | "django==2.1.11", 11 | "django-environ==0.4.5", 12 | "djangorestframework==3.9.1", 13 | "django-filter==2.0.0", 14 | "django-cors-headers==2.4.0", 15 | "requests==2.20.1", 16 | "coreapi==2.3.3", 17 | "whitenoise==4.1.2", 18 | "pytz==2018.7", 19 | ], 20 | python_requires=">=3.4", 21 | ) 22 | --------------------------------------------------------------------------------