├── .agignore ├── .circleci └── config.yml ├── .coveragerc ├── .editorconfig ├── .env.example ├── .gitattributes ├── .github ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── CHANGELOG.md ├── Dockerfile ├── Jenkinsfile ├── LICENSE ├── Makefile ├── Procfile ├── README.md ├── config └── gunicorn_config.py ├── database ├── migration │ ├── README │ ├── alembic.ini │ ├── env.py │ ├── script.py.mako │ └── versions │ │ └── .gitkeep └── mysql.sql ├── docker-compose.yml ├── docs ├── .gitignore ├── Makefile ├── _static │ └── .gitignore ├── api │ ├── advance.rst │ ├── application.rst │ ├── audit.rst │ ├── cluster.rst │ ├── index.rst │ ├── infra.rst │ ├── instance.rst │ ├── subscription.rst │ ├── support.rst │ ├── team.rst │ ├── token.rst │ ├── user.rst │ └── webhook.rst ├── assets │ └── index.rst ├── conf.py ├── contact │ └── index.rst ├── design │ └── index.rst ├── dev │ └── index.rst ├── index.rst ├── intro │ └── index.rst └── ops │ ├── deployment.rst │ ├── environment.rst │ └── index.rst ├── huskar_api ├── __init__.py ├── api │ ├── __init__.py │ ├── audit.py │ ├── auth.py │ ├── health_check.py │ ├── infra_config.py │ ├── instance.py │ ├── long_polling.py │ ├── middlewares │ │ ├── __init__.py │ │ ├── auth.py │ │ ├── concurrent_limit.py │ │ ├── control_access_via_api.py │ │ ├── db.py │ │ ├── error.py │ │ ├── logger.py │ │ ├── rate_limit_ip.py │ │ ├── rate_limit_user.py │ │ ├── read_only.py │ │ └── route.py │ ├── organization.py │ ├── schema │ │ ├── __init__.py │ │ ├── audit.py │ │ ├── infra.py │ │ ├── input.py │ │ ├── instance.py │ │ ├── organization.py │ │ ├── service.py │ │ ├── user.py │ │ ├── validates.py │ │ └── webhook.py │ ├── service_info.py │ ├── service_instance.py │ ├── service_link.py │ ├── service_route.py │ ├── support.py │ ├── user.py │ ├── utils.py │ ├── webhook.py │ └── well_known.py ├── app.py ├── bootstrap │ ├── __init__.py │ ├── consts.py │ └── core.py ├── cli.py ├── contrib │ ├── __init__.py │ ├── backdoor.py │ └── signal.py ├── ext.py ├── extras │ ├── __init__.py │ ├── auth.py │ ├── concurrent_limiter.py │ ├── email.py │ ├── mail_client │ │ ├── __init__.py │ │ └── abstract_client.py │ ├── marshmallow.py │ ├── monitor.py │ ├── payload.py │ ├── rate_limiter.py │ ├── raven.py │ ├── uptime.py │ └── utils.py ├── models │ ├── __init__.py │ ├── alembic.py │ ├── audit │ │ ├── __init__.py │ │ ├── action.py │ │ ├── audit.py │ │ ├── const.py │ │ ├── index.py │ │ └── rollback.py │ ├── auth │ │ ├── __init__.py │ │ ├── application.py │ │ ├── role.py │ │ ├── session.py │ │ ├── team.py │ │ └── user.py │ ├── cache │ │ ├── __init__.py │ │ ├── client.py │ │ ├── hook.py │ │ └── region.py │ ├── catalog │ │ ├── __init__.py │ │ ├── default_route.py │ │ ├── dependency.py │ │ ├── info.py │ │ ├── route.py │ │ └── schema.py │ ├── comment.py │ ├── const.py │ ├── container │ │ ├── __init__.py │ │ ├── common.py │ │ └── management.py │ ├── dataware │ │ ├── __init__.py │ │ └── zookeeper │ │ │ ├── __init__.py │ │ │ └── client.py │ ├── db.py │ ├── exceptions.py │ ├── infra │ │ ├── __init__.py │ │ ├── downstream.py │ │ └── utils.py │ ├── instance │ │ ├── __init__.py │ │ ├── management.py │ │ └── schema.py │ ├── manifest.py │ ├── route │ │ ├── __init__.py │ │ ├── hijack.py │ │ ├── hijack_stage.py │ │ ├── management.py │ │ ├── resolver.py │ │ └── utils.py │ ├── signals.py │ ├── tree │ │ ├── __init__.py │ │ ├── cleaner.py │ │ ├── common.py │ │ ├── extra.py │ │ ├── holder.py │ │ ├── hub.py │ │ └── watcher.py │ ├── utils.py │ ├── webhook │ │ ├── __init__.py │ │ ├── notify.py │ │ └── webhook.py │ └── znode.py ├── scripts │ ├── __init__.py │ ├── db.py │ ├── infra.py │ └── vacuum.py ├── service │ ├── __init__.py │ ├── admin │ │ ├── __init__.py │ │ ├── application_auth.py │ │ ├── exc.py │ │ └── user.py │ ├── comment.py │ ├── config.py │ ├── data.py │ ├── exc.py │ ├── organization │ │ ├── __init__.py │ │ └── exc.py │ ├── service.py │ ├── switch.py │ └── utils.py ├── settings.py ├── switch.py ├── templates │ ├── .editorconfig │ ├── docs-assets-index.rst │ ├── email-base-components.html │ ├── email-base-layout.html │ ├── email-debug.html │ ├── email-infra-config-create.html │ ├── email-password-reset.html │ ├── email-permission-dismiss.html │ ├── email-permission-grant.html │ └── email-signup.html └── wsgi.py ├── jenkins.dockerfile ├── manage.sh ├── pytest.ini ├── requirements-dev.in ├── requirements-dev.txt ├── requirements.in ├── requirements.txt ├── run.py ├── service_check.py ├── tests ├── __init__.py ├── conftest.py ├── test_api │ ├── __init__.py │ ├── conftest.py │ ├── test_application.py │ ├── test_application.yaml │ ├── test_application_auth.py │ ├── test_application_auth.yaml │ ├── test_audit_rollback.py │ ├── test_audit_rollback.yaml │ ├── test_audit_view.py │ ├── test_audit_view.yaml │ ├── test_config_and_switch_read_only.py │ ├── test_config_and_switch_read_only.yaml │ ├── test_db_tester.py │ ├── test_graceful_startup.py │ ├── test_health_check.py │ ├── test_http_concurrent_limit.py │ ├── test_http_concurrent_limit.yaml │ ├── test_http_control_access_via_api.py │ ├── test_http_control_access_via_api.yaml │ ├── test_http_error.py │ ├── test_http_log.py │ ├── test_http_rate_limit.py │ ├── test_http_rate_limit.yaml │ ├── test_huskar_admin.py │ ├── test_infra_config.py │ ├── test_infra_downstream.py │ ├── test_instance.py │ ├── test_instance.yaml │ ├── test_long_polling.py │ ├── test_long_polling.yaml │ ├── test_service.py │ ├── test_service.yaml │ ├── test_service_info.py │ ├── test_service_info.yaml │ ├── test_service_link.py │ ├── test_service_link.yaml │ ├── test_service_route.py │ ├── test_service_route.yaml │ ├── test_session_auth.py │ ├── test_support_blacklist.py │ ├── test_support_container.py │ ├── test_support_container.yaml │ ├── test_support_route_program.py │ ├── test_support_whomami.py │ ├── test_team.py │ ├── test_team_audit.py │ ├── test_token.py │ ├── test_user.py │ ├── test_user.yaml │ ├── test_utils.py │ ├── test_validation.py │ ├── test_validation.yaml │ ├── test_webhook.py │ ├── test_webhook.yaml │ └── test_wellkown.py ├── test_bootstrap.py ├── test_cli.py ├── test_ext.py ├── test_extras │ ├── __init__.py │ ├── test_auth.py │ ├── test_email.py │ ├── test_email_snapshots │ │ ├── email-debug-0.html │ │ ├── email-infra-config-create-0.html │ │ ├── email-infra-config-create-1.html │ │ ├── email-infra-config-create-2.html │ │ ├── email-infra-config-create-3.html │ │ ├── email-infra-config-create-4.html │ │ ├── email-infra-config-create-5.html │ │ ├── email-infra-config-create-6.html │ │ ├── email-password-reset-0.html │ │ ├── email-permission-dismiss-0.html │ │ ├── email-permission-grant-0.html │ │ └── email-signup-0.html │ ├── test_monitor.py │ ├── test_raven.py │ ├── test_uptime.py │ └── test_utils.py ├── test_models │ ├── __init__.py │ ├── conftest.py │ ├── test_alembic.py │ ├── test_audit │ │ ├── __init__.py │ │ ├── test_action.py │ │ ├── test_audit.py │ │ └── test_index.py │ ├── test_auth │ │ ├── __init__.py │ │ ├── test_application.py │ │ ├── test_session.py │ │ ├── test_team.py │ │ ├── test_user.py │ │ └── test_user.yaml │ ├── test_cache.py │ ├── test_catalog.py │ ├── test_comment.py │ ├── test_container.py │ ├── test_container.yaml │ ├── test_db.py │ ├── test_extras │ │ └── __init__.py │ ├── test_infra │ │ ├── __init__.py │ │ ├── test_downstream.py │ │ └── test_utils.py │ ├── test_instance.py │ ├── test_instance.yaml │ ├── test_route.py │ ├── test_route_hijack.py │ ├── test_route_hijack.yaml │ ├── test_tree │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_cleaner.py │ │ ├── test_common.py │ │ ├── test_extra.py │ │ ├── test_holder.py │ │ └── test_watcher.py │ ├── test_utils.py │ ├── test_utils.yaml │ ├── test_webhook │ │ ├── __init__.py │ │ ├── test_notify.py │ │ └── test_webhook.py │ └── test_znode.py ├── test_scripts │ ├── __init__.py │ ├── test_db.py │ ├── test_infra.py │ ├── test_vacuum.py │ └── test_vacuum.yaml ├── test_wsgi.py └── utils.py └── tools ├── ci └── run.sh ├── git-hooks ├── pre-commit └── pre-push ├── huskar-lint └── huskar_lint.py ├── pylint ├── py2pytest.py └── pylintrc └── zookeeper-lint ├── testteam_shredder.py └── zookeeper_lint.py /.agignore: -------------------------------------------------------------------------------- 1 | database/migration/versions 2 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Python CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-python/ for more details 4 | # 5 | version: 2 6 | jobs: 7 | build: 8 | machine: 9 | image: ubuntu-1604:201903-01 10 | 11 | working_directory: ~/repo 12 | 13 | steps: 14 | - checkout 15 | 16 | - run: 17 | name: install dependencies 18 | command: | 19 | set -x 20 | 21 | pip install --user codecov 22 | cp .env.example .env 23 | 24 | docker-compose pull 25 | docker-compose build 26 | 27 | # run tests! 28 | - run: 29 | name: run tests 30 | command: | 31 | set -x 32 | 33 | docker-compose up -d 34 | docker-compose ps 35 | 36 | docker-compose run --rm wsgi tools/ci/run.sh initdb 37 | docker-compose run --rm wsgi tools/ci/run.sh test -v --cov=huskar_api tests/ 38 | 39 | python -m codecov 40 | 41 | - store_artifacts: 42 | path: test-reports 43 | destination: test-reports 44 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | concurrency = gevent 3 | branch = True 4 | 5 | [report] 6 | omit = 7 | huskar_api/contrib/* 8 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | end_of_line = lf 8 | insert_final_newline = true 9 | 10 | [*.py] 11 | indent_style = space 12 | indent_size = 4 13 | 14 | [*.{yml,yaml,json}] 15 | indent_style = space 16 | indent_size = 2 17 | 18 | [Makefile] 19 | indent_style = tab 20 | indent_size = 4 21 | 22 | [Dockerfile,*.dockerfile] 23 | indent_style = space 24 | indent_size = 4 25 | 26 | [Vagrantfile] 27 | indent_style = space 28 | indent_size = 2 29 | language = ruby 30 | 31 | [Jenkinsfile] 32 | indent_style = space 33 | indent_size = 2 34 | language = groovy 35 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | HUSKAR_API_DEBUG=true 2 | HUSKAR_API_TESTING=true 3 | HUSKAR_API_SECRET_KEY=foobar 4 | HUSKAR_API_ZK_SERVERS=127.0.0.1:2181,127.0.0.2:2181 5 | DATABASE_DEFAULT_URL=mysql+pymysql://root@localhost:3306/huskar?charset=utf8mb4 6 | REDIS_DEFAULT_URL=redis://localhost:6379 7 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | tests/test_extras/test_email_snapshots/*.html linguist-generated=true 2 | docs/assets/index.rst linguist-generated=true 3 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing Guide 2 | 3 | 1. Keep your branch up-to-date 4 | - `git fetch upstream` 5 | - `git rebase upstream/master` 6 | 2. Squash or reword commits for clear history 7 | - `git rebase -i --autosquash upstream/master` 8 | 3. Create a pull request 9 | - Edit the description 10 | - Invite reviewers 11 | - Assign someone who can merge pull requests 12 | - Choose labels and milestone 13 | - If there is a topic-related issue, attach it 14 | 15 | **IMPORTANT** The `master` branch should **be always deployable**. 16 | Once you break the `master` via a pull request, choose a quickest way, 17 | **revert it** or **fix it** as soon as possible. 18 | 19 | The regression test must be included in all hotfix pull requests. 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Context 2 | 3 | 4 | 5 | ## Description 6 | 7 | 8 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Description 2 | 3 | 4 | 5 | ### Reference 6 | 7 | 11 | 12 | ### Todo 13 | 14 | 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *.swp 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | /env/ 12 | env_new 13 | /build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | /lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | target/ 60 | 61 | # PyCharm 62 | .idea/ 63 | .python-version 64 | 65 | # rope 66 | .ropeproject/ 67 | 68 | # Jenkins 69 | junit.xml 70 | 71 | # dotenv 72 | .env 73 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Changlog 2 | ========= 3 | 4 | 0.242.4 (2019-10-31) 5 | --------------------- 6 | 7 | * First import. 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:2.7.10-slim 2 | 3 | RUN apt-get update && apt-get install -y --no-install-recommends \ 4 | build-essential \ 5 | git \ 6 | libpq-dev \ 7 | librabbitmq-dev \ 8 | libncurses5-dev \ 9 | mariadb-client \ 10 | shellcheck \ 11 | golang-go 12 | 13 | ENV GOPATH "/opt/gopath" 14 | RUN go get -u github.com/client9/misspell/cmd/misspell 15 | 16 | RUN pip install -U virtualenv && virtualenv /opt/huskar_api 17 | ENV VIRTUAL_ENV "/opt/huskar_api" 18 | 19 | ENV PATH "$VIRTUAL_ENV/bin:$GOPATH/bin:$PATH" 20 | 21 | ADD . /srv/huskar_api 22 | WORKDIR /srv/huskar_api 23 | 24 | RUN pip install --no-cache-dir -r requirements-dev.txt 25 | 26 | ENTRYPOINT ["./manage.sh"] 27 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent { 3 | dockerfile { 4 | filename 'jenkins.dockerfile' 5 | args '-v /data/jenkins-volumes/pip:/home/jenkins/.cache/pip' 6 | } 7 | } 8 | environment { 9 | PYTHONUNBUFFERED = '1' 10 | GEVENT_RESOLVER = 'block' 11 | PIP_INDEX_URL = 'https://pypi.doubanio.com/simple/' 12 | DATABASE_DEFAULT_URL = "mysql+pymysql://root@127.0.0.1:3306/huskar_api?charset=utf8mb4" 13 | HUSKAR_API_DB_URL = "mysql+pymysql://root@127.0.0.1:3306/huskar_api?charset=utf8mb4" 14 | REDIS_DEFAULT_URL = "redis://127.0.0.1:6379" 15 | HUSKAR_API_REDIS_URL = "redis://127.0.0.1:6379" 16 | 17 | HUSKAR_API_DEBUG = 'true' 18 | HUSKAR_API_TESTING = 'true' 19 | HUSKAR_API_SECRET_KEY = 'test-secret-key' 20 | HUSKAR_API_ZK_SERVERS = "127.0.0.1:2181" 21 | } 22 | stages { 23 | stage('Install') { 24 | steps { 25 | sh 'mysqld_safe --user=jenkins --skip-grant-tables &' 26 | sh 'redis-server &' 27 | sh 'zookeeper-server start-foreground &' 28 | 29 | sh 'make install-deps' 30 | sh './manage.sh initdb' 31 | } 32 | } 33 | stage('Test') { 34 | steps { 35 | sh './manage.sh lint' 36 | sh './manage.sh testonly tests -xv --junitxml=junit.xml --cov=huskar_api --cov-report term-missing' 37 | sh 'coverage xml' 38 | sh 'coverage html' 39 | sh 'coverage report --show-missing' 40 | sh 'test "$(coverage report | tail -n1 | awk \'{print $6}\')" = "100%"' 41 | } 42 | } 43 | stage('Build Doc') { 44 | steps { 45 | sh 'make -C docs html' 46 | archiveArtifacts artifacts: 'docs/_build/html/**', fingerprint: true 47 | } 48 | when { 49 | anyOf { 50 | branch 'master' 51 | changeset '**/*.rst' 52 | changelog 'Docs:.+' 53 | changeRequest title: 'Docs:.+', comparator: 'REGEXP' 54 | changeRequest branch: 'docs/*', comparator: 'GLOB' 55 | } 56 | } 57 | } 58 | } 59 | post { 60 | success { 61 | junit 'junit.xml' 62 | cobertura coberturaReportFile: 'coverage.xml' 63 | publishHTML(target: [ 64 | allowMissing: false, 65 | alwaysLinkToLastBuild: false, 66 | keepAll: true, 67 | reportDir: 'htmlcov', 68 | reportFiles: 'index.html', 69 | reportName: 'Coverage Report' 70 | ]) 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) <2019> 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: help install-deps compile-deps prepare-deps git-hooks pow 2 | 3 | help: 4 | @echo "Available commands: help install-deps compile-deps" 5 | 6 | install-deps: prepare-deps 7 | pip-sync requirements.txt requirements-dev.txt 8 | 9 | compile-deps: prepare-deps 10 | pip-compile --no-index --no-emit-trusted-host requirements.in 11 | pip-compile --no-index --no-emit-trusted-host requirements-dev.in 12 | 13 | prepare-deps: 14 | @[ -n "$${VIRTUAL_ENV}" ] || (echo >&2 "Please activate virtualenv."; false) 15 | pip install -U pip==19.2.3 setuptools==41.4.0 wheel==0.33.6 pip-tools==4.2.0 16 | 17 | git-hooks: 18 | ln -sf `pwd`/tools/git-hooks/* .git/hooks/ 19 | 20 | pow: 21 | echo "http://$$(docker-compose port wsgi 5000)" > ~/.pow/huskar.test 22 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: gunicorn -c config/gunicorn_config.py huskar_api.wsgi:app 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Huskar API 2 | ========== 3 | 4 | [![CircleCI](https://circleci.com/gh/huskar-org/huskar/tree/master.svg?style=svg)](https://circleci.com/gh/huskar-org/huskar/tree/master) 5 | [![codecov](https://codecov.io/gh/huskar-org/huskar/branch/master/graph/badge.svg)](https://codecov.io/gh/huskar-org/huskar) 6 | 7 | HTTP API of Huskar. 8 | 9 | Demo 10 | ------ 11 | 12 | * 面板: https://demo.huskar.org (用户名: `huskar` 密码: `test`) 13 | * API: https://api.demo.huskar.org 14 | * 备注: 每天凌晨三点自动重置数据。 15 | 16 | How to start 17 | ------------ 18 | 19 | $ cp .env.example .env && vim .env 20 | 21 | Starting the API server in local environment: 22 | 23 | $ . path/to/venv/activate 24 | $ make install-deps 25 | $ honcho start 26 | 27 | Starting the API server in [Docker](https://www.docker.com/products/docker): 28 | 29 | $ docker-compose run --rm wsgi initdb # initialize database 30 | $ docker-compose run --rm wsgi initadmin # initialize administrator 31 | $ docker-compose up wsgi # start web server 32 | 33 | 34 | Development FAQ 35 | --------------- 36 | 37 | Using the ZooKeeper CLI: 38 | 39 | $ zkCli -server $(docker-compose port zookeeper 2181) 40 | 41 | Using the MySQL CLI: 42 | 43 | $ mycli mysql://root@$(docker-compose port mysql 3306)/huskar_api 44 | 45 | Updating dependencies: 46 | 47 | $ docker-compose run --rm wsgi make compile-deps 48 | $ git add -p requirements* 49 | 50 | Running tests: 51 | 52 | $ docker-compose run --rm wsgi testall -xv 53 | $ docker-compose run --rm wsgi test test_foo.py -xv 54 | 55 | Maintaining database schema: 56 | 57 | $ docker-compose run --rm wsgi alembic upgrade head 58 | $ vim huskar_api/models/foobar.py 59 | $ docker-compose run --rm wsgi alembic revision --autogenerate -m 'add an index of foo' 60 | $ vim database/migration/versions/xxxxxxx.py 61 | $ docker-compose run --rm wsgi alembic upgrade head 62 | $ docker-compose run --rm wsgi dumpdb 63 | $ git add database 64 | 65 | Updating snapshot of email template in tests: 66 | 67 | $ docker-compose run --rm wsgi python run.py tests.test_extras.test_email:gen 68 | -------------------------------------------------------------------------------- /config/gunicorn_config.py: -------------------------------------------------------------------------------- 1 | bind = "0.0.0.0:5000" 2 | worker_class = "gevent" 3 | -------------------------------------------------------------------------------- /database/migration/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. -------------------------------------------------------------------------------- /database/migration/alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # path to migration scripts 5 | script_location = database/migration 6 | 7 | # template used to generate migration files 8 | # file_template = %%(rev)s_%%(slug)s 9 | 10 | # max length of characters to apply to the 11 | # "slug" field 12 | #truncate_slug_length = 40 13 | 14 | # set to 'true' to run the environment during 15 | # the 'revision' command, regardless of autogenerate 16 | # revision_environment = false 17 | 18 | # set to 'true' to allow .pyc and .pyo files without 19 | # a source .py file to be detected as revisions in the 20 | # versions/ directory 21 | # sourceless = false 22 | 23 | # version location specification; this defaults 24 | # to database/migration/versions. When using multiple version 25 | # directories, initial revisions must be specified with --version-path 26 | # version_locations = %(here)s/bar %(here)s/bat database/migration/versions 27 | 28 | # the output encoding used when revision files 29 | # are written from script.py.mako 30 | # output_encoding = utf-8 31 | 32 | # Logging configuration 33 | [loggers] 34 | keys = root,sqlalchemy,alembic 35 | 36 | [handlers] 37 | keys = console 38 | 39 | [formatters] 40 | keys = generic 41 | 42 | [logger_root] 43 | level = WARN 44 | handlers = console 45 | qualname = 46 | 47 | [logger_sqlalchemy] 48 | level = WARN 49 | handlers = 50 | qualname = sqlalchemy.engine 51 | 52 | [logger_alembic] 53 | level = INFO 54 | handlers = 55 | qualname = alembic 56 | 57 | [handler_console] 58 | class = StreamHandler 59 | args = (sys.stderr,) 60 | level = NOTSET 61 | formatter = generic 62 | 63 | [formatter_generic] 64 | format = %(levelname)-5.5s [%(name)s] %(message)s 65 | datefmt = %H:%M:%S 66 | -------------------------------------------------------------------------------- /database/migration/env.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | 3 | from logging.config import fileConfig 4 | 5 | from alembic import context 6 | from huskar_api.models.db import db_manager 7 | 8 | from huskar_api.models.alembic import get_metadata 9 | 10 | # this is the Alembic Config object, which provides 11 | # access to the values within the .ini file in use. 12 | config = context.config 13 | 14 | # Interpret the config file for Python logging. 15 | # This line sets up loggers basically. 16 | fileConfig(config.config_file_name) 17 | 18 | # add your model's MetaData object here 19 | # for 'autogenerate' support 20 | target_metadata = get_metadata() 21 | 22 | # other values from the config, defined by the needs of env.py, 23 | # can be acquired: 24 | # my_important_option = config.get_main_option("my_important_option") 25 | # ... etc. 26 | 27 | 28 | def run_migrations_offline(): 29 | """Run migrations in 'offline' mode. 30 | 31 | This configures the context with just a URL 32 | and not an Engine, though an Engine is acceptable 33 | here as well. By skipping the Engine creation 34 | we don't even need a DBAPI to be available. 35 | 36 | Calls to context.execute() here emit the given string to the 37 | script output. 38 | 39 | """ 40 | engine = db_manager.get_session('default').get_bind() 41 | context.configure(url=engine.url, target_metadata=target_metadata) 42 | 43 | with context.begin_transaction(): 44 | context.run_migrations() 45 | 46 | 47 | def run_migrations_online(): 48 | """Run migrations in 'online' mode. 49 | 50 | In this scenario we need to create an Engine 51 | and associate a connection with the context. 52 | 53 | """ 54 | connectable = db_manager.get_session('default').get_bind() 55 | 56 | with connectable.connect() as connection: 57 | context.configure( 58 | connection=connection, 59 | target_metadata=target_metadata, 60 | compare_type=True, 61 | ) 62 | 63 | with context.begin_transaction(): 64 | context.run_migrations() 65 | 66 | 67 | if context.is_offline_mode(): 68 | run_migrations_offline() 69 | else: 70 | run_migrations_online() 71 | -------------------------------------------------------------------------------- /database/migration/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | 9 | from alembic import op 10 | import sqlalchemy as sa 11 | ${imports if imports else ""} 12 | 13 | # revision identifiers, used by Alembic. 14 | revision = ${repr(up_revision)} 15 | down_revision = ${repr(down_revision)} 16 | branch_labels = ${repr(branch_labels)} 17 | depends_on = ${repr(depends_on)} 18 | 19 | 20 | def upgrade(): 21 | ${upgrades if upgrades else "pass"} 22 | 23 | 24 | def downgrade(): 25 | ${downgrades if downgrades else "pass"} 26 | -------------------------------------------------------------------------------- /database/migration/versions/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/database/migration/versions/.gitkeep -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | wsgi: 4 | build: 5 | context: . 6 | command: gunicorn -c config/gunicorn_config.py huskar_api.wsgi:app 7 | cap_add: 8 | - ALL 9 | ports: 10 | - "5000" 11 | depends_on: 12 | - mysql 13 | - redis 14 | - zookeeper 15 | volumes: 16 | - ".:/srv/huskar_api" 17 | - "/root/.cache" 18 | links: 19 | - "mysql:mysql" 20 | - "redis:redis" 21 | - "zookeeper:zookeeper" 22 | environment: 23 | PYTHONUNBUFFERED: "1" 24 | PYTHONPATH: "/srv/huskar_api" 25 | GEVENT_RESOLVER: "block" 26 | DATABASE_DEFAULT_URL: "mysql+pymysql://root@mysql:3306/huskar_api?charset=utf8mb4" 27 | HUSKAR_API_DB_URL: "mysql+pymysql://root@mysql:3306/huskar_api?charset=utf8mb4" 28 | REDIS_DEFAULT_URL: "redis://redis:6379" 29 | HUSKAR_API_REDIS_URL: "redis://redis:6379" 30 | HUSKAR_API_ZK_SERVERS: "zookeeper:2181" 31 | env_file: 32 | - .env 33 | mysql: 34 | image: mysql:5.7 35 | environment: 36 | - MYSQL_DATABASE=huskar_api 37 | - MYSQL_ALLOW_EMPTY_PASSWORD=1 38 | ports: 39 | - "3306" 40 | redis: 41 | image: redis:3.0.7 42 | ports: 43 | - "6379" 44 | zookeeper: 45 | image: zookeeper:3.4.14 46 | ports: 47 | - "2181" 48 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | venv 2 | -------------------------------------------------------------------------------- /docs/_static/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/docs/_static/.gitignore -------------------------------------------------------------------------------- /docs/api/advance.rst: -------------------------------------------------------------------------------- 1 | .. _advance: 2 | 3 | Advance Management 4 | ================== 5 | 6 | .. _service_link: 7 | 8 | Service Link 9 | ------------ 10 | 11 | The **Service Link** is a simple way to redirect traffic of a cluster to 12 | another one in the side of service provider. 13 | 14 | .. autoflask:: huskar_api.wsgi:app 15 | :endpoints: api.service_link 16 | :groupby: view 17 | 18 | 19 | .. _service_route: 20 | 21 | Service Route 22 | ------------- 23 | 24 | The **Service Route** is another way to manage the traffic of clusters. You 25 | could demand to redirect traffic which going to specific destination 26 | application, in the side of service consumer. 27 | 28 | .. autoflask:: huskar_api.wsgi:app 29 | :endpoints: api.service_route 30 | :groupby: view 31 | 32 | .. autoflask:: huskar_api.wsgi:app 33 | :endpoints: api.service_default_route 34 | :groupby: view 35 | 36 | 37 | .. _service_info: 38 | 39 | Service Info 40 | ------------ 41 | 42 | The API of **Service Info** is used for the management of application-scope and 43 | cluster-scope information, which is read by sidecar for health check. 44 | 45 | The ``cluster_name`` is an optional component of URL. If you don't specify it, 46 | the API works on application-scope. Otherwise the API works on cluster-scope. 47 | 48 | Instead of using the API always, we recommend you consider trying the 49 | `Web Console`_. 50 | 51 | .. autoflask:: huskar_api.wsgi:app 52 | :endpoints: api.service_info,api.cluster_info 53 | :groupby: view 54 | 55 | .. _`Web Console`: https://example.com/application/foo.bar/service?info 56 | -------------------------------------------------------------------------------- /docs/api/application.rst: -------------------------------------------------------------------------------- 1 | .. _application: 2 | 3 | Application Management 4 | ====================== 5 | 6 | An application of Huskar is an organization for containing service, switch and 7 | config. It includes a name (a.k.a appid) and a related team. 8 | 9 | The team admin have default authority (``read`` and ``write``) on applications 10 | inside their team. You could see :ref:`team` also. 11 | 12 | There is a serial of API to view and manage applications. 13 | 14 | Basic Management 15 | ---------------- 16 | 17 | .. _application_list: 18 | 19 | .. autoflask:: huskar_api.wsgi:app 20 | :endpoints: api.application 21 | :groupby: view 22 | 23 | .. _application_item: 24 | 25 | .. autoflask:: huskar_api.wsgi:app 26 | :endpoints: api.application_item 27 | :groupby: view 28 | 29 | .. _application_auth: 30 | 31 | Authority Management 32 | -------------------- 33 | 34 | There are two types of authority, ``read`` and ``write``. 35 | 36 | The ``read`` authority allows people to read secret area of applications, 37 | including *switch*, *config* and *audit log*. All authenticated users can read 38 | public area, including *service registry* for now, without any authority. 39 | 40 | The ``write`` authority is required for creating or updating anything in 41 | public area and secret area, unless the authenticated user is a 42 | :ref:`site admin ` or :ref:`team admin `. 43 | 44 | .. autoflask:: huskar_api.wsgi:app 45 | :endpoints: api.application_auth 46 | :groupby: view 47 | -------------------------------------------------------------------------------- /docs/api/audit.rst: -------------------------------------------------------------------------------- 1 | .. _audit: 2 | 3 | Audit Log 4 | ========= 5 | 6 | You can view the audit log via the `Web Console`_. It is based on following 7 | API. 8 | 9 | .. _`Web Console`: https://example.com/audit 10 | .. _audit_schema: 11 | 12 | Schema 13 | ------ 14 | 15 | There is the response schema of audit log entities. 16 | 17 | =============== =================== ===================================== 18 | Name Type Example 19 | --------------- ------------------- ------------------------------------- 20 | id :js:class:`Integer` ``10001`` 21 | user :js:class:`Object` :ref:`User Schema ` 22 | remote_addr :js:class:`String` ``"10.0.0.1"`` 23 | action_name :js:class:`String` ``"CREATE_TEAM"`` 24 | action_data :js:class:`String` ``"{"team_name": "base"}"`` 25 | created_at :js:class:`String` ``"1993-05-01T12:00:00+08:00"`` 26 | rollback_to :js:class:`Object` ``null`` or 27 | :ref:`Audit Log Schema` 28 | =============== =================== ===================================== 29 | 30 | 31 | API 32 | --- 33 | 34 | .. autoflask:: huskar_api.wsgi:app 35 | :endpoints: api.audit_site,api.audit_team,api.audit_application 36 | :groupby: view 37 | -------------------------------------------------------------------------------- /docs/api/cluster.rst: -------------------------------------------------------------------------------- 1 | .. _cluster: 2 | 3 | Cluster Management 4 | ================== 5 | 6 | .. autoflask:: huskar_api.wsgi:app 7 | :endpoints: api.service_cluster,api.switch_cluster,api.config_cluster 8 | :groupby: view 9 | -------------------------------------------------------------------------------- /docs/api/index.rst: -------------------------------------------------------------------------------- 1 | .. _api: 2 | 3 | HTTP API 4 | ======== 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | token 10 | user 11 | team 12 | application 13 | cluster 14 | instance 15 | subscription 16 | advance 17 | audit 18 | webhook 19 | infra 20 | support 21 | -------------------------------------------------------------------------------- /docs/api/infra.rst: -------------------------------------------------------------------------------- 1 | .. _infra: 2 | 3 | Infrastructure 4 | ============== 5 | 6 | .. _infra_config: 7 | 8 | Universal Configuration 9 | ----------------------- 10 | 11 | The universal infrastructure configuration follows `a spec `_ (a.k.a Naming Service). 12 | 13 | It gives applications ability to discover and configure an infrastructure 14 | client with a pre-defined key. For example:: 15 | 16 | RedisClient redis = RedisClientRegistry.get('r100010') 17 | 18 | There are high-level management API to register and configure infrastructure. 19 | If you are a SDK developer, please don't use those management API in your 20 | client-side code. Look for :ref:`long-polling` instead which provides stronger 21 | guarantee. 22 | 23 | .. autoflask:: huskar_api.wsgi:app 24 | :endpoints: api.infra_config 25 | :groupby: view 26 | 27 | .. _infra_config_downstream: 28 | 29 | It is possible to look up the downstream applications of infrastructures also. 30 | 31 | .. autoflask:: huskar_api.wsgi:app 32 | :endpoints: api.infra_config_downstream 33 | :groupby: view 34 | -------------------------------------------------------------------------------- /docs/api/support.rst: -------------------------------------------------------------------------------- 1 | .. _support: 2 | 3 | Internal Support 4 | ================ 5 | 6 | The following APIs are designed for internal usage. We use them to support 7 | some special situation. 8 | 9 | Please **do not use them** if you don't know what are them. 10 | 11 | Internal API 12 | ----------------- 13 | 14 | .. autoflask:: huskar_api.wsgi:app 15 | :endpoints: api.whoami 16 | :groupby: view 17 | 18 | .. autoflask:: huskar_api.wsgi:app 19 | :endpoints: api.team_application_token 20 | :groupby: view 21 | 22 | .. autoflask:: huskar_api.wsgi:app 23 | :endpoints: api.internal_container_registry 24 | :groupby: view 25 | 26 | .. autoflask:: huskar_api.wsgi:app 27 | :endpoints: api.internal_route_program 28 | :groupby: view 29 | 30 | .. autoflask:: huskar_api.wsgi:app 31 | :endpoints: api.service_weight 32 | :groupby: view 33 | 34 | .. autoflask:: huskar_api.wsgi:app 35 | :endpoints: api.well_known_common 36 | :groupby: view 37 | 38 | OPS Internal API 39 | ---------------- 40 | 41 | .. autoflask:: huskar_api.wsgi:app 42 | :endpoints: api.internal_blacklist 43 | :groupby: view 44 | -------------------------------------------------------------------------------- /docs/api/team.rst: -------------------------------------------------------------------------------- 1 | .. _team: 2 | 3 | Team Management 4 | =============== 5 | 6 | Basic Management 7 | ---------------- 8 | 9 | .. autoflask:: huskar_api.wsgi:app 10 | :endpoints: api.team 11 | :groupby: view 12 | 13 | .. _team_admin: 14 | 15 | Admin Management 16 | ---------------- 17 | 18 | .. autoflask:: huskar_api.wsgi:app 19 | :endpoints: api.team_admin 20 | :groupby: view 21 | -------------------------------------------------------------------------------- /docs/api/token.rst: -------------------------------------------------------------------------------- 1 | .. _token: 2 | 3 | Authorization and Token 4 | ======================= 5 | 6 | A token is a credential of Huskar API. It provides information about 7 | "who are you". All tokens are `JSON Web Token `_ compatible. 8 | 9 | For now, there are two types of token. The **user token** (human token) 10 | points to a employee. The **app token** (application token) points to an 11 | application. 12 | 13 | The user tokens usually have an optional expiration time. But the app tokens 14 | are always immortal. 15 | 16 | Our deployment may deny requests to the bare API URL which use user tokens and 17 | requests to the Web management console which use app tokens. 18 | 19 | .. _how-to-use-token: 20 | 21 | Using Token 22 | ----------- 23 | 24 | The token could be placed in the request header :http:header:`Authorization`. 25 | For example: 26 | 27 | .. code-block:: sh 28 | 29 | HUSKAR_TOKEN="xxxx" 30 | curl http://example.com -H Authorization:$HUSKAR_TOKEN 31 | 32 | .. _how-to-get-token: 33 | 34 | Getting Token 35 | ------------- 36 | 37 | There are two different APIs for getting :ref:`user-token` and 38 | :ref:`app-token`. 39 | 40 | .. _user-token: 41 | 42 | User Token 43 | ~~~~~~~~~~ 44 | 45 | .. autoflask:: huskar_api.wsgi:app 46 | :endpoints: api.huskar_token 47 | 48 | .. _app-token: 49 | 50 | Application Token 51 | ~~~~~~~~~~~~~~~~~ 52 | 53 | .. autoflask:: huskar_api.wsgi:app 54 | :endpoints: api.application_token 55 | -------------------------------------------------------------------------------- /docs/api/user.rst: -------------------------------------------------------------------------------- 1 | .. _user: 2 | 3 | User Management 4 | =============== 5 | 6 | .. _user_schema: 7 | 8 | Basic Schema 9 | ------------ 10 | 11 | There is the response schema of user entities. 12 | 13 | =============== =================== ================================ 14 | Name Type Example 15 | --------------- ------------------- -------------------------------- 16 | id :js:class:`Integer` ``10001`` 17 | username :js:class:`String` ``"san.zhang"`` 18 | email :js:class:`String` ``"san.zhang@example.com"`` 19 | is_active :js:class:`Boolean` ``true`` 20 | is_admin :js:class:`Boolean` ``false`` 21 | is_application :js:class:`Boolean` ``false`` 22 | created_at :js:class:`String` ``"1993-05-01T12:00:00+08:00"`` 23 | updated_at :js:class:`String` ``"1993-05-01T12:00:00+08:00"`` 24 | =============== =================== ================================ 25 | 26 | Basic Management 27 | ---------------- 28 | 29 | You could create, modify and delete users via following API. 30 | 31 | .. autoflask:: huskar_api.wsgi:app 32 | :endpoints: api.user 33 | :groupby: view 34 | 35 | Password Retrieval 36 | ------------------ 37 | 38 | The users have ability to reset their password by validating their email. 39 | 40 | .. autoflask:: huskar_api.wsgi:app 41 | :endpoints: api.password_reset 42 | :groupby: view 43 | 44 | .. _site_admin: 45 | 46 | Admin Management 47 | ---------------- 48 | 49 | The site admin has highest authority in Huskar API. The users could be granted 50 | to site admin or dismissed from site admin via following API. 51 | 52 | .. autoflask:: huskar_api.wsgi:app 53 | :endpoints: api.huskar_admin 54 | :groupby: view 55 | -------------------------------------------------------------------------------- /docs/assets/index.rst: -------------------------------------------------------------------------------- 1 | .. DO NOT EDIT (auto generated) 2 | 3 | Assets 4 | ====== 5 | 6 | Email Snapshots 7 | --------------- 8 | 9 | * Huskar Debug 10 | * :download:`快照范例 1 <../../tests/test_extras/test_email_snapshots/email-debug-0.html>` 11 | * Huskar 基础资源绑定通知 12 | * :download:`快照范例 1 <../../tests/test_extras/test_email_snapshots/email-infra-config-create-0.html>` 13 | * :download:`快照范例 2 <../../tests/test_extras/test_email_snapshots/email-infra-config-create-1.html>` 14 | * :download:`快照范例 3 <../../tests/test_extras/test_email_snapshots/email-infra-config-create-2.html>` 15 | * :download:`快照范例 4 <../../tests/test_extras/test_email_snapshots/email-infra-config-create-3.html>` 16 | * :download:`快照范例 5 <../../tests/test_extras/test_email_snapshots/email-infra-config-create-4.html>` 17 | * :download:`快照范例 6 <../../tests/test_extras/test_email_snapshots/email-infra-config-create-5.html>` 18 | * :download:`快照范例 7 <../../tests/test_extras/test_email_snapshots/email-infra-config-create-6.html>` 19 | * Huskar 密码重置 20 | * :download:`快照范例 1 <../../tests/test_extras/test_email_snapshots/email-password-reset-0.html>` 21 | * Huskar 权限变更 22 | * :download:`快照范例 1 <../../tests/test_extras/test_email_snapshots/email-permission-dismiss-0.html>` 23 | * Huskar 权限变更 24 | * :download:`快照范例 1 <../../tests/test_extras/test_email_snapshots/email-permission-grant-0.html>` 25 | * Huskar 用户创建 26 | * :download:`快照范例 1 <../../tests/test_extras/test_email_snapshots/email-signup-0.html>` 27 | -------------------------------------------------------------------------------- /docs/contact/index.rst: -------------------------------------------------------------------------------- 1 | .. _contact: 2 | 3 | 联系我们 4 | ======== 5 | 6 | 需求、建议或其他需要联系我们的, 请通过创建 issue 的方式联系支持; 7 | -------------------------------------------------------------------------------- /docs/dev/index.rst: -------------------------------------------------------------------------------- 1 | Internal API 2 | ============ 3 | 4 | Common 5 | ------ 6 | 7 | .. automodule:: huskar_api.models 8 | :members: 9 | 10 | .. automodule:: huskar_api.models.znode 11 | :members: 12 | 13 | Authorization 14 | ------------- 15 | 16 | .. automodule:: huskar_api.models.auth.application 17 | :members: 18 | 19 | .. automodule:: huskar_api.models.auth.team 20 | :members: 21 | 22 | .. automodule:: huskar_api.models.auth.user 23 | :members: 24 | 25 | .. automodule:: huskar_api.models.auth.session 26 | :members: 27 | 28 | Audit Log 29 | --------- 30 | 31 | We have defined three level of audit log, ``site``, ``team`` and ``application``, 32 | the content of audit log is a ``Dict``, it must have a field named ``action_data`` to 33 | be used to describe the action of this audit log. 34 | 35 | .. Note:: If the level is ``application``, there must be a ``string`` field ``application_name`` 36 | or a ``list`` field ``application_names`` in the value of ``action_data``. 37 | 38 | 39 | .. automodule:: huskar_api.models.audit.audit 40 | :members: 41 | 42 | .. automodule:: huskar_api.models.audit.action 43 | :members: 44 | 45 | .. automodule:: huskar_api.models.audit.index 46 | :members: 47 | 48 | .. automodule:: huskar_api.models.audit.const 49 | :members: 50 | 51 | Tree Watch 52 | ---------- 53 | 54 | .. automodule:: huskar_api.models.tree.hub 55 | :members: 56 | 57 | .. automodule:: huskar_api.models.tree.holder 58 | :members: 59 | 60 | .. automodule:: huskar_api.models.tree.watcher 61 | :members: 62 | 63 | .. automodule:: huskar_api.models.tree.common 64 | :members: 65 | 66 | Service Management 67 | ------------------ 68 | 69 | .. automodule:: huskar_api.models.catalog 70 | :members: 71 | 72 | .. automodule:: huskar_api.models.instance 73 | :members: 74 | 75 | .. automodule:: huskar_api.models.manifest 76 | :members: 77 | 78 | .. automodule:: huskar_api.models.comment 79 | :members: 80 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Huskar API Document 2 | =================== 3 | 4 | This project provides HTTP API for service discovery and configuration 5 | management. 6 | 7 | Contents 8 | -------- 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | 13 | intro/index 14 | design/index 15 | api/index 16 | dev/index 17 | assets/index 18 | ops/index 19 | contact/index 20 | 21 | Resource Links 22 | -------------- 23 | 24 | API 25 | ~~~ 26 | 27 | - Codebase: https://github.com/huskar-org/huskar 28 | - Document: http://example.com/huskar_api/ 29 | 30 | Web Admin 31 | ~~~~~~~~~ 32 | 33 | - Codebase: https://github.com/huskar-org/huskar-fe 34 | - Document: https://example.com/huskar 35 | 36 | .. _language_sdk: 37 | 38 | Language Client 39 | ~~~~~~~~~~~~~~~ 40 | 41 | - Go: https://github.com/huskar-org/huskar-go 42 | - Python: https://github.com/huskar-org/huskar-python 43 | - Java: https://github.com/huskar-org/huskar-java 44 | 45 | -------------------------------------------------------------------------------- /docs/intro/index.rst: -------------------------------------------------------------------------------- 1 | 简介 2 | ==== 3 | 4 | 什么是 Huskar 5 | ------------- 6 | 7 | Huskar 是个解决服务发现、服务治理和配置管理等痛点的一站式解决方案. 8 | 9 | .. _intro: 10 | 11 | 如何接入和使用 Huskar 12 | --------------------- 13 | 14 | # TODO 15 | -------------------------------------------------------------------------------- /docs/ops/deployment.rst: -------------------------------------------------------------------------------- 1 | .. _devops.deployment: 2 | 3 | 部署 4 | ==== 5 | 6 | 依赖环境 7 | -------- 8 | 9 | 一个 Huskar API 实例的运行, 至少会依赖 ZooKeeper, MySQL 和 Redis 三种基础设施. 10 | 11 | 部署环境中需要准备好: 12 | 13 | - 一个可访问的 ZooKeeper 集群 14 | - 一个可访问的 MySQL 实例 15 | - 一个可访问的 Redis 实例 16 | 17 | 自举 18 | ---- 19 | 20 | 在一个全新的数据中心部署 Huskar API 时, 因为 Huskar API 本身也依赖 ZooKeeper 做配置管理, 21 | 故而会有 Bootstrap 问题. 一个可选的方案是通过 ``zkCli`` 访问 ZooKeeper 集群, 22 | 手动写入下述 Huskar API 赖以启动的配置, 23 | 然后启动 Huskar API 实例 (``PREFIX`` 为 ``/huskar/config/arch.huskar_api/overall``): 24 | 25 | ================================== =========================================== 26 | Key Value 27 | ================================== =========================================== 28 | ``{PREFIX}/SECRET_KEY`` A random string 29 | ``{PREFIX}/DATABASE_URL`` mysql url 30 | ``{PREFIX}/REDIS_URL`` redis url 31 | ``{PREFIX}/HUSKAR_API_ZK_SERVICES`` zk hosts 32 | ================================== =========================================== 33 | 34 | 不过 *更加方便的一种方式* 是使用环境变量注入配置, 在 shell 中启动临时 Huskar API 实例: 35 | 36 | .. code-block:: sh 37 | 38 | export HUSKAR_API_SECRET_KEY= 39 | export HUSKAR_API_DATABASE_URL=mysql+pymysql://root@localhost:3306/huskar_api?charset=utf8 40 | export HUSKAR_API_REDIS_URL=redis://localhost:6379 41 | export HUSKAR_API_ZK_SERVICES=127.0.0.1:2181 42 | 43 | source .venv/bin/activate # 进入 virtualenv 44 | ./manage.sh initadmin # 创建管理员用户 (如果是全新的 MySQL 数据库) 45 | gunicorn -k gevent -b 0.0.0.0:5000 huskar_api.wsgi:app 46 | 47 | 临时实例启动后, 可发布 ``arch.huskar_fe`` 以启用 Web 面板, 使用面板将相关配置写入 48 | ``arch.huskar_api`` 之下. 完成后 `Ctrl-C` 关闭临时实例, 启动正式实例. 49 | 50 | 服务验证 51 | -------- 52 | 53 | 验证一个 Huskar API 实例是否正常部署的最简单方法是使用健康检查接口:: 54 | 55 | curl http://localhost:5000/api/health_check 56 | 57 | 健康检查接口返回 HTTP 200 说明至少 Gunicorn 实例正常启动, ZooKeeper 可以正常连接. 58 | 如果还需要验证 MySQL 等其他设施是否配置正确, 则需要通过功能 API 或访问 Huskar 59 | Web 面板. 60 | 61 | .. _huskar-api: http://example.com/huskar-api 62 | .. _huskar-fe: http://example.com/huskar-fe 63 | -------------------------------------------------------------------------------- /docs/ops/environment.rst: -------------------------------------------------------------------------------- 1 | .. _devops.environment: 2 | 3 | 环境信息 4 | ======== 5 | -------------------------------------------------------------------------------- /docs/ops/index.rst: -------------------------------------------------------------------------------- 1 | .. _devops: 2 | 3 | 运维开发 4 | ======== 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | environment 10 | deployment 11 | -------------------------------------------------------------------------------- /huskar_api/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from orphanage import exit_when_orphaned 4 | 5 | 6 | exit_when_orphaned() 7 | 8 | __version__ = '0.242.4' 9 | -------------------------------------------------------------------------------- /huskar_api/api/health_check.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from flask.views import MethodView 4 | 5 | from .utils import api_response 6 | 7 | 8 | class HealthCheckView(MethodView): 9 | def get(self): 10 | return api_response('ok') 11 | -------------------------------------------------------------------------------- /huskar_api/api/middlewares/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/huskar_api/api/middlewares/__init__.py -------------------------------------------------------------------------------- /huskar_api/api/middlewares/auth.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | from flask import Blueprint, request, g, abort 6 | 7 | from huskar_api import settings 8 | from huskar_api.ext import db_tester 9 | from huskar_api.extras.monitor import monitor_client 10 | from huskar_api.models.auth import SessionAuth 11 | from huskar_api.models.const import MM_REASON_TESTER, MM_REASON_STARTUP 12 | from huskar_api.extras.uptime import process_uptime 13 | from huskar_api.service.admin.application_auth import ( 14 | is_application_blacklisted) 15 | 16 | 17 | bp = Blueprint('middlewares.auth', __name__) 18 | logger = logging.getLogger(__name__) 19 | 20 | 21 | @bp.before_app_request 22 | def check_blacklist(): 23 | if request.remote_addr not in settings.AUTH_IP_BLACKLIST: 24 | return 25 | abort(403, 'The IP address is blacklisted') 26 | 27 | 28 | @bp.before_app_request 29 | def authenticate(): 30 | token = request.headers.get('Authorization', '').strip() 31 | g.auth = SessionAuth.from_token(token) 32 | g.auth.load_user() 33 | if not db_tester.ok: 34 | g.auth.enter_minimal_mode(MM_REASON_TESTER) 35 | if (settings.MM_GRACEFUL_STARTUP_TIME and 36 | process_uptime() <= settings.MM_GRACEFUL_STARTUP_TIME): 37 | g.auth.enter_minimal_mode(MM_REASON_STARTUP) 38 | 39 | 40 | @bp.before_app_request 41 | def check_application_blacklist(): 42 | if is_application_blacklisted(g.auth.username): 43 | abort(403, 'application: {} is blacklisted'.format(g.auth.username)) 44 | 45 | 46 | @bp.before_app_request 47 | def detect_token_abuse(): 48 | frontend_name = request.headers.get('X-Frontend-Name') 49 | if (g.auth.is_application and 50 | frontend_name and frontend_name == settings.ADMIN_FRONTEND_NAME): 51 | abort(403, 'Using application token in web is not permitted.') 52 | 53 | 54 | @bp.after_app_request 55 | def track_user_qps(response): 56 | if not request.endpoint: 57 | return response 58 | 59 | if g.get('auth'): 60 | name = g.auth.username 61 | kind = 'app' if g.auth.is_application else 'user' 62 | else: 63 | name = 'anonymous' 64 | kind = 'anonymous' 65 | tags = dict(kind=kind, name=name) 66 | if kind == 'app': 67 | tags.update(appid=name) 68 | monitor_client.increment('qps.all', tags=tags) 69 | monitor_client.increment('qps.url', tags=dict( 70 | endpoint=request.endpoint, method=request.method, **tags)) 71 | 72 | return response 73 | 74 | 75 | @bp.after_app_request 76 | def indicate_minimal_mode(response): 77 | auth = g.get('auth') 78 | if auth is not None and auth.is_minimal_mode: 79 | response.headers['X-Minimal-Mode'] = u'1' 80 | response.headers['X-Minimal-Mode-Reason'] = \ 81 | unicode(auth.minimal_mode_reason or u'') 82 | return response 83 | -------------------------------------------------------------------------------- /huskar_api/api/middlewares/concurrent_limit.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | from flask import Blueprint, request, g, abort 6 | 7 | from huskar_api import settings 8 | from huskar_api.extras.concurrent_limiter import ( 9 | check_new_request, release_request, ConcurrencyExceededError) 10 | from huskar_api.switch import switch, SWITCH_ENABLE_CONCURRENT_LIMITER 11 | 12 | bp = Blueprint('middlewares.concurrent_limit', __name__) 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | @bp.before_app_request 17 | def check_concurrent_limit(): 18 | if not switch.is_switched_on(SWITCH_ENABLE_CONCURRENT_LIMITER): 19 | return 20 | 21 | if g.get('auth'): 22 | anonymous = False 23 | username = g.auth.username 24 | else: 25 | anonymous = True 26 | username = request.remote_addr 27 | config = get_limiter_config( 28 | settings.CONCURRENT_LIMITER_SETTINGS, username, anonymous=anonymous) 29 | if not config: 30 | return 31 | 32 | ttl, capacity = config['ttl'], config['capacity'] 33 | try: 34 | result = check_new_request(username, ttl, capacity) 35 | except ConcurrencyExceededError: 36 | abort(429, 'Too Many Requests, only allow handling {} requests ' 37 | 'in {} seconds'.format(capacity, ttl)) 38 | else: 39 | if result is not None: 40 | key, sub_item = result 41 | g.concurrent_limiter_data = {'key': key, 'sub_item': sub_item} 42 | 43 | 44 | @bp.after_app_request 45 | def release_concurrent_limiter_data(response): 46 | if (g.get('concurrent_limiter_data') and 47 | (response.status_code != 200 or 48 | request.endpoint != 'api.long_polling')): 49 | data = g.concurrent_limiter_data 50 | release_request(data['key'], data['sub_item']) 51 | g.concurrent_limiter_data = None 52 | 53 | return response 54 | 55 | 56 | def get_limiter_config(configs, username, anonymous): 57 | if username in configs: 58 | return configs[username] 59 | if anonymous and '__anonymous__' in configs: 60 | return configs['__anonymous__'] 61 | return configs.get('__default__') 62 | -------------------------------------------------------------------------------- /huskar_api/api/middlewares/db.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | from flask import Blueprint 6 | 7 | from huskar_api import settings 8 | from huskar_api.ext import sentry 9 | from huskar_api.models import DBSession 10 | 11 | 12 | bp = Blueprint('middlewares.db', __name__) 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | @bp.after_app_request 17 | def close_session(response): 18 | try: 19 | DBSession.remove() 20 | except Exception: 21 | logger.exception('Failed to close database during request teardown') 22 | sentry.captureException() 23 | if settings.TESTING: 24 | raise 25 | return response 26 | -------------------------------------------------------------------------------- /huskar_api/api/middlewares/error.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from flask import Blueprint, json 4 | from werkzeug.exceptions import HTTPException, InternalServerError 5 | from marshmallow.exceptions import ValidationError 6 | 7 | # TODO Do not use this base exception in future 8 | from huskar_api.service.exc import HuskarApiException 9 | from huskar_api.api.utils import api_response 10 | 11 | 12 | bp = Blueprint('middlewares.error', __name__) 13 | 14 | 15 | def http_errorhandler(fn): 16 | def iter_derived_classes(base_class): 17 | for class_ in base_class.__subclasses__(): 18 | yield class_ 19 | for derived_class in iter_derived_classes(class_): 20 | yield derived_class 21 | 22 | for http_error in iter_derived_classes(HTTPException): 23 | if http_error is InternalServerError: 24 | continue 25 | bp.app_errorhandler(http_error)(fn) 26 | return fn 27 | 28 | 29 | @http_errorhandler 30 | def handle_http_error(error): 31 | status = error.name.replace(u' ', '') 32 | description = error.description 33 | 34 | if isinstance(error, KeyError) and error.args: 35 | description = u'"%s" is required field.' % error.args[0] 36 | 37 | return api_response(status=status, message=description), error.code 38 | 39 | 40 | @bp.app_errorhandler(HuskarApiException) 41 | def handle_huskar_api_error(error): 42 | status = error.__class__.__name__ 43 | description = ( 44 | next(iter(error.args), None) or getattr(error, 'message', None) or u'') 45 | return api_response(status=status, message=description), 400 46 | 47 | 48 | @bp.app_errorhandler(ValidationError) 49 | def handle_marshmallow_validation_error(error): 50 | description = json.dumps(error.messages) 51 | return api_response(status='ValidationError', message=description), 400 52 | 53 | 54 | @bp.app_errorhandler(InternalServerError) 55 | def handle_unknown_error(error): 56 | return handle_http_error(InternalServerError()) 57 | -------------------------------------------------------------------------------- /huskar_api/api/middlewares/logger.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import json 4 | import time 5 | import logging 6 | 7 | from flask import request, g, Blueprint 8 | 9 | from huskar_api.extras.monitor import monitor_client 10 | 11 | 12 | bp = Blueprint('middlewares.logger', __name__) 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | COMMON_SENSITIVE_FIELDS = frozenset([ 17 | 'password', 18 | 'old_password', 19 | 'new_password', 20 | 'value', 21 | ]) 22 | INFRA_CONFIG_SENSITIVE_FIELDS = frozenset([ 23 | 'master', 24 | 'slave', 25 | 'url', 26 | ]) 27 | 28 | 29 | def get_request_user(): 30 | if g.auth: 31 | return '%s %s' % (g.auth.username, request.remote_addr) 32 | else: 33 | return 'anonymous_user %s' % request.remote_addr 34 | 35 | 36 | def get_request_args(): 37 | args = {} 38 | args.update(request.view_args.items()) 39 | args.update(request.values.items()) 40 | json_body = request.get_json(silent=True) or {} 41 | if isinstance(json_body, dict): 42 | args.update(json_body) 43 | 44 | sensitive_fields = set(COMMON_SENSITIVE_FIELDS) 45 | if request.endpoint == 'api.infra_config': 46 | sensitive_fields.update(INFRA_CONFIG_SENSITIVE_FIELDS) 47 | 48 | return {k: v for k, v in args.items() if k not in sensitive_fields} 49 | 50 | 51 | @bp.before_app_request 52 | def start_profiling(): 53 | g._start_timestamp = int(time.time() * 1000) 54 | 55 | 56 | @bp.after_app_request 57 | def record_access_log(response): 58 | if not g.get('_start_timestamp') or request.endpoint in ( 59 | None, 'api.health_check'): 60 | return response 61 | 62 | time_usage = int(time.time() * 1000) - g._start_timestamp 63 | is_ok = (response.status_code // 100) in (2, 3) 64 | api_status = getattr(response, '_api_status', 'unknown') 65 | 66 | # use JSON string to follow standard Tokenizer of ELK 67 | # elastic.co/guide/cn/elasticsearch/guide/current/standard-tokenizer.html 68 | logger.info( 69 | 'Call %s: %s call <%s %s> %s, time: %sms, soa_mode: %s, cluster: %s, ' 70 | 'status: %s, status_code: %s', 71 | 'Ok' if is_ok else 'Failed', 72 | get_request_user(), request.method, request.path, 73 | json.dumps(get_request_args()), time_usage, g.get('route_mode'), 74 | g.get('cluster_name') or 'unknown', api_status, response.status_code) 75 | 76 | tags = dict(method=request.method, 77 | endpoint=request.endpoint, 78 | status=api_status, 79 | status_code=str(response.status_code)) 80 | monitor_client.timing('api_response', time_usage, tags=tags) 81 | 82 | return response 83 | -------------------------------------------------------------------------------- /huskar_api/api/middlewares/rate_limit_ip.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | from flask import Blueprint, request, abort 6 | 7 | from huskar_api import settings 8 | from huskar_api.extras.rate_limiter import ( 9 | check_new_request, RateExceededError) 10 | from huskar_api.switch import switch, SWITCH_ENABLE_RATE_LIMITER 11 | 12 | 13 | bp = Blueprint('middlewares.rate_limit_ip', __name__) 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | @bp.before_app_request 18 | def check_rate_limit(): 19 | if not switch.is_switched_on(SWITCH_ENABLE_RATE_LIMITER): 20 | return 21 | 22 | remote_addr = request.remote_addr 23 | config = get_limiter_config(settings.RATE_LIMITER_SETTINGS, remote_addr) 24 | if not config: 25 | return 26 | 27 | rate, capacity = config['rate'], config['capacity'] 28 | try: 29 | check_new_request(remote_addr, rate, capacity) 30 | except RateExceededError: 31 | abort(429, 'Too Many Requests, the rate limit is {}/s'.format(rate)) 32 | 33 | 34 | def get_limiter_config(configs, remote_addr): 35 | if remote_addr in configs: 36 | return configs[remote_addr] 37 | if '__anonymous__' in configs: 38 | return configs['__anonymous__'] 39 | return configs.get('__default__') 40 | -------------------------------------------------------------------------------- /huskar_api/api/middlewares/rate_limit_user.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | from flask import Blueprint, g, abort 6 | 7 | from huskar_api import settings 8 | from huskar_api.extras.rate_limiter import ( 9 | check_new_request, RateExceededError) 10 | from huskar_api.switch import switch, SWITCH_ENABLE_RATE_LIMITER 11 | 12 | 13 | bp = Blueprint('middlewares.rate_limit_user', __name__) 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | @bp.before_app_request 18 | def check_rate_limit(): 19 | if not switch.is_switched_on(SWITCH_ENABLE_RATE_LIMITER): 20 | return 21 | if not g.get('auth'): 22 | return 23 | 24 | username = g.auth.username 25 | config = get_limiter_config(settings.RATE_LIMITER_SETTINGS, username) 26 | if not config: 27 | return 28 | 29 | rate, capacity = config['rate'], config['capacity'] 30 | try: 31 | check_new_request(username, rate, capacity) 32 | except RateExceededError: 33 | abort(429, 'Too Many Requests, the rate limit is {}/s'.format(rate)) 34 | 35 | 36 | def get_limiter_config(configs, username): 37 | if username in configs: 38 | return configs[username] 39 | return configs.get('__default__') 40 | -------------------------------------------------------------------------------- /huskar_api/api/middlewares/read_only.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | from flask import Blueprint, request 6 | 7 | from huskar_api import settings 8 | from huskar_api.api.utils import ( 9 | api_response, config_and_switch_readonly_endpoints) 10 | from huskar_api.switch import ( 11 | switch, SWITCH_ENABLE_CONFIG_AND_SWITCH_WRITE) 12 | 13 | bp = Blueprint('middlewares.read_only', __name__) 14 | logger = logging.getLogger(__name__) 15 | READ_METHOD_SET = frozenset({'GET', 'HEAD', 'OPTION'}) 16 | 17 | 18 | @bp.before_app_request 19 | def check_config_and_switch_read_only(): 20 | method = request.method 21 | view_args = request.view_args 22 | appid = view_args and view_args.get('application_name') 23 | 24 | response = api_response( 25 | message='Config and switch write inhibit', 26 | status="Forbidden") 27 | response.status_code = 403 28 | 29 | if method in READ_METHOD_SET: 30 | return 31 | if request.endpoint not in config_and_switch_readonly_endpoints: 32 | return 33 | if appid and appid in settings.CONFIG_AND_SWITCH_READONLY_BLACKLIST: 34 | return response 35 | if switch.is_switched_on(SWITCH_ENABLE_CONFIG_AND_SWITCH_WRITE, True): 36 | return 37 | if appid and appid in settings.CONFIG_AND_SWITCH_READONLY_WHITELIST: 38 | return 39 | return response 40 | -------------------------------------------------------------------------------- /huskar_api/api/middlewares/route.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from flask import Blueprint, request, g, abort 4 | 5 | from huskar_api.models.const import ROUTE_MODES 6 | from huskar_api.extras.monitor import monitor_client 7 | 8 | 9 | bp = Blueprint('middlewares.route', __name__) 10 | 11 | 12 | @bp.before_app_request 13 | def collect_route_mode(): 14 | frontend_name = request.headers.get('X-Frontend-Name') 15 | mode = request.headers.get('X-SOA-Mode') 16 | if mode and mode not in ROUTE_MODES: 17 | abort(400, u'X-SOA-Mode must be one of %s' % u'/'.join(ROUTE_MODES)) 18 | if not mode: 19 | mode = 'unknown' 20 | g.route_mode = mode 21 | if not frontend_name and g.auth.username and g.auth.is_application: 22 | monitor_client.increment('route_mode.qps', tags=dict( 23 | mode=mode, from_user=g.auth.username, appid=g.auth.username)) 24 | 25 | 26 | @bp.before_app_request 27 | def collect_application_name(): 28 | g.cluster_name = request.headers.get('X-Cluster-Name') 29 | if g.auth.username and g.auth.is_application: 30 | monitor_client.increment('route_mode.cluster', tags=dict( 31 | from_cluster=g.cluster_name or 'unknown', 32 | from_user=g.auth.username, appid=g.auth.username)) 33 | if g.auth and g.auth.is_application and g.route_mode == 'route': 34 | g.application_name = g.auth.username 35 | if not g.cluster_name: 36 | abort(400, u'X-Cluster-Name is required while X-SOA-Mode is route') 37 | else: 38 | g.application_name = None 39 | -------------------------------------------------------------------------------- /huskar_api/api/schema/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from marshmallow import ValidationError 4 | 5 | from huskar_api.switch import switch, SWITCH_VALIDATE_SCHEMA 6 | from .user import UserSchema 7 | from .instance import InstanceSchema 8 | from .audit import AuditLogSchema 9 | from .input import EventSubscribeSchema, validate_email 10 | from .organization import ApplicationSchema, TeamSchema, ApplicationAuthSchema 11 | from .infra import InfraDownstreamSchema 12 | from .service import ServiceInstanceValueSchema 13 | from .webhook import WebhookSchema 14 | 15 | 16 | __all__ = ['user_schema', 'instance_schema', 'audit_log_schema', 17 | 'application_schema', 'team_schema', 'service_value_schema', 18 | 'event_subscribe_schema', 'validate_email', 'validate_fields', 19 | 'webhook_schema'] 20 | 21 | user_schema = UserSchema(strict=True) 22 | instance_schema = InstanceSchema(strict=True) 23 | audit_log_schema = AuditLogSchema(strict=True) 24 | application_schema = ApplicationSchema(strict=True) 25 | application_auth_schema = ApplicationAuthSchema(strict=True) 26 | team_schema = TeamSchema(strict=True) 27 | event_subscribe_schema = EventSubscribeSchema(strict=True) 28 | service_value_schema = ServiceInstanceValueSchema(strict=True) 29 | webhook_schema = WebhookSchema(strict=True) 30 | 31 | infra_downstream_schema = InfraDownstreamSchema(strict=True) 32 | 33 | 34 | def validate_fields(schema, data, optional_fields=(), partial=True): 35 | """validate fields value but which field name in `optional_fields` 36 | and the value is None. 37 | """ 38 | if not switch.is_switched_on(SWITCH_VALIDATE_SCHEMA, True): 39 | return 40 | 41 | fields = set(data) 42 | if not fields.issubset(schema.fields): 43 | raise ValidationError( 44 | 'The set of fields "%s" is not a subset of %s' 45 | % (fields, schema)) 46 | 47 | data = {k: v for k, v in data.items() 48 | if not (k in optional_fields and v is None)} 49 | schema.validate(data, partial=partial) 50 | -------------------------------------------------------------------------------- /huskar_api/api/schema/audit.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from marshmallow import Schema, fields 4 | 5 | from huskar_api.extras.marshmallow import LocalDateTime 6 | from .user import UserSchema 7 | 8 | 9 | class AuditLogSchema(Schema): 10 | id = fields.Integer() 11 | user = fields.Nested(UserSchema) 12 | remote_addr = fields.String() 13 | action_name = fields.String() 14 | action_data = fields.String() 15 | created_at = LocalDateTime() 16 | rollback_to = fields.Nested('self', exclude=['rollback_to']) 17 | -------------------------------------------------------------------------------- /huskar_api/api/schema/infra.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from marshmallow import Schema, fields 4 | 5 | from huskar_api.extras.marshmallow import LocalDateTime, NamedTuple 6 | 7 | 8 | class InfraDownstreamSchema(Schema): 9 | id = fields.Integer() 10 | user_application_name = fields.String() 11 | user_infra_type = fields.String() 12 | user_infra_name = fields.String() 13 | user_scope_pair = NamedTuple(['type', 'name']) 14 | user_field_name = fields.String() 15 | version = fields.Integer() 16 | created_at = LocalDateTime() 17 | updated_at = LocalDateTime() 18 | -------------------------------------------------------------------------------- /huskar_api/api/schema/input.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from marshmallow import Schema, fields, validates 4 | from marshmallow.validate import Email 5 | from .validates import application_name_validate, cluster_name_validate 6 | 7 | _email_validator = Email() 8 | 9 | 10 | class Clusters(Schema): 11 | application_name = fields.String( 12 | required=True, 13 | validate=application_name_validate, 14 | ) 15 | clusters = fields.List( 16 | fields.String( 17 | required=True, 18 | validate=cluster_name_validate 19 | ) 20 | ) 21 | 22 | 23 | class EventSubscribeSchema(Schema): 24 | service = fields.Dict() 25 | config = fields.Dict() 26 | switch = fields.Dict() 27 | service_info = fields.Dict() 28 | 29 | _clusters_schema = Clusters(strict=True) 30 | 31 | def _validate_clusters(self, value): 32 | for application_name, clusters in value.items(): 33 | self._clusters_schema.validate(dict( 34 | application_name=application_name, 35 | clusters=clusters 36 | )) 37 | 38 | @validates('service') 39 | def validate_service(self, value): 40 | self._validate_clusters(value) 41 | 42 | @validates('config') 43 | def validate_config(self, value): 44 | self._validate_clusters(value) 45 | 46 | @validates('switch') 47 | def validate_switch(self, value): 48 | self._validate_clusters(value) 49 | 50 | 51 | def validate_email(email): 52 | _email_validator(email) 53 | -------------------------------------------------------------------------------- /huskar_api/api/schema/instance.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from marshmallow import Schema, fields, validate 4 | from .validates import ( 5 | application_name_validate, cluster_name_validate, 6 | key_name_validate, value_validate) 7 | 8 | 9 | class InstanceSchema(Schema): 10 | # The length of the application is not restricted before, 11 | # so we relaxed the check of the length 12 | application = fields.String( 13 | required=True, 14 | validate=application_name_validate) 15 | cluster = fields.String(required=True, validate=cluster_name_validate) 16 | key = fields.String(required=True, validate=key_name_validate) 17 | value = fields.String(required=True, validate=value_validate) 18 | comment = fields.String(validate.Length(max=2048)) 19 | -------------------------------------------------------------------------------- /huskar_api/api/schema/organization.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import collections 4 | 5 | from marshmallow import Schema, fields, pre_dump, post_dump 6 | 7 | from huskar_api.models.route import lookup_route_stage 8 | from huskar_api.service.admin.application_auth import ( 9 | is_application_blacklisted, is_application_deprecated) 10 | from .user import UserSchema 11 | from .validates import application_name_validate, team_name_validate 12 | 13 | 14 | class TeamSchema(Schema): 15 | name = fields.String(required=True, validate=team_name_validate) 16 | 17 | 18 | class ApplicationSchema(Schema): 19 | name = fields.String(required=True, validate=application_name_validate) 20 | team_name = fields.String(required=True, validate=team_name_validate) 21 | team_desc = fields.String() 22 | route_stage = fields.Method('_route_stage') 23 | is_deprecated = fields.Method('_is_deprecated') 24 | is_blacklisted = fields.Method('_is_blacklisted') 25 | 26 | DataWrapper = collections.namedtuple('DataWrapper', [ 27 | 'route_stage_table', 'name', 'team_name', 'team_desc', 28 | ]) 29 | 30 | @pre_dump(pass_many=True) 31 | def fill_route_stage(self, data, many): 32 | rs = lookup_route_stage() 33 | if many: 34 | return [self.DataWrapper(rs, **item) for item in data] 35 | return self.DataWrapper(rs, **data) 36 | 37 | @post_dump(pass_many=True) 38 | def strip_blacklisted_items(self, data, many): 39 | if many: 40 | return [item for item in data if not item['is_blacklisted']] 41 | return data 42 | 43 | def _route_stage(self, obj): 44 | return obj.route_stage_table.get(obj.name, {}) 45 | 46 | def _is_deprecated(self, obj): 47 | return is_application_deprecated(obj.name) 48 | 49 | def _is_blacklisted(self, obj): 50 | return is_application_blacklisted(obj.name) 51 | 52 | 53 | class ApplicationAuthSchema(Schema): 54 | authority = fields.String() 55 | user = fields.Nested(UserSchema) 56 | username = fields.FormattedString('{user.username}') 57 | -------------------------------------------------------------------------------- /huskar_api/api/schema/service.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from ipaddress import IPv4Address 4 | from marshmallow import ( 5 | Schema, fields, validate, validates, pre_load, post_load, ValidationError) 6 | from marshmallow.utils import ensure_text_type 7 | 8 | from huskar_api.extras.marshmallow import NestedDict 9 | 10 | 11 | class ServiceInstanceValueSchema(Schema): 12 | ip = fields.String( 13 | required=True, 14 | error_messages={ 15 | 'required': 'ip must be provided in instance value' 16 | }) 17 | port = NestedDict( 18 | fields.Integer(validate=validate.Range(min=1, max=65535)), 19 | required=True, 20 | error_messages={ 21 | 'required': 'port must be provided in instance value', 22 | 'invalid': 'port must be a dict, eg: {{"port":{{"main": 8080}}' 23 | }) 24 | state = fields.String(validate=validate.OneOf( 25 | ['up', 'down'], error='state must be "up" or "down".')) 26 | meta = fields.Dict(required=False) 27 | # TODO: remove these fields below 28 | idc = fields.String(required=False) 29 | cluster = fields.String(required=False) 30 | name = fields.String(required=False) 31 | 32 | @validates('ip') 33 | def validate_ip(self, value): 34 | try: 35 | IPv4Address(unicode(value)) 36 | except ValueError: 37 | raise ValidationError('illegal IP address') 38 | 39 | @validates('port') 40 | def validate_port(self, value): 41 | if 'main' not in value: 42 | raise ValidationError('main port is required') 43 | 44 | @pre_load 45 | def ensure_meta_dict(self, data): 46 | data['meta'] = data.get('meta') or {} 47 | return data 48 | 49 | @post_load 50 | def coerce_meta_value(self, data): 51 | meta = data.setdefault('meta', {}) 52 | for key, value in meta.iteritems(): 53 | meta[key] = ensure_text_type(value or '') 54 | return data 55 | -------------------------------------------------------------------------------- /huskar_api/api/schema/user.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from marshmallow import Schema, fields, validate 4 | 5 | from huskar_api.extras.marshmallow import LocalDateTime 6 | from .validates import user_name_validate 7 | 8 | 9 | class UserSchema(Schema): 10 | id = fields.Integer() 11 | username = fields.String(required=True, validate=user_name_validate) 12 | email = fields.String(required=True, validate=validate.Email()) 13 | is_active = fields.Boolean() 14 | is_admin = fields.Boolean() 15 | is_application = fields.Boolean() 16 | last_login = LocalDateTime() 17 | created_at = LocalDateTime() 18 | updated_at = LocalDateTime() 19 | 20 | # backward compatibility 21 | huskar_admin = fields.Boolean(attribute='is_admin') 22 | -------------------------------------------------------------------------------- /huskar_api/api/schema/validates.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from marshmallow import validate 4 | from marshmallow.exceptions import ValidationError 5 | 6 | 7 | application_name_validate = validate.Regexp( 8 | r'^[a-zA-Z0-9][a-zA-Z0-9_\-.]{0,126}[a-zA-Z0-9]$', 9 | error=(u'AppID({input}) should consist by most 128 characters ' 10 | u'of numbers, lowercase letters and underscore.'), 11 | ) 12 | team_name_validate = validate.Regexp( 13 | r'^[^_][a-zA-Z0-9_\-\.]{1,32}$', 14 | error=(u'Team({input}) should consist by most 32 characters of numbers, ' 15 | u' letters and underscores.') 16 | ) 17 | cluster_name_validate = validate.Regexp( 18 | r'^(?!^\.+$)([a-zA-Z0-9_\-.]{1,64})$', 19 | error=( 20 | u'Cluster({input}) should consist by most 64 characters of numbers, ' 21 | u'letters and underscores, and not starts with dots.') 22 | ) 23 | user_name_validate = validate.Regexp( 24 | r'^[a-zA-Z0-9_\-.]{1,128}$', 25 | error=(u'Username({input}) should consist by most 128 characters ' 26 | u'of numbers, lowercase letters and underscore.'), 27 | ) 28 | 29 | 30 | def key_name_validate(value): 31 | regex_validate = validate.Regexp( 32 | r'^(?!^\.+$)\S+$', 33 | error=u'Key({input}) should not starts with dots or contains CRLF.' 34 | ) 35 | value = regex_validate(value) 36 | if not all(0x00 < ord(c) < 0x7F for c in unicode(value)): 37 | raise ValidationError( 38 | u'Key({}) contains unicode characters.'.format(value)) 39 | 40 | 41 | def value_validate(value): 42 | if not value: 43 | raise ValidationError(u"Value can't be a empty string.") 44 | -------------------------------------------------------------------------------- /huskar_api/api/schema/webhook.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from marshmallow import Schema, fields, validate 4 | from huskar_api.models.audit import action_types 5 | from huskar_api.models.webhook import Webhook 6 | 7 | 8 | class WebhookSchema(Schema): 9 | webhook_url = fields.URL(required=True) 10 | webhook_type = fields.Integer( 11 | validate=validate.OneOf( 12 | Webhook.HOOK_TYPES, 13 | error='invalid webhook type' 14 | ) 15 | ) 16 | event_list = fields.List( 17 | fields.String( 18 | validate=validate.OneOf( 19 | action_types.action_map, 20 | error='not a valid event type' 21 | ), 22 | ) 23 | ) 24 | -------------------------------------------------------------------------------- /huskar_api/api/well_known.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from flask.views import MethodView 4 | 5 | from huskar_api import settings 6 | from .utils import api_response 7 | 8 | 9 | class WellKnownCommonView(MethodView): 10 | def get(self): 11 | """Gets the common well-known data. 12 | 13 | An example of response:: 14 | 15 | { 16 | "status": "SUCCESS", 17 | "data": { 18 | "framework_versions": { 19 | "latest": { 20 | } 21 | }, 22 | "idc_list": ["alta", "altb"], 23 | "ezone_list": ["alta1", "altb1"], 24 | "route_default_hijack_mode": { 25 | "alta1": "S", 26 | "altb1": "D", 27 | "altc1": "S" 28 | }, 29 | "force_routing_clusters": { 30 | } 31 | } 32 | } 33 | """ 34 | route_default_hijack_mode = settings.ROUTE_EZONE_DEFAULT_HIJACK_MODE 35 | data = { 36 | 'framework_versions': settings.FRAMEWORK_VERSIONS, 37 | 'idc_list': settings.ROUTE_IDC_LIST, 38 | 'ezone_list': settings.ROUTE_EZONE_LIST, 39 | 'route_default_hijack_mode': route_default_hijack_mode, 40 | 'force_routing_clusters': settings.FORCE_ROUTING_CLUSTERS, 41 | } 42 | return api_response(data) 43 | -------------------------------------------------------------------------------- /huskar_api/app.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from flask import Flask 4 | from werkzeug.utils import import_string 5 | 6 | from huskar_api import settings 7 | 8 | 9 | extensions = [ 10 | 'huskar_api.ext:babel', 11 | 'huskar_api.ext:sentry', 12 | 'huskar_api.ext:db_tester', 13 | ] 14 | 15 | blueprints = [ 16 | ('huskar_api.api.middlewares.rate_limit_ip:bp', None), 17 | ('huskar_api.api.middlewares.auth:bp', None), 18 | ('huskar_api.api.middlewares.concurrent_limit:bp', None), 19 | ('huskar_api.api.middlewares.rate_limit_user:bp', None), 20 | ('huskar_api.api.middlewares.route:bp', None), 21 | ('huskar_api.api.middlewares.error:bp', None), 22 | ('huskar_api.api.middlewares.db:bp', None), 23 | ('huskar_api.api.middlewares.logger:bp', None), 24 | ('huskar_api.api.middlewares.read_only:bp', None), 25 | ('huskar_api.api.middlewares.control_access_via_api:bp', None), 26 | 27 | ('huskar_api.api:bp', '/api'), 28 | ] 29 | 30 | 31 | def create_app(): 32 | app = Flask(__name__) 33 | app.config['DEBUG'] = settings.DEBUG 34 | app.config['SECRET_KEY'] = settings.SECRET_KEY 35 | app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = False 36 | app.config['SENTRY_DSN'] = settings.SENTRY_DSN 37 | app.config['BABEL_DEFAULT_LOCALE'] = settings.DEFAULT_LOCALE 38 | app.config['BABEL_DEFAULT_TIMEZONE'] = settings.DEFAULT_TIMEZONE 39 | app.config['LOGGER_HANDLER_POLICY'] = 'never' 40 | app.logger.propagate = True 41 | 42 | for extension_qualname in extensions: 43 | extension = import_string(extension_qualname) 44 | extension.init_app(app) 45 | 46 | for blueprint_qualname, url_prefix in blueprints: 47 | blueprint = import_string(blueprint_qualname) 48 | app.register_blueprint(blueprint, url_prefix=url_prefix) 49 | 50 | return app 51 | -------------------------------------------------------------------------------- /huskar_api/bootstrap/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .core import Bootstrap 4 | 5 | __all__ = ['Bootstrap'] 6 | -------------------------------------------------------------------------------- /huskar_api/bootstrap/consts.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import os 4 | 5 | VAR_RUN_DIR = '/data/run' 6 | HUSKAR_CACHE_DIR = os.path.join(VAR_RUN_DIR, 'huskar_python_cache') 7 | 8 | ENV_DEV = 'dev' 9 | ENV_TESTING = 'testing' 10 | ENV_PROD = 'prod' 11 | -------------------------------------------------------------------------------- /huskar_api/cli.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function 2 | 3 | import sys 4 | 5 | from flask_script import Manager, prompt_pass 6 | from flask_script.commands import ShowUrls, Clean 7 | 8 | from .app import create_app 9 | from .models.auth import User 10 | 11 | 12 | manager = Manager(create_app()) 13 | manager.add_command(ShowUrls()) 14 | manager.add_command(Clean()) 15 | 16 | 17 | @manager.command 18 | def initadmin(): 19 | """Creates an initial user.""" 20 | admin_user = User.get_by_name('admin') 21 | if admin_user: 22 | print('The user "admin" exists', file=sys.stderr) 23 | sys.exit(1) 24 | 25 | password = None 26 | while not password: 27 | password = prompt_pass('Password', default='').strip() 28 | admin_user = User.create_normal('admin', password, is_active=True) 29 | admin_user.grant_admin() 30 | 31 | 32 | def main(): 33 | manager.run() 34 | -------------------------------------------------------------------------------- /huskar_api/contrib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/huskar_api/contrib/__init__.py -------------------------------------------------------------------------------- /huskar_api/contrib/backdoor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from gevent import Greenlet 4 | from gevent.backdoor import BackdoorServer 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class ServeBackdoor(Greenlet): 10 | __instance = None 11 | 12 | def __init__(self, addr, *args, **kwargs): 13 | if self.__class__.__instance is not None: 14 | raise RuntimeError( 15 | 'only one backdoor server allowed to be running' 16 | ) 17 | self.addr = addr 18 | self.server = BackdoorServer(addr) 19 | Greenlet.__init__(self, *args, **kwargs) 20 | 21 | @classmethod 22 | def get_instance(cls): 23 | return cls.__instance 24 | 25 | # pylint: disable=E0202 26 | def _run(self): 27 | cls = self.__class__ 28 | try: 29 | cls.__instance = self 30 | logger.info("starting backdoor server on %r...", self.addr) 31 | self.server.serve_forever() 32 | finally: 33 | logger.info('backdoor server on %r stopped', self.addr) 34 | cls.__instance = None 35 | -------------------------------------------------------------------------------- /huskar_api/contrib/signal.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | import sys 5 | 6 | import os 7 | import signal 8 | import gevent 9 | 10 | from huskar_api.contrib.backdoor import ServeBackdoor 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | def handle_sig(signum, handler): 16 | signal.signal(signum, handler) 17 | logger.info('handling signal: %s', signum) 18 | 19 | 20 | def ignore_sig(signum): 21 | signal.signal(signum, signal.SIG_IGN) 22 | logger.info('ignoring %s', signum) 23 | 24 | 25 | def _start_backdoor_server(signum, frame): 26 | host = os.environ.get('HUSKAR_API_BS_HOST', '127.0.0.1') 27 | port = os.environ.get('HUSKAR_API_BS_PORT', 4455) 28 | addr = (host, int(port)) 29 | 30 | try: 31 | server = ServeBackdoor(addr) 32 | server.start() 33 | except: # noqa 34 | e = sys.exc_info()[1] 35 | logger.info('failed to start backdoor server on %r: %s', addr, e) 36 | return 37 | 38 | _handle_ttou() 39 | 40 | 41 | def _stop_backdoor_server(signum, frame): 42 | def _stop(): 43 | server = ServeBackdoor.get_instance() 44 | if server: 45 | logger.info('stopping backdoor server on %r...', server.addr) 46 | server.kill() 47 | 48 | ignore_sig(signal.SIGTTOU) 49 | gevent.spawn(_stop()) 50 | 51 | 52 | def handle_ttin(): 53 | handle_sig(signal.SIGTTIN, _start_backdoor_server) 54 | 55 | 56 | def _handle_ttou(): 57 | handle_sig(signal.SIGTTIN, _stop_backdoor_server) 58 | -------------------------------------------------------------------------------- /huskar_api/extras/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/huskar_api/extras/__init__.py -------------------------------------------------------------------------------- /huskar_api/extras/concurrent_limiter.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | import time 5 | import uuid 6 | 7 | from huskar_api.models import redis_client 8 | 9 | logger = logging.getLogger(__name__) 10 | SCRIPT = ''' 11 | local key = KEYS[1] 12 | 13 | local capacity = tonumber(ARGV[1]) 14 | local timestamp = tonumber(ARGV[2]) 15 | local id = ARGV[3] 16 | 17 | local count = redis.call("zcard", key) 18 | local allowed = count < capacity 19 | 20 | if allowed then 21 | redis.call("zadd", key, timestamp, id) 22 | end 23 | 24 | return { allowed, count } 25 | ''' 26 | PREFIX = 'huskar_api.concurrent_requests_limiter' 27 | 28 | 29 | def check_new_request(identity, ttl, capacity): 30 | timestamp = int(time.time()) 31 | key = '{}.{}'.format(PREFIX, identity) 32 | sub_item = str(uuid.uuid4()) 33 | keys = [key] 34 | keys_and_args = keys + [capacity, timestamp, sub_item] 35 | try: 36 | redis_client.zremrangebyscore(key, '-inf', timestamp - ttl) 37 | allowed, count = redis_client.eval(SCRIPT, len(keys), *keys_and_args) 38 | except Exception as e: 39 | logger.warning('check new request for concurrent limit error: %s', e) 40 | return 41 | 42 | if allowed != 1: 43 | raise ConcurrencyExceededError() 44 | 45 | return key, sub_item 46 | 47 | 48 | def release_request(key, sub_item): 49 | try: 50 | redis_client.zrem(key, sub_item) 51 | except Exception as e: 52 | logger.warning('release request for concurrent limit error: %s', e) 53 | 54 | 55 | def release_after_iterator_end(data, iterator): 56 | try: 57 | for item in iterator: 58 | yield item 59 | finally: 60 | if data: 61 | release_request(data['key'], data['sub_item']) 62 | 63 | 64 | class ConcurrencyExceededError(Exception): 65 | pass 66 | -------------------------------------------------------------------------------- /huskar_api/extras/email.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from __future__ import absolute_import 4 | 5 | import logging 6 | 7 | from enum import Enum 8 | from flask import render_template 9 | 10 | from huskar_api import settings 11 | from huskar_api.extras.mail_client import AbstractMailClient 12 | 13 | __all__ = ['EmailTemplate', 'EmailDeliveryError', 'deliver_email'] 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | class EmailTemplate(Enum): 19 | DEBUG = ('email-debug.html', u'Huskar Debug', {'foo'}) 20 | SIGNUP = ( 21 | 'email-signup.html', u'Huskar 用户创建', {'username', 'password'}) 22 | PASSWORD_RESET = ( 23 | 'email-password-reset.html', u'Huskar 密码重置', 24 | {'username', 'token', 'expires_in'}) 25 | PERMISSION_GRANT = ( 26 | 'email-permission-grant.html', u'Huskar 权限变更', 27 | {'username', 'application_name', 'authority'}) 28 | PERMISSION_DISMISS = ( 29 | 'email-permission-dismiss.html', u'Huskar 权限变更', 30 | {'username', 'application_name', 'authority'}) 31 | INFRA_CONFIG_CREATE = ( 32 | 'email-infra-config-create.html', u'Huskar 基础资源绑定通知', 33 | {'infra_name', 'infra_type', 'application_name', 'is_authorized'}) 34 | 35 | 36 | class EmailDeliveryError(Exception): 37 | pass 38 | 39 | 40 | def render_email_template(template, **kwargs): 41 | template = EmailTemplate(template) 42 | filename, subject, _ = template.value 43 | context = dict(kwargs) 44 | context['title'] = subject 45 | context['core_config'] = { 46 | 'env': settings.ENV, 47 | } 48 | context['settings'] = settings 49 | return render_template(template.value, **context) 50 | 51 | 52 | def deliver_email(template, receiver, arguments, cc=None, client=None): 53 | cc = cc or [] 54 | _, subject, required_arguments = template.value 55 | if set(arguments) != required_arguments: 56 | raise ValueError('Invalid arguments') 57 | 58 | if client and isinstance(client, AbstractMailClient): 59 | message = render_email_template(template, **arguments) 60 | client.deliver_email(receiver, subject, message, cc) 61 | -------------------------------------------------------------------------------- /huskar_api/extras/mail_client/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from .abstract_client import AbstractMailClient 3 | 4 | __all__ = ['AbstractMailClient'] 5 | -------------------------------------------------------------------------------- /huskar_api/extras/mail_client/abstract_client.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | 4 | class AbstractMailClient(object): 5 | __metaclass__ = abc.ABCMeta 6 | 7 | @abc.abstractmethod 8 | def deliver_email(self, receiver, subject, message, cc): 9 | pass # pragma: no cover 10 | -------------------------------------------------------------------------------- /huskar_api/extras/monitor.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | 4 | class MonitorClient(object): 5 | def __init__(self): 6 | pass 7 | 8 | def timing(self, name, time, tags=None, upper_enable=True): 9 | pass 10 | 11 | def increment(self, name, sample_rate=1, tags=None): 12 | pass 13 | 14 | def payload(self, name, data_length=0, tags=None): 15 | pass 16 | 17 | 18 | monitor_client = MonitorClient() 19 | -------------------------------------------------------------------------------- /huskar_api/extras/payload.py: -------------------------------------------------------------------------------- 1 | from huskar_api.extras.monitor import monitor_client 2 | 3 | 4 | def zk_payload(payload_data, payload_type): 5 | monitor_client.payload("zookeeper.payload", 6 | tags=dict(payload_type=payload_type), 7 | data_length=len( 8 | payload_data) if payload_data else 0) 9 | -------------------------------------------------------------------------------- /huskar_api/extras/rate_limiter.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | import time 5 | 6 | from huskar_api.models import redis_client 7 | 8 | logger = logging.getLogger(__name__) 9 | SCRIPT = ''' 10 | local tokens_key = KEYS[1] 11 | local timestamp_key = KEYS[2] 12 | 13 | local rate = tonumber(ARGV[1]) 14 | local capacity = tonumber(ARGV[2]) 15 | local now = tonumber(ARGV[3]) 16 | local requested = tonumber(ARGV[4]) 17 | 18 | local fill_time = capacity/rate 19 | local ttl = math.floor(fill_time+0.999) 20 | 21 | local last_tokens = tonumber(redis.call("get", tokens_key)) 22 | if last_tokens == nil then 23 | last_tokens = capacity 24 | end 25 | 26 | local last_refreshed = tonumber(redis.call("get", timestamp_key)) 27 | if last_refreshed == nil then 28 | last_refreshed = 0 29 | end 30 | 31 | local delta = math.max(0, now-last_refreshed) 32 | local filled_tokens = math.min(capacity, last_tokens+(delta*rate)) 33 | local allowed = filled_tokens >= requested 34 | local new_tokens = filled_tokens 35 | if allowed then 36 | new_tokens = filled_tokens - requested 37 | end 38 | 39 | redis.call("setex", tokens_key, ttl, new_tokens) 40 | redis.call("setex", timestamp_key, ttl, now) 41 | 42 | return { allowed, new_tokens } 43 | ''' 44 | PREFIX = 'huskar_api.request_rate_limiter' 45 | 46 | 47 | def check_new_request(identity, rate, capacity, requested=1): 48 | prefix = '{%s}' % ('{}.{}'.format(PREFIX, identity)) 49 | keys = ['{}.tokens'.format(prefix), '{}.timestamp'.format(prefix)] 50 | keys_and_args = keys + [rate, capacity, int(time.time()), requested] 51 | try: 52 | allowed, tokens_left = redis_client.eval( 53 | SCRIPT, len(keys), *keys_and_args) 54 | except Exception as e: 55 | logger.warning('check new request for rate limit error: %s', e) 56 | return 57 | 58 | if allowed != 1: 59 | raise RateExceededError() 60 | 61 | 62 | class RateExceededError(Exception): 63 | pass 64 | -------------------------------------------------------------------------------- /huskar_api/extras/raven.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | import raven 6 | 7 | from huskar_api import settings 8 | from huskar_api.switch import ( 9 | switch, SWITCH_ENABLE_SENTRY_MESSAGE, SWITCH_ENABLE_SENTRY_EXCEPTION) 10 | 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | raven_client = raven.Client( 15 | dsn=settings.SENTRY_DSN) if settings.SENTRY_DSN else None 16 | 17 | 18 | def capture_message(*args, **kwargs): 19 | if not switch.is_switched_on(SWITCH_ENABLE_SENTRY_MESSAGE): 20 | return 21 | try: 22 | if raven_client: 23 | raven_client.captureMessage(*args, **kwargs) 24 | else: 25 | logger.warn('Ignored capture_message %r %r', args, kwargs) 26 | except Exception as e: 27 | logger.warn('Failed to send event to sentry: %r', e, exc_info=True) 28 | 29 | 30 | def capture_exception(*args, **kwargs): 31 | if not switch.is_switched_on(SWITCH_ENABLE_SENTRY_EXCEPTION): 32 | return 33 | try: 34 | if raven_client: 35 | raven_client.captureException(*args, **kwargs) 36 | else: 37 | logger.warn('Ignored capture_exception with %r %r', args, kwargs) 38 | except Exception as e: 39 | logger.warn('Failed to send event to sentry: %r', e, exc_info=True) 40 | -------------------------------------------------------------------------------- /huskar_api/extras/uptime.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import time 4 | 5 | 6 | _start_time = time.time() 7 | 8 | 9 | def process_uptime(): 10 | """Gets the uptime of current process.""" 11 | return time.time() - _start_time 12 | -------------------------------------------------------------------------------- /huskar_api/models/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import datetime 4 | 5 | from huskar_sdk_v2.bootstrap.client import BaseClient 6 | from huskar_sdk_v2.consts import BASE_PATH 7 | from sqlalchemy import text, Column, DateTime 8 | from sqlalchemy.ext.declarative import declared_attr 9 | 10 | from huskar_api import settings 11 | from huskar_api.settings import ZK_SETTINGS 12 | from huskar_api.models.cache import Cache, cache_mixin 13 | from huskar_api.models.db import model_base, db_manager 14 | from .utils import make_cache_decorator 15 | 16 | __all__ = ['huskar_client', 'DBSession', 'DeclarativeBase', 'TimestampMixin', 17 | 'cache_manager', 'cache_on_arguments', 'CacheMixin'] 18 | 19 | #: The client of Huskar SDK which manages the ZooKeeper sessions 20 | huskar_client = BaseClient( 21 | ZK_SETTINGS['servers'], ZK_SETTINGS['username'], ZK_SETTINGS['password'], 22 | base_path=BASE_PATH, max_retries=-1) 23 | huskar_client.start(ZK_SETTINGS['start_timeout']) 24 | 25 | #: The scoped session factory of SQLAlchemy 26 | DBSession = db_manager.get_session('default') 27 | 28 | #: The base class of SQLAlchemy declarative model 29 | DeclarativeBase = model_base() 30 | DeclarativeBase.__table_args__ = { 31 | 'mysql_character_set': 'utf8mb4', 'mysql_collate': 'utf8mb4_bin'} 32 | 33 | cache_manager = Cache(settings.CACHE_SETTINGS['default']) 34 | 35 | #: The Redis raw client 36 | redis_client = cache_manager.make_client(raw=True) 37 | #: The decorator of Redis cache 38 | cache_on_arguments = make_cache_decorator(cache_manager.make_client(raw=True)) 39 | #: The mixin class of Redis cache 40 | CacheMixin = cache_mixin( 41 | cache=cache_manager.make_client(namespace='%s:v2' % __name__), 42 | session=DBSession) 43 | CacheMixin.TABLE_CACHE_EXPIRATION_TIME = settings.TABLE_CACHE_EXPIRATION_TIME 44 | 45 | 46 | class TimestampMixin(object): 47 | """The mixin class of timestamp field. 48 | 49 | For the requirement of DBA, all new models which use MySQL should include 50 | this class as one of their bases. 51 | """ 52 | 53 | @declared_attr 54 | def created_at(cls): 55 | return Column( 56 | DateTime, nullable=False, default=datetime.datetime.now, 57 | server_default=text('CURRENT_TIMESTAMP'), index=True) 58 | 59 | @declared_attr 60 | def updated_at(cls): 61 | return Column( 62 | DateTime, nullable=False, server_default=text( 63 | 'CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'), 64 | index=True) 65 | -------------------------------------------------------------------------------- /huskar_api/models/alembic.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | # Include all models 4 | import huskar_api.models.auth 5 | import huskar_api.models.comment 6 | import huskar_api.models.audit 7 | import huskar_api.models.infra 8 | import huskar_api.models.webhook 9 | 10 | 11 | def get_metadata(): 12 | return huskar_api.models.DeclarativeBase.metadata 13 | -------------------------------------------------------------------------------- /huskar_api/models/audit/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | from .action import action_types, action_creator 6 | from .rollback import action_rollback 7 | from .audit import AuditLog 8 | 9 | 10 | __all__ = ['action_types', 'action_creator', 'AuditLog', 'logger', 11 | 'action_rollback'] 12 | 13 | 14 | logger = logging.getLogger(__name__) 15 | -------------------------------------------------------------------------------- /huskar_api/models/audit/const.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | 4 | # Generic types 5 | TYPE_SITE = 0 6 | TYPE_TEAM = 1 7 | TYPE_APPLICATION = 2 8 | 9 | # Key types 10 | TYPE_CONFIG = 10 11 | TYPE_SWITCH = 11 12 | TYPE_SERVICE = 12 13 | 14 | 15 | # User types 16 | NORMAL_USER = 0 17 | APPLICATION_USER = 1 18 | 19 | # Severity of action 20 | SEVERITY_NORMAL = 0 21 | SEVERITY_DANGEROUS = 1 22 | -------------------------------------------------------------------------------- /huskar_api/models/auth/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .team import Team, TeamAdmin 4 | from .user import User 5 | from .application import Application, ApplicationAuth 6 | from .session import SessionAuth 7 | from .role import Authority 8 | 9 | 10 | __all__ = ['Team', 'TeamAdmin', 'User', 'Application', 'ApplicationAuth', 11 | 'SessionAuth', 'Authority'] 12 | -------------------------------------------------------------------------------- /huskar_api/models/auth/role.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from enum import Enum, unique 4 | 5 | 6 | @unique 7 | class Authority(Enum): 8 | READ = u'read' 9 | WRITE = u'write' 10 | ADMIN = u'admin' 11 | -------------------------------------------------------------------------------- /huskar_api/models/catalog/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .schema import ServiceInfo, ClusterInfo 4 | 5 | 6 | __all__ = ['ServiceInfo', 'ClusterInfo'] 7 | -------------------------------------------------------------------------------- /huskar_api/models/catalog/dependency.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from marshmallow import Schema, fields, validates, ValidationError 4 | 5 | from huskar_api.models.znode import ZnodeModel 6 | 7 | 8 | class DependencySchemaMixin(Schema): 9 | dependency = fields.Dict() 10 | 11 | @validates('dependency') 12 | def validate_dependency(self, value): 13 | for application_name, cluster_names in value.iteritems(): 14 | if (not isinstance(application_name, unicode) or 15 | not application_name): 16 | raise ValidationError('Invalid application name') 17 | if not isinstance(cluster_names, list): 18 | raise ValidationError('Invalid cluster list') 19 | if not all(isinstance(n, unicode) for n in cluster_names): 20 | raise ValidationError('Invalid cluster name') 21 | 22 | 23 | class DependencyMixin(ZnodeModel): 24 | def get_dependency(self): 25 | data = self.data or {} 26 | return data.get('dependency', {}) 27 | 28 | def freeze_dependency(self): 29 | return _freeze_dependency(self.get_dependency()) 30 | 31 | def add_dependency(self, application_name, cluster_name): 32 | dependency = self._initialize() 33 | cluster_names = set(dependency.get(application_name, [])) 34 | cluster_names.add(cluster_name) 35 | dependency[application_name] = sorted(cluster_names) 36 | 37 | def discard_dependency(self, application_name, cluster_name): 38 | dependency = self._initialize() 39 | cluster_names = set(dependency.get(application_name, [])) 40 | cluster_names.discard(cluster_name) 41 | dependency[application_name] = sorted(cluster_names) 42 | 43 | def _initialize(self): 44 | data = self.setdefault({}) 45 | dependency = data.setdefault('dependency', {}) 46 | return dependency 47 | 48 | 49 | def _freeze_dependency(dependency): 50 | return frozenset((k, frozenset(v)) for k, v in dependency.iteritems() if v) 51 | -------------------------------------------------------------------------------- /huskar_api/models/catalog/info.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from marshmallow import Schema, fields 4 | 5 | from huskar_api.models.znode import ZnodeModel 6 | 7 | 8 | class InfoSchemaMixin(Schema): 9 | info = fields.Dict() 10 | 11 | 12 | class InfoMixin(ZnodeModel): 13 | 14 | def get_info(self): 15 | data = self.data or {} 16 | return data.get('info', {}) 17 | 18 | def set_info(self, new_data): 19 | data = self.setdefault({}) 20 | if new_data: 21 | data['info'] = new_data 22 | else: 23 | data.pop('info', None) 24 | -------------------------------------------------------------------------------- /huskar_api/models/catalog/route.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from marshmallow import Schema, fields 4 | 5 | from huskar_api.models.znode import ZnodeModel 6 | from huskar_api.models.const import ROUTE_LINKS_DELIMITER 7 | 8 | 9 | class RouteSchemaMixin(Schema): 10 | link = fields.List(fields.String()) 11 | route = fields.Dict() 12 | 13 | 14 | class RouteMixin(ZnodeModel): 15 | def get_route(self): 16 | data = self.data or {} 17 | return data.get('route', {}) 18 | 19 | def set_route(self, route_key, cluster_name): 20 | data = self.setdefault({}) 21 | route = data.setdefault('route', {}) 22 | route[route_key] = cluster_name 23 | 24 | def discard_route(self, route_key): 25 | data = self.setdefault({}) 26 | route = data.setdefault('route', {}) 27 | return route.pop(route_key, None) 28 | 29 | def get_link(self): 30 | data = self.data or {} 31 | link = data.get('link', []) 32 | return ROUTE_LINKS_DELIMITER.join(sorted(link)) or None 33 | 34 | def set_link(self, link): 35 | data = self.setdefault({}) 36 | data['link'] = sorted(frozenset(link.split(ROUTE_LINKS_DELIMITER))) 37 | 38 | def delete_link(self): 39 | data = self.setdefault({}) 40 | data['link'] = [] 41 | -------------------------------------------------------------------------------- /huskar_api/models/catalog/schema.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from marshmallow import Schema, fields, pre_load 4 | 5 | from huskar_api.models.znode import ZnodeModel 6 | from huskar_api.models.exceptions import MalformedDataError 7 | from .route import RouteSchemaMixin, RouteMixin 8 | from .default_route import DefaultRouteSchemaMixin, DefaultRouteMixin 9 | from .dependency import DependencySchemaMixin, DependencyMixin 10 | from .info import InfoMixin, InfoSchemaMixin 11 | 12 | 13 | class DataFixMixin(Schema): 14 | """Fix the exception of marshmallow while "None" is accepted.""" 15 | 16 | @pre_load 17 | def process_none(self, data): 18 | if data is None: 19 | return {} 20 | return data 21 | 22 | 23 | class ServiceInfoSchema(InfoSchemaMixin, DependencySchemaMixin, 24 | DefaultRouteSchemaMixin, DataFixMixin, Schema): 25 | _version = fields.Constant('1', dump_only=True) 26 | 27 | 28 | class ClusterInfoSchema(InfoSchemaMixin, RouteSchemaMixin, 29 | DataFixMixin, Schema): 30 | _version = fields.Constant('1', dump_only=True) 31 | 32 | 33 | class DummyFactoryMixin(object): 34 | @classmethod 35 | def make_dummy(cls, data, **kwargs): 36 | """Make a dummy instance to support read-only operations.""" 37 | # TODO Could we find a better way? 38 | instance = cls(client=None, **kwargs) 39 | instance.load = None 40 | instance.save = None 41 | if data: 42 | try: 43 | instance.data, _ = cls.MARSHMALLOW_SCHEMA.loads(data) 44 | except cls._MALFORMED_DATA_EXCEPTIONS as e: 45 | raise MalformedDataError(instance, e) 46 | return instance 47 | 48 | 49 | class ServiceInfo(InfoMixin, DependencyMixin, DefaultRouteMixin, 50 | DummyFactoryMixin, ZnodeModel): 51 | """The application-level control info. 52 | 53 | :param type_name: The catalog type (service, switch or config). 54 | :param application_name: The application name (a.k.a appid). 55 | """ 56 | 57 | PATH_PATTERN = u'/huskar/{type_name}/{application_name}' 58 | MARSHMALLOW_SCHEMA = ServiceInfoSchema(strict=True) 59 | 60 | 61 | class ClusterInfo(InfoMixin, RouteMixin, DummyFactoryMixin, ZnodeModel): 62 | """The cluster-level control info. 63 | 64 | :param type_name: The catalog type (service, switch or config). 65 | :param application_name: The application name (a.k.a appid). 66 | :param cluster_name: The cluster name. 67 | """ 68 | 69 | PATH_PATTERN = u'/huskar/{type_name}/{application_name}/{cluster_name}' 70 | MARSHMALLOW_SCHEMA = ClusterInfoSchema(strict=True) 71 | -------------------------------------------------------------------------------- /huskar_api/models/const.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from huskar_api import settings 4 | 5 | 6 | ENV_DEV = 'dev' 7 | 8 | USER_AGENT = u'%s/%s' % ( 9 | settings.APP_NAME, settings.APP_COMMIT[:8] if settings.APP_COMMIT 10 | else u'unknown-version') 11 | 12 | ROUTE_DEFAULT_INTENT = 'direct' 13 | ROUTE_MODE_ROUTE = 'route' 14 | ROUTE_MODES = ('orig', 'prefix', ROUTE_MODE_ROUTE) 15 | ROUTE_LINKS_DELIMITER = u'+' 16 | 17 | MM_REASON_AUTH = 'auth' 18 | MM_REASON_SWITCH = 'switch' 19 | MM_REASON_TESTER = 'tester' 20 | MM_REASON_STARTUP = 'startup' 21 | 22 | SELF_APPLICATION_NAME = settings.APP_NAME 23 | 24 | # Spec http://example.com/design/infra_key.html 25 | INFRA_CONFIG_KEYS = { 26 | 'database': 'FX_DATABASE_SETTINGS', # Final 27 | 'redis': 'FX_REDIS_SETTINGS', # Final 28 | 'amqp': 'FX_AMQP_SETTINGS', # Final 29 | 'es': 'FX_ES_SETTINGS', # Draft 30 | 'mongo': 'FX_MONGO_SETTINGS', # Draft 31 | 'oss': 'FX_OSS_SETTINGS', # Draft 32 | 'kafka': 'FX_KAFKA_SETTINGS', # Draft 33 | } 34 | 35 | # scope types 36 | SCOPE_SITE = 0 37 | SCOPE_TEAM = 1 38 | SCOPE_APPLICATION = 2 39 | SCOPE_SCENE = 3 40 | 41 | EXTRA_SUBDOMAIN_SERVICE_INFO = 'service_info' 42 | 43 | MAGIC_CONFIG_KEYS = { 44 | 'batch_config.inclusive_keys': 'HUSKAR_BATCH_CONFIG_INCLUSIVE_KEYS', 45 | } 46 | RELEASE_WINDOW_BYPASS_VALUE = 'bypass' 47 | -------------------------------------------------------------------------------- /huskar_api/models/container/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .common import is_container_id 4 | from .management import ContainerManagement 5 | 6 | 7 | __all__ = ['ContainerManagement', 'is_container_id'] 8 | -------------------------------------------------------------------------------- /huskar_api/models/container/common.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import re 3 | 4 | container_regex = re.compile(r"^[a-z0-9]{64}$") 5 | 6 | 7 | def is_container_id(key): 8 | """Checks an instance key is a container id or not. 9 | 10 | :param key: The instance key. 11 | :returns: ``True`` for possible container id. 12 | """ 13 | if re.match(container_regex, key): 14 | return True 15 | return False 16 | -------------------------------------------------------------------------------- /huskar_api/models/dataware/__init__.py: -------------------------------------------------------------------------------- 1 | # TODO Expecting a volunteer to delete this package totally 2 | -------------------------------------------------------------------------------- /huskar_api/models/dataware/zookeeper/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .client import HuskarClient 4 | 5 | 6 | config_client = HuskarClient('config') 7 | switch_client = HuskarClient('switch') 8 | service_client = HuskarClient('service') 9 | -------------------------------------------------------------------------------- /huskar_api/models/dataware/zookeeper/client.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | # TODO Expecting a volunteer to delete this module totally 4 | 5 | from __future__ import absolute_import 6 | 7 | from huskar_sdk_v2.consts import BASE_PATH 8 | from huskar_sdk_v2.utils import combine 9 | from kazoo.exceptions import NoNodeError, NodeExistsError, BadVersionError 10 | 11 | from huskar_api.extras.payload import zk_payload 12 | from huskar_api.models import huskar_client 13 | from huskar_api.models.exceptions import OutOfSyncError 14 | from huskar_api.service.exc import DataExistsError 15 | 16 | 17 | class HuskarClient(object): 18 | ''' 19 | base operations on config/service/switch 20 | ''' 21 | 22 | def __init__(self, sub_domain): 23 | self.sub_domain = sub_domain 24 | 25 | @property 26 | def raw_client(self): 27 | return huskar_client.client 28 | 29 | def get_path(self, application, cluster, key=None): 30 | return combine(BASE_PATH, self.sub_domain, application, cluster, key) 31 | 32 | def get(self, application=None, cluster=None, key=None): 33 | # TODO [refactor] those should be different functions 34 | if application and cluster and key: # application+cluster+key 35 | path = self.get_path(application, cluster, key) 36 | try: 37 | value, _ = self.raw_client.get(path) 38 | return value 39 | except NoNodeError: 40 | return None 41 | else: # pragma: no cover 42 | raise NotImplementedError() 43 | 44 | def set(self, application, cluster, key, value, version=None): 45 | value = str(value) if value else '' 46 | path = self.get_path(application, cluster, key) 47 | try: 48 | if version is None: 49 | self.raw_client.set(path, value) 50 | else: 51 | self.raw_client.set(path, value, version) 52 | zk_payload(payload_data=value, payload_type='set') 53 | except NoNodeError: 54 | self.raw_client.create(path, value, makepath=True) 55 | zk_payload(payload_data=value, payload_type='create') 56 | except BadVersionError as e: 57 | raise OutOfSyncError(e) 58 | 59 | def delete(self, application, cluster=None, key=None, strict=False): 60 | path = self.get_path(application, cluster, key) 61 | self.raw_client.delete(path, recursive=True) 62 | 63 | def create_if_not_exist(self, application, cluster=None, strict=False): 64 | path = self.get_path(application, cluster) 65 | try: 66 | self.raw_client.create(path, b'', makepath=True) 67 | zk_payload(payload_data=b'', payload_type='create') 68 | except NodeExistsError: 69 | if strict: 70 | target = 'application' if cluster is None else 'cluster' 71 | raise DataExistsError('%s exists already' % target) 72 | -------------------------------------------------------------------------------- /huskar_api/models/exceptions.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | 4 | class HuskarException(Exception): 5 | """The base class of domain exceptions.""" 6 | 7 | 8 | class NameOccupiedError(HuskarException): 9 | """The resource name has been occupied.""" 10 | 11 | 12 | class ContainerUnboundError(HuskarException): 13 | """The container resource has been unbound""" 14 | 15 | 16 | class MalformedDataError(HuskarException): 17 | """The data is malformed in upstream.""" 18 | 19 | def __init__(self, info, *args, **kwargs): 20 | super(MalformedDataError, self).__init__(*args, **kwargs) 21 | self.info = info 22 | 23 | 24 | class TreeTimeoutError(HuskarException): 25 | """The initialization of tree holder is timeout.""" 26 | 27 | 28 | class OutOfSyncError(HuskarException): 29 | """The the local data is outdated.""" 30 | 31 | 32 | class NotEmptyError(HuskarException): 33 | """The deleting resource is not empty.""" 34 | 35 | 36 | class AuditLogTooLongError(HuskarException): 37 | """The audit log is too long.""" 38 | 39 | 40 | class AuditLogLostError(HuskarException): 41 | """The audit log could not be committed to database.""" 42 | 43 | 44 | class EmptyClusterError(HuskarException): 45 | pass 46 | 47 | 48 | class InfraNameNotExistError(HuskarException): 49 | pass 50 | 51 | 52 | class DataConflictError(HuskarException): 53 | pass 54 | -------------------------------------------------------------------------------- /huskar_api/models/infra/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .downstream import InfraDownstream 4 | from .utils import extract_application_names 5 | 6 | 7 | __all__ = ['extract_application_names', 'InfraDownstream'] 8 | -------------------------------------------------------------------------------- /huskar_api/models/infra/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import re 4 | 5 | 6 | rfc1738_pattern = re.compile(r''' 7 | (?P[\w\+]+):// 8 | (?: 9 | (?P[^:/]*) 10 | (?::(?P.*))? 11 | @)? 12 | (?: 13 | (?: 14 | \[(?P[^/]+)\] | 15 | (?P[^/:]+) 16 | )? 17 | (?::(?P[^/]*))? 18 | )? 19 | (?:/(?P.*))? 20 | ''', re.X) 21 | 22 | 23 | def parse_rfc1738_args(url): 24 | """Parse URL with the RFC 1738.""" 25 | m = rfc1738_pattern.match(url) 26 | if m is None: 27 | raise ValueError('Cannot parse RFC 1738 URL: {!r}'.format(url)) 28 | return m.groupdict() 29 | 30 | 31 | def extract_application_name(url): 32 | """Parses the Sam URL and returns its application name. 33 | 34 | :param url: The URL string. 35 | :returns: The application name, or ``None`` if this is not a Sam URL. 36 | """ 37 | try: 38 | args = parse_rfc1738_args(url) 39 | except ValueError: 40 | return 41 | scheme = args['name'] or '' 42 | if scheme.startswith('sam+'): 43 | return args['ipv4host'] or args['ipv6host'] 44 | 45 | 46 | def extract_application_names(urls): 47 | """Parses the Sam URLs and returns names of valid applications. 48 | 49 | :param urls: The list or dictionary of Sam URLs. 50 | :returns: The list or dictionary of application names. 51 | """ 52 | if isinstance(urls, dict): 53 | iterator = ( 54 | (key, extract_application_name(url)) 55 | for key, url in urls.iteritems()) 56 | return {key: name for key, name in iterator if name} 57 | iterator = (extract_application_name(url) for url in urls) 58 | return [name for name in iterator if name] 59 | -------------------------------------------------------------------------------- /huskar_api/models/instance/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .management import InstanceManagement 4 | from .schema import InfraInfo 5 | 6 | 7 | __all__ = ['InstanceManagement', 'InfraInfo'] 8 | -------------------------------------------------------------------------------- /huskar_api/models/manifest.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from huskar_sdk_v2.utils import combine 4 | from huskar_sdk_v2.consts import BASE_PATH 5 | 6 | from huskar_api.models import huskar_client 7 | from huskar_api.models.znode import ZnodeList 8 | 9 | 10 | __all__ = ['application_manifest'] 11 | 12 | 13 | class ApplicationManifest(object): 14 | """The manifest of all applications in ZooKeeper. 15 | 16 | This model serves the minimal mode of Huskar API. Once the database falls 17 | in a system outage, the API will provide application list here instead. 18 | """ 19 | 20 | def __init__(self, huskar_client): 21 | self._lists = [ 22 | ZnodeList(huskar_client.client, combine(BASE_PATH, 'service')), 23 | ZnodeList(huskar_client.client, combine(BASE_PATH, 'switch')), 24 | ZnodeList(huskar_client.client, combine(BASE_PATH, 'config')), 25 | ] 26 | 27 | def start(self): 28 | for l in self._lists: 29 | l.start() 30 | 31 | def check_is_application(self, name): 32 | return any(name in l.children for l in self._lists) 33 | 34 | def as_list(self): 35 | return sorted({c for l in self._lists for c in l.children}) 36 | 37 | 38 | application_manifest = ApplicationManifest(huskar_client) 39 | application_manifest.start() 40 | -------------------------------------------------------------------------------- /huskar_api/models/route/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .management import RouteManagement 4 | from .resolver import ClusterResolver 5 | from .hijack_stage import lookup_route_stage 6 | 7 | 8 | __all__ = ['RouteManagement', 'ClusterResolver', 'lookup_route_stage'] 9 | -------------------------------------------------------------------------------- /huskar_api/models/route/hijack_stage.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from flask import json 4 | 5 | from huskar_sdk_v2.consts import CONFIG_SUBDOMAIN 6 | 7 | from huskar_api.settings import APP_NAME 8 | 9 | 10 | __all__ = ['lookup_route_stage'] 11 | 12 | 13 | def lookup_route_stage(): 14 | from huskar_api.models import huskar_client 15 | from huskar_api.models.instance import InstanceManagement 16 | 17 | stage_table = {} 18 | im = InstanceManagement(huskar_client, APP_NAME, CONFIG_SUBDOMAIN) 19 | cluster_list = im.list_cluster_names() 20 | for cluster_name in cluster_list: 21 | instance, _ = im.get_instance(cluster_name, 'ROUTE_HIJACK_LIST') 22 | data = json.loads(instance.data) if instance.data else {} 23 | for application_name, stage in data.items(): 24 | t = stage_table.setdefault(application_name, {}) 25 | t[cluster_name] = stage 26 | return stage_table 27 | -------------------------------------------------------------------------------- /huskar_api/models/route/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import collections 4 | 5 | from huskar_sdk_v2.consts import OVERALL 6 | 7 | from huskar_api import settings 8 | from huskar_api.models.const import ROUTE_DEFAULT_INTENT 9 | 10 | RouteKey = collections.namedtuple('RouteKey', 'application_name intent') 11 | 12 | 13 | def make_route_key(application_name, intent=None): 14 | if intent and intent != ROUTE_DEFAULT_INTENT: 15 | return u'{0}@{1}'.format(application_name, intent) 16 | return application_name 17 | 18 | 19 | def parse_route_key(route_key): 20 | args = route_key.split('@', 1) 21 | if len(args) == 1: 22 | return RouteKey(args[0], ROUTE_DEFAULT_INTENT) 23 | return RouteKey(*args) 24 | 25 | 26 | def try_to_extract_ezone(cluster_name, default=OVERALL): 27 | for ezone in settings.ROUTE_EZONE_LIST: 28 | if cluster_name.startswith(u'{0}-'.format(ezone)): 29 | return ezone 30 | return default 31 | -------------------------------------------------------------------------------- /huskar_api/models/signals.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from blinker import Namespace 4 | 5 | 6 | namespace = Namespace() 7 | 8 | team_will_be_archived = namespace.signal('team_will_be_archived') 9 | team_will_be_deleted = namespace.signal('team_will_be_deleted') 10 | session_load_user_failed = namespace.signal('session_load_user_failed') 11 | new_action_detected = namespace.signal('conecerd_action_detected') 12 | user_grant_admin = namespace.signal('user_grant_admin') 13 | user_dismiss_admin = namespace.signal('user_dismiss_admin') 14 | -------------------------------------------------------------------------------- /huskar_api/models/tree/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .hub import TreeHub 4 | from .cleaner import TreeHolderCleaner 5 | 6 | 7 | __all__ = ['TreeHub', 'TreeHolderCleaner'] 8 | -------------------------------------------------------------------------------- /huskar_api/models/tree/hub.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | from gevent.lock import Semaphore 6 | 7 | from .holder import TreeHolder 8 | from .watcher import TreeWatcher 9 | 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | class TreeHub(object): 15 | """The hub for holding multiple trees.""" 16 | 17 | def __init__(self, huskar_client, startup_max_concurrency=None): 18 | self.base_path = huskar_client.base_path 19 | self.client = huskar_client.client 20 | self.tree_map = {} 21 | self.tree_holder_class = TreeHolder 22 | self.tree_watcher_class = TreeWatcher 23 | self.lock = Semaphore() 24 | if startup_max_concurrency: 25 | self.throttle = Semaphore(startup_max_concurrency) 26 | else: 27 | self.throttle = None 28 | 29 | def get_tree_holder(self, application_name, type_name): 30 | """Gets a tree holder which specified by its type and application. 31 | 32 | Example:: 33 | 34 | holder = hub.get_tree_holder('switch', 'base.foo') 35 | 36 | If the tree holder does not exist, it will be created firstly. 37 | 38 | :returns: A :class:`TreeHolder` instance. 39 | """ 40 | with self.lock: 41 | key = (application_name, type_name) 42 | if key not in self.tree_map: 43 | holder = self.tree_holder_class( 44 | self, application_name, type_name, self.throttle) 45 | holder.start() 46 | self.tree_map[key] = holder 47 | return self.tree_map[key] 48 | 49 | def release_tree_holder(self, application_name, type_name): 50 | """Releases the tree holder. 51 | 52 | This method should be called after the 53 | :exc:`huskar_api.models.exceptions.TreeTimeoutError` raised. 54 | """ 55 | holder = self.tree_map.pop((application_name, type_name), None) 56 | if holder is not None: 57 | holder.close() 58 | 59 | return holder 60 | 61 | def make_watcher(self, *args, **kwargs): 62 | """Creates a watcher and binds it to tree holders of this instance. 63 | 64 | :returns: A :class:`TreeWatcher` instance. 65 | """ 66 | return self.tree_watcher_class(self, *args, **kwargs) 67 | -------------------------------------------------------------------------------- /huskar_api/models/webhook/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .webhook import Webhook 4 | from .notify import notifier 5 | 6 | notifier.start() 7 | 8 | __all__ = ['Webhook', 'notifier'] 9 | -------------------------------------------------------------------------------- /huskar_api/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/huskar_api/scripts/__init__.py -------------------------------------------------------------------------------- /huskar_api/scripts/db.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from copy import deepcopy 4 | from subprocess import check_output 5 | 6 | from pkg_resources import resource_string, resource_filename 7 | from sqlalchemy.engine import create_engine 8 | from huskar_api import settings 9 | from huskar_api.models import DBSession 10 | 11 | 12 | __all__ = ['initdb', 'dumpdb'] 13 | 14 | 15 | SCHEMA_FILE = ('huskar_api', '../database/mysql.sql') 16 | 17 | 18 | def initdb(): 19 | if not settings.IS_IN_DEV: 20 | raise RuntimeError('Should never use this in production environment') 21 | 22 | engine = get_engine() 23 | database_name = quote(engine.dialect, engine.url.database) 24 | schema_ddl = resource_string(*SCHEMA_FILE) 25 | 26 | anonymous_url = deepcopy(engine.url) 27 | anonymous_url.database = None 28 | anonymous_url.query = {} 29 | anonymous_engine = create_engine(anonymous_url) 30 | 31 | with anonymous_engine.connect() as connection: 32 | connection.execute( 33 | 'drop database if exists {0}'.format(database_name)) 34 | connection.execute( 35 | 'create database {0} character set utf8mb4 ' 36 | 'collate utf8mb4_bin'.format(database_name)) 37 | 38 | with anonymous_engine.connect() as connection: 39 | connection.execute('use {0}'.format(database_name)) 40 | connection.execute(schema_ddl) 41 | 42 | 43 | def dumpdb(): 44 | ddl = mysqldump_output('--no-data') 45 | sql = mysqldump_output('--tables', 'alembic_version', '--no-create-info') 46 | with open(resource_filename(*SCHEMA_FILE), 'w') as schema_file: 47 | schema_file.writelines([ddl.strip(), '\n\n', sql.strip(), '\n']) 48 | 49 | 50 | def get_engine(): 51 | session = DBSession() 52 | return session.engines['master'] 53 | 54 | 55 | def mysqldump_output(*args): 56 | engine = get_engine() 57 | process_args = [ 58 | 'mysqldump', 59 | '--host=%s' % engine.url.host, 60 | '--port=%s' % engine.url.port, 61 | '--user=%s' % engine.url.username, 62 | ] 63 | if engine.url.password: # pragma: no cover 64 | process_args.append('--password=%s' % engine.url.password) 65 | process_args.append(engine.url.database) 66 | process_args.extend(args) 67 | return check_output(process_args) 68 | 69 | 70 | def quote(dialect, literal): 71 | return dialect.preparer(dialect).quote(literal) 72 | -------------------------------------------------------------------------------- /huskar_api/scripts/vacuum.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | from huskar_sdk_v2.consts import ( 6 | SERVICE_SUBDOMAIN, SWITCH_SUBDOMAIN, CONFIG_SUBDOMAIN) 7 | 8 | from huskar_api import settings 9 | from huskar_api.models import huskar_client 10 | from huskar_api.models.manifest import application_manifest 11 | from huskar_api.models.instance import InstanceManagement 12 | from huskar_api.models.container import ContainerManagement 13 | from huskar_api.models.exceptions import ( 14 | NotEmptyError, OutOfSyncError, MalformedDataError) 15 | 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | def _vacuum_empty_clusters(type_name): 21 | for application_name in application_manifest.as_list(): 22 | if application_name in settings.AUTH_APPLICATION_BLACKLIST: 23 | continue 24 | logger.info('[%s] Check application %s', type_name, application_name) 25 | im = InstanceManagement(huskar_client, application_name, type_name) 26 | try: 27 | for cluster_name in im.list_cluster_names(): 28 | ident = (application_name, type_name, cluster_name) 29 | try: 30 | im.delete_cluster(cluster_name) 31 | except OutOfSyncError: 32 | logger.info('Skip %r because of changed version.', ident) 33 | except NotEmptyError as e: 34 | logger.info( 35 | 'Skip %r because %s.', ident, e.args[0].lower()) 36 | except MalformedDataError: 37 | logger.info('Skip %r because of unrecognized data.', ident) 38 | else: 39 | logger.info('Okay %r is gone.', ident) 40 | except Exception as e: 41 | logger.exception('Skip %s because %s.', application_name, e) 42 | 43 | 44 | def vacuum_empty_clusters(): 45 | logger.info('Begin to vacuum empty clusters') 46 | for type_name in ( 47 | SERVICE_SUBDOMAIN, SWITCH_SUBDOMAIN, CONFIG_SUBDOMAIN): 48 | _vacuum_empty_clusters(type_name) 49 | logger.info('Done to vacuum empty clusters') 50 | 51 | 52 | def vacuum_stale_barriers(): 53 | logger.info('Begin to vacuum stale container barriers') 54 | vacuum_iterator = ContainerManagement.vacuum_stale_barriers(huskar_client) 55 | for container_id, is_stale in vacuum_iterator: 56 | if is_stale: 57 | logger.info('Delete stale barrier of container %s', container_id) 58 | else: 59 | logger.info('Skip barrier of container %s', container_id) 60 | logger.info('Done to vacuum stale container barriers') 61 | -------------------------------------------------------------------------------- /huskar_api/service/__init__.py: -------------------------------------------------------------------------------- 1 | # TODO Expecting a volunteer to delete this package totally 2 | 3 | from .config import ConfigData 4 | from .switch import SwitchData 5 | from .service import ServiceData 6 | 7 | 8 | config = ConfigData() 9 | switch = SwitchData() 10 | service = ServiceData() 11 | -------------------------------------------------------------------------------- /huskar_api/service/admin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/huskar_api/service/admin/__init__.py -------------------------------------------------------------------------------- /huskar_api/service/admin/application_auth.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | from flask import g 6 | 7 | from huskar_api import settings 8 | from huskar_api.models.auth import Application, Authority 9 | from huskar_api.service.organization.exc import ApplicationNotExistedError 10 | from .exc import NoAuthError 11 | 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | def check_application_auth(application_name, authority): 17 | assert authority in Authority 18 | 19 | if g.auth.is_minimal_mode: 20 | check_application(application_name) 21 | if authority == Authority.READ: 22 | return True 23 | if g.auth.is_admin or g.auth.is_application: 24 | return True 25 | else: 26 | application = check_application(application_name) 27 | if application.check_auth(authority, g.auth.id): 28 | return True 29 | if (application.domain_name in settings.AUTH_PUBLIC_DOMAIN and 30 | authority == Authority.READ): 31 | return True 32 | 33 | raise NoAuthError('{} has no {} authority on {}'.format( 34 | g.auth.username, authority.value, application_name)) 35 | 36 | 37 | def check_application(application_name): 38 | if is_application_blacklisted(application_name): 39 | raise ApplicationNotExistedError( 40 | 'application: {} is blacklisted'.format(application_name)) 41 | 42 | if g.auth.is_minimal_mode: 43 | return 44 | application = Application.get_by_name(application_name) 45 | if application is None: 46 | raise ApplicationNotExistedError( 47 | "application: {} doesn't exist".format(application_name)) 48 | return application 49 | 50 | 51 | def is_application_blacklisted(application_name): 52 | return application_name in settings.AUTH_APPLICATION_BLACKLIST 53 | 54 | 55 | def is_application_deprecated(application_name): 56 | return application_name in settings.LEGACY_APPLICATION_LIST 57 | -------------------------------------------------------------------------------- /huskar_api/service/admin/exc.py: -------------------------------------------------------------------------------- 1 | from huskar_api.service.exc import HuskarApiException 2 | 3 | 4 | class NoAuthError(HuskarApiException): 5 | pass 6 | 7 | 8 | class UserNotExistedError(HuskarApiException): 9 | pass 10 | 11 | 12 | class AuthorityNotExistedError(HuskarApiException): 13 | pass 14 | 15 | 16 | class LoginError(HuskarApiException): 17 | pass 18 | 19 | 20 | class AuthorityExistedError(HuskarApiException): 21 | pass 22 | -------------------------------------------------------------------------------- /huskar_api/service/admin/user.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import uuid 4 | import datetime 5 | 6 | from flask import abort 7 | from werkzeug.security import safe_str_cmp 8 | 9 | from huskar_api.models import DBSession, cache_manager 10 | from huskar_api.models.auth import User 11 | from huskar_api.extras.email import deliver_email, EmailTemplate 12 | 13 | 14 | _PASSWORD_RESET_KEY = '%s:reset_password:{username}:token' % __name__ 15 | _PASSWORD_RESET_DURATION = datetime.timedelta(minutes=10) 16 | 17 | _redis_client = cache_manager.make_client(namespace='%s:v1' % __name__) 18 | 19 | 20 | # TODO deprecate 21 | def request_to_reset_password(username): 22 | user = User.get_by_name(username) 23 | if not user or user.is_application: 24 | abort(404, u'user {0} not found'.format(username)) 25 | if not user.email: 26 | abort(403, u'user {0} does not have email'.format(username)) 27 | 28 | # Generate and record the token 29 | token = uuid.uuid4() 30 | _redis_client.set( 31 | raw_key=_PASSWORD_RESET_KEY.format(username=username), 32 | val=token.hex, expiration_time=_PASSWORD_RESET_DURATION) 33 | 34 | deliver_email(EmailTemplate.PASSWORD_RESET, user.email, { 35 | 'username': user.username, 36 | 'token': token, 37 | 'expires_in': _PASSWORD_RESET_DURATION, 38 | }) 39 | return user, token 40 | 41 | 42 | # TODO deprecate 43 | def reset_password(username, token, new_password): 44 | key = _PASSWORD_RESET_KEY.format(username=username) 45 | expected_token = _redis_client.get(key) 46 | if expected_token and safe_str_cmp(token.hex, expected_token): 47 | _redis_client.delete(key) 48 | user = User.get_by_name(username) 49 | if user is None or user.is_application: 50 | abort(404, u'user {0} not found'.format(username)) 51 | user.change_password(new_password) 52 | else: 53 | abort(403, u'token is expired') 54 | return user 55 | 56 | 57 | # TODO deprecate 58 | def change_email(user, new_email): 59 | with DBSession().close_on_exit(False): 60 | user.email = new_email 61 | -------------------------------------------------------------------------------- /huskar_api/service/comment.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from huskar_api.models.comment import set_comment, get_comment 4 | 5 | 6 | def save(key_type, application, cluster, key, comment=u''): 7 | set_comment(application, cluster, key_type, key, comment) 8 | 9 | 10 | def delete(key_type, application, cluster, key): 11 | set_comment(application, cluster, key_type, key, None) 12 | 13 | 14 | def get(key_type, application, cluster, key): 15 | return get_comment(application, cluster, key_type, key) 16 | -------------------------------------------------------------------------------- /huskar_api/service/config.py: -------------------------------------------------------------------------------- 1 | # coding=utf8 2 | from .data import DataBase 3 | from huskar_api.models.dataware.zookeeper import config_client 4 | 5 | 6 | class ConfigData(DataBase): 7 | 8 | client = config_client 9 | -------------------------------------------------------------------------------- /huskar_api/service/data.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | from huskar_sdk_v2.utils import encode_key 6 | 7 | from huskar_api.models import huskar_client 8 | from huskar_api.models.instance import InstanceManagement 9 | from huskar_api.models.exceptions import NotEmptyError 10 | from .exc import DataNotExistsError, DataNotEmptyError 11 | from .utils import check_cluster_name_in_creation 12 | 13 | 14 | log = logging.getLogger(__name__) 15 | 16 | 17 | # TODO [refactor] clean up the usage and re-design the api 18 | class DataBase(object): 19 | """ ServiceBase for `Config`, `Switch` and `Service`. """ 20 | 21 | client = None 22 | 23 | @classmethod 24 | def get_value(cls, application, cluster, key): 25 | return cls.client.get( 26 | application=application, cluster=cluster, key=encode_key(key)) 27 | 28 | @classmethod 29 | def create(cls, application, cluster, key, value, version=None): 30 | cls.check_cluster_name_in_creation(application, cluster) 31 | cls.client.set(application=application, 32 | cluster=cluster, 33 | key=encode_key(key), 34 | value=unicode(value).encode('utf-8'), 35 | version=version) 36 | 37 | @classmethod 38 | def delete(cls, application, cluster, key, strict=False): 39 | cls.client.delete(application=application, 40 | cluster=cluster, 41 | key=encode_key(key), 42 | strict=strict) 43 | 44 | @classmethod 45 | def create_cluster(cls, application, cluster, strict=False): 46 | cls.check_cluster_name_in_creation(application, cluster) 47 | cls.client.create_if_not_exist(application, cluster, strict) 48 | 49 | @classmethod 50 | def delete_cluster(cls, application, cluster, strict=False): 51 | data_type = cls.client.sub_domain 52 | im = InstanceManagement(huskar_client, application, data_type) 53 | try: 54 | cluster_info = im.delete_cluster(cluster) 55 | except NotEmptyError as e: 56 | if strict: 57 | raise DataNotEmptyError(*e.args) 58 | else: 59 | if cluster_info is None and strict: 60 | raise DataNotExistsError('cluster does not exist') 61 | 62 | @classmethod 63 | def check_cluster_name_in_creation(cls, application, cluster): 64 | if not cls.exists(application, cluster): 65 | check_cluster_name_in_creation(cluster, application) 66 | 67 | @classmethod 68 | def exists(cls, application, cluster, key=None): 69 | # TODO: Move this out of service module 70 | path = cls.client.get_path(application, cluster, key=key) 71 | return cls.client.raw_client.exists(path) 72 | -------------------------------------------------------------------------------- /huskar_api/service/exc.py: -------------------------------------------------------------------------------- 1 | # TODO Remove this module in future 2 | 3 | 4 | class HuskarApiException(Exception): 5 | pass 6 | 7 | 8 | class ServiceValueError(ValueError): 9 | """The input value is invalid for service registry.""" 10 | 11 | 12 | class ServiceLinkExisted(HuskarApiException): 13 | pass 14 | 15 | 16 | class ServiceLinkError(HuskarApiException): 17 | pass 18 | 19 | 20 | class DataExistsError(HuskarApiException): 21 | message = 'The data you tried to add is already exist.' 22 | 23 | 24 | class DataNotExistsError(HuskarApiException): 25 | message = 'The data is not exist.' 26 | 27 | 28 | class DataNotEmptyError(HuskarApiException): 29 | message = 'The target is not empty.' 30 | 31 | 32 | class DuplicatedEZonePrefixError(HuskarApiException): 33 | message = 'The target should not contain duplicated E-Zone prefix.' 34 | 35 | 36 | class ClusterNameUnsupportedError(HuskarApiException): 37 | message = 'Cluster name are not allowed in Huskar.' 38 | -------------------------------------------------------------------------------- /huskar_api/service/organization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/huskar_api/service/organization/__init__.py -------------------------------------------------------------------------------- /huskar_api/service/organization/exc.py: -------------------------------------------------------------------------------- 1 | from huskar_api.service.exc import HuskarApiException 2 | 3 | 4 | class ApplicationNotExistedError(HuskarApiException): 5 | pass 6 | 7 | 8 | class ApplicationExistedError(HuskarApiException): 9 | pass 10 | -------------------------------------------------------------------------------- /huskar_api/service/switch.py: -------------------------------------------------------------------------------- 1 | # coding=utf8 2 | from huskar_api.models.dataware.zookeeper import switch_client 3 | 4 | from .data import DataBase 5 | 6 | 7 | class SwitchData(DataBase): 8 | 9 | client = switch_client 10 | -------------------------------------------------------------------------------- /huskar_api/service/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from huskar_api.settings import ROUTE_INTENT_LIST 4 | from huskar_api.models.utils import normalize_cluster_name 5 | from .exc import DuplicatedEZonePrefixError, ClusterNameUnsupportedError 6 | 7 | 8 | def check_cluster_name(cluster_name, application_name="unknown"): 9 | if normalize_cluster_name(cluster_name) != cluster_name: 10 | raise DuplicatedEZonePrefixError( 11 | 'Cluster name should not contain duplicated E-Zone prefix.') 12 | 13 | return cluster_name 14 | 15 | 16 | def check_cluster_name_in_creation(cluster_name, application_name="unknown"): 17 | if cluster_name in ROUTE_INTENT_LIST: 18 | raise ClusterNameUnsupportedError( 19 | 'Cluster name "{}" are not allowed in Huskar.'.format( 20 | cluster_name 21 | ) 22 | ) 23 | 24 | if normalize_cluster_name(cluster_name) != cluster_name: 25 | raise DuplicatedEZonePrefixError( 26 | 'Cluster name should not contain duplicated E-Zone prefix.') 27 | 28 | return cluster_name 29 | -------------------------------------------------------------------------------- /huskar_api/switch.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from huskar_api.settings import bootstrap 4 | 5 | switch = bootstrap.get_huskar_switch() 6 | 7 | SWITCH_ENABLE_ROUTE_FORCE_CLUSTERS = 'switch_enable_route_force_clusters' 8 | SWITCH_ENABLE_MINIMAL_MODE = 'enable_minimal_mode' 9 | SWITCH_ENABLE_AUDIT_LOG = 'enable_audit_log' 10 | SWITCH_ENABLE_SENTRY_MESSAGE = 'enable_sentry_message' 11 | SWITCH_ENABLE_SENTRY_EXCEPTION = 'enable_sentry_exception' 12 | SWITCH_VALIDATE_SCHEMA = 'validate_schema' 13 | SWITCH_ENABLE_WEBHOOK_NOTIFY = 'enable_webhook_notify' 14 | SWITCH_ENABLE_ROUTE_HIJACK = 'enable_route_hijack' 15 | SWITCH_ENABLE_DECLARE_UPSTREAM = 'enable_declare_upstream' 16 | SWITCH_DETECT_BAD_ROUTE = 'detect_bad_route' 17 | SWITCH_ENABLE_EMAIL = 'enable_email' 18 | SWITCH_ENABLE_CONFIG_PREFIX_BLACKLIST = 'enable_config_prefix_blacklist' 19 | SWITCH_ENABLE_META_MESSAGE_CANARY = 'enable_meta_message_canary' 20 | SWITCH_ENABLE_LONG_POLLING_MAX_LIFE_SPAN = 'enable_long_polling_max_life_span' 21 | SWITCH_ENABLE_RATE_LIMITER = 'enable_rate_limiter' 22 | SWITCH_ENABLE_CONCURRENT_LIMITER = 'enable_concurrent_limiter' 23 | SWITCH_ENABLE_ROUTE_HIJACK_WITH_LOCAL_EZONE = ( 24 | 'enable_route_hijack_with_local_ezone') 25 | SWITCH_ENABLE_TREE_HOLDER_CLEANER_CLEAN = 'enable_tree_holder_cleaner_clean' 26 | SWITCH_ENABLE_TREE_HOLDER_CLEANER_TRACK = ( 27 | 'enable_tree_holder_cleaner_track') 28 | SWITCH_ENABLE_CONFIG_AND_SWITCH_WRITE = 'enable_config_and_switch_write' 29 | SWITCH_DISABLE_FETCH_VIA_API = 'disable_fetch_via_api' 30 | SWITCH_DISABLE_UPDATE_VIA_API = 'disable_update_via_api' 31 | -------------------------------------------------------------------------------- /huskar_api/templates/.editorconfig: -------------------------------------------------------------------------------- 1 | [*.html] 2 | indent_style = space 3 | indent_size = 2 4 | language = jinja 5 | -------------------------------------------------------------------------------- /huskar_api/templates/docs-assets-index.rst: -------------------------------------------------------------------------------- 1 | Assets 2 | ====== 3 | 4 | Email Snapshots 5 | --------------- 6 | 7 | {% for group in indices|groupby(1) -%} 8 | * {{ group.grouper[1] }} 9 | {%- for id, _, relpath in group.list %} 10 | * :download:`快照范例 {{ id + 1 }} <{{ relpath }}>` 11 | {%- endfor %} 12 | {% endfor %} 13 | -------------------------------------------------------------------------------- /huskar_api/templates/email-base-components.html: -------------------------------------------------------------------------------- 1 | {% macro button(text, href) %} 2 | 3 | 4 | 5 | 14 | 15 | 16 |
6 | 7 | 8 | 9 | 10 | 11 | 12 |
{{ text }}
13 |
17 | {% endmacro %} 18 | 19 | {% macro permission_label(code, universe=True) %} 20 | {%- if code == 'read' -%} 21 | 读取 22 | {%- elif code == 'write' -%} 23 | {%- if universe -%} 24 | 读取和写入 25 | {%- else -%} 26 | 写入 27 | {%- endif -%} 28 | {%- elif code == 'admin' -%} 29 | {%- if universe -%} 30 | 读取、写入和管理 31 | {%- else -%} 32 | 管理 33 | {%- endif -%} 34 | {%- endif %} 35 | {% endmacro %} 36 | 37 | {% macro current_user_label(code) %} 38 | {%- if g.auth and g.auth.username -%} 39 | {{- g.auth.username -}} 40 | {%- else -%} 41 | 系统 42 | {%- endif %} 43 | {% endmacro %} 44 | 45 | {% macro infra_type_label(type_name) %} 46 | {%- if type_name == 'database' -%} 47 | 关系型数据库 48 | {%- elif type_name == 'redis' %} 49 | Redis 集群 50 | {%- elif type_name == 'amqp' %} 51 | AMQP 消息队列 52 | {%- elif type_name == 'es' %} 53 | ElasticSearch 集群 54 | {%- elif type_name == 'oss' %} 55 | 对象存储资源 56 | {%- elif type_name == 'kafka' %} 57 | Kafka 消息队列 58 | {%- else -%} 59 | 未知类型的基础资源 60 | {%- endif -%} 61 | {% endmacro %} 62 | -------------------------------------------------------------------------------- /huskar_api/templates/email-debug.html: -------------------------------------------------------------------------------- 1 | DEBUG-1: {{ foo }} 2 | DEBUG-2: 中文 3 | -------------------------------------------------------------------------------- /huskar_api/templates/email-infra-config-create.html: -------------------------------------------------------------------------------- 1 | {% extends 'email-base-layout.html' %} 2 | {% import 'email-base-components.html' as c %} 3 | 4 | {%- block preheader -%} 5 | 您的服务 {{ application_name }} 在 {{ core_config.env }} 6 | 环境绑定了新的{{ c.infra_type_label(infra_type) }} 7 | {%- endblock -%} 8 | 9 | {%- block content -%} 10 |

Hi there,

11 |

12 | {{ c.current_user_label() }} 在您的服务 13 | {{ application_name }}（环境: {{ core_config.env }}） 14 | 上绑定了新的{{ c.infra_type_label(infra_type) }}，框架和 15 | SDK 接入该资源的时所用的名字 (code name) 是 {{ infra_name }} 16 |

17 | {%- endblock -%} 18 | -------------------------------------------------------------------------------- /huskar_api/templates/email-password-reset.html: -------------------------------------------------------------------------------- 1 | {% extends 'email-base-layout.html' %} 2 | {% import 'email-base-components.html' as c %} 3 | 4 | {%- block preheader -%} 5 | 确认 {{ username }} 的 Huskar 密码重置请求 6 | {%- endblock -%} 7 | 8 | {%- block content -%} 9 |

Hi there,

10 |

11 | 我们收到了 {{ core_config.env }} 环境 Huskar 账号 {{ username }} 的密码重置请求. 12 | 如果该请求是您发起的, 请点击链接重置您的密码: 13 |

14 | {{ c.button('重置密码', settings.ADMIN_RESET_PASSWORD_URL.format(username=username, token=token.hex)) }} 15 |

该链接会在{{ expires_in | timedeltaformat }}内过期.

16 |

17 | 如果该请求不是由您发起, 请忽略该邮件. 如果有其他疑问, 请发送邮件至 18 | huskar@example.com 19 | 联系支持. 20 |

21 | {%- endblock -%} 22 | -------------------------------------------------------------------------------- /huskar_api/templates/email-permission-dismiss.html: -------------------------------------------------------------------------------- 1 | {% extends 'email-base-layout.html' %} 2 | {% import 'email-base-components.html' as c %} 3 | 4 | {%- block preheader -%} 5 | Huskar 权限变更: 取消授权 {{ username }} {{ c.permission_label(authority, universe=False) }} {{ application_name}} 6 | {%- endblock -%} 7 | 8 | {%- block content -%} 9 |

Hi there,

10 |

11 | 您的 Huskar 权限已发生变更, {{ c.current_user_label() }} 12 | 解除了您在 {{ core_config.env }} 环境 13 | {{ c.permission_label(authority, universe=False) }} 应用 14 | {{ application_name}} 的权限, 权限变更已即时生效. 15 |

16 |

17 | 如果对此变更有疑虑, 请向变更人询问. 对于其他疑问, 请发送邮件至 18 | huskar@example.com 19 | 联系支持. 20 |

21 | {%- endblock -%} 22 | -------------------------------------------------------------------------------- /huskar_api/templates/email-permission-grant.html: -------------------------------------------------------------------------------- 1 | {% extends 'email-base-layout.html' %} 2 | {% import 'email-base-components.html' as c %} 3 | 4 | {%- block preheader -%} 5 | Huskar 权限变更: 授权 {{ username }} {{ c.permission_label(authority) }} {{ application_name}} 6 | {%- endblock -%} 7 | 8 | {%- block content -%} 9 |

Hi there,

10 |

11 | 您的 Huskar 权限已发生变更, {{ c.current_user_label() }} 12 | 授权您在 {{ core_config.env }} 环境 13 | {{ c.permission_label(authority) }} 应用 {{ application_name}}, 14 | 权限已即时生效. 15 |

16 | {{ c.button('立即查看', settings.ADMIN_HOME_URL) }} 17 |

18 | 如果您遗忘了该环境的用户名或密码, 请在登录页面点击忘记密码链接, 自助重置. 19 | 20 | 如有其他疑问, 请发送邮件至 21 | huskar@example.com 22 | 联系支持. 23 |

24 | {%- endblock -%} 25 | -------------------------------------------------------------------------------- /huskar_api/templates/email-signup.html: -------------------------------------------------------------------------------- 1 | {% extends 'email-base-layout.html' %} 2 | {% import 'email-base-components.html' as c %} 3 | 4 | {%- block preheader -%} 5 | 已为 {{ username }} 创建新的 Huskar 帐号 6 | {%- endblock -%} 7 | 8 | {%- block content -%} 9 |

Hi there,

10 |

已创建新的 Huskar 账号, 信息如下:

11 |

12 | 环境: {{ core_config.env }} 13 |
14 | 用户名: {{ username }} 15 |
16 | 初始密码: {{ password }} 17 |

18 | {{ c.button('点击登录', settings.ADMIN_HOME_URL) }} 19 |

20 | 为了信息安全, 请尽快修改初始密码并妥善保管。 21 |

22 |

23 | 如有疑问, 请发送邮件至 24 | huskar@example.com 25 | 联系支持. 26 |

27 | {%- endblock -%} 28 | -------------------------------------------------------------------------------- /huskar_api/wsgi.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import gevent.monkey; gevent.monkey.patch_all() # noqa 4 | 5 | from werkzeug.contrib.fixers import ProxyFix 6 | from .app import create_app 7 | 8 | 9 | app = create_app() 10 | app.wsgi_app = ProxyFix(app.wsgi_app) 11 | -------------------------------------------------------------------------------- /jenkins.dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 2 | 3 | RUN curl -s https://mirror.go-repo.io/centos/go-repo.repo > /etc/yum.repos.d/go-repo.repo && \ 4 | curl -s https://www.apache.org/dist/bigtop/stable/repos/centos7/bigtop.repo > /etc/yum.repos.d/bigtop.repo && \ 5 | curl -s https://downloads.mariadb.com/MariaDB/mariadb_repo_setup | bash && \ 6 | yum install -y epel-release && \ 7 | yum install -y \ 8 | gcc gcc-c++ make git \ 9 | postgresql-devel \ 10 | librabbitmq-devel \ 11 | mariadb-client \ 12 | python \ 13 | python-virtualenv \ 14 | java \ 15 | golang \ 16 | mariadb-server \ 17 | redis \ 18 | zookeeper 19 | ENV GOPATH "/opt/gopath" 20 | RUN go get -v -u github.com/client9/misspell/cmd/misspell && \ 21 | virtualenv /opt/venv && \ 22 | /opt/venv/bin/pip install -U pip setuptools wheel 23 | RUN yum install -y ShellCheck 24 | 25 | ENV VIRTUAL_ENV "/opt/venv" 26 | ENV PATH "/opt/venv/bin:/opt/gopath/bin:/usr/local/sbin:/usr/sbin:/sbin:/usr/local/bin:/usr/bin:/bin:/usr/libexec" 27 | 28 | RUN useradd -u 1060 -m jenkins && \ 29 | printf 'unset JAVA_HOME\n' > /usr/lib/zookeeper/conf/java.env && \ 30 | mkdir -p \ 31 | /var/{lib,log,run}/{mysql,mariadb,redis,zookeeper} && \ 32 | chown -R jenkins:jenkins \ 33 | /var/{lib,log,run}/{mysql,mariadb,redis,zookeeper} \ 34 | /opt/{venv,gopath} 35 | -------------------------------------------------------------------------------- /manage.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | wrapper=(env) 5 | cli=(python run.py huskar_api.cli:main) 6 | if [ -n "$(command -v honcho)" ]; then 7 | wrapper=(honcho run) 8 | cli=(honcho run python run.py huskar_api.cli:main) 9 | fi 10 | 11 | clear_py_cache() { 12 | find . -name '*.pyc' -delete 13 | find . -name '*.pyo' -delete 14 | find . -name __pycache__ -delete 15 | } 16 | 17 | case $1 in 18 | bash) 19 | exec /bin/bash 20 | ;; 21 | alembic) 22 | exec "${wrapper[@]}" alembic --config=database/migration/alembic.ini "${@:2}" 23 | ;; 24 | initdb) 25 | exec "${wrapper[@]}" python run.py huskar_api.scripts.db:initdb 26 | ;; 27 | dumpdb) 28 | exec "${wrapper[@]}" python run.py huskar_api.scripts.db:dumpdb 29 | ;; 30 | initadmin) 31 | exec "${wrapper[@]}" python run.py huskar_api.cli:main initadmin 32 | ;; 33 | testall) 34 | clear_py_cache 35 | "$0" lint 36 | "pytest" tests "${@:2}" 37 | "pytest" coverage html 38 | ;; 39 | testonly) 40 | clear_py_cache 41 | exec pytest "${@:2}" 42 | ;; 43 | test) 44 | clear_py_cache 45 | "$0" lint 46 | exec "${wrapper[@]}" pytest "${@:2}" 47 | ;; 48 | lint) 49 | if [ -n "$(command -v misspell)" ]; then 50 | find huskar_api docs tools tests \ 51 | \( \ 52 | -name '*.rst' -or \ 53 | -name '*.py' -or \ 54 | -name '*.yml' \ 55 | \) \ 56 | -exec misspell -error {} + 57 | else 58 | echo "misspell is not installed." >&2 59 | fi 60 | if [ -n "$(command -v shellcheck)" ]; then 61 | find . -type f -name '*.sh' -exec shellcheck {} + 62 | else 63 | echo "shellcheck is not installed." >&2 64 | fi 65 | flake8 --exclude=docs,.venv,database/migration/versions . 66 | ;; 67 | make) 68 | exec "${@}" 69 | ;; 70 | docs) 71 | exec make \ 72 | 'SPHINXBUILD=sphinx-autobuild' \ 73 | 'SPHINXOPTS=-i *.swp -s 3' \ 74 | -C docs html 75 | ;; 76 | ""|help|--help|-h) 77 | "${cli[@]}" -- --help 78 | printf '\n' 79 | printf 'extra commands:\n' 80 | printf ' alembic\t\tMigrates the database schema.\n' 81 | printf ' initdb\t\tDrops and creates all database tables.\n' 82 | printf ' dumpdb\t\tDumps the schema and alembic version of database.\n' 83 | printf ' testall\t\tRuns testing for all modules.\n' 84 | printf ' test\t\tRuns testing for specified arguments.\n' 85 | printf ' docs\t\tBuilds docs with auto-reload support.\n' 86 | ;; 87 | *) exec "$@" 88 | esac 89 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = --durations=10 --cov=huskar_api --cov-report=xml --cov-report=term-missing --no-cov-on-fail --xpara 3 | -------------------------------------------------------------------------------- /requirements-dev.in: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | 3 | sphinx>=1.5 4 | sphinx-rtd-theme>=0.1 5 | sphinx-autobuild>=0.6 6 | sphinxcontrib-httpdomain>=1.7.0,<1.8 7 | 8 | pytest 9 | pytest-flask 10 | pytest-cov 11 | pytest-mock 12 | pytest-faker 13 | pytest-xpara[yaml] 14 | flake8 15 | freezegun 16 | requests-mock 17 | 18 | pylint 19 | -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | flask==0.12.4 2 | flask-script==2.0.5 3 | flask-babel==0.11.1 4 | babel==2.3.4 5 | certifi>=2018.8.24 6 | orphanage>=0.1,<0.2 7 | requests==2.20.0 8 | 9 | sqlalchemy==0.9.3 10 | alembic>=0.7,<0.8 11 | raven>=5.2,<6.0 12 | 13 | python-decouple>=3.0,<3.1 14 | python-dateutil>=2.6,<2.7 15 | marshmallow>=2.9,<2.10 16 | ipaddress>=1.0,<1.1 17 | enum34>=1.1.6,<1.2 18 | more-itertools>=3.2,<4.0 19 | 20 | huskar-sdk-v2[bootstrap]==0.18.0 21 | https://github.com/huskar-org/kazoo/archive/2.0.post5.zip#egg=kazoo 22 | https://github.com/huskar-org/gevent/raw/1.0.2b1/dist/gevent-1.0.2b1.tar.gz#egg=gevent 23 | https://github.com/huskar-org/doctor/archive/v0.2.1.zip#egg=doctor 24 | dogpile.cache==0.5.4 25 | decorator==3.4.0 26 | meepo2==0.2.5 27 | blinker==1.3 28 | pymysql==0.6.2 29 | redis==2.10.5 30 | psutil==5.0.0 31 | gunicorn==19.3.0 32 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --no-emit-trusted-host --no-index --output-file=requirements.txt requirements.in 6 | # 7 | alembic==0.7.7 8 | atomicfile==1.0 # via huskar-sdk-v2 9 | babel==2.3.4 10 | blinker==1.3 11 | certifi==2018.8.24 12 | cffi==1.11.2 # via orphanage 13 | chardet==3.0.4 # via requests 14 | click==5.1 # via flask 15 | decorator==3.4.0 16 | https://github.com/huskar-org/doctor/archive/v0.2.1.zip#egg=doctor 17 | dogpile.cache==0.5.4 18 | dogpile.core==0.4.1 # via dogpile.cache 19 | enum34==1.1.6 20 | flask-babel==0.11.1 21 | flask-script==2.0.5 22 | flask==0.12.4 23 | https://github.com/huskar-org/gevent/raw/1.0.2b1/dist/gevent-1.0.2b1.tar.gz#egg=gevent 24 | greenlet==0.4.10 25 | gunicorn==19.3.0 26 | huskar-sdk-v2[bootstrap]==0.18.0 27 | idna==2.6 # via requests 28 | ipaddress==1.0.17 29 | itsdangerous==0.24 # via flask 30 | jinja2==2.7.3 # via flask, flask-babel 31 | https://github.com/huskar-org/kazoo/archive/2.0.post5.zip#egg=kazoo 32 | mako==1.0.4 # via alembic 33 | markupsafe==0.23 # via jinja2, mako 34 | marshmallow==2.9.1 35 | meepo2==0.2.5 36 | more-itertools==3.2.0 37 | mysql-replication==0.5 # via meepo2 38 | orphanage==0.1.0 39 | psutil==5.0.0 40 | pycparser==2.18 # via cffi 41 | pyketama==0.2.1 # via meepo2 42 | pymysql==0.6.2 43 | python-dateutil==2.6.0 44 | python-decouple==3.0 45 | pytz==2016.6.1 # via babel 46 | pyzmq==14.7.0 # via meepo2 47 | raven==5.2.0 48 | redis==2.10.5 49 | requests==2.20.0 50 | simplejson==3.7.3 # via huskar-sdk-v2 51 | six==1.10.0 # via more-itertools, mysql-replication, python-dateutil 52 | sqlalchemy==0.9.3 53 | urllib3==1.24.1 # via requests 54 | werkzeug==0.11.11 # via flask 55 | -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import gevent.monkey; gevent.monkey.patch_all() # noqa 4 | 5 | import sys 6 | import os 7 | import click 8 | 9 | 10 | def import_obj(obj_path, hard=False): 11 | """ 12 | import_obj imports an object by uri, example:: 13 | 14 | >>> import_obj("module:main") 15 | 16 | 17 | :param obj_path: a string represents the object uri. 18 | ;param hard: a boolean value indicates whether to raise an exception on 19 | import failures. 20 | """ 21 | try: 22 | # ``__import__`` of Python 2.x could not resolve unicode, so we need 23 | # to ensure the type of ``module`` and ``obj`` is native str. 24 | module, obj = str(obj_path).rsplit(':', 1) 25 | m = __import__(module, globals(), locals(), [obj], 0) 26 | return getattr(m, obj) 27 | except (ValueError, AttributeError, ImportError): 28 | if hard: 29 | raise 30 | 31 | 32 | @click.command( 33 | context_settings={ 34 | "ignore_unknown_options": True, 35 | "allow_extra_args": True 36 | }, 37 | add_help_option=True) 38 | @click.argument('script_or_uri', required=True) 39 | @click.pass_context 40 | @click.option('-i', '--force-import', type=bool, default=False, is_flag=True, 41 | help=("force to import as an object, don't try as executing" 42 | " a script")) 43 | def run(ctx, script_or_uri, force_import): 44 | """ 45 | Run a script or uri. 46 | """ 47 | group_name = ctx.parent.command.name + ' ' if ctx.parent else '' 48 | prog_name = "{}{}".format(group_name, ctx.command.name) 49 | 50 | sys.argv = [prog_name] + ctx.args 51 | try: 52 | ret = None 53 | 54 | entry = import_obj(script_or_uri, hard=force_import) 55 | if entry: 56 | ret = entry() 57 | # Backward compatibility: if ret is int, 58 | # means it's cli return code. 59 | if isinstance(entry, int): 60 | sys.exit(ret) 61 | else: 62 | execfile(script_or_uri, { 63 | '__name__': '__main__', 64 | '__file__': os.path.realpath(script_or_uri), 65 | }) 66 | except SystemExit: 67 | raise 68 | except BaseException: 69 | raise 70 | 71 | 72 | cmds = [run] 73 | 74 | 75 | if __name__ == '__main__': 76 | # pylint: disable=E1120 77 | run() 78 | -------------------------------------------------------------------------------- /service_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import absolute_import, print_function 4 | 5 | from sqlalchemy import func 6 | from huskar_sdk_v2.consts import OVERALL 7 | 8 | from huskar_api import settings 9 | from huskar_api.models import DBSession, cache_manager 10 | from huskar_api.models.auth import ApplicationAuth 11 | from huskar_api.models.catalog import ServiceInfo 12 | from huskar_api.models.dataware.zookeeper import switch_client, config_client 13 | 14 | 15 | def check_settings(): 16 | for intent in settings.ROUTE_INTENT_LIST: 17 | cluster_name = settings.ROUTE_DEFAULT_POLICY.get(intent) 18 | assert cluster_name, 'Incomplete ROUTE_DEFAULT_POLICY: %s' % intent 19 | ServiceInfo.check_default_route_args(OVERALL, intent, cluster_name) 20 | 21 | 22 | def check_mysql(): 23 | if is_minimal_mode(): 24 | print('minimal mode detected') 25 | return 26 | db = DBSession() 27 | assert db.query(func.count(ApplicationAuth.id)).scalar(), 'mysql not ok' 28 | 29 | 30 | def check_zookeeper(): 31 | value_cluster = config_client.get( 32 | settings.APP_NAME, 33 | settings.CLUSTER, 34 | 'SECRET_KEY') 35 | value_overall = config_client.get( 36 | settings.APP_NAME, 37 | 'overall', 38 | 'SECRET_KEY') 39 | assert any((value_cluster, value_overall)), 'zk not ok' 40 | 41 | 42 | def check_redis(): 43 | if is_minimal_mode(): 44 | print('minimal mode detected') 45 | return 46 | client = cache_manager.make_client(raw=True) 47 | client.set('huskar_service_check', 'hello') 48 | assert client.get('huskar_service_check') == 'hello', 'redis not ok' 49 | 50 | 51 | def is_minimal_mode(): 52 | value_cluster = switch_client.get( 53 | settings.APP_NAME, 54 | settings.CLUSTER, 55 | 'enable_minimal_mode') 56 | value_overall = switch_client.get( 57 | settings.APP_NAME, 58 | 'overall', 59 | 'enable_minimal_mode') 60 | return float(value_cluster or value_overall or 0) > 0 61 | 62 | 63 | def main(): 64 | check_settings() 65 | check_mysql() 66 | check_zookeeper() 67 | check_redis() 68 | 69 | 70 | if __name__ == '__main__': 71 | main() 72 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/tests/test_api/__init__.py -------------------------------------------------------------------------------- /tests/test_api/test_audit_rollback.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | test_rollback_infra_config_change: 4 | args: _action_type,_scope_type,_scope_name,_old_value,_new_value,_expected_value,_expected_action_type 5 | data: 6 | - _action_type: 'UPDATE_INFRA_CONFIG' 7 | _scope_type: 'idcs' 8 | _scope_name: 'alta1' 9 | _old_value: null 10 | _new_value: {'url': 'sam+redis://redis.foobar/overall.alta'} 11 | _expected_action_type: 'DELETE_INFRA_CONFIG' 12 | _expected_value: null 13 | - _action_type: 'UPDATE_INFRA_CONFIG' 14 | _scope_type: 'idcs' 15 | _scope_name: 'altb1' 16 | _old_value: {'url': 'sam+redis://redis.foobar/overall.altb'} 17 | _new_value: {'url': 'sam+redis://redis.foobar/overall.altb', 'connect_timeout': 10} 18 | _expected_action_type: 'UPDATE_INFRA_CONFIG' 19 | _expected_value: {'url': 'sam+redis://redis.foobar/overall.altb'} 20 | - _action_type: 'DELETE_INFRA_CONFIG' 21 | _scope_type: 'idcs' 22 | _scope_name: 'altb1' 23 | _old_value: {'url': 'sam+redis://redis.foobar/overall.altb'} 24 | _new_value: null, 25 | _expected_action_type: 'UPDATE_INFRA_CONFIG' 26 | _expected_value: {'url': 'sam+redis://redis.foobar/overall.altb'} 27 | -------------------------------------------------------------------------------- /tests/test_api/test_audit_view.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | test_fetch_with_date: 4 | args: timedelta,result_len 5 | data: 6 | - timedelta: -1 7 | result_len: 0 8 | - timedelta: 1 9 | result_len: 0 10 | - timedelta: 0 11 | result_len: 2 12 | 13 | test_get_audit_instance_timeline: 14 | args: instance_type,cluster_name,instance_key,prepare_data,expected_audit_num 15 | data: 16 | - instance_type: "config" 17 | cluster_name: "foo" 18 | instance_key: "bar" 19 | prepare_data: 20 | - created_date: "2017-12-12" 21 | audit_num: 3 22 | - created_date: "2017-12-15" 23 | audit_num: 5 24 | expected_audit_num: 8 25 | - instance_type: "switch" 26 | cluster_name: "foo" 27 | instance_key: "bar" 28 | prepare_data: 29 | - created_date: "2017-12-12" 30 | audit_num: 10 31 | - created_date: "2017-12-15" 32 | audit_num: 12 33 | expected_audit_num: 20 34 | - instance_type: "service" 35 | cluster_name: "foo" 36 | instance_key: "bar" 37 | prepare_data: 38 | - created_date: "2017-12-12" 39 | audit_num: 3 40 | - created_date: "2017-12-15" 41 | audit_num: 5 42 | expected_audit_num: 8 43 | -------------------------------------------------------------------------------- /tests/test_api/test_config_and_switch_read_only.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from pytest import fixture, mark 4 | 5 | from huskar_api import settings 6 | from huskar_api.switch import SWITCH_ENABLE_CONFIG_AND_SWITCH_WRITE 7 | from ..utils import assert_response_ok 8 | 9 | 10 | @fixture 11 | def test_application_name(test_application): 12 | return test_application.application_name 13 | 14 | 15 | def test_update_config_and_switch_readonly_whitelist(): 16 | settings.update_config_and_switch_readonly_whitelist(['foo.test']) 17 | assert settings.CONFIG_AND_SWITCH_READONLY_WHITELIST == \ 18 | frozenset(['foo.test', 'arch.huskar_api']) 19 | 20 | settings.update_config_and_switch_readonly_whitelist(['arch.huskar_api']) 21 | assert settings.CONFIG_AND_SWITCH_READONLY_WHITELIST == \ 22 | frozenset(['arch.huskar_api']) 23 | 24 | 25 | def test_update_config_and_switch_readonly_blacklist(): 26 | settings.update_config_and_switch_readonly_blacklist(['foo.test']) 27 | assert settings.CONFIG_AND_SWITCH_READONLY_BLACKLIST == \ 28 | frozenset(['foo.test']) 29 | 30 | 31 | @mark.xparametrize 32 | def test_config_and_switch_readonly_middleware( 33 | client, mock_switches, test_application_name, 34 | test_application_token, _path, _method, _read_only, 35 | _data, _status, _in_whitelist, _in_blacklist): 36 | mock_switches({SWITCH_ENABLE_CONFIG_AND_SWITCH_WRITE: _read_only}) 37 | headers = {'Authorization': test_application_token} 38 | 39 | settings.update_config_and_switch_readonly_whitelist([]) 40 | if _in_whitelist: 41 | settings.update_config_and_switch_readonly_whitelist( 42 | [test_application_name]) 43 | 44 | settings.update_config_and_switch_readonly_blacklist([]) 45 | if _in_blacklist: 46 | settings.update_config_and_switch_readonly_blacklist( 47 | [test_application_name]) 48 | 49 | r = client.open( 50 | method=_method, path=_path % test_application_name, 51 | data=_data, headers=headers, 52 | ) 53 | 54 | if _status: 55 | assert_response_ok(r) 56 | else: 57 | assert r.status_code == 403 58 | -------------------------------------------------------------------------------- /tests/test_api/test_config_and_switch_read_only.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | test_config_and_switch_readonly_middleware: 4 | args: _path,_method,_read_only,_data,_status,_in_whitelist,_in_blacklist 5 | data: 6 | - _path: '/api/config/%s/overall' 7 | _method: 'GET' 8 | _read_only: True 9 | _data: 10 | key: 'foo' 11 | _status: True 12 | _in_whitelist: False 13 | _in_blacklist: False 14 | - _path: '/api/config/%s/overall' 15 | _method: 'GET' 16 | _read_only: False 17 | _data: 18 | key: 'foo' 19 | _status: True 20 | _in_whitelist: False 21 | _in_blacklist: False 22 | - _path: '/api/config/%s/overall' 23 | _method: 'POST' 24 | _read_only: True 25 | _data: 26 | key: 'foo' 27 | value: 'test' 28 | _status: True 29 | _in_whitelist: False 30 | _in_blacklist: False 31 | - _path: '/api/config/%s/overall' 32 | _method: 'POST' 33 | _read_only: False 34 | _data: 35 | key: 'foo' 36 | value: 'test' 37 | _status: False 38 | _in_whitelist: False 39 | _in_blacklist: False 40 | - _path: '/api/config/%s/overall' 41 | _method: 'POST' 42 | _read_only: False 43 | _data: 44 | key: 'foo' 45 | value: 'test' 46 | _status: True 47 | _in_whitelist: True 48 | _in_blacklist: False 49 | - _path: '/api/config/%s/overall' 50 | _method: 'POST' 51 | _read_only: False 52 | _data: 53 | key: 'foo' 54 | value: 'test' 55 | _status: False 56 | _in_whitelist: False 57 | _in_blacklist: True 58 | - _path: '/api/config/%s/overall' 59 | _method: 'POST' 60 | _read_only: True 61 | _data: 62 | key: 'foo' 63 | value: 'test' 64 | _status: False 65 | _in_whitelist: False 66 | _in_blacklist: True 67 | - _path: '/api/service/%s/overall' 68 | _method: 'POST' 69 | _read_only: False 70 | _data: 71 | key: '169.254.1.2_5000' 72 | value: '{"ip": "169.254.1.2","port":{"main": 5000},"state":"up","other":"test"}' 73 | _status: True 74 | _in_whitelist: False 75 | _in_blacklist: False 76 | -------------------------------------------------------------------------------- /tests/test_api/test_db_tester.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import datetime 4 | 5 | from pytest import fixture 6 | from flask import g 7 | from freezegun import freeze_time 8 | from sqlalchemy.exc import SQLAlchemyError 9 | from redis.exceptions import RedisError 10 | 11 | from huskar_api.app import create_app 12 | 13 | 14 | @fixture 15 | def app(): 16 | app = create_app() 17 | app.config['PROPAGATE_EXCEPTIONS'] = False 18 | 19 | @app.route('/api/minimal-mode') 20 | def minimal_mode(): 21 | return unicode(g.auth.is_minimal_mode) 22 | 23 | @app.route('/api/mysql') 24 | def mysql_error(): 25 | raise SQLAlchemyError() 26 | 27 | @app.route('/api/redis') 28 | def redis_error(): 29 | raise RedisError() 30 | 31 | return app 32 | 33 | 34 | def test_minimal_mode(client): 35 | r = client.get('/api/minimal-mode') 36 | assert r.data == u'False' 37 | 38 | for _ in xrange(5): 39 | client.get('/api/mysql') 40 | client.get('/api/redis') 41 | 42 | r = client.get('/api/minimal-mode') 43 | assert r.headers['X-Minimal-Mode'] == '1' 44 | assert r.headers['X-Minimal-Mode-Reason'] == 'tester' 45 | assert r.data == u'True' 46 | 47 | with freeze_time() as frozen_time: 48 | for _ in xrange(10): 49 | client.get('/api/minimal-mode') 50 | frozen_time.tick(delta=datetime.timedelta(seconds=120)) 51 | r = client.get('/api/minimal-mode') 52 | assert r.headers.get('X-Minimal-Mode') is None 53 | assert r.headers.get('X-Minimal-Mode-Reason') is None 54 | assert r.data == u'False' 55 | -------------------------------------------------------------------------------- /tests/test_api/test_graceful_startup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import datetime 4 | 5 | from pytest import fixture 6 | from flask import g 7 | from freezegun import freeze_time 8 | 9 | from huskar_api.app import create_app 10 | 11 | 12 | @fixture 13 | def app(): 14 | app = create_app() 15 | app.config['PROPAGATE_EXCEPTIONS'] = False 16 | 17 | @app.route('/api/minimal-mode') 18 | def minimal_mode(): 19 | return unicode(g.auth.is_minimal_mode) 20 | 21 | return app 22 | 23 | 24 | def test_graceful_startup(mocker, client): 25 | mocker.patch('huskar_api.settings.MM_GRACEFUL_STARTUP_TIME', 600) 26 | 27 | r = client.get('/api/minimal-mode') 28 | assert r.headers['X-Minimal-Mode'] == '1' 29 | assert r.headers['X-Minimal-Mode-Reason'] == 'startup' 30 | assert r.data == u'True' 31 | 32 | with freeze_time() as frozen_time: 33 | frozen_time.tick(delta=datetime.timedelta(seconds=600)) 34 | r = client.get('/api/minimal-mode') 35 | assert r.headers.get('X-Minimal-Mode') is None 36 | assert r.headers.get('X-Minimal-Mode-Reason') is None 37 | assert r.data == u'False' 38 | -------------------------------------------------------------------------------- /tests/test_api/test_health_check.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from ..utils import assert_response_ok 4 | 5 | 6 | def test_ok(client): 7 | r = client.get('/api/health_check') 8 | assert_response_ok(r) 9 | assert r.json['data'] == 'ok' 10 | -------------------------------------------------------------------------------- /tests/test_api/test_http_concurrent_limit.yaml: -------------------------------------------------------------------------------- 1 | 2 | test_anonymous_with_concurrent_limit: 3 | args: configs 4 | data: 5 | - configs: { 6 | '__anonymous__': { 7 | 'ttl': 10, 8 | 'capacity': 1, 9 | } 10 | } 11 | - configs: { 12 | '127.0.0.1': { 13 | 'ttl': 10, 14 | 'capacity': 1, 15 | } 16 | } 17 | - configs: { 18 | '__default__': { 19 | 'ttl': 10, 20 | 'capacity': 1, 21 | } 22 | } 23 | dataids: 24 | - __anonymous__ 25 | - ip 26 | - __default__ 27 | 28 | test_login_with_concurrent_limit: 29 | args: configs,use_username 30 | data: 31 | - configs: { 32 | '__anonymous__': { 33 | 'ttl': 100, 34 | 'capacity': 300, 35 | }, 36 | '127.0.0.1': { 37 | 'ttl': 100, 38 | 'capacity': 300, 39 | }, 40 | '__default__': { 41 | 'ttl': 100, 42 | 'capacity': 300, 43 | } 44 | } 45 | use_username: true 46 | - configs: { 47 | '__anonymous__': { 48 | 'ttl': 100, 49 | 'capacity': 300, 50 | }, 51 | '127.0.0.1': { 52 | 'ttl': 100, 53 | 'capacity': 300, 54 | }, 55 | '__default__': { 56 | 'ttl': 100, 57 | 'capacity': 1, 58 | } 59 | } 60 | use_username: false 61 | dataids: 62 | - username_config 63 | - __default__ 64 | -------------------------------------------------------------------------------- /tests/test_api/test_http_rate_limit.yaml: -------------------------------------------------------------------------------- 1 | 2 | test_anonymous_with_rate_limit: 3 | args: configs 4 | data: 5 | - configs: { 6 | '__anonymous__': { 7 | 'rate': 1, 8 | 'capacity': 3, 9 | } 10 | } 11 | - configs: { 12 | '127.0.0.1': { 13 | 'rate': 1, 14 | 'capacity': 3, 15 | } 16 | } 17 | - configs: { 18 | '__default__': { 19 | 'rate': 1, 20 | 'capacity': 3, 21 | } 22 | } 23 | dataids: 24 | - __anonymous__ 25 | - ip 26 | - __default__ 27 | 28 | test_logged_with_rate_limit: 29 | args: configs,use_username 30 | data: 31 | - configs: { 32 | '__anonymous__': { 33 | 'rate': 100, 34 | 'capacity': 300, 35 | }, 36 | '127.0.0.1': { 37 | 'rate': 100, 38 | 'capacity': 300, 39 | }, 40 | '__default__': { 41 | 'rate': 100, 42 | 'capacity': 300, 43 | } 44 | } 45 | use_username: true 46 | - configs: { 47 | '__anonymous__': { 48 | 'rate': 100, 49 | 'capacity': 300, 50 | }, 51 | '127.0.0.1': { 52 | 'rate': 100, 53 | 'capacity': 300, 54 | }, 55 | '__default__': { 56 | 'rate': 1, 57 | 'capacity': 3, 58 | } 59 | } 60 | use_username: false 61 | dataids: 62 | - username_config 63 | - __default__ 64 | -------------------------------------------------------------------------------- /tests/test_api/test_long_polling.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | test_session_with_max_life_span: 4 | args: switch_on,max_life_span,jitter,life_span,exclude,timeout 5 | data: 6 | - switch_on: false 7 | max_life_span: 5 8 | jitter: 3 9 | life_span: 1 10 | exclude: false 11 | timeout: 2 12 | - switch_on: true 13 | max_life_span: 5 14 | jitter: 3 15 | life_span: 1 16 | exclude: true 17 | timeout: 2 18 | - switch_on: true 19 | max_life_span: 3 20 | jitter: 2 21 | life_span: 0 22 | exclude: false 23 | timeout: 6 24 | - switch_on: true 25 | max_life_span: 1 26 | jitter: 1 27 | life_span: 4 28 | exclude: false 29 | timeout: 3 30 | - switch_on: true 31 | max_life_span: 1 32 | jitter: 1 33 | life_span: 4 34 | exclude: true 35 | timeout: 5 36 | - switch_on: true 37 | max_life_span: 5 38 | jitter: 3 39 | life_span: 1 40 | exclude: false 41 | timeout: 3 42 | - switch_on: true 43 | max_life_span: 5 44 | jitter: 3 45 | life_span: 1 46 | exclude: true 47 | timeout: 3 48 | dataids: 49 | - switch_off_no_effect 50 | - switch_on_but_exclude_no_effect 51 | - switch_on_default_life_span_effect 52 | - switch_on_large_life_span_effect 53 | - switch_on_large_life_span_but_exclude_no_effect 54 | - switch_on_small_life_span_no_effect 55 | - switch_on_small_life_span_exclude_no_effect 56 | 57 | test_enable_force_cluster_route: 58 | args: from_cluster_name,req_dest_cluster,route_dest_cluster,force_dest_cluster,rule,intent,use_route 59 | data: 60 | - from_cluster_name: 'altc1-test-pre' 61 | req_dest_cluster: 'altc1-channel-stable-1' 62 | route_dest_cluster: 'altc1-test' 63 | force_dest_cluster: 'altc1-test-pre' 64 | rule: {'altc1-test-pre': 'altc1-test-pre', 65 | 'altc1-test-pre@direct': 'altc1-test-pre',} 66 | intent: 'direct' 67 | use_route: false 68 | - from_cluster_name: 'altc1-test-pre' 69 | req_dest_cluster: 'direct' 70 | route_dest_cluster: 'altc1-test' 71 | force_dest_cluster: 'altc1-test-pre' 72 | rule: {'altc1-test-pre': 'altc1-test-pre', 73 | 'altc1-test-pre@direct': 'altc1-test-pre',} 74 | intent: 'direct' 75 | use_route: true 76 | -------------------------------------------------------------------------------- /tests/test_api/test_support_container.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | test_list_registry: 4 | args: preset,result 5 | data: 6 | - preset: [] 7 | result: [] 8 | - preset: 9 | - ['base.foo', 'alpha_stable'] 10 | result: 11 | - application_name: 'base.foo' 12 | cluster_name: 'alpha_stable' 13 | - preset: 14 | - ['base.foo', 'alpha_stable'] 15 | - ['base.bar', 'alpha_dev'] 16 | result: 17 | - application_name: 'base.bar' 18 | cluster_name: 'alpha_dev' 19 | - application_name: 'base.foo' 20 | cluster_name: 'alpha_stable' 21 | -------------------------------------------------------------------------------- /tests/test_api/test_support_whomami.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from ..utils import assert_response_ok 4 | 5 | 6 | def test_whoami( 7 | client, test_application, test_application_token, 8 | test_user, admin_user, test_token, admin_token): 9 | r = client.get('/api/whoami', headers={ 10 | 'Authorization': test_application_token}) 11 | assert_response_ok(r) 12 | assert r.json['data'] == { 13 | 'is_anonymous': False, 14 | 'is_application': True, 15 | 'is_minimal_mode': False, 16 | 'is_admin': False, 17 | 'username': test_application.application_name, 18 | } 19 | 20 | r = client.get('/api/whoami', headers={'Authorization': test_token}) 21 | assert_response_ok(r) 22 | assert r.json['data'] == { 23 | 'is_anonymous': False, 24 | 'is_application': False, 25 | 'is_minimal_mode': False, 26 | 'is_admin': False, 27 | 'username': test_user.username, 28 | } 29 | 30 | r = client.get('/api/whoami', headers={'Authorization': admin_token}) 31 | assert_response_ok(r) 32 | assert r.json['data'] == { 33 | 'is_anonymous': False, 34 | 'is_application': False, 35 | 'is_minimal_mode': False, 36 | 'is_admin': True, 37 | 'username': admin_user.username, 38 | } 39 | 40 | r = client.get('/api/whoami') 41 | assert_response_ok(r) 42 | assert r.json['data'] == { 43 | 'is_anonymous': True, 44 | 'is_application': False, 45 | 'is_minimal_mode': False, 46 | 'is_admin': False, 47 | 'username': '', 48 | } 49 | -------------------------------------------------------------------------------- /tests/test_api/test_validation.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from pytest import raises, mark 4 | 5 | from huskar_api.api.schema import ( 6 | instance_schema, validate_fields, ValidationError, 7 | service_value_schema, event_subscribe_schema 8 | ) 9 | 10 | 11 | # TODO: move validation tests to this file 12 | 13 | def test_validate_does_not_exists_field(): 14 | # the set of data keys is a subset of schema fields 15 | validate_fields(instance_schema, {'application': 'test'}) 16 | 17 | # the set of data keys is not subset of schema fields 18 | with raises(ValidationError) as ex: 19 | validate_fields(instance_schema, {'test': 'test'}) 20 | assert ex.value.message == ( 21 | 'The set of fields "set([\'test\'])" is not a subset of ' 22 | '' 23 | ) 24 | 25 | 26 | @mark.xparametrize 27 | def test_service_meta(meta, expected_meta): 28 | data = service_value_schema.load({ 29 | 'ip': '127.0.0.1', 30 | 'port': {'main': 5000}, 31 | 'meta': meta 32 | }).data 33 | assert data['meta'] == expected_meta 34 | 35 | 36 | @mark.xparametrize 37 | def test_invalid_meta(meta): 38 | with raises(ValidationError) as ex: 39 | service_value_schema.load({ 40 | 'ip': '127.0.0.1', 41 | 'port': {'main': 500}, 42 | 'meta': meta 43 | }) 44 | assert ex.value.message == {'meta': [u'Not a valid mapping type.']} 45 | 46 | 47 | @mark.xparametrize 48 | def test_long_polling_validation(application, clusters): 49 | subscription = { 50 | 'service': {application: clusters}, 51 | 'config': {application: clusters}, 52 | 'switch': {application: clusters} 53 | } 54 | data = event_subscribe_schema.load(subscription).data 55 | assert data == subscription 56 | 57 | 58 | @mark.xparametrize 59 | def test_invalid_long_polling_input(application, clusters): 60 | with raises(ValidationError): 61 | subscription = { 62 | 'service': {application: clusters}, 63 | 'config': {application: clusters}, 64 | 'switch': {application: clusters} 65 | } 66 | event_subscribe_schema.load(subscription) 67 | 68 | 69 | @mark.xparametrize 70 | def test_instance(fields, optional_fields): 71 | validate_fields(instance_schema, fields, optional_fields) 72 | 73 | 74 | @mark.xparametrize 75 | def test_invalid_instance_fields(fields, optional_fields, error): 76 | with raises(ValidationError) as ex: 77 | validate_fields(instance_schema, fields, optional_fields) 78 | assert ex.value.message == error 79 | -------------------------------------------------------------------------------- /tests/test_api/test_wellkown.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from huskar_api import settings 4 | from ..utils import assert_response_ok 5 | 6 | 7 | def test_wellknown_common(mocker, client): 8 | framework_versions = { 9 | 'latest': { 10 | 'test_1': '233', 11 | 'test_2': '666', 12 | } 13 | } 14 | route_ezone_default_hijack_mode = { 15 | 'alta1': 'S', 16 | 'altb1': 'D', 17 | 'altc1': 'D', 18 | } 19 | idc_list = ['adca', 'alta', 'altb'] 20 | ezone_list = ['alta1', 'altb1', 'altc1'] 21 | force_routing_clusters = {"alta-test@direct": "alta-test"} 22 | mocker.patch.object(settings, 'FRAMEWORK_VERSIONS', framework_versions) 23 | mocker.patch.object(settings, 'ROUTE_IDC_LIST', idc_list) 24 | mocker.patch.object(settings, 'ROUTE_EZONE_LIST', ezone_list) 25 | mocker.patch.object(settings, 'ROUTE_EZONE_DEFAULT_HIJACK_MODE', 26 | route_ezone_default_hijack_mode) 27 | mocker.patch.object(settings, 'FORCE_ROUTING_CLUSTERS', 28 | force_routing_clusters) 29 | 30 | r = client.get('/api/.well-known/common') 31 | assert_response_ok(r) 32 | data = r.json['data'] 33 | assert data == { 34 | 'framework_versions': framework_versions, 35 | 'idc_list': idc_list, 36 | 'ezone_list': ezone_list, 37 | 'route_default_hijack_mode': route_ezone_default_hijack_mode, 38 | 'force_routing_clusters': force_routing_clusters, 39 | } 40 | 41 | 42 | def test_update_framework_versions(): 43 | framework_versions = { 44 | 'latest': { 45 | 'test_1': '233', 46 | 'test_2': '666', 47 | } 48 | } 49 | try: 50 | assert settings.FRAMEWORK_VERSIONS == {} 51 | settings.update_framework_versions(framework_versions) 52 | assert settings.FRAMEWORK_VERSIONS == framework_versions 53 | finally: 54 | settings.update_framework_versions({}) 55 | -------------------------------------------------------------------------------- /tests/test_bootstrap.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from gevent import sleep 4 | 5 | from huskar_api.switch import switch 6 | 7 | 8 | def test_switch_change_on_fly(zk): 9 | switch_name = 'test_switch_change_on_fly' 10 | path = '/huskar/switch/arch.huskar_api/overall/%s' % switch_name 11 | zk.ensure_path(path) 12 | zk.set(path, '0') 13 | sleep(1) 14 | assert switch.is_switched_on(switch_name, default=None) is False 15 | 16 | # update 17 | zk.set(path, '100') 18 | sleep(1) 19 | assert switch.is_switched_on(switch_name, default=None) is True 20 | -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import pytest 4 | 5 | import huskar_api.cli 6 | from huskar_api.models.auth import User 7 | 8 | 9 | @pytest.fixture(scope='function') 10 | def admin_user(db): 11 | admin_user = User.create_normal('admin', password='admin', is_active=True) 12 | admin_user.grant_admin() 13 | return admin_user 14 | 15 | 16 | def test_cli_entry(mocker): 17 | run = mocker.patch.object(huskar_api.cli.manager, 'run') 18 | huskar_api.cli.main() 19 | run.assert_called_once() 20 | 21 | 22 | def test_initdb_with_admin_present(mocker, admin_user): 23 | prompt_pass = mocker.patch.object(huskar_api.cli, 'prompt_pass') 24 | 25 | with pytest.raises(SystemExit): 26 | huskar_api.cli.initadmin() 27 | 28 | prompt_pass.assert_not_called() 29 | 30 | user = User.get_by_name('admin') 31 | assert user is admin_user 32 | 33 | 34 | def test_initdb(mocker): 35 | prompt_pass = mocker.patch.object(huskar_api.cli, 'prompt_pass') 36 | prompt_pass.side_effect = ['', __name__] 37 | create_user = mocker.spy(User, 'create_normal') 38 | 39 | try: 40 | huskar_api.cli.initadmin() 41 | except SystemExit: 42 | pytest.fail('unexpected sys.exit') 43 | 44 | assert len(prompt_pass.mock_calls) == 2 45 | assert len(create_user.mock_calls) == 1 46 | 47 | user = User.get_by_name('admin') 48 | assert user is not None 49 | assert user.check_password(__name__) 50 | assert user.is_admin 51 | -------------------------------------------------------------------------------- /tests/test_ext.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from huskar_api.ext import EnhancedSentry 4 | 5 | 6 | def test_sentry_init_app_client_none(mocker): 7 | mocked_client = mocker.MagicMock(return_value=None) 8 | mocked_client.__bool__ = mocker.MagicMock(return_value=False) 9 | mocker.patch('raven.contrib.flask.make_client', mocked_client) 10 | sentry = EnhancedSentry() 11 | sentry.register_signal = False 12 | sentry.client = None 13 | sentry.init_app(mocker.MagicMock(__name__='app')) 14 | assert not sentry.client 15 | -------------------------------------------------------------------------------- /tests/test_extras/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/tests/test_extras/__init__.py -------------------------------------------------------------------------------- /tests/test_extras/test_auth.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from huskar_api.extras.email import EmailDeliveryError 4 | from huskar_api.extras.auth import ( 5 | ensure_owners, AppInfo, Department, Owner, NameOccupiedError) 6 | from huskar_api.models.auth import User 7 | 8 | 9 | def test_ensure_ensure_owners_send_mail_failed(mocker): 10 | deliver_email = mocker.patch('huskar_api.extras.auth.deliver_email') 11 | deliver_email.side_effect = EmailDeliveryError() 12 | 13 | prefix = 'test_ensure_ensure_owners_send_mail_failed' 14 | application_name = '{}_app'.format(prefix) 15 | owner = Owner( 16 | '{}_user'.format(prefix), 17 | '{}_user@a.com'.format(prefix), 18 | 'owner', 19 | ) 20 | department = Department( 21 | '1', '{}_team'.format(prefix), '2', '{}_team'.format(prefix)) 22 | appinfo = AppInfo( 23 | department=department, 24 | application_name=application_name, 25 | owners=[owner]) 26 | 27 | assert department.team_name == '{}-{}'.format( 28 | department.parent_id, department.child_id) 29 | assert department.team_desc == '{}-{}'.format( 30 | department.parent_name, department.child_name) 31 | 32 | assert len(list(ensure_owners(appinfo))) == 0 33 | 34 | assert User.get_by_name('{}_user'.format(prefix)) is not None 35 | deliver_email.assert_called_once() 36 | 37 | mocker.patch.object(owner, 'ensure', 38 | mocker.MagicMock(side_effect=NameOccupiedError)) 39 | assert len(list(ensure_owners(appinfo))) == 0 40 | 41 | 42 | def test_department(): 43 | parent_name = '' 44 | child_name = '233_team_child' 45 | department = Department(None, parent_name, 233, child_name) 46 | assert department.team_name == '233' 47 | assert department.team_desc == child_name 48 | assert department.parent_name == parent_name 49 | -------------------------------------------------------------------------------- /tests/test_extras/test_email_snapshots/email-debug-0.html: -------------------------------------------------------------------------------- 1 | DEBUG-1: bar 2 | DEBUG-2: 中文 -------------------------------------------------------------------------------- /tests/test_extras/test_monitor.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from huskar_api.extras.monitor import MonitorClient 4 | 5 | 6 | def test_for_cov(): 7 | c = MonitorClient() 8 | assert c.increment('test') is None 9 | assert c.timing('test', 233) is None 10 | assert c.payload('test') is None 11 | -------------------------------------------------------------------------------- /tests/test_extras/test_raven.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import raven 4 | 5 | from pytest import fixture 6 | 7 | from huskar_api.switch import ( 8 | switch, SWITCH_ENABLE_SENTRY_MESSAGE, SWITCH_ENABLE_SENTRY_EXCEPTION) 9 | from huskar_api.extras import raven as huskar_raven 10 | from huskar_api.extras.raven import capture_message, capture_exception 11 | 12 | 13 | @fixture 14 | def turn_off(mocker): 15 | def turn_off(name): 16 | def is_switched_on(switch_name, default=True): 17 | if switch_name == name: 18 | return False 19 | return default 20 | return mocker.patch.object(switch, 'is_switched_on', is_switched_on) 21 | return turn_off 22 | 23 | 24 | @fixture 25 | def sentry_client(mocker): 26 | c = mocker.patch.object( 27 | huskar_raven, 'raven_client', 28 | raven.Client(dns='gevent+http://foo:bar@example.com/1')) 29 | return c 30 | 31 | 32 | def test_message_on(sentry_client, mocker): 33 | func = mocker.patch.object( 34 | sentry_client, 'captureMessage', autospec=True) 35 | capture_message('foobar') 36 | func.assert_called_once_with('foobar') 37 | 38 | 39 | def test_message_off(sentry_client, mocker, turn_off): 40 | func = mocker.patch.object( 41 | sentry_client, 'captureMessage', autospec=True) 42 | turn_off(SWITCH_ENABLE_SENTRY_MESSAGE) 43 | capture_message('foobar') 44 | assert not func.called 45 | 46 | 47 | def test_exception_on(sentry_client, mocker): 48 | func = mocker.patch.object( 49 | sentry_client, 'captureException', autospec=True) 50 | mocker.patch('huskar_api.extras.raven.raven_client', sentry_client) 51 | capture_exception() 52 | func.assert_called_once_with() 53 | 54 | 55 | def test_exception_off(sentry_client, mocker, turn_off): 56 | func = mocker.patch.object( 57 | sentry_client, 'captureException', autospec=True) 58 | turn_off(SWITCH_ENABLE_SENTRY_EXCEPTION) 59 | capture_exception() 60 | assert not func.called 61 | 62 | 63 | def test_ignore_send_error(mocker): 64 | def is_switched_on(switch_name, default=True): 65 | return default 66 | 67 | mocker.patch.object(switch, 'is_switched_on', is_switched_on) 68 | 69 | mocker.patch('huskar_api.extras.raven.raven_client', 70 | mocker.MagicMock( 71 | captureMessage=mocker.MagicMock( 72 | side_effect=Exception), 73 | captureException=mocker.MagicMock( 74 | side_effect=Exception))) 75 | 76 | assert capture_message('test') is None 77 | assert capture_exception('error') is None 78 | -------------------------------------------------------------------------------- /tests/test_extras/test_uptime.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function 2 | 3 | from freezegun import freeze_time 4 | 5 | from huskar_api.extras.uptime import process_uptime 6 | 7 | 8 | def test_process_uptime(): 9 | uptime = process_uptime() 10 | assert uptime >= 0 11 | with freeze_time() as frozen_datetime: 12 | assert int(process_uptime()) == int(uptime) 13 | frozen_datetime.tick() 14 | assert int(process_uptime()) == int(uptime) + 1 15 | -------------------------------------------------------------------------------- /tests/test_models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/tests/test_models/__init__.py -------------------------------------------------------------------------------- /tests/test_models/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/tests/test_models/conftest.py -------------------------------------------------------------------------------- /tests/test_models/test_alembic.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from huskar_api.models.alembic import get_metadata 4 | 5 | 6 | def test_metadata(): 7 | assert get_metadata() 8 | -------------------------------------------------------------------------------- /tests/test_models/test_audit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/tests/test_models/test_audit/__init__.py -------------------------------------------------------------------------------- /tests/test_models/test_audit/test_action.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import copy 4 | 5 | from pytest import raises 6 | 7 | from huskar_api.models.audit import action_types, action_creator 8 | 9 | 10 | def test_action_types(): 11 | assert action_types.CREATE_TEAM == 1001 12 | assert action_types[1001] == 'CREATE_TEAM' 13 | 14 | with raises(AttributeError): 15 | action_types.DISCARD_TYPE 16 | with raises(AttributeError): 17 | action_types._DISCARD_TYPE 18 | with raises(KeyError): 19 | action_types[-1] 20 | 21 | with raises(AttributeError): 22 | action_types.create_team 23 | with raises(KeyError): 24 | action_types['1001'] 25 | 26 | with raises(AttributeError): 27 | action_types.CREATE_TEAM = 1001 28 | 29 | 30 | def test_action_creator(): 31 | creator = copy.deepcopy(action_creator) 32 | 33 | @creator(10010) 34 | def make_china_unicom(action_type, telephone): 35 | return {'telephone': telephone}, [] 36 | 37 | with raises(KeyError): 38 | creator.make_action(10086) 39 | 40 | with raises(TypeError): 41 | creator.make_action(10010) 42 | 43 | action = creator.make_action(10010, telephone='10010') 44 | assert len(action) == 3 45 | assert action[0] == 10010 46 | assert action[1] == {'telephone': '10010'} 47 | assert action[2] == [] 48 | -------------------------------------------------------------------------------- /tests/test_models/test_audit/test_index.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import datetime 4 | 5 | from pytest import raises 6 | 7 | from huskar_api.models.audit.index import AuditIndex, AuditIndexInstance 8 | from huskar_api.models.audit.const import TYPE_SITE, TYPE_TEAM, TYPE_CONFIG 9 | 10 | 11 | def test_audit_index(db): 12 | # The AuditIndex is an internal model. So we test it simply. 13 | date = datetime.date.today() 14 | assert AuditIndex.get_audit_ids(TYPE_SITE, 0) == [] 15 | assert AuditIndex.get_audit_ids(TYPE_TEAM, 1) == [] 16 | assert AuditIndex.get_audit_ids(TYPE_TEAM, 3) == [] 17 | assert AuditIndex.get_audit_ids_by_date(TYPE_SITE, 0, date) == [] 18 | assert AuditIndex.get_audit_ids_by_date(TYPE_TEAM, 1, date) == [] 19 | assert AuditIndex.get_audit_ids_by_date(TYPE_TEAM, 3, date) == [] 20 | 21 | created_at = datetime.datetime.now() 22 | date = created_at.date() 23 | with db.close_on_exit(False): 24 | AuditIndex.create(db, 1, created_at, TYPE_SITE, 0) 25 | AuditIndex.create(db, 1, created_at, TYPE_TEAM, 1) 26 | AuditIndex.create(db, 2, created_at, TYPE_TEAM, 1) 27 | with raises(AssertionError): 28 | AuditIndex.create(db, 2, created_at, TYPE_SITE, 1) 29 | AuditIndex.flush_cache(date, TYPE_SITE, 0) 30 | AuditIndex.flush_cache(date, TYPE_TEAM, 1) 31 | AuditIndex.flush_cache(date, TYPE_TEAM, 1) 32 | 33 | assert AuditIndex.get_audit_ids(TYPE_SITE, 0) == [1] 34 | assert AuditIndex.get_audit_ids(TYPE_TEAM, 1) == [2, 1] 35 | assert AuditIndex.get_audit_ids(TYPE_TEAM, 3) == [] 36 | assert AuditIndex.get_audit_ids_by_date(TYPE_SITE, 0, date) == [1] 37 | assert AuditIndex.get_audit_ids_by_date(TYPE_TEAM, 1, date) == [2, 1] 38 | assert AuditIndex.get_audit_ids_by_date(TYPE_TEAM, 3, date) == [] 39 | 40 | 41 | def test_audit_instance_index(db): 42 | application_id = 1 43 | cluster_name = 'bar' 44 | key = 'test' 45 | now = datetime.datetime.now() 46 | 47 | assert AuditIndexInstance.get_audit_ids( 48 | TYPE_CONFIG, application_id, cluster_name, key) == [] 49 | with db.close_on_exit(False): 50 | AuditIndexInstance.create( 51 | db, 1, now, TYPE_CONFIG, application_id, cluster_name, key) 52 | with raises(AssertionError): 53 | AuditIndexInstance.create( 54 | db, 1, now, application_id, cluster_name, key, -1) 55 | 56 | AuditIndexInstance.flush_cache( 57 | now.date(), TYPE_CONFIG, application_id, cluster_name, key) 58 | assert AuditIndexInstance.get_audit_ids( 59 | TYPE_CONFIG, application_id, cluster_name, key) == [1] 60 | -------------------------------------------------------------------------------- /tests/test_models/test_auth/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/tests/test_models/test_auth/__init__.py -------------------------------------------------------------------------------- /tests/test_models/test_auth/test_session.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from pytest import fixture 4 | 5 | from huskar_api import settings 6 | from huskar_api.models.auth import User 7 | from huskar_api.models.auth.session import SessionAuth 8 | 9 | 10 | @fixture 11 | def session_auth(): 12 | return SessionAuth('san.zhang') 13 | 14 | 15 | @fixture 16 | def create_user(db): 17 | def create(username): 18 | user = User(username=username, password='*') 19 | db.add(user) 20 | db.commit() 21 | User.get(user.id, force=True) # touch cache 22 | return user 23 | 24 | return create 25 | 26 | 27 | def test_minimal_mode_metrics(session_auth, monitor_client): 28 | assert session_auth.is_minimal_mode is False 29 | assert session_auth.minimal_mode_reason is None 30 | monitor_client.increment.assert_not_called() 31 | 32 | session_auth.enter_minimal_mode('tester') 33 | session_auth.enter_minimal_mode() # ignored 34 | session_auth.enter_minimal_mode() # ignored 35 | assert session_auth.is_minimal_mode is True 36 | assert session_auth.minimal_mode_reason == 'tester' 37 | monitor_client.increment.assert_called_once_with('minimal_mode.qps', 1) 38 | 39 | 40 | def test_switch_as(create_user): 41 | user_foo = create_user('foo') 42 | auth = SessionAuth(user_foo.username) 43 | assert repr(auth) == 'SessionAuth(%r)' % 'foo' 44 | auth.load_user() 45 | assert auth.id == user_foo.id 46 | 47 | user_bar = create_user('bar') 48 | with auth.switch_as(user_bar.username): 49 | assert auth.id == user_bar.id 50 | 51 | assert auth.id == user_foo.id 52 | 53 | 54 | def test_update_admin_emergency_user_list(): 55 | old_data = settings.ADMIN_EMERGENCY_USER_LIST 56 | new_data = ['a', 'foo', 'foobar'] 57 | try: 58 | settings.update_admin_emergency_user_list(new_data) 59 | assert settings.ADMIN_EMERGENCY_USER_LIST == frozenset(new_data) 60 | finally: 61 | settings.ADMIN_EMERGENCY_USER_LIST = old_data 62 | -------------------------------------------------------------------------------- /tests/test_models/test_auth/test_user.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | test_change_password: 4 | args: input_password,hashed_password 5 | data: 6 | - input_password: 'x' 7 | hashed_password: '54a2f7f92a5f975d8096af77a126edda7da60c5aa872ef1b871701ae' 8 | - input_password: 'fainiung2uoP4cho7joh1ahCaeTho6qu' 9 | hashed_password: 'a336148a6338aa0b17861dcb0bb554f9c1654d15a458e5324dd679a2' 10 | - input_password: '蛤' 11 | hashed_password: '0c5f75f952b01236f59457927493cd961bf676bd8a0ca646daae2a86' 12 | dataids: 13 | - sample_1 14 | - sample_2 15 | - sample_3 16 | 17 | test_check_password: 18 | args: present_password,input_password 19 | data: 20 | - present_password: '0808f64e60d58979fcb676c96ec938270dea42445aeefcd3a4e6f8db' 21 | input_password: 22 | correct: 'foo' 23 | incorrect: 'bar' 24 | - present_password: '07daf010de7f7f0d8d76a76eb8d1eb40182c8d1e7a3877a6686c9bf0' 25 | input_password: 26 | correct: 'bar' 27 | incorrect: '' 28 | - present_password: '0c5f75f952b01236f59457927493cd961bf676bd8a0ca646daae2a86' 29 | input_password: 30 | correct: '蛤' 31 | incorrect: '赛艇' 32 | dataids: 33 | - sample_1 34 | - sample_2 35 | - sample_3 36 | -------------------------------------------------------------------------------- /tests/test_models/test_comment.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from huskar_api.models.comment import set_comment, get_comment, Comment 4 | 5 | 6 | def test_create_comment(db): 7 | assert db.query(Comment).count() == 0 8 | set_comment('base.foo', 'alta-stable', 'config', 'DB_URL', u'\u86e4') 9 | 10 | assert db.query(Comment).count() == 1 11 | comment = db.query(Comment).first() 12 | assert comment.application == 'base.foo' 13 | assert comment.cluster == 'alta-stable' 14 | assert comment.key_type == 'config' 15 | assert comment.key_name == 'DB_URL' 16 | assert comment.key_comment == u'\u86e4' 17 | 18 | 19 | def test_override_comment(db): 20 | assert db.query(Comment).count() == 0 21 | set_comment('base.foo', 'alta-stable', 'config', 'DB_URL', u'\u86e4') 22 | set_comment('base.foo', 'alta-stable', 'config', 'DB_URL', u'+1s') 23 | 24 | assert db.query(Comment).count() == 1 25 | comment = db.query(Comment).first() 26 | assert comment.application == 'base.foo' 27 | assert comment.cluster == 'alta-stable' 28 | assert comment.key_type == 'config' 29 | assert comment.key_name == 'DB_URL' 30 | assert comment.key_comment == u'+1s' 31 | 32 | 33 | def test_delete_comment(db): 34 | assert db.query(Comment).count() == 0 35 | 36 | set_comment('base.foo', 'alta-stable', 'config', 'DB_URL', u'\u86e4') 37 | assert db.query(Comment).count() == 1 38 | 39 | set_comment('base.foo', 'alta-stable', 'config', 'DB_URL', None) 40 | assert db.query(Comment).count() == 0 41 | 42 | set_comment('base.foo', 'alta-stable', 'switch', 'DB_URL', None) 43 | assert db.query(Comment).count() == 0 44 | 45 | 46 | def test_get_comment(db): 47 | stmt = Comment.__table__.insert().values( 48 | application='base.foo', 49 | cluster='test', 50 | key_type='switch', 51 | key_name='k', 52 | key_comment=u'\u957f\u8005' 53 | ) 54 | db.execute(stmt) 55 | db.commit() 56 | 57 | assert get_comment('base.foo', 'test', 'config', 'k') == u'' 58 | assert get_comment('base.foo', 'test', 'switch', 'k') == u'\u957f\u8005' 59 | assert get_comment('base.foo', 'test', 'switch', 'K') == u'' 60 | -------------------------------------------------------------------------------- /tests/test_models/test_extras/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/tests/test_models/test_extras/__init__.py -------------------------------------------------------------------------------- /tests/test_models/test_infra/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | -------------------------------------------------------------------------------- /tests/test_models/test_infra/test_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from pytest import mark 4 | 5 | from huskar_api.models.infra import extract_application_names 6 | from huskar_api.models.infra.utils import extract_application_name 7 | 8 | 9 | @mark.parametrize('url,result', [ 10 | ('', None), 11 | ('mysql://localhost:3306/sample', None), 12 | ('sam+mysql:///overall', None), 13 | ('sam+mysql://dal.test.auto/overall', 'dal.test.auto'), 14 | ('sam+mysql://user:pass@dal.test.auto/overall', 'dal.test.auto'), 15 | ('sam+amqp://user:@#$deX^h&@rabbitmq.100010/vhost/overall', 16 | 'rabbitmq.100010'), 17 | ]) 18 | def test_extract_application_name(url, result): 19 | assert extract_application_name(url) == result 20 | 21 | 22 | def test_extract_application_names(): 23 | assert extract_application_names([ 24 | '', 25 | 'mysql://localhost:3306/sample', 26 | 'sam+mysql:///overall', 27 | 'sam+mysql://dal.test.auto/overall', 28 | 'sam+mysql://user:pass@dal.test.auto/overall', 29 | ]) == [ 30 | 'dal.test.auto', 31 | 'dal.test.auto', 32 | ] 33 | 34 | assert extract_application_names({ 35 | 'u1': '', 36 | 'u2': 'mysql://localhost:3306/sample', 37 | 'u3': 'sam+mysql:///overall', 38 | 'u4': 'sam+mysql://dal.test.auto/overall', 39 | 'u5': 'sam+mysql://user:pass@dal.test.auto/overall', 40 | }) == { 41 | 'u4': 'dal.test.auto', 42 | 'u5': 'dal.test.auto', 43 | } 44 | -------------------------------------------------------------------------------- /tests/test_models/test_tree/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/tests/test_models/test_tree/__init__.py -------------------------------------------------------------------------------- /tests/test_models/test_tree/conftest.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from pytest import fixture 4 | 5 | from huskar_api.models import huskar_client 6 | from huskar_api.models.tree import TreeHub 7 | 8 | 9 | @fixture 10 | def test_application_name(faker): 11 | return faker.uuid4() 12 | 13 | 14 | @fixture 15 | def hub(): 16 | return TreeHub(huskar_client) 17 | 18 | 19 | @fixture 20 | def holder(hub, test_application_name): 21 | holder = hub.get_tree_holder(test_application_name, 'config') 22 | holder.block_until_initialized(5) 23 | return holder 24 | 25 | 26 | @fixture 27 | def service_holder(hub, test_application_name): 28 | holder = hub.get_tree_holder(test_application_name, 'service') 29 | holder.block_until_initialized(5) 30 | return holder 31 | -------------------------------------------------------------------------------- /tests/test_models/test_tree/test_common.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import functools 4 | 5 | from huskar_api.models.tree.common import parse_path, ClusterMap 6 | 7 | 8 | def test_path(): 9 | p = functools.partial(parse_path, '/huskar') 10 | 11 | path = p('/huskar/service') 12 | assert not path.is_none() 13 | assert path.get_level() == 1 14 | assert path.type_name == 'service' 15 | 16 | path = p('/huskar/service/base.foo') 17 | assert not path.is_none() 18 | assert path.get_level() == 2 19 | assert path.type_name == 'service' 20 | assert path.application_name == 'base.foo' 21 | 22 | path = p('/huskar/service/base.foo/stable') 23 | assert not path.is_none() 24 | assert path.get_level() == 3 25 | assert path.type_name == 'service' 26 | assert path.application_name == 'base.foo' 27 | assert path.cluster_name == 'stable' 28 | 29 | path = p('/huskar/service/base.foo/stable/10.0.0.1_5000') 30 | assert not path.is_none() 31 | assert path.get_level() == 4 32 | assert path.type_name == 'service' 33 | assert path.application_name == 'base.foo' 34 | assert path.cluster_name == 'stable' 35 | assert path.data_name == '10.0.0.1_5000' 36 | 37 | path = p('/huskar/service/base.foo/stable/10.0.0.1_5000/runtime') 38 | assert path.is_none() 39 | 40 | path = p('/huskar-service') 41 | assert path.is_none() 42 | 43 | path = p('/') 44 | assert path.is_none() 45 | 46 | 47 | def test_cluster_map(): 48 | cluster_map = ClusterMap() 49 | 50 | # Empty-tolerance 51 | cluster_map.register('foo', None) 52 | assert cluster_map.cluster_names == {} 53 | assert cluster_map.resolved_names == {} 54 | 55 | # Empty-tolerance 56 | cluster_map.deregister('foo') 57 | assert cluster_map.cluster_names == {} 58 | assert cluster_map.resolved_names == {} 59 | 60 | # Register symlink or route 61 | cluster_map.register('foo', 'bar') 62 | cluster_map.register('baz', 'bar') 63 | cluster_map.register('s', 'e') 64 | assert cluster_map.cluster_names == {'foo': 'bar', 'baz': 'bar', 's': 'e'} 65 | assert cluster_map.resolved_names == {'bar': {'foo', 'baz'}, 'e': {'s'}} 66 | 67 | # Deregister symlink or route 68 | cluster_map.deregister('baz') 69 | cluster_map.deregister('s') 70 | assert cluster_map.cluster_names == {'foo': 'bar'} 71 | assert cluster_map.resolved_names == {'bar': {'foo'}, 'e': set()} 72 | 73 | # Register multiplex symlink 74 | cluster_map.register('baz', 'bar+foo') 75 | assert cluster_map.cluster_names == {'foo': 'bar', 'baz': 'bar+foo'} 76 | assert cluster_map.resolved_names == { 77 | 'bar': {'foo', 'baz'}, 'foo': {'baz'}, 'e': set()} 78 | 79 | # Deregister multiplex symlink 80 | cluster_map.deregister('baz') 81 | assert cluster_map.cluster_names == {'foo': 'bar'} 82 | assert cluster_map.resolved_names == { 83 | 'bar': {'foo'}, 'foo': set(), 'e': set()} 84 | -------------------------------------------------------------------------------- /tests/test_models/test_tree/test_extra.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import json 4 | 5 | from pytest import fixture 6 | 7 | from huskar_api.models.tree.common import Path 8 | from huskar_api.models.tree.extra import extra_handlers 9 | from huskar_api.models.tree.watcher import TreeWatcher 10 | 11 | 12 | @fixture 13 | def watcher(hub): 14 | return TreeWatcher(hub) 15 | 16 | 17 | def test_service_info_handler_with_invalid_path( 18 | zk, watcher, test_application_name): 19 | base_path = '/huskar/service/%s' % test_application_name 20 | instance_path = '%s/stable/192.168.10.1_8080' % base_path 21 | zk.create( 22 | instance_path, json.dumps({'ip': '191.168.10.1_8080'}), 23 | makepath=True) 24 | path = Path.parse(instance_path) 25 | 26 | update_handler = extra_handlers['service_info', 'update'] 27 | assert update_handler(watcher, path) == {} 28 | 29 | all_handler = extra_handlers['service_info', 'all'] 30 | assert all_handler(watcher, path) == {} 31 | -------------------------------------------------------------------------------- /tests/test_models/test_webhook/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/tests/test_models/test_webhook/__init__.py -------------------------------------------------------------------------------- /tests/test_scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/huskar-org/huskar/395775c59c7da97c46efe9756365cad028b7c95a/tests/test_scripts/__init__.py -------------------------------------------------------------------------------- /tests/test_scripts/test_db.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from pytest import fixture, raises 4 | 5 | import huskar_api.scripts.db 6 | 7 | 8 | @fixture(scope='function') 9 | def get_engine(mocker, monkeypatch, db): 10 | engine = db.engines['master'] 11 | new_database = '%s_test_db_script' % engine.url.database 12 | monkeypatch.setattr(engine.url, 'database', new_database) 13 | return mocker.spy(huskar_api.scripts.db, 'get_engine') 14 | 15 | 16 | def test_initdb(db, get_engine): 17 | db.close() 18 | 19 | assert get_engine.call_count == 0 20 | 21 | with db.close_on_exit(True): 22 | before_tables = db.execute('show tables').fetchall() 23 | 24 | huskar_api.scripts.db.initdb() 25 | 26 | with db.close_on_exit(True): 27 | after_tables = db.execute('show tables').fetchall() 28 | 29 | assert before_tables == after_tables 30 | assert get_engine.call_count > 0 31 | 32 | 33 | def test_initdb_in_production(get_engine, mocker): 34 | mocker.patch('huskar_api.settings.IS_IN_DEV', False) 35 | 36 | with raises(RuntimeError): 37 | huskar_api.scripts.db.initdb() 38 | assert get_engine.call_count == 0 39 | 40 | 41 | def test_dumpdb(db, mocker): 42 | db.close() 43 | 44 | schema_open = mocker.patch('huskar_api.scripts.db.open') 45 | schema_file = schema_open() 46 | schema_file.__enter__ = mocker.Mock() 47 | schema_file.__enter__.return_value = schema_file 48 | schema_file.__exit__ = mocker.Mock() 49 | 50 | huskar_api.scripts.db.dumpdb() 51 | 52 | schema_file.writelines.assert_called_once() 53 | -------------------------------------------------------------------------------- /tests/test_scripts/test_vacuum.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | test_cluster_data: 4 | args: before,should_exist,should_not_exist 5 | data: 6 | - before: 7 | - path: '/huskar_{base_path}/service/{test_application}/foo' 8 | data: null 9 | - path: '/huskar_{base_path}/service/{test_application}/bar/x' 10 | data: null 11 | - path: '/huskar_{base_path}/service/foo /bar' 12 | data: null 13 | - path: '/huskar_{base_path}/service/black_appid/bar' 14 | data: null 15 | should_exist: 16 | - path: '/huskar_{base_path}/service/{test_application}/bar' 17 | - path: '/huskar_{base_path}/service/black_appid/bar' 18 | - path: '/huskar_{base_path}/service/foo /bar' 19 | should_not_exist: 20 | - path: '/huskar_{base_path}/service/{test_application}/foo' 21 | - before: 22 | - path: '/huskar_{base_path}/service/{test_application}/foo' 23 | data: 'xxx' 24 | - path: '/huskar_{base_path}/service/{test_application}/bar' 25 | data: '{}' 26 | - path: '/huskar_{base_path}/service/{test_application}/baz' 27 | data: '{"link":[]}' 28 | - path: '/huskar_{base_path}/service/{test_application}/biu' 29 | data: '{"link":["foo"]}' 30 | - path: '/huskar_{base_path}/service/{test_application}/boo' 31 | data: '1' 32 | should_exist: 33 | - path: '/huskar_{base_path}/service/{test_application}/foo' 34 | - path: '/huskar_{base_path}/service/{test_application}/biu' 35 | - path: '/huskar_{base_path}/service/{test_application}/boo' 36 | should_not_exist: 37 | - path: '/huskar_{base_path}/service/{test_application}/bar' 38 | - path: '/huskar_{base_path}/service/{test_application}/baz' 39 | - before: 40 | - path: '/huskar_{base_path}/service/{test_application}/foo' 41 | data: '{"info": {"balance_policy":"Random"}}' 42 | - path: '/huskar_{base_path}/service/{test_application}/bar' 43 | data: '{"info": {"balance_policy":"Random"}, "link":[]}' 44 | - path: '/huskar_{base_path}/service/{test_application}/baz' 45 | data: '{"link":[]}' 46 | should_exist: 47 | - path: '/huskar_{base_path}/service/{test_application}/foo' 48 | - path: '/huskar_{base_path}/service/{test_application}/bar' 49 | should_not_exist: 50 | - path: '/huskar_{base_path}/service/{test_application}/baz' 51 | - before: 52 | - path: '/huskar_{base_path}/service/{test_application}/foo' 53 | data: '{"route":{"foo":"bar"}}' 54 | - path: '/huskar_{base_path}/service/{test_application}/bar' 55 | data: '{"route":{}}' 56 | should_exist: 57 | - path: '/huskar_{base_path}/service/{test_application}/foo' 58 | should_not_exist: 59 | - path: '/huskar_{base_path}/service/{test_application}/bar' 60 | -------------------------------------------------------------------------------- /tests/test_wsgi.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from pytest import raises 4 | from decouple import UndefinedValueError 5 | 6 | import huskar_api.wsgi 7 | import huskar_api.settings 8 | 9 | 10 | def test_wsgi_entry(): 11 | assert callable(huskar_api.wsgi.app) 12 | 13 | 14 | def test_settings(): 15 | with raises(UndefinedValueError): 16 | huskar_api.settings.config.get('NOT_EXISTS') 17 | assert huskar_api.settings.config.get('NOT_EXISTS', default=None) is None 18 | assert huskar_api.settings.config_repository.get('NOT_EXISTS') is None 19 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | 4 | def assert_response_ok(response): 5 | """Test whether the response is mean success 6 | 7 | :param response: Response instance from ``pytest_flask.fixtures.client`` 8 | :raise AssertionError: If test failed, will raise ``AssertionError`` 9 | """ 10 | assert response.status_code == 200, response.data 11 | assert response.json['status'] == 'SUCCESS' 12 | assert response.json['message'] == '' 13 | 14 | 15 | def assert_response_status_code(response, status_code): 16 | assert response.status_code == status_code 17 | 18 | 19 | def assert_semaphore_is_zero(semaphore, x): 20 | assert not semaphore.locked() 21 | for _ in range(x): 22 | semaphore.acquire() 23 | try: 24 | assert semaphore.locked() 25 | finally: 26 | for _ in range(x): 27 | semaphore.release() 28 | -------------------------------------------------------------------------------- /tools/ci/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | set -x 6 | 7 | sleep 10 8 | 9 | ./manage.sh "$@" 10 | -------------------------------------------------------------------------------- /tools/git-hooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # USAGE: 4 | # 1) Install jshint via `npm i -g jshint` 5 | # 2) Make this file executable with chmod +x 6 | # 3) Put it in project's .git/hooks/ and name it pre-commit 7 | 8 | pylintconf=$PWD/tools/pylint/pylintrc 9 | 10 | export PYTHONPATH="$PWD/tools/pylint:$PYTHONPATH" 11 | 12 | if git rev-parse --verify HEAD >/dev/null 2>&1 13 | then 14 | against=HEAD 15 | else 16 | # Initial commit: diff against an empty tree object 17 | against=4b825dc642cb6eb9a060e54bf8d69288fbee4904 18 | fi 19 | 20 | py_flag=0 21 | flake8_flag=0 22 | 23 | pyfiles=$(git diff-index --name-only --diff-filter=ACMR ${against} -- \ 24 | | grep '.py$') 25 | 26 | if test -z "$pyfiles" 27 | then 28 | echo "no diff py file for pylint" 29 | elif which pylint >/dev/null 30 | then 31 | for file in $pyfiles 32 | do 33 | flake8 $file || flake8_flag=1 34 | pylint --rcfile="${pylintconf}" $file 35 | py_flag=$(($? & 2)) 36 | done 37 | else 38 | echo "Y U NO USE PYLINT" 39 | fi 40 | 41 | all_flag=`expr $py_flag + $flake8_flag` 42 | exit $all_flag 43 | -------------------------------------------------------------------------------- /tools/git-hooks/pre-push: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # An example hook script to verify what is about to be pushed. Called by "git 4 | # push" after it has checked the remote status, but before anything has been 5 | # pushed. If this script exits with a non-zero status nothing will be pushed. 6 | # 7 | # This hook is called with the following parameters: 8 | # 9 | # $1 -- Name of the remote to which the push is being done 10 | # $2 -- URL to which the push is being done 11 | # 12 | # If pushing without using a named remote those arguments will be equal. 13 | # 14 | # Information about the commits which are being pushed is supplied as lines to 15 | # the standard input in the form: 16 | # 17 | # 18 | # 19 | # This sample shows how to prevent push of commits where the log message starts 20 | # with "WIP" (work in progress). 21 | 22 | remote="$1" 23 | url="$2" 24 | 25 | if echo $url | grep -q "example.com" 26 | then 27 | echo "!-- DO NOT PUSH TO UPSTREAM --!" 28 | echo "Send Pull Request instead" 29 | exit 1 30 | fi 31 | -------------------------------------------------------------------------------- /tools/huskar-lint/huskar_lint.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import sys 4 | import itertools 5 | 6 | from huskar_api.models import DBSession 7 | from huskar_api.models.auth import User 8 | 9 | 10 | CODES = { 11 | 'E-HUSKAR001': 'Unknown email domain', 12 | 'E-HUSKAR002': 'Mismatched email and username', 13 | } 14 | 15 | 16 | def check_users(db): 17 | for user in db.query(User).all(): 18 | if user.email is not None: 19 | if not user.email.endswith(u'@example.com'): 20 | yield 'E-HUSKAR001', u'%d\t%s\t%s' % ( 21 | user.id, user.email, user.username) 22 | elif user.email != u'%s@example.com' % user.username: 23 | yield 'E-HUSKAR002', u'%d\t%s\t%s' % ( 24 | user.id, user.email, user.username) 25 | 26 | 27 | def main(): 28 | db = DBSession() 29 | linters = [ 30 | check_users(db), 31 | ] 32 | 33 | for code, info in itertools.chain.from_iterable(linters): 34 | desc = CODES[code] 35 | print(u'%s\t%s\t%s' % (code, desc, info), file=sys.stderr) 36 | else: 37 | sys.exit(0) 38 | 39 | sys.exit(1) 40 | 41 | 42 | if __name__ == '__main__': 43 | main() 44 | -------------------------------------------------------------------------------- /tools/pylint/py2pytest.py: -------------------------------------------------------------------------------- 1 | """Astroid hooks for pytest 2 | 3 | """ 4 | 5 | from astroid import MANAGER 6 | from astroid import nodes 7 | from astroid.builder import AstroidBuilder 8 | 9 | 10 | MODULE_TRANSFORMS = {} 11 | 12 | 13 | def transform(module): 14 | try: 15 | tr = MODULE_TRANSFORMS[module.name] 16 | except KeyError: 17 | pass 18 | else: 19 | tr(module) 20 | 21 | 22 | def pytest_transform(module): 23 | fake = AstroidBuilder(MANAGER).string_build(''' 24 | 25 | try: 26 | import _pytest.mark 27 | import _pytest.recwarn 28 | import _pytest.runner 29 | import _pytest.python 30 | except ImportError: 31 | pass 32 | else: 33 | deprecated_call = _pytest.recwarn.deprecated_call 34 | exit = _pytest.runner.exit 35 | fail = _pytest.runner.fail 36 | fixture = _pytest.python.fixture 37 | importorskip = _pytest.runner.importorskip 38 | mark = _pytest.mark.MarkGenerator() 39 | raises = _pytest.python.raises 40 | skip = _pytest.runner.skip 41 | yield_fixture = _pytest.python.yield_fixture 42 | 43 | ''') 44 | 45 | for item_name, item in fake.locals.items(): 46 | module.locals[item_name] = item 47 | 48 | 49 | MODULE_TRANSFORMS['pytest'] = pytest_transform 50 | MODULE_TRANSFORMS['py.test'] = pytest_transform 51 | 52 | MANAGER.register_transform(nodes.Module, transform) 53 | 54 | 55 | def register(linter): 56 | pass 57 | -------------------------------------------------------------------------------- /tools/zookeeper-lint/testteam_shredder.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | from huskar_sdk_v2.consts import ( 6 | SERVICE_SUBDOMAIN, SWITCH_SUBDOMAIN, CONFIG_SUBDOMAIN) 7 | 8 | from huskar_api.models import huskar_client 9 | from huskar_api.models.auth import Application 10 | from huskar_api.models.manifest import application_manifest 11 | 12 | 13 | NAME_PREFIXES = ('testteam',) 14 | 15 | 16 | logger = logging.getLogger(__name__) 17 | logger.addHandler(logging.StreamHandler()) 18 | logger.propagate = False 19 | 20 | 21 | def collect_applications(): 22 | names = set() 23 | names.update(x.application_name for x in Application.get_all()) 24 | names.update(application_manifest.as_list()) 25 | return sorted(n for n in names if n.startswith(NAME_PREFIXES)) 26 | 27 | 28 | def destroy_application(name): 29 | application = Application.get_by_name(name) 30 | if application is not None: 31 | Application.delete(application.id) 32 | logger.info('Removed application from DBMS: %s', name) 33 | 34 | for type_name in (SERVICE_SUBDOMAIN, SWITCH_SUBDOMAIN, CONFIG_SUBDOMAIN): 35 | path = '/huskar/%s/%s' % (type_name, name) 36 | if huskar_client.client.exists(path): 37 | huskar_client.client.delete(path, recursive=True) 38 | logger.info('Removed application from ZooKeeper: %s', path) 39 | 40 | 41 | def main(): 42 | application_names = collect_applications() 43 | logger.info('Collected %d applications', len(application_names)) 44 | for application_name in application_names: 45 | logger.info('Processing %s', application_name) 46 | destroy_application(application_name) 47 | 48 | 49 | if __name__ == '__main__': 50 | main() 51 | --------------------------------------------------------------------------------