├── .dive-ci ├── .dockerignore ├── .env ├── .github.env.gpg ├── .github └── workflows │ ├── cd.yml │ ├── ci-live.yml │ └── ci.yml ├── .gitignore ├── .prettierignore ├── .yamllint ├── CODEOWNERS ├── Gruntfile.js ├── LICENSE.txt ├── MANIFEST.in ├── README.rst ├── babel.cfg ├── docker-compose.yml ├── docker ├── app │ ├── Dockerfile │ ├── run-celery.sh │ ├── run-ci.sh │ └── run-gunicorn.sh ├── ci │ ├── Dockerfile │ └── requirements.txt ├── client │ ├── Dockerfile │ ├── client.env │ ├── run-celery.sh │ ├── run-crontab.sh │ ├── run-gunicorn.sh │ ├── run-lokole.sh │ └── webapp.env ├── docker-compose.prod.yml ├── docker-compose.setup.yml ├── docker-compose.test.yml ├── docker-compose.tools.yml ├── integtest │ ├── 0-wait-for-services.sh │ ├── 1-register-client.sh │ ├── 2-client-uploads-emails.sh │ ├── 3-receive-email-for-client.sh │ ├── 4-client-downloads-emails.sh │ ├── 5-assert-on-results.sh │ ├── 6-receive-service-email.sh │ ├── Dockerfile │ ├── files │ │ ├── .gitignore │ │ ├── echo-service-email.mime │ │ ├── emails.jsonl │ │ ├── inbound-email-2.mime │ │ ├── inbound-email.mime │ │ ├── wikipedia-service-email.mime │ │ └── zzusers.jsonl │ ├── requirements.txt │ ├── tests.sh │ └── utils.sh ├── nginx │ ├── Dockerfile │ ├── nginx.conf.mustache │ ├── run-nginx.sh │ ├── server.conf.mustache │ └── static │ │ ├── favicon.ico │ │ └── robots.txt ├── setup │ ├── Dockerfile │ ├── arm.parameters.json │ ├── arm.template.json │ ├── renew-cert.sh │ ├── requirements.txt │ ├── setup-dns.sh │ ├── setup.sh │ ├── upgrade-helm.sh │ ├── upgrade-vm.sh │ ├── upgrade.sh │ ├── utils.sh │ └── vm.sh └── statuspage │ └── Dockerfile ├── helm ├── .gitignore └── opwen_cloudserver │ ├── Chart.yaml │ ├── templates │ ├── _helpers.tpl │ ├── api-autoscaler.yaml │ ├── api-deployment.yaml │ ├── api-service.yaml │ ├── cluster-issuer.yaml │ ├── ingress.yaml │ ├── nginx-deployment.yaml │ ├── nginx-service.yaml │ ├── webapp-deployment.yaml │ ├── webapp-service.yaml │ ├── worker-autoscaler.yaml │ └── worker-deployment.yaml │ └── values.yaml ├── install.py ├── makefile ├── opwen_email_client ├── __init__.py ├── domain │ ├── __init__.py │ ├── email │ │ ├── __init__.py │ │ ├── attachment.py │ │ ├── client.py │ │ ├── sql_store.py │ │ ├── store.py │ │ ├── sync.py │ │ └── user_store.py │ ├── modem │ │ ├── __init__.py │ │ ├── e303.py │ │ ├── e3131.py │ │ └── e353.py │ └── sim │ │ ├── __init__.py │ │ └── hologram.py ├── util │ ├── __init__.py │ ├── network.py │ ├── os.py │ ├── serialization.py │ ├── sqlalchemy.py │ └── wtforms.py └── webapp │ ├── __init__.py │ ├── actions.py │ ├── cache.py │ ├── commands.py │ ├── config.py │ ├── forms │ ├── __init__.py │ ├── email.py │ ├── login.py │ ├── register.py │ └── settings.py │ ├── ioc.py │ ├── jinja.py │ ├── login.py │ ├── mkwvconf.py │ ├── security.py │ ├── session.py │ ├── static │ ├── css │ │ ├── _base_email.css │ │ ├── about.css │ │ ├── email_new.css │ │ ├── home.css │ │ └── settings.css │ ├── favicon.ico │ ├── images │ │ └── logo.png │ └── js │ │ ├── _base.js │ │ ├── _base_email.js │ │ ├── email_new.js │ │ ├── register.js │ │ └── settings.js │ ├── tasks.py │ ├── templates │ ├── _base.html │ ├── _base_email.html │ ├── about.html │ ├── email.html │ ├── email_new.html │ ├── email_search.html │ ├── emails │ │ ├── account_finalized.html │ │ ├── forward.html │ │ └── reply.html │ ├── home.html │ ├── macros │ │ ├── email.html │ │ ├── flag.html │ │ ├── form.html │ │ ├── messages.html │ │ └── nav.html │ ├── news.html │ ├── register.html │ ├── security │ │ ├── change_password.html │ │ ├── login_user.html │ │ └── register_user.html │ ├── settings.html │ └── users.html │ ├── translations │ ├── fr │ │ └── LC_MESSAGES │ │ │ └── messages.po │ └── ln │ │ └── LC_MESSAGES │ │ └── messages.po │ └── views.py ├── opwen_email_server ├── __init__.py ├── actions.py ├── config.py ├── constants │ ├── __init__.py │ ├── events.py │ ├── github.py │ ├── mailbox.py │ ├── sendgrid.py │ └── sync.py ├── integration │ ├── __init__.py │ ├── azure.py │ ├── celery.py │ ├── cli.py │ ├── connexion.py │ ├── webapp.py │ └── wsgi.py ├── mailers │ ├── __init__.py │ ├── echo.py │ └── wikipedia.py ├── services │ ├── __init__.py │ ├── auth.py │ ├── dns.py │ ├── sendgrid.py │ └── storage.py ├── swagger │ ├── client-metrics.yaml │ ├── client-read.yaml │ ├── client-register.yaml │ ├── client-write.yaml │ ├── email-receive.yaml │ └── healthcheck.yaml └── utils │ ├── __init__.py │ ├── collections.py │ ├── email_parser.py │ ├── log.py │ ├── path.py │ ├── serialization.py │ ├── string.py │ ├── temporary.py │ └── unique.py ├── opwen_statuspage ├── config-overrides.js ├── package-lock.json ├── package.json ├── public │ └── index.html ├── src │ ├── App.js │ ├── Button.js │ ├── ClientStats.js │ ├── ErrorNotification.js │ ├── Grid.js │ ├── Header.js │ ├── PingStats.js │ ├── Settings.js │ ├── index.js │ └── react-app-env.d.ts └── tsconfig.json ├── package.json ├── requirements-dev.txt ├── requirements-webapp.txt ├── requirements.txt ├── secrets └── .gitignore ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── files │ ├── opwen_email_client │ │ └── compressedpackages │ │ │ ├── .gitignore │ │ │ └── sync.tar.gz │ └── opwen_email_server │ │ └── utils │ │ └── test_email_parser │ │ ├── email-attachment.mime │ │ ├── email-ccbcc.mime │ │ ├── email-cid.mime │ │ ├── email-html.mime │ │ ├── large.png │ │ ├── small.png │ │ └── test_image.png ├── opwen_email_client │ ├── __init__.py │ ├── domain │ │ ├── __init__.py │ │ └── email │ │ │ ├── __init__.py │ │ │ ├── test_sql_store.py │ │ │ ├── test_store.py │ │ │ └── test_sync.py │ ├── util │ │ ├── __init__.py │ │ ├── test_os.py │ │ ├── test_serialization.py │ │ └── test_wtforms.py │ └── webapp │ │ ├── __init__.py │ │ ├── base.py │ │ ├── test_mkwvconf.py │ │ └── test_views.py └── opwen_email_server │ ├── __init__.py │ ├── helpers.py │ ├── integration │ ├── __init__.py │ └── test_celery.py │ ├── mailers │ ├── __init__.py │ └── test_wikipedia.py │ ├── services │ ├── __init__.py │ ├── test_auth.py │ ├── test_dns.py │ ├── test_sendgrid.py │ └── test_storage.py │ ├── test_actions.py │ ├── test_config.py │ ├── test_swagger.py │ └── utils │ ├── __init__.py │ ├── test_collections.py │ ├── test_email_parser.py │ ├── test_path.py │ ├── test_serialization.py │ ├── test_string.py │ ├── test_temporary.py │ └── test_unique.py └── yarn.lock /.dive-ci: -------------------------------------------------------------------------------- 1 | rules: 2 | lowestEfficiency: 0.95 3 | highestWastedBytes: "disabled" 4 | highestUserWastedPercent: "disabled" 5 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !.env 3 | !.yamllint 4 | !*.yml 5 | !.prettierignore 6 | !docker/ 7 | !helm/ 8 | !opwen_email_client/ 9 | opwen_email_client/webapp/static/css/*.min.css 10 | opwen_email_client/webapp/static/flags/ 11 | opwen_email_client/webapp/static/fonts/ 12 | opwen_email_client/webapp/static/js/*.min.js 13 | !opwen_email_server/ 14 | !opwen_statuspage/ 15 | opwen_statuspage/build/ 16 | opwen_statuspage/node_modules/ 17 | !tests/ 18 | !requirements*.txt 19 | !setup.cfg 20 | !install.py 21 | !MANIFEST.in 22 | !setup.py 23 | !README.rst 24 | !package.json 25 | !yarn.lock 26 | !Gruntfile.js 27 | !babel.cfg 28 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | STATUSPAGE_PORT=3000 2 | APP_PORT=8080 3 | CLIENT_PORT=5000 4 | BUILD_TAG=development 5 | BUILD_TARGET=builder 6 | DOCKER_REPO=ascoderu 7 | WEBAPP_WORKERS=1 8 | SERVER_WORKERS=1 9 | QUEUE_WORKERS=1 10 | NGINX_WORKERS=1 11 | LOKOLE_LOG_LEVEL=INFO 12 | LOKOLE_QUEUE_BROKER_SCHEME=amqp 13 | LOKOLE_EMAIL_SERVER_QUEUES_SAS_NAME=ascoderu 14 | LOKOLE_EMAIL_SERVER_QUEUES_SAS_KEY=123456 15 | LOKOLE_EMAIL_SERVER_QUEUES_NAMESPACE=rabbitmq 16 | LOKOLE_SENDGRID_KEY= 17 | LOKOLE_RESOURCE_SUFFIX= 18 | REGISTRATION_CREDENTIALS=admin:password 19 | TEST_STEP_DELAY=10 20 | LIVE= 21 | 22 | CLOUDBROWSER_PORT=10001 23 | AZURITE_PORT=10000 24 | AZURITE_ACCOUNT=lokolestorage 25 | AZURITE_KEY=c2VjcmV0a2V5 26 | AZURITE_HOST=azurite:10000 27 | AZURITE_SECURE=False 28 | 29 | APPINSIGHTS_INSTRUMENTATIONKEY=a314c6f7-776c-4e82-a677-8a3ecfb4669f 30 | 31 | FLOWER_PORT=5555 32 | RABBITMQ_PORT=5556 33 | -------------------------------------------------------------------------------- /.github.env.gpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ascoderu/lokole/1b2a7b18f472952df06f9938ebd01a0ecd749a79/.github.env.gpg -------------------------------------------------------------------------------- /.github/workflows/cd.yml: -------------------------------------------------------------------------------- 1 | name: CD 2 | 3 | on: 4 | release: 5 | types: 6 | - created 7 | 8 | jobs: 9 | release: 10 | runs-on: ubuntu-18.04 11 | 12 | steps: 13 | - uses: actions/checkout@v2 14 | - run: make github-env 15 | env: 16 | GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} 17 | - run: | 18 | make \ 19 | -e BUILD_TARGET=runtime \ 20 | -e DOCKER_TAG="${GITHUB_REF##*/}" \ 21 | release-docker deploy-docker release-pypi deploy-pypi 22 | -------------------------------------------------------------------------------- /.github/workflows/ci-live.yml: -------------------------------------------------------------------------------- 1 | name: CI Live 2 | 3 | on: 4 | pull_request_target: 5 | types: [labeled] 6 | 7 | jobs: 8 | test-live: 9 | runs-on: ubuntu-18.04 10 | if: contains(github.event.pull_request.labels.*.name, 'safe to test') 11 | 12 | steps: 13 | - uses: actions/checkout@v3 14 | with: 15 | ref: ${{ github.event.pull_request.head.sha }} 16 | - run: make github-env 17 | env: 18 | GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} 19 | - run: | 20 | make \ 21 | -e BUILD_TARGET=runtime \ 22 | -e REGISTRATION_CREDENTIALS="$GITHUB_AUTH_TOKEN" \ 23 | -e LOKOLE_QUEUE_BROKER_SCHEME=azureservicebus \ 24 | -e LOKOLE_RESOURCE_SUFFIX="$SUFFIX" \ 25 | -e APPINSIGHTS_INSTRUMENTATIONKEY="$SUFFIX" \ 26 | -e AZURITE_ACCOUNT="$TEST_AZURE_STORAGE_ACCOUNT" \ 27 | -e AZURITE_KEY="$TEST_AZURE_STORAGE_KEY" \ 28 | -e AZURITE_HOST="" \ 29 | -e AZURITE_SECURE="True" \ 30 | -e TEST_STEP_DELAY=90 \ 31 | -e LIVE="True" \ 32 | build start integration-tests 33 | - run: make status 34 | if: ${{ failure() }} 35 | - run: make clean-storage stop 36 | if: ${{ always() }} 37 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - opened 7 | - reopened 8 | - synchronize 9 | 10 | jobs: 11 | test-local: 12 | runs-on: ubuntu-18.04 13 | 14 | steps: 15 | - uses: actions/checkout@v3 16 | - run: | 17 | make \ 18 | -e BUILD_TARGET=runtime \ 19 | -e REGISTRATION_CREDENTIALS=admin:password \ 20 | -e LOKOLE_QUEUE_BROKER_SCHEME=amqp \ 21 | build start integration-tests 22 | - run: make status 23 | if: ${{ failure() }} 24 | - run: make stop 25 | if: ${{ always() }} 26 | 27 | test-unit: 28 | runs-on: ubuntu-18.04 29 | 30 | steps: 31 | - uses: actions/checkout@v3 32 | - run: | 33 | make \ 34 | -e BUILD_TARGET=runtime \ 35 | -e LOKOLE_SENDGRID_KEY= \ 36 | -e LOKOLE_QUEUE_BROKER_SCHEME= \ 37 | ci build verify-build 38 | - run: bash <(curl -s https://codecov.io/bash) 39 | if: ${{ success() }} 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .vscode/ 3 | 4 | *.pyc 5 | __pycache__/ 6 | /build/ 7 | /dist/ 8 | /*.egg-info/ 9 | venv*/ 10 | .coverage 11 | coverage.xml 12 | cover/ 13 | htmlcov/ 14 | .mypy_cache/ 15 | dive.log 16 | 17 | serviceprincipal.json 18 | .github.env 19 | gpg-passphrase.txt 20 | 21 | node_modules/ 22 | opwen_email_client/webapp/static/css/*.min.css 23 | opwen_email_client/webapp/static/js/*.min.js 24 | opwen_email_client/webapp/static/fonts 25 | opwen_email_client/webapp/static/flags 26 | 27 | venv/ 28 | requirements.txt.out 29 | *.mo 30 | *.pot 31 | /state/ 32 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | *.min.js 2 | 3 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | extends: "default" 2 | 3 | rules: 4 | line-length: "disable" 5 | document-start: "disable" 6 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Docs: 2 | # https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners 3 | # https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/about-protected-branches#require-pull-request-reviews-before-merging 4 | 5 | /makefile @ascoderu/lokole-dev 6 | /.github.env.gpg @ascoderu/lokole-dev 7 | /docker-compose.yml @ascoderu/lokole-dev 8 | /docker/ @ascoderu/lokole-dev 9 | /.github @ascoderu/lokole-dev -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements*.txt 2 | include README.rst 3 | recursive-include opwen_email_client/webapp/static * 4 | recursive-include opwen_email_client/webapp/templates *.html 5 | recursive-include opwen_email_client/webapp/translations *.po 6 | recursive-include opwen_email_client/webapp/translations *.mo 7 | -------------------------------------------------------------------------------- /babel.cfg: -------------------------------------------------------------------------------- 1 | [python: **.py] 2 | [jinja2: **/templates/**.html] 3 | -------------------------------------------------------------------------------- /docker/app/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG PYTHON_VERSION=3.9 2 | FROM python:${PYTHON_VERSION} AS builder 3 | 4 | RUN apt-get update \ 5 | && apt-get install -y --no-install-recommends mobile-broadband-provider-info=20201225-1 \ 6 | && rm -rf /var/lib/apt/lists/* 7 | 8 | WORKDIR /app 9 | 10 | COPY requirements-dev.txt ./ 11 | RUN pip install --no-cache-dir -r requirements-dev.txt \ 12 | && rm -rf /tmp/pip-ephem-wheel-cache* 13 | 14 | COPY requirements-webapp.txt ./ 15 | RUN pip install --no-cache-dir -r requirements-webapp.txt \ 16 | && rm -rf /tmp/pip-ephem-wheel-cache* 17 | 18 | COPY requirements.txt ./ 19 | RUN pip install --no-cache-dir -r requirements.txt \ 20 | && pip wheel --no-cache-dir -r requirements.txt -w /deps \ 21 | && rm -rf /tmp/pip-ephem-wheel-cache* 22 | 23 | COPY . . 24 | 25 | FROM python:${PYTHON_VERSION}-slim AS runtime 26 | 27 | RUN groupadd -r opwen \ 28 | && useradd -r -s /bin/false -g opwen opwen 29 | 30 | COPY --from=builder /deps /deps 31 | # hadolint ignore=DL3013 32 | RUN pip --no-cache-dir -q install /deps/*.whl 33 | 34 | USER opwen 35 | WORKDIR /app 36 | 37 | COPY --from=builder /app/docker/app/run-celery.sh ./docker/app/run-celery.sh 38 | COPY --from=builder /app/docker/app/run-gunicorn.sh ./docker/app/run-gunicorn.sh 39 | COPY --from=builder /app/opwen_email_server ./opwen_email_server 40 | -------------------------------------------------------------------------------- /docker/app/run-celery.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [[ "${CELERY_QUEUE_NAMES}" = "all" ]]; then 4 | CELERY_QUEUE_NAMES="$(python -m opwen_email_server.integration.cli print-queues --separator=,)" 5 | fi 6 | 7 | exec celery \ 8 | --app="opwen_email_server.integration.celery" \ 9 | worker \ 10 | --without-gossip \ 11 | --without-heartbeat \ 12 | --without-mingle \ 13 | --concurrency="${QUEUE_WORKERS}" \ 14 | --loglevel="${LOKOLE_LOG_LEVEL}" \ 15 | --queues="${CELERY_QUEUE_NAMES}" 16 | -------------------------------------------------------------------------------- /docker/app/run-ci.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | scriptdir="$(dirname "$0")" 6 | cd "${scriptdir}/../.." 7 | 8 | flake8 opwen_email_server opwen_email_client 9 | isort --check-only opwen_email_server opwen_email_client 10 | yapf --recursive --parallel --diff opwen_email_server opwen_email_client tests 11 | bandit --recursive opwen_email_server opwen_email_client 12 | mypy opwen_email_server opwen_email_client 13 | 14 | coverage run -m nose2 -v 15 | coverage xml 16 | coverage report 17 | 18 | if [[ -n "$1" ]]; then 19 | echo "$1" 20 | cat coverage.xml 21 | fi 22 | -------------------------------------------------------------------------------- /docker/app/run-gunicorn.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | declare -a api_spec_paths 4 | 5 | case "${CONNEXION_SPEC}" in 6 | file:*) 7 | specs="${CONNEXION_SPEC:5}" 8 | IFS="," read -r -a api_spec_paths <<<"${specs}" 9 | ;; 10 | dir:*) 11 | specs="${CONNEXION_SPEC:4}" 12 | mapfile -t api_spec_paths < <(find "${specs}" -type f -name '*.yaml') 13 | ;; 14 | esac 15 | 16 | apis="" 17 | for api_spec_path in "${api_spec_paths[@]}"; do 18 | if [[ ! -f "${api_spec_path}" ]]; then 19 | echo "Unable to start server: connexion spec file ${api_spec_path} does not exist" >&2 20 | exit 1 21 | fi 22 | apis="${apis},'${api_spec_path}'" 23 | done 24 | apis="[${apis:1:${#apis}-1}]" 25 | 26 | if [[ "${LOKOLE_STORAGE_PROVIDER}" = "LOCAL" ]]; then 27 | mkdir -p "${LOKOLE_EMAIL_SERVER_AZURE_BLOBS_NAME}" 28 | mkdir -p "${LOKOLE_EMAIL_SERVER_AZURE_TABLES_NAME}" 29 | mkdir -p "${LOKOLE_CLIENT_AZURE_STORAGE_NAME}" 30 | fi 31 | 32 | exec gunicorn \ 33 | --workers="${SERVER_WORKERS}" \ 34 | --log-level="${LOKOLE_LOG_LEVEL}" \ 35 | --bind="0.0.0.0:${PORT}" \ 36 | "opwen_email_server.integration.wsgi:build_app(apis=${apis}, ui=${TESTING_UI})" 37 | -------------------------------------------------------------------------------- /docker/ci/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG PYTHON_VERSION=3.9 2 | FROM python:${PYTHON_VERSION} AS builder 3 | 4 | ARG HADOLINT_VERSION=v1.17.1 5 | RUN wget -q -O /usr/bin/hadolint "https://github.com/hadolint/hadolint/releases/download/${HADOLINT_VERSION}/hadolint-Linux-$(uname -m)" \ 6 | && chmod +x /usr/bin/hadolint \ 7 | && hadolint --version 8 | 9 | ARG SHELLCHECK_VERSION=v0.7.1 10 | RUN wget -q -O /tmp/shellcheck.tar.xz "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.$(uname -m).tar.xz" \ 11 | && tar -xJf /tmp/shellcheck.tar.xz -C /usr/bin --strip-components=1 "shellcheck-${SHELLCHECK_VERSION}/shellcheck" \ 12 | && rm /tmp/shellcheck.tar.xz \ 13 | && shellcheck --version 14 | 15 | ARG HELM_VERSION=3.2.1 16 | RUN wget -q "https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz" \ 17 | && tar xf "helm-v${HELM_VERSION}-linux-amd64.tar.gz" \ 18 | && mv "linux-amd64/helm" /usr/local/bin/helm \ 19 | && chmod +x /usr/local/bin/helm \ 20 | && rm -rf "linux-amd64" "helm-v${HELM_VERSION}-linux-amd64.tar.gz" 21 | 22 | ARG KUBEVAL_VERSION=0.14.0 23 | RUN wget -q https://github.com/instrumenta/kubeval/releases/download/${KUBEVAL_VERSION}/kubeval-linux-amd64.tar.gz \ 24 | && tar xf kubeval-linux-amd64.tar.gz \ 25 | && cp kubeval /usr/local/bin \ 26 | && rm kubeval-linux-amd64.tar.gz \ 27 | && kubeval --version 28 | 29 | ARG SHFMT_VERSION=3.1.1 30 | RUN wget -q https://github.com/mvdan/sh/releases/download/v${SHFMT_VERSION}/shfmt_v${SHFMT_VERSION}_linux_amd64 \ 31 | && mv shfmt_v${SHFMT_VERSION}_linux_amd64 shfmt \ 32 | && cp shfmt /usr/local/bin \ 33 | && rm shfmt \ 34 | && chmod +x /usr/local/bin/shfmt \ 35 | && shfmt -version 36 | 37 | COPY docker/ci/requirements.txt ./ 38 | RUN pip install --no-cache-dir -r requirements.txt 39 | 40 | WORKDIR /app 41 | 42 | COPY . . 43 | 44 | SHELL ["/bin/bash", "-o", "pipefail", "-c"] 45 | 46 | RUN find . -type f -regex '.*\.ya?ml' ! -path '*/helm/*' | while read -r file; do \ 47 | if ! yamllint "${file}"; then \ 48 | echo "Failed yamllint: ${file}" >&2; \ 49 | exit 1; \ 50 | fi; \ 51 | done 52 | 53 | RUN find . -type f -name Dockerfile | while read -r file; do \ 54 | if ! hadolint "${file}"; then \ 55 | echo "Failed hadolint: ${file}" >&2; \ 56 | exit 1; \ 57 | fi; \ 58 | done 59 | 60 | RUN find . -type f -name '*.sh' | while read -r file; do \ 61 | if ! shellcheck "${file}"; then \ 62 | echo "Failed shellcheck: ${file}" >&2; \ 63 | exit 1; \ 64 | fi; \ 65 | done 66 | 67 | RUN helm lint --strict ./helm/opwen_cloudserver \ 68 | && helm template ./helm/opwen_cloudserver > helm.yaml \ 69 | && kubeval --ignore-missing-schemas helm.yaml \ 70 | && rm helm.yaml 71 | 72 | RUN shfmt -d -i 2 -ci . 73 | -------------------------------------------------------------------------------- /docker/ci/requirements.txt: -------------------------------------------------------------------------------- 1 | yamllint==1.25.0 2 | -------------------------------------------------------------------------------- /docker/client/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG NODE_VERSION=12 2 | ARG PYTHON_VERSION=3.9 3 | FROM node:${NODE_VERSION} AS yarn 4 | 5 | WORKDIR /app 6 | 7 | COPY package.json yarn.lock ./ 8 | RUN yarn install 9 | 10 | COPY Gruntfile.js .prettierignore ./ 11 | COPY opwen_email_client/webapp/static opwen_email_client/webapp/static 12 | RUN yarn run lint \ 13 | && yarn run build 14 | 15 | FROM python:${PYTHON_VERSION} AS builder 16 | 17 | WORKDIR /app 18 | 19 | COPY requirements-dev.txt ./ 20 | RUN pip install --no-cache-dir -r requirements-dev.txt 21 | 22 | COPY requirements-webapp.txt ./ 23 | RUN pip install --no-cache-dir -r requirements-webapp.txt 24 | 25 | COPY requirements.txt ./ 26 | RUN pip install --no-cache-dir -r requirements.txt 27 | 28 | ENV OPWEN_SESSION_KEY=changeme 29 | ENV OPWEN_SETTINGS=/app/docker/client/webapp.env 30 | 31 | COPY --from=yarn /app/opwen_email_client/webapp/static/ /app/opwen_email_client/webapp/static/ 32 | COPY . . 33 | 34 | FROM builder AS compiler 35 | 36 | ARG VERSION=0.0.0 37 | 38 | RUN pybabel extract -F babel.cfg -k lazy_gettext -o babel.pot opwen_email_client/webapp \ 39 | && pybabel compile -d opwen_email_client/webapp/translations 40 | 41 | RUN sed -i "s|^__version__ = '[^']*'|__version__ = '${VERSION}'|g" opwen_email_client/__init__.py \ 42 | && sed -i "s|^__version__ = '[^']*'|__version__ = '${VERSION}'|g" opwen_email_server/__init__.py \ 43 | && python setup.py sdist \ 44 | && cp "dist/opwen_email_client-${VERSION}.tar.gz" dist/pkg.tar.gz 45 | 46 | FROM python:${PYTHON_VERSION}-slim AS runtime 47 | 48 | # hadolint ignore=DL3010 49 | COPY --from=compiler /app/dist/pkg.tar.gz /app/dist/pkg.tar.gz 50 | 51 | # hadolint ignore=DL3013 52 | RUN pip install --no-cache-dir "/app/dist/pkg.tar.gz[opwen_email_server]" \ 53 | && rm -rf /tmp/pip-ephem-wheel-cache* 54 | 55 | COPY --from=compiler /app/docker/client/run-*.sh /app/docker/client/ 56 | COPY --from=compiler /app/docker/client/*.env /app/docker/client/ 57 | 58 | ENV OPWEN_SESSION_KEY=changeme 59 | ENV OPWEN_SETTINGS=/app/docker/client/webapp.env 60 | -------------------------------------------------------------------------------- /docker/client/client.env: -------------------------------------------------------------------------------- 1 | OPWEN_SIM_TYPE=Ethernet 2 | -------------------------------------------------------------------------------- /docker/client/run-celery.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | exec celery \ 4 | --app="opwen_email_client.webapp.tasks" \ 5 | worker \ 6 | --loglevel="${LOKOLE_LOG_LEVEL}" \ 7 | --concurrency="${QUEUE_WORKERS}" 8 | -------------------------------------------------------------------------------- /docker/client/run-crontab.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | exec celery \ 4 | --app=opwen_email_client.webapp.tasks \ 5 | beat \ 6 | --loglevel="${LOKOLE_LOG_LEVEL}" 7 | -------------------------------------------------------------------------------- /docker/client/run-gunicorn.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | exec gunicorn \ 4 | --workers="${WEBAPP_WORKERS}" \ 5 | --log-level="${LOKOLE_LOG_LEVEL}" \ 6 | --bind="0.0.0.0:${WEBAPP_PORT}" \ 7 | opwen_email_client.webapp:app 8 | -------------------------------------------------------------------------------- /docker/client/run-lokole.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | scriptdir="$(dirname "$0")" 6 | 7 | export FLASK_APP="opwen_email_client.webapp:app" 8 | 9 | if [[ -n "${LOKOLE_ADMIN_NAME}" ]] && [[ -n "${LOKOLE_ADMIN_PASSWORD}" ]]; then 10 | ( 11 | flask manage createadmin --name="${LOKOLE_ADMIN_NAME}" --password="${LOKOLE_ADMIN_PASSWORD}" 12 | ) 13 | fi 14 | 15 | "${scriptdir}/run-celery.sh" & 16 | celery_pid="$!" 17 | 18 | "${scriptdir}/run-crontab.sh" & 19 | crontab_pid="$!" 20 | 21 | "${scriptdir}/run-gunicorn.sh" & 22 | gunicorn_pid="$!" 23 | 24 | while :; do 25 | if [[ ! -e "/proc/${celery_pid}" ]]; then 26 | echo "celery crashed" >&2 27 | exit 1 28 | elif [[ ! -e "/proc/${crontab_pid}" ]]; then 29 | echo "crontab crashed" >&2 30 | exit 2 31 | elif [[ ! -e "/proc/${gunicorn_pid}" ]]; then 32 | echo "gunicorn crashed" >&2 33 | exit 3 34 | else 35 | sleep 10 36 | fi 37 | done 38 | -------------------------------------------------------------------------------- /docker/client/webapp.env: -------------------------------------------------------------------------------- 1 | OPWEN_APP_ROOT=/web 2 | OPWEN_STATE_DIRECTORY=/tmp 3 | OPWEN_MAX_UPLOAD_SIZE_MB=1 4 | OPWEN_SIM_TYPE=LocalOnly 5 | OPWEN_CLIENT_NAME=web 6 | OPWEN_ROOT_DOMAIN=lokole.ca 7 | LOKOLE_IOC=opwen_email_server.integration.webapp.AzureIoc 8 | OPWEN_EMAILS_PER_PAGE=5 9 | OPWEN_CAN_REGISTER_USER=False 10 | OPWEN_CAN_CHANGE_PASSWORD=False 11 | OPWEN_CAN_SEARCH_EMAIL=False 12 | -------------------------------------------------------------------------------- /docker/docker-compose.prod.yml: -------------------------------------------------------------------------------- 1 | version: '3.4' 2 | 3 | x-shared-secret-environment: 4 | &shared-secret-environment 5 | environment: 6 | PORT: 8888 7 | HOST: 0.0.0.0 8 | WEBAPP_PORT: 8080 9 | LOKOLE_STORAGE_PROVIDER: AZURE_BLOBS 10 | LOKOLE_QUEUE_BROKER_SCHEME: azureservicebus 11 | CONNEXION_SPEC: dir:/app/opwen_email_server/swagger 12 | CELERY_QUEUE_NAMES: all 13 | TESTING_UI: "False" 14 | LOKOLE_LOG_LEVEL: INFO 15 | WEBAPP_WORKERS: 3 16 | SERVER_WORKERS: 4 17 | QUEUE_WORKERS: 5 18 | env_file: 19 | - ../secrets/azure.env 20 | - ../secrets/cloudflare.env 21 | - ../secrets/users.env 22 | - ../secrets/sendgrid.env 23 | volumes: 24 | - /tmp:/tmp 25 | restart: always 26 | 27 | services: 28 | 29 | webapp: 30 | image: ascoderu/opwenwebapp:latest 31 | command: ["/app/docker/client/run-gunicorn.sh"] 32 | <<: *shared-secret-environment 33 | ports: 34 | - 8080:8080 35 | 36 | api: 37 | image: ascoderu/opwenserver_app:latest 38 | command: ["/app/docker/app/run-gunicorn.sh"] 39 | <<: *shared-secret-environment 40 | ports: 41 | - 8888:8888 42 | 43 | worker: 44 | image: ascoderu/opwenserver_app:latest 45 | command: ["/app/docker/app/run-celery.sh"] 46 | <<: *shared-secret-environment 47 | -------------------------------------------------------------------------------- /docker/docker-compose.setup.yml: -------------------------------------------------------------------------------- 1 | version: '3.4' 2 | 3 | services: 4 | 5 | setup: 6 | image: ${DOCKER_REPO}/opwenserver_setup:${BUILD_TAG} 7 | build: 8 | context: . 9 | dockerfile: docker/setup/Dockerfile 10 | -------------------------------------------------------------------------------- /docker/docker-compose.test.yml: -------------------------------------------------------------------------------- 1 | version: '3.4' 2 | 3 | services: 4 | 5 | integtest: 6 | image: ${DOCKER_REPO}/opwenserver_integtest:${BUILD_TAG} 7 | build: 8 | context: . 9 | dockerfile: docker/integtest/Dockerfile 10 | environment: 11 | REGISTRATION_CREDENTIALS: ${REGISTRATION_CREDENTIALS} 12 | APPINSIGHTS_INSTRUMENTATIONKEY: ${APPINSIGHTS_INSTRUMENTATIONKEY} 13 | AZURITE_ACCOUNT: ${AZURITE_ACCOUNT} 14 | AZURITE_KEY: ${AZURITE_KEY} 15 | AZURITE_HOST: ${AZURITE_HOST} 16 | TEST_STEP_DELAY: ${TEST_STEP_DELAY} 17 | LIVE: ${LIVE} 18 | volumes: 19 | - /var/run/docker.sock:/var/run/docker.sock 20 | 21 | ci: 22 | image: ${DOCKER_REPO}/opwenserver_ci:${BUILD_TAG} 23 | build: 24 | context: . 25 | dockerfile: docker/ci/Dockerfile 26 | -------------------------------------------------------------------------------- /docker/docker-compose.tools.yml: -------------------------------------------------------------------------------- 1 | version: '3.4' 2 | 3 | services: 4 | 5 | flower: 6 | image: mher/flower:latest 7 | depends_on: 8 | - rabbitmq 9 | command: ["--address=0.0.0.0", "--port=5555", "--broker=amqp://${LOKOLE_EMAIL_SERVER_QUEUES_SAS_NAME}:${LOKOLE_EMAIL_SERVER_QUEUES_SAS_KEY}@rabbitmq"] 10 | ports: 11 | - ${FLOWER_PORT}:5555 12 | 13 | cloudbrowser: 14 | image: cwolff/django-cloud-browser:latest 15 | depends_on: 16 | - azurite 17 | environment: 18 | CLOUD_BROWSER_DATASTORE: "ApacheLibcloud" 19 | CLOUD_BROWSER_APACHE_LIBCLOUD_PROVIDER: "AZURE_BLOBS" 20 | CLOUD_BROWSER_APACHE_LIBCLOUD_ACCOUNT: "${AZURITE_ACCOUNT}" 21 | CLOUD_BROWSER_APACHE_LIBCLOUD_SECRET_KEY: "${AZURITE_KEY}" 22 | CLOUD_BROWSER_APACHE_LIBCLOUD_HOST: "azurite" 23 | CLOUD_BROWSER_APACHE_LIBCLOUD_PORT: "10000" 24 | CLOUD_BROWSER_APACHE_LIBCLOUD_SECURE: "${AZURITE_SECURE}" 25 | ports: 26 | - ${CLOUDBROWSER_PORT}:8000 27 | -------------------------------------------------------------------------------- /docker/integtest/0-wait-for-services.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | 4 | scriptdir="$(dirname "$0")" 5 | # shellcheck disable=SC1090 6 | . "${scriptdir}/utils.sh" 7 | 8 | readonly polling_interval_seconds=2 9 | readonly max_retries=30 10 | 11 | wait_for_rabbitmq() { 12 | local rabbitmq 13 | local i 14 | 15 | if [[ "${LIVE}" = "True" ]]; then 16 | log "Skipping waiting for rabbitmq for live test" 17 | return 18 | fi 19 | 20 | rabbitmq="$(get_container rabbitmq)" 21 | 22 | for i in $(seq 1 "${max_retries}"); do 23 | if docker exec "${rabbitmq}" rabbitmqctl wait -q -P 1 -t "${polling_interval_seconds}"; then 24 | log "Rabbitmq is running" 25 | return 26 | fi 27 | log "Waiting for rabbitmq (${i}/${max_retries})" 28 | done 29 | 30 | exit 1 31 | } 32 | 33 | wait_for_appinsights() { 34 | local i 35 | 36 | if [[ "${LIVE}" = "True" ]]; then 37 | log "Skipping waiting for appinsights for live test" 38 | return 39 | fi 40 | 41 | for i in $(seq 1 "${max_retries}"); do 42 | if [[ \ 43 | "$(az storage container exists \ 44 | --name "${APPINSIGHTS_INSTRUMENTATIONKEY}" \ 45 | --connection-string "$(az_connection_string)" \ 46 | --output tsv)" = "True" ]] \ 47 | ; then 48 | log "Appinsights is running" 49 | return 50 | fi 51 | log "Waiting for appinsights (${i}/${max_retries})" 52 | sleep "${polling_interval_seconds}s" 53 | done 54 | 55 | exit 3 56 | } 57 | 58 | wait_for_api() { 59 | local i 60 | 61 | for i in $(seq 1 "${max_retries}"); do 62 | if curl -fs "http://nginx:8888/healthcheck/ping" >/dev/null; then 63 | log "Api is running" 64 | return 65 | fi 66 | log "Waiting for api (${i}/${max_retries})" 67 | sleep "${polling_interval_seconds}s" 68 | done 69 | 70 | exit 4 71 | } 72 | 73 | wait_for_webapp() { 74 | local i 75 | 76 | for i in $(seq 1 "${max_retries}"); do 77 | if curl -fs "http://nginx:8888/web/healthcheck/ping" >/dev/null; then 78 | log "Webapp is running" 79 | return 80 | fi 81 | log "Waiting for webapp (${i}/${max_retries})" 82 | sleep "${polling_interval_seconds}s" 83 | done 84 | 85 | exit 4 86 | } 87 | 88 | wait_for_client() { 89 | local i 90 | 91 | for i in $(seq 1 "${max_retries}"); do 92 | if curl -fs "http://client:5000/healthcheck/ping" >/dev/null; then 93 | log "Client is running" 94 | return 95 | fi 96 | log "Waiting for client (${i}/${max_retries})" 97 | sleep "${polling_interval_seconds}s" 98 | done 99 | 100 | exit 5 101 | } 102 | 103 | wait_for_rabbitmq 104 | wait_for_appinsights 105 | wait_for_api 106 | wait_for_webapp 107 | wait_for_client 108 | -------------------------------------------------------------------------------- /docker/integtest/1-register-client.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | 4 | scriptdir="$(dirname "$0")" 5 | out_dir="${scriptdir}/files/test.out" 6 | mkdir -p "${out_dir}" 7 | # shellcheck disable=SC1090 8 | . "${scriptdir}/utils.sh" 9 | 10 | readonly polling_interval_seconds=2 11 | readonly max_retries=150 12 | 13 | wait_for_client() { 14 | local client="$1" 15 | local i 16 | 17 | for i in $(seq 1 "${max_retries}"); do 18 | if authenticated_request "http://nginx:8888/api/email/register/developer${client}.lokole.ca" >"${out_dir}/register${client}.json"; then 19 | log "Client ${client} is registered" 20 | return 21 | fi 22 | log "Waiting for client ${client} registration (${i}/${max_retries})" 23 | sleep "${polling_interval_seconds}" 24 | done 25 | 26 | exit 1 27 | } 28 | 29 | # workflow 3: register a new client called "developer" 30 | # normally this endpoint would be called during a new lokole device setup 31 | authenticated_request \ 32 | "http://nginx:8888/api/email/register/" \ 33 | -H "Content-Type: application/json" \ 34 | -d '{"domain":"developer1.lokole.ca"}' 35 | 36 | wait_for_client 1 37 | 38 | # registering a client with bad credentials should fail 39 | if REGISTRATION_CREDENTIALS="baduser:badpassword" authenticated_request \ 40 | "http://nginx:8888/api/email/register/" \ 41 | -H "Content-Type: application/json" \ 42 | -d '{"domain":"hacker.lokole.ca"}' \ 43 | ; then 44 | echo "Was able to register a client with bad basic auth credentials" >&2 45 | exit 4 46 | fi 47 | 48 | if REGISTRATION_CREDENTIALS="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" authenticated_request \ 49 | "http://nginx:8888/api/email/register/" \ 50 | -H "Content-Type: application/json" \ 51 | -d '{"domain":"hacker.lokole.ca"}' \ 52 | ; then 53 | echo "Was able to register a client with bad bearer credentials" >&2 54 | exit 4 55 | fi 56 | 57 | # also register another client to simulate multi-client emails 58 | authenticated_request \ 59 | "http://nginx:8888/api/email/register/" \ 60 | -H "Content-Type: application/json" \ 61 | -d '{"domain":"developer2.lokole.ca"}' 62 | 63 | wait_for_client 2 64 | 65 | # after creating a client, creating the same one again should fail but we should be able to delete it 66 | authenticated_request \ 67 | "http://nginx:8888/api/email/register/" \ 68 | -H "Content-Type: application/json" \ 69 | -d '{"domain":"developer3.lokole.ca"}' 70 | 71 | wait_for_client 3 72 | 73 | if authenticated_request \ 74 | "http://nginx:8888/api/email/register/" \ 75 | -H "Content-Type: application/json" \ 76 | -d '{"domain":"developer3.lokole.ca"}' \ 77 | ; then 78 | echo "Was able to register a duplicate client" >&2 79 | exit 5 80 | fi 81 | 82 | authenticated_request \ 83 | "http://nginx:8888/api/email/register/developer3.lokole.ca" \ 84 | -X DELETE 85 | -------------------------------------------------------------------------------- /docker/integtest/2-client-uploads-emails.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | 4 | scriptdir="$(dirname "$0")" 5 | in_dir="${scriptdir}/files" 6 | out_dir="${scriptdir}/files/test.out" 7 | mkdir -p "${out_dir}" 8 | # shellcheck disable=SC1090 9 | . "${scriptdir}/utils.sh" 10 | 11 | emails_to_send="${in_dir}/client-emails.tar.gz" 12 | tar -czf "${emails_to_send}" -C "${in_dir}" emails.jsonl zzusers.jsonl 13 | 14 | client_id="$(jq -r '.client_id' <"${out_dir}/register1.json")" 15 | resource_container="$(jq -r '.resource_container' <"${out_dir}/register1.json")" 16 | resource_id="$(uuidgen).tar.gz" 17 | 18 | # workflow 1: send emails written on the client to the world 19 | # first we simulate the client uploading its emails to the shared blob storage 20 | az_storage upload "${resource_container}" "${resource_id}" "${emails_to_send}" 21 | 22 | # the client then calls the server to trigger the delivery of the emails 23 | curl -fs \ 24 | -H "Content-Type: application/json" \ 25 | -d '{"resource_id":"'"${resource_id}"'"}' \ 26 | "http://nginx:8888/api/email/upload/${client_id}" 27 | -------------------------------------------------------------------------------- /docker/integtest/3-receive-email-for-client.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | 4 | scriptdir="$(dirname "$0")" 5 | in_dir="${scriptdir}/files" 6 | out_dir="${scriptdir}/files/test.out" 7 | mkdir -p "${out_dir}" 8 | # shellcheck disable=SC1090 9 | . "${scriptdir}/utils.sh" 10 | 11 | email_to_receive="${in_dir}/inbound-email.mime" 12 | 13 | if [[ -n "$1" ]]; then 14 | client_id="$1" 15 | else 16 | client_id="$(jq -r '.client_id' <"${out_dir}/register1.json")" 17 | fi 18 | 19 | # workflow 2a: receive an email directed at one of the clients 20 | # this simulates sendgrid delivering an email to the service 21 | http --ignore-stdin --check-status -f POST \ 22 | "http://nginx:8888/api/email/sendgrid/${client_id}" \ 23 | "dkim={@sendgrid.com : pass}" \ 24 | "SPF=pass" \ 25 | "email=@${email_to_receive}" 26 | 27 | # simulate delivery of the same email to the second mailbox 28 | http --ignore-stdin --check-status -f POST \ 29 | "http://nginx:8888/api/email/sendgrid/${client_id}" \ 30 | "dkim={@sendgrid.com : pass}" \ 31 | "SPF=pass" \ 32 | "email=@${email_to_receive}" 33 | 34 | # simulate delivery of another email 35 | http --ignore-stdin --check-status -f POST \ 36 | "http://nginx:8888/api/email/sendgrid/${client_id}" \ 37 | "dkim={@sendgrid.com : pass}" \ 38 | "SPF=pass" \ 39 | "email=@${in_dir}/inbound-email-2.mime" 40 | -------------------------------------------------------------------------------- /docker/integtest/4-client-downloads-emails.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | 4 | scriptdir="$(dirname "$0")" 5 | out_dir="${scriptdir}/files/test.out" 6 | mkdir -p "${out_dir}" 7 | # shellcheck disable=SC1090 8 | . "${scriptdir}/utils.sh" 9 | 10 | declare -A num_emails_expected_for_client 11 | num_emails_expected_for_client[1]=2 12 | num_emails_expected_for_client[2]=1 13 | 14 | for i in 1 2; do 15 | 16 | client_id="$(jq -r '.client_id' <"${out_dir}/register${i}.json")" 17 | resource_container="$(jq -r '.resource_container' <"${out_dir}/register${i}.json")" 18 | 19 | # workflow 2b: deliver emails written by the world to a lokole client 20 | # first the client makes a request to ask the server to package up all the 21 | # emails sent from the world to the client during the last period; the server 22 | # will package up the emails and store them on the shared blob storage 23 | curl -fs \ 24 | -H "Accept: application/json" \ 25 | "http://nginx:8888/api/email/download/${client_id}" | 26 | tee "${out_dir}/download${i}.json" 27 | 28 | resource_id="$(jq -r '.resource_id' <"${out_dir}/download${i}.json")" 29 | 30 | # now we simulate the client downloading the package from the shared blob storage 31 | az_storage download "${resource_container}" "${resource_id}" "${out_dir}/downloaded${i}.tar.gz" 32 | 33 | tar xzf "${out_dir}/downloaded${i}.tar.gz" -C "${out_dir}" 34 | 35 | num_emails_actual="$(wc -l "${out_dir}/emails.jsonl" | cut -d' ' -f1)" 36 | num_emails_expected="${num_emails_expected_for_client[${i}]}" 37 | 38 | if [[ "${num_emails_actual}" -ne "${num_emails_expected}" ]]; then 39 | echo "Got ${num_emails_actual} emails but expected ${num_emails_expected}" >&2 40 | exit 1 41 | fi 42 | 43 | rm "${out_dir}/emails.jsonl" 44 | 45 | done 46 | -------------------------------------------------------------------------------- /docker/integtest/5-assert-on-results.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | 4 | scriptdir="$(dirname "$0")" 5 | out_dir="${scriptdir}/files/test.out" 6 | mkdir -p "${out_dir}" 7 | # shellcheck disable=SC1090 8 | . "${scriptdir}/utils.sh" 9 | 10 | num_exceptions="$(az storage blob list \ 11 | --prefix "Microsoft.ApplicationInsights.Exception/" \ 12 | --container-name "${APPINSIGHTS_INSTRUMENTATIONKEY}" \ 13 | --connection-string "$(az_connection_string)" \ 14 | --output tsv | wc -l)" 15 | num_exceptions_expected=0 16 | 17 | if [[ "${num_exceptions}" -ne "${num_exceptions_expected}" ]]; then 18 | echo "Got ${num_exceptions} exceptions but expected ${num_exceptions_expected}" >&2 19 | exit 2 20 | fi 21 | 22 | num_users="$(authenticated_request 'http://nginx:8888/api/email/metrics/users/developer1.lokole.ca' | jq -r '.users')" 23 | num_users_expected=1 24 | 25 | if [[ "${num_users}" -ne "${num_users_expected}" ]]; then 26 | echo "Got ${num_users} users but expected ${num_users_expected}" >&2 27 | exit 3 28 | fi 29 | -------------------------------------------------------------------------------- /docker/integtest/6-receive-service-email.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | 4 | scriptdir="$(dirname "$0")" 5 | in_dir="${scriptdir}/files" 6 | out_dir="${scriptdir}/files/test.out" 7 | mkdir -p "${out_dir}" 8 | # shellcheck disable=SC1090 9 | . "${scriptdir}/utils.sh" 10 | 11 | echo_email_to_receive="${in_dir}/echo-service-email.mime" 12 | wikipedia_email_to_receive="${in_dir}/wikipedia-service-email.mime" 13 | 14 | #receive an email directed at the service endpoint 15 | http --ignore-stdin --check-status -f POST \ 16 | "http://nginx:8888/api/email/sendgrid/service" \ 17 | "dkim={@sendgrid.com : pass}" \ 18 | "SPF=pass" \ 19 | "email=@${echo_email_to_receive}" 20 | 21 | http --ignore-stdin --check-status -f POST \ 22 | "http://nginx:8888/api/email/sendgrid/service" \ 23 | "dkim={@sendgrid.com : pass}" \ 24 | "SPF=pass" \ 25 | "email=@${wikipedia_email_to_receive}" 26 | -------------------------------------------------------------------------------- /docker/integtest/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/azure-cli:2.0.32 2 | 3 | RUN apk add -q --no-cache \ 4 | curl=7.59.0-r0 \ 5 | docker=1.11.2-r1 \ 6 | jq=1.5-r2 \ 7 | util-linux=2.28-r3 8 | 9 | WORKDIR /app 10 | 11 | COPY docker/integtest/requirements.txt ./ 12 | RUN pip3 install --no-cache-dir -r requirements.txt 13 | 14 | COPY .env . 15 | COPY docker/integtest/ ./ 16 | 17 | CMD ["./tests.sh"] 18 | -------------------------------------------------------------------------------- /docker/integtest/files/.gitignore: -------------------------------------------------------------------------------- 1 | test.out/ 2 | client-emails.tar.gz 3 | -------------------------------------------------------------------------------- /docker/integtest/files/echo-service-email.mime: -------------------------------------------------------------------------------- 1 | Received: by mx0028p1mdw1.sendgrid.net with SMTP id Yt3NEnbnLU Mon, 13 Feb 2017 06:25:41 +0000 (UTC) 2 | Received: from mail-yw0-f176.google.com (mail-yw0-f176.google.com [209.85.161.176]) by mx0028p1mdw1.sendgrid.net (Postfix) with ESMTPS id C726D640B63; Mon, 13 Feb 2017 06:25:41 +0000 (UTC) 3 | Received: by mail-yw0-f176.google.com with SMTP id w75so45612320ywg.1; Sun, 12 Feb 2017 22:25:41 -0800 (PST) 4 | DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=mime-version:from:date:message-id:subject:to; bh=ViHLGS6kOdo9Q9CkDDSSSS3bgKuN0a+UXhwMw06ak4Q=; b=f3WGzjgLe0tPG2edhiHxiCEZatThUga/qJFnWZNyY4lEVjRM9l3qn1BZ4ITawT9tDK LS6qFx//6in7u0rV0YKoa8TfScUFOpPHGCmq1Wxdp7mrWP7GDuCOz3LzyXQsrBe/erGy YEjAVU876sWJ109mcMcmbgOL1SD3d4ak+8GVBSC8oMKPj5XWZsET7WmsonhKf5PHE9IW eJHKqdOkxiPbmDutVx7uS1Bi5u4d9UYPhgxFwAK9lWyJ/Esw6yffjlrUvmQCPibSCxRv o979yY6FyJXDJ82l4ErntcOloFNpzWZ89WkRhb1aBLUoZs3402s6D3wC0ljpmvneIAkw 3D6A== 5 | X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:mime-version:from:date:message-id:subject:to; bh=ViHLGS6kOdo9Q9CkDDSSSS3bgKuN0a+UXhwMw06ak4Q=; b=sWY7uU6kK3dg62wVuxcLsRYLg3eGcoLuoLjL0Ju/sl9rGqSDxVc2saIS0ThfUaHlfZ g1zvF+rBoxa7v9jk7MhEw3izW01WXDMm0w2JGc1QLTo3ZM2xW9Clss63R3ZtNKabuyhd 77NHAgbarmQGW5XuqwS1Fy0NMWHkAlLsZd2AnkNb6gCI/VHCCv/oem19bWvNWwRTPBYE cQDPJfzRiUzRPNZPLtlL5ybd2yyb4lcuG+2QoQV8uxPsKS4eDOjNmM76UWZ9s/Ul/mR+ Qbyui7suOO0vPy8GFJHPV9X2ffLqesafTAetCj3LClCdLIdfQDaK86mmVHOT6zldeCTa HH6Q== 6 | X-Gm-Message-State: AMke39n2h/OZU6fwgOdDltzsKqISVbe3ez6t19OeVrg2sT3pDRhSSQiIcwGzKjdWOD/oX96rQlTi0O9t9yhUfA== 7 | X-Received: by 10.129.81.4 with SMTP id f4mr15409224ywb.239.1486967141412; Sun, 12 Feb 2017 22:25:41 -0800 (PST) 8 | MIME-Version: 1.0 9 | Received: by 10.129.156.139 with HTTP; Sun, 12 Feb 2017 22:25:01 -0800 (PST) 10 | From: Clemens Wolff 11 | Date: Sun, 12 Feb 2017 22:25:01 -0800 12 | Message-ID: 13 | Subject: Service Email 14 | To: echo@bot.lokole.ca, 15 | Content-Type: multipart/alternative; boundary=001a1146392641b94705486384bf 16 | 17 | --001a1146392641b94705486384bf 18 | Content-Type: text/plain; charset=UTF-8 19 | 20 | This is an email that should be echoed. 21 | 22 | --001a1146392641b94705486384bf 23 | Content-Type: text/html; charset=UTF-8 24 | 25 |
Body of the message.
26 | 27 | --001a1146392641b94705486384bf-- 28 | -------------------------------------------------------------------------------- /docker/integtest/files/emails.jsonl: -------------------------------------------------------------------------------- 1 | {"from":"clemens@developer1.lokole.ca","to":["clemens.wolff@gmail.com","laura.barluzzi@gmail.com"],"subject":"First test email sent from Lokole client","body":"Some content","_uid":"ed262575-73db-49cc-9c87-e2e95f8ac9cc","sent_at":"2019-10-26 11:23"} 2 | {"from":"laura@developer1.lokole.ca","to":["clemens.wolff@gmail.com"],"subject":"Second test email sent from Lokole client","body":"Some more content","_uid":"d03fff22-5c51-4f10-8111-5b24686964ba","attachments":[{"filename":"test.txt","content":"dGVzdCBhdHRhY2htZW50"}],"sent_at":"2019-10-25 05:00"} 3 | {"from":"Clemens@developer1.lokole.ca","to":["clemens@gmail.com","laura.barluzzi@gmail.com"],"subject":"Third test email sent from Lokole client","body":"Some extra content","_uid":"43f619b3-64ba-4ea3-963d-740cf0e58989","sent_at":"2019-10-26 12:34"} 4 | -------------------------------------------------------------------------------- /docker/integtest/files/inbound-email-2.mime: -------------------------------------------------------------------------------- 1 | Received: by mx0028p1mdw1.sendgrid.net with SMTP id Yt3NEnbnLU Mon, 13 Feb 2017 06:25:41 +0000 (UTC) 2 | Received: from mail-yw0-f176.google.com (mail-yw0-f176.google.com [209.85.161.176]) by mx0028p1mdw1.sendgrid.net (Postfix) with ESMTPS id C726D640B63; Mon, 13 Feb 2017 06:25:41 +0000 (UTC) 3 | Received: by mail-yw0-f176.google.com with SMTP id w75so45612320ywg.1; Sun, 12 Feb 2017 22:25:41 -0800 (PST) 4 | DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=mime-version:from:date:message-id:subject:to; bh=ViHLGS6kOdo9Q9CkDDSSSS3bgKuN0a+UXhwMw06ak4Q=; b=f3WGzjgLe0tPG2edhiHxiCEZatThUga/qJFnWZNyY4lEVjRM9l3qn1BZ4ITawT9tDK LS6qFx//6in7u0rV0YKoa8TfScUFOpPHGCmq1Wxdp7mrWP7GDuCOz3LzyXQsrBe/erGy YEjAVU876sWJ109mcMcmbgOL1SD3d4ak+8GVBSC8oMKPj5XWZsET7WmsonhKf5PHE9IW eJHKqdOkxiPbmDutVx7uS1Bi5u4d9UYPhgxFwAK9lWyJ/Esw6yffjlrUvmQCPibSCxRv o979yY6FyJXDJ82l4ErntcOloFNpzWZ89WkRhb1aBLUoZs3402s6D3wC0ljpmvneIAkw 3D6A== 5 | X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:mime-version:from:date:message-id:subject:to; bh=ViHLGS6kOdo9Q9CkDDSSSS3bgKuN0a+UXhwMw06ak4Q=; b=sWY7uU6kK3dg62wVuxcLsRYLg3eGcoLuoLjL0Ju/sl9rGqSDxVc2saIS0ThfUaHlfZ g1zvF+rBoxa7v9jk7MhEw3izW01WXDMm0w2JGc1QLTo3ZM2xW9Clss63R3ZtNKabuyhd 77NHAgbarmQGW5XuqwS1Fy0NMWHkAlLsZd2AnkNb6gCI/VHCCv/oem19bWvNWwRTPBYE cQDPJfzRiUzRPNZPLtlL5ybd2yyb4lcuG+2QoQV8uxPsKS4eDOjNmM76UWZ9s/Ul/mR+ Qbyui7suOO0vPy8GFJHPV9X2ffLqesafTAetCj3LClCdLIdfQDaK86mmVHOT6zldeCTa HH6Q== 6 | X-Gm-Message-State: AMke39n2h/OZU6fwgOdDltzsKqISVbe3ez6t19OeVrg2sT3pDRhSSQiIcwGzKjdWOD/oX96rQlTi0O9t9yhUfA== 7 | X-Received: by 10.129.81.4 with SMTP id f4mr15409224ywb.239.1486967141412; Sun, 12 Feb 2017 22:25:41 -0800 (PST) 8 | MIME-Version: 1.0 9 | Received: by 10.129.156.139 with HTTP; Sun, 12 Feb 2017 22:25:01 -0800 (PST) 10 | From: Clemens Wolff 11 | Date: Sun, 12 Feb 2017 22:25:01 -0800 12 | Message-ID: 13 | Subject: Two recipients 14 | To: Clemens@developer1.lokole.ca, LaUrA@developer1.lokole.ca 15 | Content-Type: multipart/alternative; boundary=001a1146392641b94705486384bf 16 | 17 | --001a1146392641b94705486384bf 18 | Content-Type: text/plain; charset=UTF-8 19 | 20 | Body of the message. 21 | 22 | --001a1146392641b94705486384bf 23 | Content-Type: text/html; charset=UTF-8 24 | 25 |
Body of the message.
26 | 27 | --001a1146392641b94705486384bf-- 28 | -------------------------------------------------------------------------------- /docker/integtest/files/wikipedia-service-email.mime: -------------------------------------------------------------------------------- 1 | Received: by mx0028p1mdw1.sendgrid.net with SMTP id Yt3NEnbnLU Mon, 13 Feb 2017 06:25:41 +0000 (UTC) 2 | Received: from mail-yw0-f176.google.com (mail-yw0-f176.google.com [209.85.161.176]) by mx0028p1mdw1.sendgrid.net (Postfix) with ESMTPS id C726D640B63; Mon, 13 Feb 2017 06:25:41 +0000 (UTC) 3 | Received: by mail-yw0-f176.google.com with SMTP id w75so45612320ywg.1; Sun, 12 Feb 2017 22:25:41 -0800 (PST) 4 | DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=mime-version:from:date:message-id:subject:to; bh=ViHLGS6kOdo9Q9CkDDSSSS3bgKuN0a+UXhwMw06ak4Q=; b=f3WGzjgLe0tPG2edhiHxiCEZatThUga/qJFnWZNyY4lEVjRM9l3qn1BZ4ITawT9tDK LS6qFx//6in7u0rV0YKoa8TfScUFOpPHGCmq1Wxdp7mrWP7GDuCOz3LzyXQsrBe/erGy YEjAVU876sWJ109mcMcmbgOL1SD3d4ak+8GVBSC8oMKPj5XWZsET7WmsonhKf5PHE9IW eJHKqdOkxiPbmDutVx7uS1Bi5u4d9UYPhgxFwAK9lWyJ/Esw6yffjlrUvmQCPibSCxRv o979yY6FyJXDJ82l4ErntcOloFNpzWZ89WkRhb1aBLUoZs3402s6D3wC0ljpmvneIAkw 3D6A== 5 | X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:mime-version:from:date:message-id:subject:to; bh=ViHLGS6kOdo9Q9CkDDSSSS3bgKuN0a+UXhwMw06ak4Q=; b=sWY7uU6kK3dg62wVuxcLsRYLg3eGcoLuoLjL0Ju/sl9rGqSDxVc2saIS0ThfUaHlfZ g1zvF+rBoxa7v9jk7MhEw3izW01WXDMm0w2JGc1QLTo3ZM2xW9Clss63R3ZtNKabuyhd 77NHAgbarmQGW5XuqwS1Fy0NMWHkAlLsZd2AnkNb6gCI/VHCCv/oem19bWvNWwRTPBYE cQDPJfzRiUzRPNZPLtlL5ybd2yyb4lcuG+2QoQV8uxPsKS4eDOjNmM76UWZ9s/Ul/mR+ Qbyui7suOO0vPy8GFJHPV9X2ffLqesafTAetCj3LClCdLIdfQDaK86mmVHOT6zldeCTa HH6Q== 6 | X-Gm-Message-State: AMke39n2h/OZU6fwgOdDltzsKqISVbe3ez6t19OeVrg2sT3pDRhSSQiIcwGzKjdWOD/oX96rQlTi0O9t9yhUfA== 7 | X-Received: by 10.129.81.4 with SMTP id f4mr15409224ywb.239.1486967141412; Sun, 12 Feb 2017 22:25:41 -0800 (PST) 8 | MIME-Version: 1.0 9 | Received: by 10.129.156.139 with HTTP; Sun, 12 Feb 2017 22:25:01 -0800 (PST) 10 | From: Clemens Wolff 11 | Date: Sun, 12 Feb 2017 22:25:01 -0800 12 | Message-ID: 13 | Subject: jp 14 | To: wikipedia@bot.lokole.ca, 15 | Content-Type: multipart/alternative; boundary=001a1146392641b94705486384bf 16 | 17 | --001a1146392641b94705486384bf 18 | Content-Type: text/plain; charset=UTF-8 19 | 20 | linear regression 21 | 22 | --001a1146392641b94705486384bf 23 | Content-Type: text/html; charset=UTF-8 24 | 25 |
linear regression.
26 | 27 | --001a1146392641b94705486384bf-- 28 | -------------------------------------------------------------------------------- /docker/integtest/files/zzusers.jsonl: -------------------------------------------------------------------------------- 1 | {"email":"clemens@developer1.lokole.ca","password":"$2b$12$9LaXqZMPJi0PiTY.95dIQOvc8LkYQzRlg5a9pDWX47L/npaYqynU2"} 2 | -------------------------------------------------------------------------------- /docker/integtest/requirements.txt: -------------------------------------------------------------------------------- 1 | httpie==2.3.0 2 | -------------------------------------------------------------------------------- /docker/integtest/tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | 4 | scriptdir="$(dirname "$0")" 5 | # shellcheck disable=SC1090 6 | . "${scriptdir}/utils.sh" 7 | 8 | log "### 0-wait-for-services.sh" 9 | "${scriptdir}/0-wait-for-services.sh" 10 | 11 | log "### 1-register-client.sh" 12 | "${scriptdir}/1-register-client.sh" 13 | 14 | log "### 2-client-uploads-emails.sh" 15 | "${scriptdir}/2-client-uploads-emails.sh" && wait_seconds "${TEST_STEP_DELAY}" 16 | 17 | log "### 3-receive-email-for-client.sh" 18 | "${scriptdir}/3-receive-email-for-client.sh" && wait_seconds "${TEST_STEP_DELAY}" 19 | 20 | # TODO: debug failures 21 | # log "### 4-client-downloads-emails.sh" 22 | # "${scriptdir}/4-client-downloads-emails.sh" 23 | 24 | # log "### 5-assert-on-results.sh" 25 | # "${scriptdir}/5-assert-on-results.sh" 26 | 27 | # log "### 6-receive-service-email.sh" 28 | # "${scriptdir}/6-receive-service-email.sh" 29 | 30 | rm -rf "${scriptdir}/files/test.out" 31 | -------------------------------------------------------------------------------- /docker/integtest/utils.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | 4 | get_container() { 5 | docker ps --format '{{.Names}}' | grep "$1" 6 | } 7 | 8 | log() { 9 | echo "$@" >&2 10 | } 11 | 12 | authenticated_request() { 13 | local endpoint="$1" 14 | shift 15 | 16 | if [[ "${REGISTRATION_CREDENTIALS}" =~ ^[^:]+:.*$ ]]; then 17 | curl -fs "${endpoint}" -u "${REGISTRATION_CREDENTIALS}" "$@" 18 | else 19 | curl -fs "${endpoint}" -H "Authorization: Bearer ${REGISTRATION_CREDENTIALS}" "$@" 20 | fi 21 | } 22 | 23 | az_connection_string() { 24 | if [[ -z "${AZURITE_HOST}" ]]; then 25 | echo "AccountName=${AZURITE_ACCOUNT};AccountKey=${AZURITE_KEY};" 26 | else 27 | echo "AccountName=${AZURITE_ACCOUNT};AccountKey=${AZURITE_KEY};BlobEndpoint=http://${AZURITE_HOST}/${AZURITE_ACCOUNT};" 28 | fi 29 | } 30 | 31 | az_storage() { 32 | local mode="$1" 33 | local container="$2" 34 | local blob="$3" 35 | local file="$4" 36 | 37 | az storage blob "${mode}" --no-progress \ 38 | --file "${file}" \ 39 | --name "${blob}" \ 40 | --container-name "${container}" \ 41 | --connection-string "$(az_connection_string)" \ 42 | >/dev/null 43 | } 44 | 45 | wait_seconds() { 46 | local seconds="$1" 47 | 48 | printf 'Waiting' >&2 49 | while [[ "${seconds}" -gt 0 ]]; do 50 | printf '.' >&2 51 | sleep 1 52 | seconds="$((seconds - 1))" 53 | done 54 | echo >&2 55 | } 56 | -------------------------------------------------------------------------------- /docker/nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG PYTHON_VERSION=3.9 2 | FROM python:${PYTHON_VERSION} AS builder 3 | 4 | RUN curl -sSL https://git.io/get-mo -o /usr/bin/mo \ 5 | && chmod +x /usr/bin/mo 6 | 7 | FROM nginx:stable 8 | 9 | COPY --from=builder /usr/bin/mo /usr/bin/mo 10 | COPY docker/nginx/static /static 11 | COPY docker/nginx/*.mustache /app/ 12 | COPY docker/nginx/run-nginx.sh /app/run-nginx.sh 13 | 14 | RUN mkdir -p /var/cache/nginx /etc/nginx/modules-enabled /etc/nginx/sites-enabled \ 15 | && rm /etc/nginx/conf.d/default.conf \ 16 | && chown -R www-data:www-data \ 17 | /app \ 18 | /static \ 19 | /run \ 20 | /etc/nginx/modules-enabled \ 21 | /etc/nginx/sites-enabled \ 22 | /var/cache/nginx 23 | 24 | ENV DNS_RESOLVER="" 25 | ENV HOSTNAME_WEBAPP="SET_ME" 26 | ENV HOSTNAME_EMAIL_RECEIVE="SET_ME" 27 | ENV HOSTNAME_CLIENT_METRICS="SET_ME" 28 | ENV HOSTNAME_CLIENT_WRITE="SET_ME" 29 | ENV HOSTNAME_CLIENT_READ="SET_ME" 30 | ENV HOSTNAME_CLIENT_REGISTER="SET_ME" 31 | ENV PORT=8888 32 | 33 | EXPOSE ${PORT} 34 | USER www-data 35 | WORKDIR /static 36 | 37 | CMD ["/app/run-nginx.sh"] 38 | -------------------------------------------------------------------------------- /docker/nginx/nginx.conf.mustache: -------------------------------------------------------------------------------- 1 | user www-data; 2 | worker_processes {{NGINX_WORKERS}}; 3 | pid /run/nginx.pid; 4 | include /etc/nginx/modules-enabled/*.conf; 5 | 6 | events { 7 | worker_connections 1024; 8 | } 9 | 10 | http { 11 | include /etc/nginx/mime.types; 12 | default_type application/octet-stream; 13 | 14 | log_format main 15 | '$remote_addr - $remote_user [$time_local] "$request" ' 16 | '$status $body_bytes_sent "$http_referer" ' 17 | '"$http_user_agent" "$http_x_forwarded_for"'; 18 | 19 | sendfile on; 20 | tcp_nopush on; 21 | tcp_nodelay on; 22 | keepalive_timeout 65; 23 | types_hash_max_size 2048; 24 | 25 | ssl_protocols TLSv1 TLSv1.1 TLSv1.2; 26 | ssl_prefer_server_ciphers on; 27 | 28 | access_log /var/log/nginx/access.log main; 29 | error_log /var/log/nginx/error.log warn; 30 | 31 | gzip on; 32 | 33 | include /etc/nginx/sites-enabled/*; 34 | } 35 | -------------------------------------------------------------------------------- /docker/nginx/run-nginx.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | mo /app/nginx.conf 4 | mo /etc/nginx/sites-enabled/server.conf 5 | 6 | exec nginx -c "/app/nginx.conf" -p "${PWD}" -g "daemon off;" 7 | -------------------------------------------------------------------------------- /docker/nginx/server.conf.mustache: -------------------------------------------------------------------------------- 1 | upstream healthcheck_hosts { 2 | server {{HOSTNAME_EMAIL_RECEIVE}}; 3 | server {{HOSTNAME_CLIENT_METRICS}}; 4 | server {{HOSTNAME_CLIENT_WRITE}}; 5 | server {{HOSTNAME_CLIENT_READ}}; 6 | server {{HOSTNAME_CLIENT_REGISTER}}; 7 | } 8 | 9 | server { 10 | listen {{PORT}}; 11 | 12 | {{#LETSENCRYPT_DOMAIN}} 13 | server_name {{LETSENCRYPT_DOMAIN}}; 14 | listen [::]:443 ssl ipv6only=on; # managed by Certbot 15 | listen 443 ssl; # managed by Certbot 16 | ssl_certificate /etc/letsencrypt/live/mailserver.lokole.ca/fullchain.pem; # managed by Certbot 17 | ssl_certificate_key /etc/letsencrypt/live/mailserver.lokole.ca/privkey.pem; # managed by Certbot 18 | include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot 19 | ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot 20 | if ($scheme != "https") { return 301 https://$host$request_uri; } # managed by Certbot 21 | {{/LETSENCRYPT_DOMAIN}} 22 | 23 | {{#DNS_RESOLVER}} 24 | resolver {{DNS_RESOLVER}}; 25 | {{/DNS_RESOLVER}} 26 | 27 | client_max_body_size 50M; 28 | 29 | location = /favicon.ico { 30 | root {{STATIC_ROOT}}/static; 31 | } 32 | 33 | location = /robots.txt { 34 | root {{STATIC_ROOT}}/static; 35 | } 36 | 37 | location /healthcheck { 38 | proxy_pass http://healthcheck_hosts; 39 | } 40 | 41 | location /web { 42 | proxy_set_header Host $http_host; 43 | proxy_set_header X-Real-IP $remote_addr; 44 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 45 | proxy_set_header X-Forwarded-Proto $scheme; 46 | proxy_pass http://{{HOSTNAME_WEBAPP}}; 47 | } 48 | 49 | location /api/email/sendgrid { 50 | proxy_pass http://{{HOSTNAME_EMAIL_RECEIVE}}; 51 | } 52 | 53 | location /api/email/metrics { 54 | proxy_pass http://{{HOSTNAME_CLIENT_METRICS}}; 55 | } 56 | 57 | location /api/email/upload { 58 | proxy_pass http://{{HOSTNAME_CLIENT_WRITE}}; 59 | } 60 | 61 | location /api/email/download { 62 | proxy_pass http://{{HOSTNAME_CLIENT_READ}}; 63 | } 64 | 65 | location /api/email/register { 66 | proxy_pass http://{{HOSTNAME_CLIENT_REGISTER}}; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /docker/nginx/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ascoderu/lokole/1b2a7b18f472952df06f9938ebd01a0ecd749a79/docker/nginx/static/favicon.ico -------------------------------------------------------------------------------- /docker/nginx/static/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: / -------------------------------------------------------------------------------- /docker/setup/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/azure-cli:2.0.32 2 | 3 | ARG HELM_VERSION="3.2.1" 4 | ARG KUBECTL_VERSION="1.14.8" 5 | ENV KUBERNETES_VERSION="1.14.8" 6 | ENV CERT_MANAGER_VERSION="0.8.0" 7 | ENV NGINX_INGRESS_VERSION="0.3.7" 8 | 9 | RUN apk add -q --no-cache \ 10 | jq=1.5-r2 \ 11 | sshpass=1.05-r0 \ 12 | && wget -q "https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz" \ 13 | && tar xf "helm-v${HELM_VERSION}-linux-amd64.tar.gz" \ 14 | && mv "linux-amd64/helm" /usr/local/bin/helm \ 15 | && chmod +x /usr/local/bin/helm \ 16 | && rm -rf "linux-amd64" "helm-v${HELM_VERSION}-linux-amd64.tar.gz" \ 17 | && az aks install-cli --client-version "${KUBECTL_VERSION}" \ 18 | && mkdir /secrets 19 | 20 | COPY docker/setup/requirements.txt ./ 21 | RUN pip install --no-cache-dir -r requirements.txt 22 | 23 | COPY helm /app/helm 24 | COPY docker/setup/* /app/ 25 | 26 | WORKDIR /app 27 | 28 | CMD ["/app/setup.sh"] 29 | -------------------------------------------------------------------------------- /docker/setup/arm.parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "storageAccountSKU": { 6 | "value": "" 7 | }, 8 | "serviceBusSKU": { 9 | "value": "" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /docker/setup/renew-cert.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ## 3 | ## This script renews the LetsEncrypt certificate for the cluster. 4 | ## The script assumes that a kubernetes secret exists at /secrets/kube-config. 5 | ## 6 | 7 | scriptdir="$(dirname "$0")" 8 | scriptname="${BASH_SOURCE[0]}" 9 | # shellcheck disable=SC1090 10 | . "${scriptdir}/utils.sh" 11 | 12 | # 13 | # verify inputs 14 | # 15 | 16 | required_file "${scriptname}" "/secrets/kube-config" 17 | 18 | # 19 | # delete cert-manager pod: the pod will be automatically re-created which will 20 | # force a refresh of the https certificate if required 21 | # 22 | 23 | export KUBECONFIG="/secrets/kube-config" 24 | 25 | log "Looking up current cert-manager pods" 26 | kubectl get pod -l certmanager.k8s.io/acme-http01-solver=true 27 | kubectl get pod -n cert-manager 28 | 29 | log "Re-creating cert-manager pod" 30 | kubectl delete pod -l certmanager.k8s.io/acme-http01-solver=true 31 | kubectl delete pod -n cert-manager -l app=cert-manager 32 | 33 | log "Looking up new cert-manager pods" 34 | kubectl get pod -l certmanager.k8s.io/acme-http01-solver=true 35 | kubectl get pod -n cert-manager 36 | -------------------------------------------------------------------------------- /docker/setup/requirements.txt: -------------------------------------------------------------------------------- 1 | ghp-import==1.0.0 2 | twine==3.2.0 3 | -------------------------------------------------------------------------------- /docker/setup/setup-dns.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ## 3 | ## This script sets the required DNS records for a Lokole server deployment. 4 | ## The script assumes that Cloudflare credentials exists at /secrets/cloudflare.env. 5 | ## 6 | ## Required environment variables: 7 | ## 8 | ## LOKOLE_SERVER_IP 9 | ## LOKOLE_DNS_NAME 10 | ## 11 | 12 | scriptdir="$(dirname "$0")" 13 | scriptname="${BASH_SOURCE[0]}" 14 | # shellcheck disable=SC1090 15 | . "${scriptdir}/utils.sh" 16 | 17 | # 18 | # verify inputs 19 | # 20 | 21 | required_env "${scriptname}" "LOKOLE_SERVER_IP" 22 | required_env "${scriptname}" "LOKOLE_DNS_NAME" 23 | required_file "${scriptname}" "/secrets/cloudflare.env" 24 | 25 | # 26 | # configure dns 27 | # 28 | 29 | log "Setting up DNS mapping ${LOKOLE_SERVER_IP} to ${LOKOLE_DNS_NAME}" 30 | 31 | cloudflare_user="$(get_dotenv '/secrets/cloudflare.env' 'LOKOLE_CLOUDFLARE_USER')" 32 | cloudflare_key="$(get_dotenv '/secrets/cloudflare.env' 'LOKOLE_CLOUDFLARE_KEY')" 33 | 34 | cloudflare_zone_api="https://api.cloudflare.com/client/v4/zones" 35 | lokole_zone_name="${LOKOLE_DNS_NAME#*.}" 36 | cloudflare_zone_id="$(curl -sX GET "${cloudflare_zone_api}?name=${lokole_zone_name}" \ 37 | -H "X-Auth-Email: ${cloudflare_user}" \ 38 | -H "X-Auth-Key: ${cloudflare_key}" | 39 | jq -r '.result[0].id')" 40 | 41 | cloudflare_dns_api="${cloudflare_zone_api}/${cloudflare_zone_id}/dns_records" 42 | 43 | cloudflare_cname_id="$(curl -sX GET "${cloudflare_dns_api}?type=A&name=${LOKOLE_DNS_NAME}" \ 44 | -H "X-Auth-Email: ${cloudflare_user}" \ 45 | -H "X-Auth-Key: ${cloudflare_key}" | 46 | jq -r '.result[0].id')" 47 | 48 | if [[ -n "${cloudflare_cname_id}" ]] && [[ ${cloudflare_cname_id} != "null" ]]; then 49 | curl -sX PUT "${cloudflare_dns_api}/${cloudflare_cname_id}" \ 50 | -H "X-Auth-Email: ${cloudflare_user}" \ 51 | -H "X-Auth-Key: ${cloudflare_key}" \ 52 | -H "Content-Type: application/json" \ 53 | -d '{"type":"A","name":"'"${LOKOLE_DNS_NAME}"'","content":"'"${LOKOLE_SERVER_IP}"'","ttl":1,"proxied":false}' 54 | else 55 | curl -sX POST "${cloudflare_dns_api}" \ 56 | -H "X-Auth-Email: ${cloudflare_user}" \ 57 | -H "X-Auth-Key: ${cloudflare_key}" \ 58 | -H "Content-Type: application/json" \ 59 | -d '{"type":"A","name":"'"${LOKOLE_DNS_NAME}"'","content":"'"${LOKOLE_SERVER_IP}"'","ttl":1,"proxied":false}' 60 | fi 61 | -------------------------------------------------------------------------------- /docker/setup/upgrade-helm.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ## 3 | ## This script upgrades an existing production deployment. 4 | ## The script assumes that a kubernetes secret exists at /secrets/kube-config. 5 | ## 6 | ## Required environment variables: 7 | ## 8 | ## DOCKER_TAG 9 | ## HELM_NAME 10 | ## IMAGE_REGISTRY 11 | ## LOKOLE_DNS_NAME 12 | ## 13 | 14 | scriptdir="$(dirname "$0")" 15 | scriptname="${BASH_SOURCE[0]}" 16 | # shellcheck disable=SC1090 17 | . "${scriptdir}/utils.sh" 18 | 19 | # 20 | # verify inputs 21 | # 22 | 23 | required_env "${scriptname}" "DOCKER_TAG" 24 | required_env "${scriptname}" "HELM_NAME" 25 | required_env "${scriptname}" "IMAGE_REGISTRY" 26 | required_env "${scriptname}" "LOKOLE_DNS_NAME" 27 | required_file "${scriptname}" "/secrets/kube-config" 28 | 29 | # 30 | # upgrade production deployment 31 | # 32 | 33 | log "Upgrading helm deployment ${HELM_NAME}" 34 | 35 | export KUBECONFIG="/secrets/kube-config" 36 | 37 | helm_init 38 | 39 | helm upgrade "${HELM_NAME}" \ 40 | --set domain="${LOKOLE_DNS_NAME}" \ 41 | --set version.imageRegistry="${IMAGE_REGISTRY}" \ 42 | --set version.dockerTag="${DOCKER_TAG}" \ 43 | "${scriptdir}/helm/opwen_cloudserver" 44 | -------------------------------------------------------------------------------- /docker/setup/upgrade-vm.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ## 3 | ## This script upgrades an production VM. 4 | ## 5 | ## Required environment variables: 6 | ## 7 | ## LOKOLE_VM_USERNAME 8 | ## LOKOLE_VM_PASSWORD 9 | ## LOKOLE_DNS_NAME 10 | ## 11 | 12 | scriptdir="$(dirname "$0")" 13 | scriptname="${BASH_SOURCE[0]}" 14 | # shellcheck disable=SC1090 15 | . "${scriptdir}/utils.sh" 16 | 17 | # 18 | # verify inputs 19 | # 20 | 21 | required_env "${scriptname}" "LOKOLE_VM_USERNAME" 22 | required_env "${scriptname}" "LOKOLE_VM_PASSWORD" 23 | required_env "${scriptname}" "LOKOLE_DNS_NAME" 24 | 25 | # 26 | # upgrade production deployment 27 | # 28 | 29 | log "Upgrading VM ${LOKOLE_DNS_NAME}" 30 | 31 | exec sshpass -p "${LOKOLE_VM_PASSWORD}" ssh -o StrictHostKeyChecking=no "${LOKOLE_VM_USERNAME}@${LOKOLE_DNS_NAME}" <"${scriptdir}/vm.sh" 32 | -------------------------------------------------------------------------------- /docker/setup/upgrade.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | scriptdir="$(dirname "$0")" 4 | 5 | if [[ "$1" = "vm" ]]; then 6 | shift 7 | exec "${scriptdir}/upgrade-vm.sh" "$@" 8 | else 9 | exec "${scriptdir}/upgrade-helm.sh" "$@" 10 | fi 11 | -------------------------------------------------------------------------------- /docker/setup/utils.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | usage() { 4 | local script="$1" 5 | local usage 6 | 7 | usage="$(grep '^##' "${script}" | sed 's/^##//')" 8 | 9 | printf "Usage: %s\n\n%s\n" "${script}" "${usage}" 10 | } 11 | 12 | log() { 13 | local message="$1" 14 | local format="\033[1m\033[7m" 15 | local reset="\033[0m" 16 | 17 | echo -e "$(date)\t${format}${message}${reset}" 18 | } 19 | 20 | required_env() { 21 | local scriptname="$1" 22 | local envname="$2" 23 | 24 | if [[ -z "${!envname}" ]]; then 25 | echo "${envname} must be set" >&2 26 | usage "${scriptname}" 27 | exit 1 28 | fi 29 | } 30 | 31 | required_file() { 32 | local scriptname="$1" 33 | local filename="$2" 34 | 35 | if [[ ! -f "${filename}" ]]; then 36 | echo "${filename} must exist" >&2 37 | usage "${scriptname}" 38 | exit 1 39 | fi 40 | } 41 | 42 | generate_identifier() { 43 | local length="$1" 44 | 45 | tr &1 | /usr/bin/logger -t update_letsencrypt_renewal" | sudo crontab 58 | sudo chmod 0755 /etc/letsencrypt/{live,archive} 59 | 60 | # 61 | # set up app 62 | # important: remember to scp the secrets to the vm manually 63 | # 64 | git clone https://github.com/ascoderu/lokole.git 65 | docker-compose -f lokole/docker/docker-compose.prod.yml pull 66 | docker-compose -f lokole/docker/docker-compose.prod.yml up -d 67 | 68 | # 69 | # set up nginx 70 | # 71 | cat >lokole/secrets/nginx.env < package-prod.json 15 | 16 | FROM node:${NODE_VERSION}-slim AS runtime 17 | 18 | RUN npm install -g serve@11.3.0 19 | 20 | WORKDIR /app 21 | 22 | COPY --from=compiler /app/opwen_statuspage/build ./lokole/ 23 | COPY --from=compiler /app/opwen_statuspage/package-prod.json ./package.json 24 | -------------------------------------------------------------------------------- /helm/.gitignore: -------------------------------------------------------------------------------- 1 | charts/ 2 | requirements.lock 3 | -------------------------------------------------------------------------------- /helm/opwen_cloudserver/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "v1" 2 | kind: Chart 3 | name: opwen_cloudserver 4 | description: A Helm Chart for the Lokole email server by Ascoderu 5 | version: 0.0.1 6 | keywords: 7 | - ascoderu 8 | sources: 9 | - https://github.com/ascoderu/lokole 10 | -------------------------------------------------------------------------------- /helm/opwen_cloudserver/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{- define "opwen.environment.shared" -}} 2 | - name: LOKOLE_LOG_LEVEL 3 | value: {{.Values.logging.level}} 4 | - name: LOKOLE_CLIENT_AZURE_STORAGE_KEY 5 | valueFrom: 6 | secretKeyRef: 7 | name: azure 8 | key: LOKOLE_CLIENT_AZURE_STORAGE_KEY 9 | - name: LOKOLE_CLIENT_AZURE_STORAGE_NAME 10 | valueFrom: 11 | secretKeyRef: 12 | name: azure 13 | key: LOKOLE_CLIENT_AZURE_STORAGE_NAME 14 | - name: LOKOLE_EMAIL_SERVER_AZURE_BLOBS_KEY 15 | valueFrom: 16 | secretKeyRef: 17 | name: azure 18 | key: LOKOLE_EMAIL_SERVER_AZURE_BLOBS_KEY 19 | - name: LOKOLE_EMAIL_SERVER_AZURE_BLOBS_NAME 20 | valueFrom: 21 | secretKeyRef: 22 | name: azure 23 | key: LOKOLE_EMAIL_SERVER_AZURE_BLOBS_NAME 24 | - name: LOKOLE_EMAIL_SERVER_AZURE_TABLES_KEY 25 | valueFrom: 26 | secretKeyRef: 27 | name: azure 28 | key: LOKOLE_EMAIL_SERVER_AZURE_TABLES_KEY 29 | - name: LOKOLE_EMAIL_SERVER_AZURE_TABLES_NAME 30 | valueFrom: 31 | secretKeyRef: 32 | name: azure 33 | key: LOKOLE_EMAIL_SERVER_AZURE_TABLES_NAME 34 | - name: LOKOLE_EMAIL_SERVER_APPINSIGHTS_KEY 35 | valueFrom: 36 | secretKeyRef: 37 | name: azure 38 | key: LOKOLE_EMAIL_SERVER_APPINSIGHTS_KEY 39 | - name: LOKOLE_CLOUDFLARE_USER 40 | valueFrom: 41 | secretKeyRef: 42 | name: cloudflare 43 | key: LOKOLE_CLOUDFLARE_USER 44 | - name: LOKOLE_CLOUDFLARE_KEY 45 | valueFrom: 46 | secretKeyRef: 47 | name: cloudflare 48 | key: LOKOLE_CLOUDFLARE_KEY 49 | - name: LOKOLE_SENDGRID_KEY 50 | valueFrom: 51 | secretKeyRef: 52 | name: sendgrid 53 | key: LOKOLE_SENDGRID_KEY 54 | - name: LOKOLE_QUEUE_BROKER_SCHEME 55 | value: azureservicebus 56 | - name: LOKOLE_EMAIL_SERVER_QUEUES_SAS_NAME 57 | valueFrom: 58 | secretKeyRef: 59 | name: azure 60 | key: LOKOLE_EMAIL_SERVER_QUEUES_SAS_NAME 61 | - name: LOKOLE_EMAIL_SERVER_QUEUES_SAS_KEY 62 | valueFrom: 63 | secretKeyRef: 64 | name: azure 65 | key: LOKOLE_EMAIL_SERVER_QUEUES_SAS_KEY 66 | - name: LOKOLE_EMAIL_SERVER_QUEUES_NAMESPACE 67 | valueFrom: 68 | secretKeyRef: 69 | name: azure 70 | key: LOKOLE_EMAIL_SERVER_QUEUES_NAMESPACE 71 | {{- end -}} 72 | -------------------------------------------------------------------------------- /helm/opwen_cloudserver/templates/api-autoscaler.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v1 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | creationTimestamp: null 5 | name: {{ .Release.Name }}-api 6 | spec: 7 | maxReplicas: 9 8 | minReplicas: 3 9 | scaleTargetRef: 10 | apiVersion: apps/v1 11 | kind: Deployment 12 | name: {{ .Release.Name }}-api 13 | targetCPUUtilizationPercentage: 75 14 | -------------------------------------------------------------------------------- /helm/opwen_cloudserver/templates/api-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | creationTimestamp: null 5 | labels: 6 | app: {{ .Release.Name }}-api 7 | name: {{ .Release.Name }}-api 8 | spec: 9 | selector: 10 | matchLabels: 11 | app: {{ .Release.Name }}-api 12 | replicas: 1 13 | strategy: {} 14 | template: 15 | metadata: 16 | creationTimestamp: null 17 | labels: 18 | app: {{ .Release.Name }}-api 19 | spec: 20 | containers: 21 | - name: api 22 | image: {{.Values.version.imageRegistry}}/opwenserver_app:{{.Values.version.dockerTag}} 23 | command: ["/app/docker/app/run-gunicorn.sh"] 24 | env: 25 | - name: PORT 26 | value: "8080" 27 | - name: CONNEXION_SPEC 28 | value: dir:/app/opwen_email_server/swagger 29 | - name: SERVER_WORKERS 30 | value: "{{.Values.server.serverWorkers}}" 31 | - name: TESTING_UI 32 | value: "False" 33 | {{ include "opwen.environment.shared" . }} 34 | - name: LOKOLE_REGISTRATION_USERNAME 35 | valueFrom: 36 | secretKeyRef: 37 | name: users 38 | key: LOKOLE_REGISTRATION_USERNAME 39 | - name: REGISTRATION_PASSWORD 40 | valueFrom: 41 | secretKeyRef: 42 | name: users 43 | key: REGISTRATION_PASSWORD 44 | ports: 45 | - containerPort: 8080 46 | resources: 47 | limits: 48 | memory: "512Mi" 49 | cpu: "500m" 50 | requests: 51 | memory: "256Mi" 52 | cpu: "100m" 53 | restartPolicy: Always 54 | status: {} 55 | -------------------------------------------------------------------------------- /helm/opwen_cloudserver/templates/api-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | creationTimestamp: null 5 | labels: 6 | app: {{ .Release.Name }}-api 7 | name: {{ .Release.Name }}-api 8 | spec: 9 | ports: 10 | - name: "8080" 11 | port: 8080 12 | targetPort: 8080 13 | selector: 14 | app: {{ .Release.Name }}-api 15 | status: 16 | loadBalancer: {} 17 | -------------------------------------------------------------------------------- /helm/opwen_cloudserver/templates/cluster-issuer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cert-manager.io/v1alpha2 2 | kind: ClusterIssuer 3 | metadata: 4 | name: {{ .Release.Name }}-cluster-issuer 5 | spec: 6 | acme: 7 | server: {{ .Values.letsencrypt.url }} 8 | email: {{ .Values.letsencrypt.email }} 9 | privateKeySecretRef: 10 | name: {{ .Release.Name }}-tls-secret 11 | -------------------------------------------------------------------------------- /helm/opwen_cloudserver/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.k8s.io/v1beta1 2 | kind: Ingress 3 | metadata: 4 | name: {{ .Release.Name }}-ingress 5 | annotations: 6 | kubernetes.io/ingress.class: nginx 7 | certmanager.k8s.io/cluster-issuer: {{ .Release.Name }}-cluster-issuer 8 | spec: 9 | tls: 10 | - hosts: 11 | - {{.Values.domain}} 12 | secretName: {{ .Release.Name }}-tls-secret 13 | rules: 14 | - host: {{.Values.domain}} 15 | http: 16 | paths: 17 | - path: / 18 | backend: 19 | serviceName: {{ .Release.Name }}-nginx 20 | servicePort: 8888 21 | -------------------------------------------------------------------------------- /helm/opwen_cloudserver/templates/nginx-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | creationTimestamp: null 5 | labels: 6 | app: {{ .Release.Name }}-nginx 7 | name: {{ .Release.Name }}-nginx 8 | spec: 9 | selector: 10 | matchLabels: 11 | app: {{ .Release.Name }}-nginx 12 | replicas: 3 13 | strategy: {} 14 | template: 15 | metadata: 16 | creationTimestamp: null 17 | labels: 18 | app: {{ .Release.Name }}-nginx 19 | spec: 20 | containers: 21 | - name: nginx 22 | image: {{.Values.version.imageRegistry}}/opwenserver_nginx:{{.Values.version.dockerTag}} 23 | env: 24 | - name: PORT 25 | value: "8888" 26 | - name: DNS_RESOLVER 27 | value: 127.0.0.1:53 ipv6=off 28 | - name: HOSTNAME_WEBAPP 29 | value: "{{ .Release.Name }}-webapp:8080" 30 | - name: HOSTNAME_CLIENT_METRICS 31 | value: "{{ .Release.Name }}-api:8080" 32 | - name: HOSTNAME_CLIENT_READ 33 | value: "{{ .Release.Name }}-api:8080" 34 | - name: HOSTNAME_CLIENT_WRITE 35 | value: "{{ .Release.Name }}-api:8080" 36 | - name: HOSTNAME_EMAIL_RECEIVE 37 | value: "{{ .Release.Name }}-api:8080" 38 | - name: HOSTNAME_CLIENT_REGISTER 39 | value: "{{ .Release.Name }}-api:8080" 40 | ports: 41 | - containerPort: 8888 42 | resources: 43 | limits: 44 | memory: "128Mi" 45 | cpu: "100m" 46 | requests: 47 | memory: "64Mi" 48 | cpu: "50m" 49 | - name: dnsmasq 50 | image: "janeczku/go-dnsmasq:release-1.0.7" 51 | args: 52 | - --listen=127.0.0.1:53 53 | - --default-resolver 54 | - --append-search-domains 55 | - --hostsfile=/etc/hosts 56 | - --verbose 57 | restartPolicy: Always 58 | status: {} 59 | -------------------------------------------------------------------------------- /helm/opwen_cloudserver/templates/nginx-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | creationTimestamp: null 5 | labels: 6 | app: {{ .Release.Name }}-nginx 7 | name: {{ .Release.Name }}-nginx 8 | spec: 9 | ports: 10 | - name: "8888" 11 | port: 8888 12 | targetPort: 8888 13 | selector: 14 | app: {{ .Release.Name }}-nginx 15 | status: 16 | loadBalancer: {} 17 | -------------------------------------------------------------------------------- /helm/opwen_cloudserver/templates/webapp-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | creationTimestamp: null 5 | labels: 6 | app: {{ .Release.Name }}-webapp 7 | name: {{ .Release.Name }}-webapp 8 | spec: 9 | selector: 10 | matchLabels: 11 | app: {{ .Release.Name }}-webapp 12 | replicas: 3 13 | strategy: {} 14 | template: 15 | metadata: 16 | creationTimestamp: null 17 | labels: 18 | app: {{ .Release.Name }}-webapp 19 | spec: 20 | containers: 21 | - name: webapp 22 | image: {{.Values.version.imageRegistry}}/opwenwebapp:{{.Values.version.dockerTag}} 23 | env: 24 | - name: HOST 25 | value: "0.0.0.0" 26 | - name: WEBAPP_PORT 27 | value: "8080" 28 | - name: WEBAPP_WORKERS 29 | value: "{{.Values.server.webappWorkers}}" 30 | {{ include "opwen.environment.shared" . }} 31 | ports: 32 | - containerPort: 8080 33 | resources: 34 | limits: 35 | memory: "512Mi" 36 | cpu: "500m" 37 | requests: 38 | memory: "512Mi" 39 | cpu: "200m" 40 | restartPolicy: Always 41 | status: {} 42 | -------------------------------------------------------------------------------- /helm/opwen_cloudserver/templates/webapp-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | creationTimestamp: null 5 | labels: 6 | app: {{ .Release.Name }}-webapp 7 | name: {{ .Release.Name }}-webapp 8 | spec: 9 | ports: 10 | - name: "8080" 11 | port: 8080 12 | targetPort: 8080 13 | selector: 14 | app: {{ .Release.Name }}-webapp 15 | status: 16 | loadBalancer: {} 17 | -------------------------------------------------------------------------------- /helm/opwen_cloudserver/templates/worker-autoscaler.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v1 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | creationTimestamp: null 5 | name: {{ .Release.Name }}-worker 6 | spec: 7 | maxReplicas: 9 8 | minReplicas: 3 9 | scaleTargetRef: 10 | apiVersion: apps/v1 11 | kind: Deployment 12 | name: {{ .Release.Name }}-worker 13 | targetCPUUtilizationPercentage: 75 14 | -------------------------------------------------------------------------------- /helm/opwen_cloudserver/templates/worker-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | creationTimestamp: null 5 | labels: 6 | app: {{ .Release.Name }}-worker 7 | name: {{ .Release.Name }}-worker 8 | spec: 9 | selector: 10 | matchLabels: 11 | app: {{ .Release.Name }}-worker 12 | replicas: 1 13 | strategy: {} 14 | template: 15 | metadata: 16 | creationTimestamp: null 17 | labels: 18 | app: {{ .Release.Name }}-worker 19 | spec: 20 | containers: 21 | - name: worker 22 | image: {{.Values.version.imageRegistry}}/opwenserver_app:{{.Values.version.dockerTag}} 23 | command: ["/app/docker/app/run-celery.sh"] 24 | env: 25 | - name: CELERY_QUEUE_NAMES 26 | value: all 27 | - name: QUEUE_WORKERS 28 | value: "{{.Values.worker.queueWorkers}}" 29 | {{ include "opwen.environment.shared" . }} 30 | ports: 31 | - containerPort: 80 32 | resources: 33 | limits: 34 | memory: "512Mi" 35 | cpu: "500m" 36 | requests: 37 | memory: "256Mi" 38 | cpu: "100m" 39 | restartPolicy: Always 40 | status: {} 41 | -------------------------------------------------------------------------------- /helm/opwen_cloudserver/values.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "v1" 2 | 3 | kind: Values 4 | 5 | domain: mailserver.lokole.ca 6 | 7 | version: 8 | imageRegistry: ascoderu 9 | dockerTag: latest 10 | 11 | server: 12 | serverWorkers: 2 13 | webappWorkers: 2 14 | 15 | worker: 16 | queueWorkers: 1 17 | 18 | logging: 19 | level: INFO 20 | 21 | letsencrypt: 22 | email: ascoderu.opwen@gmail.com 23 | url: https://acme-v02.api.letsencrypt.org/directory 24 | -------------------------------------------------------------------------------- /opwen_email_client/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.0.0' 2 | -------------------------------------------------------------------------------- /opwen_email_client/domain/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ascoderu/lokole/1b2a7b18f472952df06f9938ebd01a0ecd749a79/opwen_email_client/domain/__init__.py -------------------------------------------------------------------------------- /opwen_email_client/domain/email/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ascoderu/lokole/1b2a7b18f472952df06f9938ebd01a0ecd749a79/opwen_email_client/domain/email/__init__.py -------------------------------------------------------------------------------- /opwen_email_client/domain/email/attachment.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta 2 | from abc import abstractmethod 3 | from base64 import b64decode 4 | from base64 import b64encode 5 | 6 | 7 | class AttachmentEncoder(metaclass=ABCMeta): 8 | 9 | @abstractmethod 10 | def encode(self, content: bytes) -> str: 11 | raise NotImplementedError # pragma: no cover 12 | 13 | @abstractmethod 14 | def decode(self, encoded: str) -> bytes: 15 | raise NotImplementedError # pragma: no cover 16 | 17 | 18 | class Base64AttachmentEncoder(AttachmentEncoder): 19 | encoding = 'utf-8' 20 | 21 | def encode(self, content): 22 | content_bytes = b64encode(content) 23 | return content_bytes.decode(self.encoding) 24 | 25 | def decode(self, encoded): 26 | content_bytes = encoded.encode(self.encoding) 27 | return b64decode(content_bytes) 28 | -------------------------------------------------------------------------------- /opwen_email_client/domain/email/client.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta 2 | from abc import abstractmethod 3 | from os import getenv 4 | from os import path 5 | from urllib.parse import urlencode 6 | 7 | from requests import get as http_get 8 | from requests import post as http_post 9 | 10 | 11 | class EmailServerClient(metaclass=ABCMeta): 12 | 13 | @abstractmethod 14 | def upload(self, resource_id: str, container: str): 15 | raise NotImplementedError # pragma: no cover 16 | 17 | @abstractmethod 18 | def download(self) -> str: 19 | raise NotImplementedError # pragma: no cover 20 | 21 | 22 | class HttpEmailServerClient(EmailServerClient): 23 | 24 | def __init__(self, compression: str, endpoint: str, client_id: str): 25 | self._compression = compression 26 | self._endpoint = endpoint 27 | self._client_id = client_id 28 | 29 | @property 30 | def _base_url(self) -> str: 31 | return '{endpoint}/api/email'.format(endpoint=self._endpoint) 32 | 33 | @property 34 | def _upload_url(self) -> str: 35 | return '{base_url}/upload/{client_id}'.format( 36 | base_url=self._base_url, 37 | client_id=self._client_id, 38 | ) 39 | 40 | @property 41 | def _download_url(self) -> str: 42 | return '{base_url}/download/{client_id}?{query}'.format( 43 | base_url=self._base_url, 44 | client_id=self._client_id, 45 | query=urlencode({ 46 | 'compression': self._compression, 47 | }), 48 | ) 49 | 50 | def upload(self, resource_id, container): 51 | payload = { 52 | 'resource_id': resource_id, 53 | } 54 | 55 | response = http_post(self._upload_url, json=payload) 56 | response.raise_for_status() 57 | 58 | def download(self): 59 | response = http_get(self._download_url) 60 | response.raise_for_status() 61 | resource_id = response.json()['resource_id'] 62 | 63 | return resource_id 64 | 65 | 66 | class LocalEmailServerClient(EmailServerClient): 67 | 68 | def download(self) -> str: 69 | root = getenv('OPWEN_REMOTE_ACCOUNT_NAME') 70 | container = getenv('OPWEN_REMOTE_RESOURCE_CONTAINER') 71 | resource_id = 'sync.tar.gz' 72 | local_file = path.join(root, container, resource_id) 73 | if not path.isfile(local_file): 74 | return '' 75 | return resource_id 76 | 77 | def upload(self, resource_id: str, container: str): 78 | print('Uploaded {}/{}'.format(container, resource_id)) 79 | -------------------------------------------------------------------------------- /opwen_email_client/domain/email/user_store.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta 2 | from abc import abstractmethod 3 | from typing import List 4 | from typing import Union 5 | 6 | from flask import Flask 7 | from flask_login import UserMixin 8 | from flask_security.datastore import Datastore as UserWriteStore 9 | from flask_security.datastore import UserDatastore as UserReadStore 10 | 11 | 12 | class User(UserMixin): 13 | 14 | @property 15 | @abstractmethod 16 | def id(self) -> Union[str, int]: 17 | raise NotImplementedError # pragma: no cover 18 | 19 | @property 20 | @abstractmethod 21 | def email(self) -> str: 22 | raise NotImplementedError # pragma: no cover 23 | 24 | @property 25 | @abstractmethod 26 | def password(self) -> str: 27 | raise NotImplementedError # pragma: no cover 28 | 29 | @property 30 | @abstractmethod 31 | def roles(self) -> List[str]: 32 | raise NotImplementedError # pragma: no cover 33 | 34 | @property 35 | @abstractmethod 36 | def active(self) -> bool: 37 | raise NotImplementedError # pragma: no cover 38 | 39 | 40 | class UserStore(metaclass=ABCMeta): 41 | 42 | def __init__(self, read: UserReadStore, write: UserWriteStore) -> None: 43 | self.r = read 44 | self.w = write 45 | 46 | def init_app(self, app: Flask) -> None: 47 | pass 48 | 49 | @abstractmethod 50 | def fetch_all(self, user: User) -> List[User]: 51 | raise NotImplementedError # pragma: no cover 52 | 53 | @abstractmethod 54 | def fetch_pending(self) -> List[User]: 55 | raise NotImplementedError # pragma: no cover 56 | -------------------------------------------------------------------------------- /opwen_email_client/domain/modem/__init__.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from subprocess import check_call # nosec 3 | from subprocess import check_output # nosec 4 | 5 | 6 | def _find_device(stdout: bytes, device_id: str): 7 | for line in stdout.splitlines(): 8 | line = line.decode('utf-8') 9 | if 'Huawei' not in line: 10 | continue 11 | if device_id in line: 12 | return True 13 | return False 14 | 15 | 16 | def modem_is_plugged(modem=None): 17 | result = check_output('/usr/bin/lsusb', shell=True) # nosec 18 | return _find_device(result, modem.uid if modem else '12d1:') 19 | 20 | 21 | def modem_is_setup(target_mode: str): 22 | result = check_output('/usr/bin/lsusb', shell=True) # nosec 23 | return _find_device(result, '12d1:{}'.format(target_mode)) 24 | 25 | 26 | def setup_modem(config: Path): 27 | check_call('/usr/sbin/usb_modeswitch --config-file="{}"'.format(config.absolute()), shell=True) # nosec 28 | -------------------------------------------------------------------------------- /opwen_email_client/domain/modem/e303.py: -------------------------------------------------------------------------------- 1 | uid = '12d1:14fe' 2 | 3 | target = '1506' 4 | 5 | modeswitch = ( 6 | 'DefaultVendor = 0x12d1\n' 7 | 'DefaultProduct = 0x14fe\n' 8 | 'TargetVendor = 0x12d1\n' 9 | 'TargetProduct = 0x1506\n' 10 | 'MessageContent = "55534243123456780000000000000011062000000101000100000000000000"\n' # noqa: E501 11 | ) 12 | -------------------------------------------------------------------------------- /opwen_email_client/domain/modem/e3131.py: -------------------------------------------------------------------------------- 1 | uid = '12d1:155b' 2 | 3 | target = '1506' 4 | 5 | modeswitch = ( 6 | 'DefaultVendor = 0x12d1\n' 7 | 'DefaultProduct = 0x155b\n' 8 | 'TargetVendor = 0x12d1\n' 9 | 'TargetProduct = 0x1506\n' 10 | 'MessageContent = "55534243123456780000000000000011062000000100000000000000000000"\n' # noqa: E501 11 | ) 12 | -------------------------------------------------------------------------------- /opwen_email_client/domain/modem/e353.py: -------------------------------------------------------------------------------- 1 | uid = '12d1:1f01' 2 | 3 | target = '1001' 4 | 5 | modeswitch = ( 6 | 'DefaultVendor = 0x12d1\n' 7 | 'DefaultProduct = 0x1f01\n' 8 | 'TargetVendor = 0x12d1\n' 9 | 'TargetProduct = 0x1001\n' 10 | 'MessageContent = "55534243123456780000000000000011060000000000000000000000000000"\n' # noqa: E501 11 | ) 12 | -------------------------------------------------------------------------------- /opwen_email_client/domain/sim/__init__.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from subprocess import Popen # nosec 3 | from time import sleep 4 | from typing import IO 5 | 6 | 7 | def _dialer_is_connected(log_path: str) -> bool: 8 | with open(log_path, 'rb') as fobj: 9 | for line in fobj: 10 | if line.startswith(b'--> secondary DNS address'): 11 | return True 12 | return False 13 | 14 | 15 | def _start_dialer(config: Path, log_file: IO) -> Popen: 16 | return Popen(['/usr/bin/wvdial', '--config', str(config.absolute())], stderr=log_file) 17 | 18 | 19 | def dialup(config: Path, log: Path, max_retries: int, poll_seconds: int) -> Popen: 20 | with log.open(mode='w+b') as log: 21 | connection = _start_dialer(config, log) 22 | 23 | while not _dialer_is_connected(log.name): 24 | if connection.poll() is not None: 25 | connection.terminate() 26 | raise ValueError('Invalid wvdial configuration') 27 | 28 | if max_retries <= 0: 29 | connection.terminate() 30 | raise ValueError('Modem taking too long to connect') 31 | 32 | sleep(poll_seconds) 33 | max_retries -= 1 34 | 35 | return connection 36 | -------------------------------------------------------------------------------- /opwen_email_client/domain/sim/hologram.py: -------------------------------------------------------------------------------- 1 | wvdial = ('[Dialer Defaults]\n' 2 | 'Init1 = ATZ\n' 3 | 'Init2 = ATQ0\n' 4 | 'Init3 = AT+CGDCONT=1,"IP","apn.konekt.io"\n' 5 | 'Phone = *99***1#\n' 6 | 'Stupid Mode = 1\n' 7 | 'Username = { }\n' 8 | 'Password = { }\n' 9 | 'Modem Type = Analog Modem\n' 10 | 'Modem = /dev/ttyUSB0\n' 11 | 'IDSN = 0\n') 12 | -------------------------------------------------------------------------------- /opwen_email_client/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ascoderu/lokole/1b2a7b18f472952df06f9938ebd01a0ecd749a79/opwen_email_client/util/__init__.py -------------------------------------------------------------------------------- /opwen_email_client/util/network.py: -------------------------------------------------------------------------------- 1 | from socket import create_connection 2 | from socket import gethostbyname 3 | 4 | 5 | def check_connection(hostname: str, port: int) -> bool: 6 | try: 7 | host = gethostbyname(hostname) 8 | with create_connection((host, port)): 9 | return True 10 | except OSError: 11 | pass 12 | return False 13 | -------------------------------------------------------------------------------- /opwen_email_client/util/os.py: -------------------------------------------------------------------------------- 1 | from fileinput import input as fileinput 2 | from gzip import GzipFile 3 | from os import listdir 4 | from os.path import isdir 5 | from os.path import join 6 | from pathlib import Path 7 | from typing import Callable 8 | from typing import Iterable 9 | from typing import Optional 10 | from typing import Union 11 | 12 | 13 | def subdirectories(root: str) -> Iterable[str]: 14 | try: 15 | return (sub for sub in listdir(root) if isdir(join(root, sub))) 16 | except OSError: 17 | return [] 18 | 19 | 20 | def replace_line(path: str, match: Callable[[str], bool], replacement: str): 21 | for line in fileinput(path, inplace=True): 22 | if match(line): 23 | end = '' 24 | if line.endswith('\r\n'): 25 | end = '\r\n' 26 | elif line.endswith('\n'): 27 | end = '\n' 28 | if replacement.endswith(end): 29 | end = '' 30 | print(replacement, end=end) 31 | else: 32 | print(line, end='') 33 | 34 | 35 | def backup(path: Union[str, Path], suffix: str = '.bak.gz') -> Optional[Path]: 36 | path = Path(path) 37 | 38 | if not path.is_file(): 39 | return None 40 | 41 | backup_path = '{}{}'.format(path, suffix) 42 | with GzipFile(backup_path, mode='ab') as fout: 43 | with path.open('rb') as fin: 44 | for line in fin: 45 | fout.write(line) 46 | 47 | return Path(backup_path) 48 | -------------------------------------------------------------------------------- /opwen_email_client/util/serialization.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta 2 | from abc import abstractmethod 3 | from base64 import b64decode 4 | from base64 import b64encode 5 | from copy import deepcopy 6 | from json import dumps 7 | from json import loads 8 | from typing import TypeVar 9 | 10 | T = TypeVar('T') 11 | 12 | 13 | class Serializer(metaclass=ABCMeta): 14 | 15 | @abstractmethod 16 | def serialize(self, obj: T, type_: str = '') -> bytes: 17 | raise NotImplementedError # pragma: no cover 18 | 19 | @abstractmethod 20 | def deserialize(self, serialized: bytes, type_: str = '') -> T: 21 | raise NotImplementedError # pragma: no cover 22 | 23 | 24 | class JsonSerializer(Serializer): 25 | _encoding = 'utf-8' 26 | _separators = (',', ':') 27 | 28 | def serialize(self, obj: dict, type_: str = '') -> bytes: 29 | if not type_ or type_ == 'email': 30 | obj = self._encode_attachments(obj) 31 | elif type_ == 'attachment': 32 | self._encode_attachment(obj) 33 | 34 | serialized = dumps(obj, separators=self._separators, sort_keys=True) 35 | return serialized.encode(self._encoding) 36 | 37 | def deserialize(self, serialized: bytes, type_: str = '') -> dict: 38 | decoded = serialized.decode(self._encoding) 39 | obj = loads(decoded) 40 | 41 | if not type_ or type_ == 'email': 42 | email = obj 43 | email = self._decode_attachments(email) 44 | return email 45 | elif type_ == 'attachment': 46 | attachment = obj 47 | self._decode_attachment(attachment) 48 | return attachment 49 | 50 | @classmethod 51 | def _encode_attachments(cls, email: dict) -> dict: 52 | attachments = email.get('attachments', []) 53 | if not attachments: 54 | return email 55 | 56 | email = deepcopy(email) 57 | for attachment in email['attachments']: 58 | cls._encode_attachment(attachment) 59 | return email 60 | 61 | @classmethod 62 | def _encode_attachment(cls, attachment: dict) -> None: 63 | content = attachment.get('content', b'') 64 | if content: 65 | attachment['content'] = b64encode(content).decode('ascii') 66 | 67 | @classmethod 68 | def _decode_attachments(cls, email: dict) -> dict: 69 | attachments = email.get('attachments', []) 70 | if not attachments: 71 | return email 72 | 73 | email = deepcopy(email) 74 | for attachment in email['attachments']: 75 | cls._decode_attachment(attachment) 76 | return email 77 | 78 | @classmethod 79 | def _decode_attachment(cls, attachment: dict) -> None: 80 | content = attachment.get('content', '') 81 | if content: 82 | attachment['content'] = b64decode(content) 83 | -------------------------------------------------------------------------------- /opwen_email_client/util/sqlalchemy.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | 3 | from sqlalchemy import create_engine 4 | from sqlalchemy.exc import IntegrityError 5 | from sqlalchemy.exc import NoResultFound 6 | from sqlalchemy.exc import SQLAlchemyError 7 | from sqlalchemy.orm import scoped_session 8 | 9 | 10 | def create_database(uri: str, base): 11 | engine = create_engine(uri) 12 | 13 | try: 14 | base.metadata.create_all(bind=engine) 15 | except SQLAlchemyError: 16 | pass 17 | 18 | return engine 19 | 20 | 21 | def get_or_create(db, model, **model_args): 22 | try: 23 | return db.query(model).filter_by(**model_args).one() 24 | except NoResultFound: 25 | pass 26 | 27 | created = model(**model_args) 28 | try: 29 | db.add(created) 30 | db.flush() 31 | return created 32 | except IntegrityError: 33 | pass 34 | 35 | db.rollback() 36 | return db.query(model).filter_by(**model_args).one() 37 | 38 | 39 | @contextmanager 40 | def session(session_maker, commit: bool = False): 41 | session_factory = scoped_session(session_maker) 42 | db = session_factory() 43 | 44 | try: 45 | yield db 46 | if commit: 47 | db.commit() 48 | except SQLAlchemyError: 49 | db.rollback() 50 | finally: 51 | db.close() 52 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/__init__.py: -------------------------------------------------------------------------------- 1 | from logging import getLogger 2 | 3 | from flask import Flask 4 | from flask_babelex import Babel 5 | 6 | from opwen_email_client.webapp.cache import cache 7 | from opwen_email_client.webapp.commands import managesbp 8 | from opwen_email_client.webapp.config import AppConfig 9 | from opwen_email_client.webapp.forms.login import RegisterForm 10 | from opwen_email_client.webapp.ioc import _new_ioc 11 | from opwen_email_client.webapp.mkwvconf import blueprint as mkwvconf 12 | from opwen_email_client.webapp.security import security 13 | 14 | app = Flask(__name__, static_url_path=AppConfig.APP_ROOT + '/static') 15 | app.config.from_object(AppConfig) 16 | 17 | app.babel = Babel(app) 18 | 19 | app.ioc = _new_ioc(AppConfig.IOC) 20 | 21 | cache.init_app(app) 22 | app.ioc.user_store.init_app(app) 23 | security.init_app(app, app.ioc.user_store.r, register_form=RegisterForm, login_form=app.ioc.login_form) 24 | 25 | app.register_blueprint(mkwvconf, url_prefix=AppConfig.APP_ROOT + '/api/mkwvconf') 26 | app.register_blueprint(managesbp) 27 | 28 | if __name__ != '__main__': 29 | gunicorn_logger = getLogger('gunicorn.error') 30 | app.logger.handlers = gunicorn_logger.handlers 31 | app.logger.setLevel(gunicorn_logger.level) 32 | 33 | from opwen_email_client.webapp import jinja # noqa: F401,E402 # isort:skip 34 | from opwen_email_client.webapp import login # noqa: F401,E402 # isort:skip 35 | from opwen_email_client.webapp import views # noqa: F401,E402 # isort:skip 36 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/cache.py: -------------------------------------------------------------------------------- 1 | from flask_caching import Cache 2 | 3 | cache = Cache() 4 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/commands.py: -------------------------------------------------------------------------------- 1 | from os import remove 2 | from pathlib import Path 3 | from threading import Event 4 | 5 | import click 6 | from flask import Blueprint 7 | from flask import current_app as app 8 | from flask_security.utils import hash_password 9 | from watchdog.events import FileSystemEvent 10 | from watchdog.events import FileSystemEventHandler 11 | from watchdog.observers import Observer 12 | 13 | from opwen_email_client.webapp.actions import RestartAppComponent 14 | from opwen_email_client.webapp.config import AppConfig 15 | 16 | managesbp = Blueprint('manage', __name__) 17 | 18 | 19 | @managesbp.cli.command('resetdb') 20 | def resetdb(): 21 | remove(AppConfig.LOCAL_EMAIL_STORE) 22 | remove(AppConfig.SQLITE_PATH) 23 | 24 | 25 | @managesbp.cli.command("restarter") 26 | @click.option('-d', '--directory', required=True) 27 | def restarter(directory): 28 | 29 | class Restarter(FileSystemEventHandler): 30 | 31 | def on_created(self, event: FileSystemEvent): 32 | restart = RestartAppComponent(restart_path=event.src_path) 33 | restart() 34 | 35 | Path(directory).mkdir(exist_ok=True, parents=True) 36 | observer = Observer() 37 | observer.schedule(Restarter(), directory) 38 | observer.start() 39 | try: 40 | Event().wait() 41 | except KeyboardInterrupt: 42 | observer.stop() 43 | observer.join() 44 | 45 | 46 | @managesbp.cli.command('createadmin') 47 | @click.option('--name', required=True) 48 | @click.option('--password', required=True) 49 | def createadmin(name, password): 50 | user_datastore = app.ioc.user_store 51 | email = '{}@{}'.format(name, AppConfig.CLIENT_EMAIL_HOST) 52 | password = hash_password(password) 53 | 54 | user = user_datastore.r.find_user(email=email) 55 | if user is None: 56 | user_datastore.w.create_user(email=email, password=password, is_admin=True) 57 | else: 58 | user.is_admin = True 59 | user.password = password 60 | user_datastore.w.put(user) 61 | 62 | user_datastore.w.commit() 63 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/forms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ascoderu/lokole/1b2a7b18f472952df06f9938ebd01a0ecd749a79/opwen_email_client/webapp/forms/__init__.py -------------------------------------------------------------------------------- /opwen_email_client/webapp/forms/login.py: -------------------------------------------------------------------------------- 1 | from flask_security import LoginForm as _LoginForm 2 | from flask_security import RegisterForm as _RegisterForm 3 | from flask_security.forms import email_required 4 | from flask_security.forms import email_validator 5 | from flask_security.forms import unique_user_email 6 | from wtforms import IntegerField 7 | from wtforms.validators import NoneOf 8 | from wtforms.validators import Regexp 9 | 10 | from opwen_email_client.util.wtforms import SuffixedStringField 11 | from opwen_email_client.webapp.config import AppConfig 12 | from opwen_email_client.webapp.config import i8n 13 | 14 | 15 | # noinspection PyClassHasNoInit 16 | class LoginForm(_LoginForm): 17 | email = SuffixedStringField(suffix='@{}'.format(AppConfig.CLIENT_EMAIL_HOST)) 18 | 19 | 20 | email_character_validator = Regexp('^[a-zA-Z0-9-.@]*$', message=i8n.EMAIL_CHARACTERS) 21 | 22 | forbidden_account_validator = NoneOf(AppConfig.FORBIDDEN_ACCOUNTS, message=i8n.FORBIDDEN_ACCOUNT) 23 | 24 | 25 | # noinspection PyClassHasNoInit 26 | class RegisterForm(_RegisterForm): 27 | email = SuffixedStringField(suffix='@{}'.format(AppConfig.CLIENT_EMAIL_HOST), 28 | validators=[ 29 | email_character_validator, 30 | forbidden_account_validator, 31 | email_required, 32 | email_validator, 33 | unique_user_email, 34 | ]) 35 | 36 | timezone_offset_minutes = IntegerField(default=0) 37 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/forms/register.py: -------------------------------------------------------------------------------- 1 | from os import getenv 2 | from pathlib import Path 3 | 4 | from flask_wtf import FlaskForm 5 | from requests import post 6 | from wtforms import StringField 7 | from wtforms import SubmitField 8 | from wtforms.validators import ValidationError 9 | 10 | from opwen_email_client.webapp.config import AppConfig 11 | from opwen_email_client.webapp.config import i8n 12 | from opwen_email_client.webapp.tasks import register 13 | 14 | 15 | class RegisterForm(FlaskForm): 16 | 17 | client_name = StringField() 18 | github_username = StringField() 19 | github_token = StringField() 20 | submit = SubmitField() 21 | 22 | def register_client(self): 23 | path = (Path(getenv('OPWEN_STATE_DIRECTORY', 'lokole/state')) / 'settings.env') 24 | self._setup_client(str(path)) 25 | 26 | def _setup_client(self, path): 27 | name = self.client_name.data.strip() 28 | token = self.github_token.data.strip() 29 | 30 | endpoint = AppConfig.EMAIL_SERVER_ENDPOINT or 'mailserver.lokole.ca' 31 | client_domain = '{}.{}'.format(name, 'lokole.ca') 32 | client_create_url = 'https://{}/api/email/register/'.format(endpoint) 33 | 34 | response = post(client_create_url, 35 | json={'domain': client_domain}, 36 | headers={'Authorization': 'Bearer {}'.format(token)}) 37 | if response.status_code != 201: 38 | raise ValidationError(i8n.FAILED_REGISTRATION) 39 | register.delay(name, token, path) 40 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/ioc.py: -------------------------------------------------------------------------------- 1 | from importlib import import_module 2 | 3 | from cached_property import cached_property 4 | 5 | from opwen_email_client.domain.email.client import HttpEmailServerClient 6 | from opwen_email_client.domain.email.client import LocalEmailServerClient 7 | from opwen_email_client.domain.email.sql_store import SqliteEmailStore 8 | from opwen_email_client.domain.email.sync import AzureSync 9 | from opwen_email_client.util.serialization import JsonSerializer 10 | from opwen_email_client.webapp.config import AppConfig 11 | from opwen_email_client.webapp.forms.login import LoginForm 12 | from opwen_email_client.webapp.login import FlaskLoginUserStore 13 | 14 | 15 | class Ioc: 16 | 17 | @cached_property 18 | def email_store(self): 19 | return SqliteEmailStore( 20 | page_size=AppConfig.EMAILS_PER_PAGE, 21 | restricted={AppConfig.NEWS_INBOX: AppConfig.NEWS_SENDERS}, 22 | database_path=AppConfig.LOCAL_EMAIL_STORE, 23 | ) 24 | 25 | @cached_property 26 | def email_sync(self): 27 | if AppConfig.TESTING: 28 | email_server_client = LocalEmailServerClient() 29 | else: 30 | if AppConfig.EMAIL_SERVER_ENDPOINT: 31 | endpoint = AppConfig.EMAIL_SERVER_ENDPOINT 32 | elif AppConfig.EMAIL_SERVER_HOSTNAME: 33 | endpoint = 'https://{}'.format(AppConfig.EMAIL_SERVER_HOSTNAME) 34 | else: 35 | endpoint = None 36 | 37 | email_server_client = HttpEmailServerClient( 38 | compression=AppConfig.COMPRESSION, 39 | endpoint=endpoint, 40 | client_id=AppConfig.CLIENT_ID, 41 | ) 42 | 43 | serializer = JsonSerializer() 44 | 45 | return AzureSync( 46 | compression=AppConfig.COMPRESSION, 47 | account_name=AppConfig.STORAGE_ACCOUNT_NAME, 48 | account_key=AppConfig.STORAGE_ACCOUNT_KEY, 49 | account_host=AppConfig.STORAGE_ACCOUNT_HOST, 50 | account_secure=AppConfig.STORAGE_ACCOUNT_SECURE, 51 | email_server_client=email_server_client, 52 | container=AppConfig.STORAGE_CONTAINER, 53 | provider=AppConfig.STORAGE_PROVIDER, 54 | serializer=serializer, 55 | ) 56 | 57 | @cached_property 58 | def user_store(self): 59 | return FlaskLoginUserStore() 60 | 61 | @cached_property 62 | def login_form(self): 63 | return LoginForm 64 | 65 | 66 | def _new_ioc(fqn: str) -> Ioc: 67 | fqn_parts = fqn.split('.') 68 | class_name = fqn_parts.pop() 69 | module_name = '.'.join(fqn_parts) 70 | 71 | module = import_module(module_name) 72 | cls = getattr(module, class_name) 73 | 74 | return cls() 75 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/jinja.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from os.path import splitext 3 | 4 | from bs4 import BeautifulSoup 5 | from flask import url_for 6 | 7 | from opwen_email_client.webapp import app 8 | 9 | 10 | @app.template_filter('asset_url') 11 | def asset_url(asset_path: str) -> str: 12 | if app.config['TESTING']: 13 | return url_for('static', filename=asset_path) 14 | 15 | asset_path, extension = splitext(asset_path) 16 | return url_for('static', filename='{}.min{}'.format(asset_path, extension)) 17 | 18 | 19 | @app.template_filter('render_body') 20 | def render_body(email: dict) -> str: 21 | body = email.get('body') 22 | if not body: 23 | return '' 24 | 25 | body = body.replace('\n', '
') 26 | 27 | soup = BeautifulSoup(body, 'html.parser') 28 | images = soup.find_all('img') 29 | if not images: 30 | return body 31 | 32 | attachments = {attachment['cid']: attachment['_uid'] for attachment in email.get('attachments', [])} 33 | for img in images: 34 | src = img.get('src') 35 | if not src: 36 | continue 37 | if src.startswith('cid:'): 38 | attachment_cid = src[4:] 39 | attachment_id = attachments.get(attachment_cid) 40 | if attachment_id: 41 | src = url_for('download_attachment', email_id=email['_uid'], attachment_id=attachment_id) 42 | del img['src'] 43 | img['data-original'] = src 44 | body = str(soup) 45 | 46 | return body 47 | 48 | 49 | @app.context_processor 50 | def _inject_format_last_login(): 51 | 52 | def format_last_login(user, current_user) -> str: 53 | if not user.last_login_at: 54 | return '' 55 | 56 | date = user.last_login_at - timedelta(minutes=current_user.timezone_offset_minutes) 57 | return date.strftime('%Y-%m-%d %H:%M') 58 | 59 | return {'format_last_login': format_last_login} 60 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/mkwvconf.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint 2 | from flask import Response 3 | from flask import jsonify 4 | from flask import request 5 | from mkwvconf import Mkwvconf 6 | from mkwvconf.mkwvconf import DEFAULT_MODEM_DEVICE 7 | from mkwvconf.mkwvconf import DEFAULT_PROFILE_NAME 8 | 9 | from opwen_email_client.webapp.cache import cache 10 | 11 | blueprint = Blueprint('mkwvconf', __name__) 12 | 13 | 14 | def create_mkwvconf() -> Mkwvconf: 15 | return Mkwvconf({ 16 | option: request.args.get(option, default) 17 | for option, default in ( 18 | ('modemDevice', DEFAULT_MODEM_DEVICE), 19 | ('profileName', DEFAULT_PROFILE_NAME), 20 | ) 21 | }) 22 | 23 | 24 | @blueprint.route('///') 25 | @cache.memoize(timeout=600) 26 | def get_config(country: str, provider: str, apn: str) -> Response: 27 | mkwvconf = create_mkwvconf() 28 | parameters = mkwvconf.getConfigParameters(country, provider, apn) 29 | config = mkwvconf.formatConfig(parameters) 30 | return jsonify({'config': config}) 31 | 32 | 33 | @blueprint.route('//') 34 | @cache.memoize(timeout=600) 35 | def list_apns(country: str, provider: str) -> Response: 36 | mkwvconf = create_mkwvconf() 37 | apns = mkwvconf.getApns(country, provider) 38 | return jsonify({'apns': apns}) 39 | 40 | 41 | @blueprint.route('/') 42 | @cache.memoize(timeout=600) 43 | def list_providers(country: str) -> Response: 44 | mkwvconf = create_mkwvconf() 45 | providers = mkwvconf.getProviders(country) 46 | return jsonify({'providers': providers}) 47 | 48 | 49 | @blueprint.route('/') 50 | @cache.memoize(timeout=600) 51 | def list_countries() -> Response: 52 | mkwvconf = create_mkwvconf() 53 | countries = mkwvconf.getCountryCodes() 54 | return jsonify({'countries': countries}) 55 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/security.py: -------------------------------------------------------------------------------- 1 | from flask_login import login_required as _login_required 2 | from flask_security import Security 3 | 4 | from opwen_email_client.webapp.config import AppConfig 5 | 6 | security = Security() 7 | 8 | 9 | def login_required(func): 10 | if AppConfig.TESTING: 11 | return func 12 | 13 | return _login_required(func) 14 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/session.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from typing import Dict 3 | from typing import Optional 4 | 5 | from flask import request 6 | from flask import session 7 | 8 | 9 | class Session(object): 10 | _current_language_key = 'current_language' 11 | _last_visited_url_key = 'last_visited_url' 12 | 13 | @classmethod 14 | def _session(cls) -> Dict[str, str]: 15 | return session 16 | 17 | @classmethod 18 | def store_last_visited_url(cls): 19 | cls._session()[cls._last_visited_url_key] = request.url 20 | 21 | @classmethod 22 | def get_last_visited_url(cls) -> Optional[str]: 23 | return cls._session().get(cls._last_visited_url_key) 24 | 25 | @classmethod 26 | def store_current_language(cls, language: str): 27 | cls._session()[cls._current_language_key] = language 28 | 29 | @classmethod 30 | def get_current_language(cls) -> str: 31 | return cls._session().get(cls._current_language_key) 32 | 33 | 34 | def track_history(func): 35 | 36 | @wraps(func) 37 | def history_tracker(*args, **kwargs): 38 | Session.store_last_visited_url() 39 | return func(*args, **kwargs) 40 | 41 | return history_tracker 42 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/static/css/_base_email.css: -------------------------------------------------------------------------------- 1 | #email-subnav { 2 | padding-bottom: 1em; 3 | } 4 | 5 | #email-search-form { 6 | padding-bottom: 1em; 7 | } 8 | 9 | .btn-group.email-actions { 10 | margin-top: 1em; 11 | } 12 | 13 | .panel-title .email-row { 14 | text-decoration: none; 15 | word-wrap: break-word; 16 | } 17 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/static/css/about.css: -------------------------------------------------------------------------------- 1 | @media screen and (max-width: 768px) { 2 | .row .about-content.text-left { 3 | text-align: center; 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/static/css/email_new.css: -------------------------------------------------------------------------------- 1 | @media only screen and (max-width: 768px) { 2 | ul.wysihtml5-toolbar { 3 | margin-bottom: 20px; 4 | } 5 | 6 | iframe.wysihtml5-sandbox { 7 | padding-top: 25px !important; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/static/css/home.css: -------------------------------------------------------------------------------- 1 | #lokole-logo { 2 | padding-top: 1em; 3 | padding-bottom: 1em; 4 | max-height: 150px; 5 | } 6 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/static/css/settings.css: -------------------------------------------------------------------------------- 1 | #wvdial { 2 | height: 20em; 3 | font-family: monospace; 4 | } 5 | -------------------------------------------------------------------------------- /opwen_email_client/webapp/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ascoderu/lokole/1b2a7b18f472952df06f9938ebd01a0ecd749a79/opwen_email_client/webapp/static/favicon.ico -------------------------------------------------------------------------------- /opwen_email_client/webapp/static/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ascoderu/lokole/1b2a7b18f472952df06f9938ebd01a0ecd749a79/opwen_email_client/webapp/static/images/logo.png -------------------------------------------------------------------------------- /opwen_email_client/webapp/static/js/_base.js: -------------------------------------------------------------------------------- 1 | (function ($, ctx, appRoot) { 2 | $(document).ready(function () { 3 | (function pollForNewEmails () { 4 | function checkForNewEmails () { 5 | $.getJSON(appRoot + '/email/unread', function (response) { 6 | if (!response.unread || document.location.pathname.startsWith('/email/') || $('.alert.new-emails').length) { 7 | return 8 | } 9 | 10 | var $flashes = $('.flashes') 11 | if (!$flashes.length) { 12 | $flashes = $('
    ') 13 | $('nav').after($flashes) 14 | } 15 | 16 | var $li = $('
  • ') 17 | var $div = $('