├── .python-version
├── AUTHORS
├── tests
├── fixtures
│ ├── email.subject
│ ├── email.b.subject
│ ├── email.text
│ ├── email.b.text
│ ├── email_globale.text
│ ├── original.txt
│ ├── email_inference.xml
│ ├── email_inference_raw.html
│ ├── email.xml
│ ├── email_with_type.xml
│ ├── email.raw.html
│ ├── email_inference.html
│ ├── email_render_with_inference.html
│ ├── email.html
│ ├── email_en_b.xml
│ ├── email_en_default.xml
│ ├── email.b.html
│ ├── email.rtl.html
│ └── email_globale.html
├── src
│ ├── ar
│ │ ├── global.xml
│ │ └── email.xml
│ ├── en
│ │ ├── global.xml
│ │ ├── email_render_with_inference.xml
│ │ ├── email_globale.xml
│ │ ├── fallback.xml
│ │ ├── email_order.xml
│ │ ├── placeholder.xml
│ │ ├── missing_placeholder.xml
│ │ ├── email_subject_resend.xml
│ │ ├── email_subjects_ab.xml
│ │ └── email.xml
│ ├── fr
│ │ ├── global.xml
│ │ ├── fallback.xml
│ │ ├── email.xml
│ │ ├── placeholder.xml
│ │ └── missing_placeholder.xml
│ └── placeholders_config.json
├── templates_html
│ ├── transactional
│ │ ├── email_render_with_inference.html
│ │ └── basic_template.html
│ ├── marketing
│ │ ├── basic_marketing_template.html
│ │ └── globale_template.html
│ ├── basic_template.css
│ └── sections
│ │ └── header-with-background.html
├── test_cmd.py
├── test_placeholder.py
├── test_fs.py
├── test_reader.py
├── test_parser.py
└── test_renderer.py
├── requirements.txt
├── setup.cfg
├── email_parser
├── utils.py
├── link_shortener.py
├── const.py
├── config.py
├── placeholder.py
├── markdown_ext.py
├── model.py
├── cmd.py
├── fs.py
├── renderer.py
├── __init__.py
└── reader.py
├── .gitignore
├── .travis.yml
├── setup.py
├── Makefile
├── CHANGELOG
├── README.md
└── LICENSE
/.python-version:
--------------------------------------------------------------------------------
1 | 3.6.8
2 |
--------------------------------------------------------------------------------
/AUTHORS:
--------------------------------------------------------------------------------
1 | ca77y (https://github.com/ca77y)
2 |
--------------------------------------------------------------------------------
/tests/fixtures/email.subject:
--------------------------------------------------------------------------------
1 | Dummy subject
2 |
--------------------------------------------------------------------------------
/tests/fixtures/email.b.subject:
--------------------------------------------------------------------------------
1 | Awesome subject
2 |
--------------------------------------------------------------------------------
/tests/fixtures/email.text:
--------------------------------------------------------------------------------
1 | Dummy content
2 |
3 | Dummy inline
4 |
--------------------------------------------------------------------------------
/tests/fixtures/email.b.text:
--------------------------------------------------------------------------------
1 | Awesome content
2 |
3 | Dummy inline
4 |
--------------------------------------------------------------------------------
/tests/fixtures/email_globale.text:
--------------------------------------------------------------------------------
1 | Dummy content
2 |
3 | Dummy inline
4 |
5 | Unsubscribe from KeepSafe updates, click here ({{unsubscribe_link}})
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | Markdown==2.6.11
2 | beautifulsoup4==4.4.1
3 | cssutils==1.0.1
4 | inlinestyler==0.2.1
5 | lxml==3.5
6 | pystache==0.5.4
7 | parse==1.8.2
8 |
--------------------------------------------------------------------------------
/tests/fixtures/original.txt:
--------------------------------------------------------------------------------
1 | Dummy subject
2 |
3 | Dummy content
4 |
5 | 
6 |
7 | 
8 |
--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
1 | [metadata]
2 | description-file = README.md
3 |
4 | [flake8]
5 | max-line-length = 120
6 | ignore = F403, F405, F401
7 |
8 | [pep8]
9 | max-line-length = 120
10 |
--------------------------------------------------------------------------------
/email_parser/utils.py:
--------------------------------------------------------------------------------
1 | from . import config
2 |
3 |
4 | def normalize_locale(locale):
5 | if locale in config.lang_mappings:
6 | return config.lang_mappings[locale]
7 | return locale
8 |
--------------------------------------------------------------------------------
/tests/src/ar/global.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
--------------------------------------------------------------------------------
/tests/src/en/global.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
--------------------------------------------------------------------------------
/tests/src/fr/global.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
--------------------------------------------------------------------------------
/tests/fixtures/email_inference.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/tests/src/en/email_render_with_inference.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .noseids
2 | temp/
3 | dist/
4 | build/
5 | target/
6 | __pycache__/
7 | *.egg-info
8 | venv/
9 | *.pyc
10 |
11 | # OSX
12 | .DS_Store
13 |
14 | # Intellij files
15 | .idea/
16 | *.iml
17 |
18 | # Sublime files
19 | *.sublime-project
20 | *.sublime-workspace
21 |
22 | # VS Code
23 | .vscode
24 | .coverage
25 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: python
2 | python:
3 | - "3.6.5"
4 | # command to install dependencies
5 | install: "pip install -r requirements-dev.txt"
6 | # command to run tests
7 | script:
8 | - flake8 email_parser tests
9 | - nosetests --with-coverage --cover-inclusive --cover-erase --cover-package=email_parser --cover-min-percentage=70
10 |
--------------------------------------------------------------------------------
/tests/fixtures/email_inference_raw.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Dummy subject
5 |
6 |
7 | {{bitmap:MY_BITMAP:max-width=160px;max-height=160px;}}
8 |
9 |
10 |
--------------------------------------------------------------------------------
/tests/fixtures/email.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
8 | {{content}}
9 |
10 | {{inline}}
11 | {{image}}
12 | {{image_absolute}}
13 |
14 |
15 |
--------------------------------------------------------------------------------
/tests/templates_html/transactional/basic_template.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | {{subject}}
5 |
6 |
7 |
8 | {{content}}
9 |
10 | {{inline}}
11 | {{image}}
12 | {{image_absolute}}
13 |
14 |
15 |
--------------------------------------------------------------------------------
/tests/templates_html/marketing/basic_marketing_template.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | {{subject}}
5 |
6 |
7 |
8 | {{content}}
9 |
10 | {{inline}}
11 | {{image}}
12 | {{image_absolute}}
13 |
14 |
15 |
--------------------------------------------------------------------------------
/tests/fixtures/email_inference.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Dummy subject
5 |
6 |
7 |
8 |
9 |
8 | {{content}}
9 |
10 | {{inline}}
11 | {{image}}
12 | {{image_absolute}}
13 | {{global_unsubscribe}}
14 |
15 |
16 |
--------------------------------------------------------------------------------
/tests/src/fr/fallback.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 | Dummy subject fr
4 | Dummy content fr
5 | [[#C0D9D9]]
6 |
7 |
8 | 
9 |
10 |
--------------------------------------------------------------------------------
/tests/fixtures/email_render_with_inference.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Awesome subject
5 |
6 |
7 |
8 |
9 |
8 |
Dummy content
9 |
10 | Dummy inline
11 |
12 | 
13 |
14 |
15 | 
16 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/tests/fixtures/email_en_b.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/tests/fixtures/email_en_default.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/tests/src/en/email_subject_resend.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 | Dummy subject
4 | Alt Dummy subject
5 | [[#C0D9D9]]
6 | Dummy content
7 | Dummy inline
8 | 
9 | 
10 |
11 |
--------------------------------------------------------------------------------
/tests/fixtures/email.b.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Awesome subject
5 |
6 |
7 |
8 |
Awesome content
9 |
10 | Dummy inline
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/tests/fixtures/email.rtl.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Dummy subject
6 |
7 |
8 |
9 |
10 |
11 | Dummy content
12 |
13 |
14 | Dummy inline
15 |
16 |
17 |
18 |
19 | 
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/email_parser/link_shortener.py:
--------------------------------------------------------------------------------
1 | import requests
2 |
3 |
4 | class NullShortener(object):
5 | name = 'null'
6 |
7 | def __init__(self, config):
8 | pass
9 |
10 | def shorten(self, link):
11 | return link
12 |
13 |
14 | class KsShortener(object):
15 | name = 'keepsafe'
16 | url = 'http://4uon.ly/url/'
17 |
18 | def __init__(self, config):
19 | pass
20 |
21 | def shorten(self, link):
22 | # TODO needs auth
23 | # TODO needs perm links
24 | res = requests.post(self.url, data={'url': link})
25 | return res.text
26 |
27 |
28 | def shortener(config):
29 | return NullShortener(config)
30 |
--------------------------------------------------------------------------------
/tests/templates_html/sections/header-with-background.html:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
6 | |
7 |
8 |
9 | |
10 | {{ content_header_bg }}
11 | |
12 |
13 |
14 | |
15 |
16 |
17 | |
18 |
--------------------------------------------------------------------------------
/tests/src/en/email_subjects_ab.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 | Dummy subject
4 | A subject
5 | B subject
6 | [[#C0D9D9]]
7 | Dummy content
8 | Dummy inline
9 | 
10 | 
11 |
12 |
--------------------------------------------------------------------------------
/tests/src/placeholders_config.json:
--------------------------------------------------------------------------------
1 | {
2 | "dummy_email": {
3 | "unsubscribe_link": 1
4 | },
5 | "email": {
6 | "unsubscribe_link": 1
7 | },
8 | "email_globale": {
9 | "unsubscribe_link": 1
10 | },
11 | "email_subject_resend": {
12 | "unsubscribe_link": 1
13 | },
14 | "email_subjects_ab": {
15 | "unsubscribe_link": 1
16 | },
17 | "fallback": {
18 | "unsubscribe_link": 1
19 | },
20 | "missing_placeholder": {
21 | "placeholder": 1,
22 | "unsubscribe_link": 1
23 | },
24 | "placeholder": {
25 | "placeholder": 1,
26 | "unsubscribe_link": 1
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/email_parser/const.py:
--------------------------------------------------------------------------------
1 | SUBJECT_EXTENSION = '.subject'
2 | TEXT_EXTENSION = '.text'
3 | HTML_EXTENSION = '.html'
4 | CSS_EXTENSION = '.css'
5 | SOURCE_EXTENSION = '.xml'
6 | SUBJECT_PLACEHOLDER = 'subject'
7 |
8 | GLOBALS_EMAIL_NAME = 'global'
9 | GLOBALS_PLACEHOLDER_PREFIX = 'global_'
10 | REPO_SRC_PATH = 'src'
11 | PLACEHOLDERS_FILENAME = 'placeholders_config.json'
12 |
13 | INLINE_TEXT_PATTERN = r'\[{2}(.+)\]{2}'
14 | IMAGE_PATTERN = ''
15 | SEGMENT_REGEX = r'\]*>'
16 | SEGMENT_NAME_REGEX = r' name="([^"]+)"'
17 |
18 | TEXT_EMAIL_PLACEHOLDER_SEPARATOR = '\n\n'
19 | HTML_PARSER = 'lxml'
20 | LOCALE_PLACEHOLDER = '{link_locale}'
21 |
22 | DEFAULT_LOCALE = 'en'
23 | DEFAULT_WORKER_POOL = 10
24 | JSON_INDENT = 4
25 |
--------------------------------------------------------------------------------
/tests/fixtures/email_globale.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Dummy subject
5 |
6 |
7 |
8 |
Dummy content
9 |
10 | Dummy inline
11 |
12 |
13 |
14 |
15 | 
16 |
17 | Unsubscribe from KeepSafe updates, click here
18 |
19 |
20 |
--------------------------------------------------------------------------------
/tests/src/en/email.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | - Dummy subject
6 | - Awesome subject
7 |
8 | [[#C0D9D9]]
9 |
10 |
11 | - Dummy content
12 | - Awesome content
13 |
14 | Dummy inline
15 | 
16 | 
17 |
18 |
--------------------------------------------------------------------------------
/email_parser/config.py:
--------------------------------------------------------------------------------
1 | from collections import namedtuple
2 |
3 | Paths = namedtuple('Paths', ['source', 'destination', 'templates', 'images', 'sections'])
4 |
5 | _default_paths = Paths('src', 'target', 'templates_html', 'templates_html/img', 'templates_html/sections')
6 | _default_pattern = '{locale}/{name}.xml'
7 | _default_base_img_path = 'http://app.getkeepsafe.com/emails/img'
8 | _default_rtl = ['ar', 'he']
9 | _default_lang_mappings = {'pt-BR': 'pt', 'zh-TW-Hant': 'zh-TW'}
10 |
11 | paths = _default_paths
12 | pattern = _default_pattern
13 | base_img_path = _default_base_img_path
14 | rtl_locales = _default_rtl
15 | lang_mappings = _default_lang_mappings
16 |
17 |
18 | def init(*,
19 | _paths=_default_paths,
20 | _pattern=_default_pattern,
21 | _base_img_path=_default_base_img_path,
22 | _rtl_locales=_default_rtl,
23 | _lang_mappings=_default_lang_mappings):
24 | global paths, pattern, base_img_path, rtl_locales, lang_mappings
25 | paths = _default_paths
26 | pattern = _pattern
27 | base_img_path = _base_img_path
28 | rtl_locales = _rtl_locales
29 | lang_mappings = _lang_mappings
30 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 | from setuptools import setup, find_packages
3 |
4 | version = '0.3.2'
5 |
6 | install_requires = [
7 | 'Markdown < 3',
8 | 'beautifulsoup4 < 5',
9 | 'inlinestyler==0.2.1',
10 | 'pystache < 0.6',
11 | 'parse < 2'
12 | ]
13 |
14 | tests_require = [
15 | 'nose',
16 | 'flake8==2.5.4',
17 | 'coverage',
18 | ]
19 |
20 | devtools_require = [
21 | 'twine',
22 | 'build',
23 | ]
24 |
25 |
26 | def read(f):
27 | return open(os.path.join(os.path.dirname(__file__), f)).read().strip()
28 |
29 | setup(
30 | name='ks-email-parser',
31 | version=version,
32 | description=('A command line tool to render HTML and text emails of markdown content.'),
33 | classifiers=[
34 | 'License :: OSI Approved :: BSD License', 'Intended Audience :: Developers', 'Programming Language :: Python'
35 | ],
36 | author='Keepsafe',
37 | author_email='support@getkeepsafe.com',
38 | url='https://github.com/KeepSafe/ks-email-parser',
39 | license='Apache',
40 | packages=find_packages(),
41 | install_requires=install_requires,
42 | tests_require=tests_require,
43 | extras_require={
44 | 'tests': tests_require,
45 | 'devtools': devtools_require,
46 | },
47 | entry_points={'console_scripts': ['ks-email-parser = email_parser.cmd:main']},
48 | include_package_data=True)
49 |
50 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | # Some simple testing tasks (sorry, UNIX only).
2 |
3 | PYTHON=venv/bin/python3
4 | PIP=venv/bin/pip
5 | EI=venv/bin/easy_install
6 | NOSE=venv/bin/nosetests
7 | FLAKE=venv/bin/flake8
8 | EMAILS_TEMPLATES_URI=git@github.com:KeepSafe/emails.git
9 | EMAILS_PATH=emails
10 | GUI_BIN=ks-email-parser
11 | FLAGS=--with-coverage --cover-inclusive --cover-erase --cover-package=email_parser --cover-min-percentage=70
12 | PYPICLOUD_HOST=pypicloud.getkeepsafe.local
13 | TWINE=./venv/bin/twine
14 |
15 |
16 | update:
17 | $(PIP) install -U pip
18 | $(PIP) install -U .
19 |
20 | env:
21 | test -d venv || python3 -m venv venv
22 |
23 | dev: env update
24 | $(PIP) install .[tests,devtools]
25 |
26 | install: env update
27 |
28 | publish:
29 | rm -rf dist
30 | $(PYTHON) -m build .
31 | $(TWINE) upload --verbose --sign --username developer --repository-url http://$(PYPICLOUD_HOST)/simple/ dist/*.whl
32 |
33 |
34 | rungui:
35 | test -e $(EMAILS_PATH) && echo Emails templates already cloned || git clone $(EMAILS_TEMPLATES_URI) $(EMAILS_PATH);
36 | $(GUI_BIN) -s $(EMAILS_PATH)/src -d $(EMAILS_PATH)/target -t $(EMAILS_PATH)/templates_html gui
37 |
38 | flake:
39 | $(FLAKE) email_parser tests
40 |
41 | test: flake
42 | $(NOSE) -s $(FLAGS)
43 |
44 | vtest:
45 | $(NOSE) -s -v $(FLAGS)
46 |
47 | testloop:
48 | while sleep 1; do $(NOSE) -s $(FLAGS); done
49 |
50 | cov cover coverage:
51 | $(NOSE) -s --with-cover --cover-html --cover-html-dir ./coverage $(FLAGS)
52 | echo "open file://`pwd`/coverage/index.html"
53 |
54 | clean:
55 | rm -rf `find . -name __pycache__`
56 | rm -f `find . -type f -name '*.py[co]' `
57 | rm -f `find . -type f -name '*~' `
58 | rm -f `find . -type f -name '.*~' `
59 | rm -f `find . -type f -name '@*' `
60 | rm -f `find . -type f -name '#*#' `
61 | rm -f `find . -type f -name '*.orig' `
62 | rm -f `find . -type f -name '*.rej' `
63 | rm -f .coverage
64 | rm -rf coverage
65 | rm -rf build
66 | rm -rf venv
67 |
68 |
69 | .PHONY: all build env linux run pep test vtest testloop cov clean
70 |
--------------------------------------------------------------------------------
/CHANGELOG:
--------------------------------------------------------------------------------
1 | CHANGES
2 | =======
3 |
4 | 0.2.9 (2016-04-14)
5 | ------------------
6 |
7 | - Link locale placeholder.
8 |
9 | 0.2.8 (2016-06-20)
10 | ------------------
11 |
12 | - Global placeholders.
13 |
14 | 0.2.6 (2016-04-09)
15 | ------------------
16 |
17 | - Warn on extra placeholders.
18 |
19 | 0.2.2 (2015-07-23)
20 | ------------------
21 |
22 | - Log to stdout by default.
23 |
24 | 0.2.1 (2015-07-23)
25 | ------------------
26 |
27 | - Validate number of placeholders too.
28 |
29 |
30 | 0.2.0 (2015-07-22)
31 | ------------------
32 |
33 | - Placeholders validation.
34 |
35 |
36 | 0.1.12 (2015-05-07)
37 | ------------------
38 |
39 | - Add --version cmd option.
40 |
41 |
42 | 0.1.11 (2015-04-14)
43 | ------------------
44 |
45 | - Set dir attribute only when possible.
46 |
47 | 0.1.10 (2015-04-14)
48 | ------------------
49 |
50 | - Set dir attribute for entire content not for each placeholder.
51 |
52 | 0.1.9 (2015-04-06)
53 | ------------------
54 |
55 | - Use href for links only if available.
56 |
57 | 0.1.8 (2015-04-06)
58 | ------------------
59 |
60 | - Improve logging for strict mode.
61 |
62 | 0.1.7 (2015-04-06)
63 | ------------------
64 |
65 | - Improve logging for strict mode.
66 |
67 | 0.1.6 (2015-04-06)
68 | ------------------
69 |
70 | - Dependencies update.
71 |
72 | 0.1.5 (2015-04-06)
73 | ------------------
74 |
75 | - Adds default parser options.
76 |
77 | 0.1.4 (2015-04-06)
78 | ------------------
79 |
80 | - Log filename if parsing fails.
81 |
82 | 0.1.3 (2015-04-06)
83 | ------------------
84 |
85 | - Fixes missing dependency.
86 |
87 | 0.1.2 (2015-04-06)
88 | ------------------
89 |
90 | - Fixes requirements parsing.
91 |
92 | 0.1.1 (2015-04-06)
93 | ------------------
94 |
95 | - Use text and href for links in text files.
96 |
97 | 0.1.0 (2015-04-06)
98 | ------------------
99 |
100 | - Refactorings.
101 |
102 |
103 | 0.0.1 (2014-09-30)
104 | ------------------
105 |
106 | - Initial release.
107 |
--------------------------------------------------------------------------------
/tests/test_cmd.py:
--------------------------------------------------------------------------------
1 | import os
2 | import tempfile
3 | import shutil
4 | from unittest import TestCase
5 |
6 | from email_parser import fs, cmd, config
7 |
8 |
9 | def read_fixture(filename):
10 | with open(os.path.join('tests/fixtures', filename)) as fp:
11 | return fp.read()
12 |
13 |
14 | class TestParser(TestCase):
15 | maxDiff = None
16 |
17 | @classmethod
18 | def setUpClass(cls):
19 | cls.root_path = tempfile.mkdtemp()
20 | shutil.copytree(os.path.join('./tests', config.paths.source), os.path.join(cls.root_path, config.paths.source))
21 | shutil.copytree(
22 | os.path.join('./tests', config.paths.templates), os.path.join(cls.root_path, config.paths.templates))
23 | cmd.parse_emails(cls.root_path)
24 |
25 | @classmethod
26 | def tearDownClass(cls):
27 | shutil.rmtree(cls.root_path)
28 |
29 | def _run_and_assert(self, actual_filename, expected_filename=None, locale='en'):
30 | expected_filename = expected_filename or actual_filename
31 | expected = read_fixture(expected_filename).strip()
32 | actual = fs.read_file(TestParser.root_path, config.paths.destination, locale, actual_filename).strip()
33 | self.assertEqual(expected, actual)
34 |
35 | def test_subject(self):
36 | self._run_and_assert('email.subject')
37 |
38 | def test_text(self):
39 | self._run_and_assert('email.text')
40 |
41 | def test_html(self):
42 | self._run_and_assert('email.html')
43 |
44 | def test_global_text(self):
45 | self._run_and_assert('email_globale.text')
46 |
47 | def test_global_html(self):
48 | self._run_and_assert('email_globale.html')
49 |
50 | def test_rtl(self):
51 | self._run_and_assert('email.html', 'email.rtl.html', 'ar')
52 |
53 | def test_template_fallback(self):
54 | expected = fs.read_file(TestParser.root_path, config.paths.destination, 'en', 'fallback.html').strip()
55 | actual = fs.read_file(TestParser.root_path, config.paths.destination, 'fr', 'fallback.html').strip()
56 | self.assertEqual(expected, actual)
57 |
--------------------------------------------------------------------------------
/email_parser/placeholder.py:
--------------------------------------------------------------------------------
1 | from collections import Counter
2 | from functools import lru_cache
3 | import re
4 | import json
5 | import logging
6 |
7 | from . import reader, fs, const
8 |
9 | logger = logging.getLogger(__name__)
10 |
11 |
12 | def _extract_placeholders(text):
13 | return Counter(m.group(1) for m in re.finditer(r'\{\{(\w+)\}\}', text))
14 |
15 |
16 | @lru_cache(maxsize=None)
17 | def expected_placeholders_file(root_path):
18 | content = fs.read_file(root_path, const.REPO_SRC_PATH, const.PLACEHOLDERS_FILENAME)
19 | return json.loads(content)
20 |
21 |
22 | def _email_placeholders(root_path, email):
23 | _, contents = reader.read(root_path, email)
24 | content = ''.join(map(lambda c: c.get_content(), contents.values()))
25 | return _extract_placeholders(content)
26 |
27 |
28 | def get_email_validation(root_path, email):
29 | email_placeholders = _email_placeholders(root_path, email)
30 | expected_placeholders = expected_placeholders_file(root_path).get(email.name, {})
31 | missing_placeholders = set(expected_placeholders) - set(email_placeholders)
32 | extra_placeholders = set(email_placeholders) - set(expected_placeholders)
33 | diff_number = []
34 | for expected_name, expected_count in expected_placeholders.items():
35 | email_count = email_placeholders.get(expected_name)
36 | if email_count and expected_count != email_count:
37 | diff_number.append({'placeholder': expected_name, 'expected_count': expected_count, 'count': email_count})
38 |
39 | valid = not missing_placeholders and not extra_placeholders and not diff_number
40 | if not valid:
41 | errors = {
42 | 'missing': list(missing_placeholders),
43 | 'extra': list(extra_placeholders),
44 | 'diff_number': diff_number
45 | }
46 | else:
47 | errors = None
48 |
49 | return {'valid': valid, 'errors': errors}
50 |
51 |
52 | def generate_config(root_path):
53 | emails = fs.emails(root_path, locale=const.DEFAULT_LOCALE)
54 | placeholders = {email.name: _email_placeholders(root_path, email) for email in emails}
55 | return placeholders
56 |
--------------------------------------------------------------------------------
/tests/test_placeholder.py:
--------------------------------------------------------------------------------
1 | from unittest import TestCase
2 | from unittest.mock import patch
3 | from collections import Counter
4 |
5 | from email_parser import placeholder, fs
6 | from email_parser.model import *
7 |
8 |
9 | class TestGenerator(TestCase):
10 | def setUp(self):
11 | super().setUp()
12 |
13 | self.patch_fs = patch('email_parser.placeholder.fs')
14 | self.mock_fs = self.patch_fs.start()
15 | self.mock_fs.read_file.return_value = 'test'
16 |
17 | self.patch_reader = patch('email_parser.placeholder.reader')
18 | self.mock_reader = self.patch_reader.start()
19 |
20 | def tearDown(self):
21 | super().tearDown()
22 | self.patch_fs.stop()
23 | self.patch_reader.stop()
24 |
25 | def test_happy_path(self):
26 | self.mock_fs.emails.return_value = iter([Email('test_name', 'en', 'path')])
27 | self.mock_reader.read.return_value = ('', {'segment': Placeholder('segment', '{{placeholder}}')})
28 | config = placeholder.generate_config('.')
29 | self.assertEqual(config, {'test_name': Counter({'placeholder': 1})})
30 |
31 | def test_no_emails(self):
32 | self.mock_fs.emails.return_value = iter([])
33 | config = placeholder.generate_config('.')
34 | self.assertEqual(config, {})
35 |
36 |
37 | class TestValidate(TestCase):
38 | def setUp(self):
39 | self.email = fs.Email('test_name', 'en', 'path')
40 | placeholder.expected_placeholders_file.cache_clear()
41 |
42 | self.patch_reader = patch('email_parser.placeholder.reader')
43 | self.mock_reader = self.patch_reader.start()
44 | self.mock_reader.read.return_value = ('', {'segment': Placeholder('segment', '{{test_placeholder}}')})
45 |
46 | self.patch_config = patch('email_parser.placeholder.expected_placeholders_file')
47 | self.mock_config = self.patch_config.start()
48 | self.mock_config.return_value = {self.email.name: {'test_placeholder': 1}}
49 |
50 | def tearDown(self):
51 | super().tearDown()
52 | self.patch_reader.stop()
53 |
54 | def test_happy_path(self):
55 | actual = placeholder.get_email_validation('.', self.email, )
56 | expected = {'valid': True, 'errors': None}
57 | self.assertEqual(expected, actual)
58 |
59 | def test_missing_placeholder(self):
60 | self.mock_reader.read.return_value = ('', {'segment': Placeholder('segment', 'content')})
61 | actual = placeholder.get_email_validation('.', self.email)
62 | expected = {'valid': False, 'errors': {'missing': ['test_placeholder'], 'extra': [], 'diff_number': []}}
63 | self.assertEqual(expected, actual)
64 |
65 | def test_extra_placeholder(self):
66 | self.mock_config.return_value = {self.email.name: {}}
67 | actual = placeholder.get_email_validation('.', self.email)
68 | expected = {'valid': False, 'errors': {'missing': [], 'extra': ['test_placeholder'], 'diff_number': []}}
69 | self.assertEqual(expected, actual)
70 |
71 | def test_diffrent_placeholder_count(self):
72 | self.mock_reader.read.return_value = ('', {'segment': Placeholder('segment',
73 | '{{test_placeholder}}{{test_placeholder}}')})
74 | actual = placeholder.get_email_validation('.', self.email)
75 | expected = {
76 | 'valid': False,
77 | 'errors': {
78 | 'missing': [],
79 | 'extra': [],
80 | 'diff_number': [{'placeholder': 'test_placeholder', 'count': 2, 'expected_count': 1}]
81 | }
82 | }
83 | self.assertEqual(expected, actual)
84 |
--------------------------------------------------------------------------------
/email_parser/markdown_ext.py:
--------------------------------------------------------------------------------
1 | from markdown.inlinepatterns import Pattern, ImagePattern, LinkPattern, LINK_RE, IMAGE_LINK_RE
2 | from markdown.blockprocessors import BlockProcessor
3 | from markdown.extensions import Extension
4 | import re
5 |
6 | from . import const
7 |
8 |
9 | class InlineBlockProcessor(BlockProcessor):
10 | """
11 | Inlines the content instead of parsing it as markdown.
12 | """
13 | RE = re.compile(const.INLINE_TEXT_PATTERN)
14 |
15 | def test(self, parent, block):
16 | return bool(self.RE.match(block))
17 |
18 | def run(self, parent, blocks):
19 | block = blocks.pop(0)
20 | m = self.RE.match(block)
21 | if m:
22 | text = m.group(1)
23 | parent.text = text
24 |
25 |
26 | class BaseUrlImagePattern(Pattern):
27 | """
28 | Adds base url to images which have relative path.
29 | """
30 |
31 | url_pattern = re.compile(
32 | r'^(?:http|ftp)s?://' # http:// or https://
33 | r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
34 | r'localhost|' # localhost...
35 | r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
36 | r'(?::\d+)?' # optional port
37 | r'(?:/?|[/?]\S+)$',
38 | re.IGNORECASE)
39 |
40 | def __init__(self, images_dir, *args):
41 | super().__init__(*args)
42 | if images_dir:
43 | self.images_dir = images_dir.strip('/')
44 | else:
45 | self.images_dir = ''
46 | self.image_pattern = ImagePattern(*args)
47 |
48 | def _is_url(self, text):
49 | url = text.strip().strip('/').split(' ')[0]
50 | return self.url_pattern.match(url)
51 |
52 | def handleMatch(self, m):
53 | if self._is_url(m.group(10)):
54 | image = m.string
55 | else:
56 | image = const.IMAGE_PATTERN.format(m.group(2), self.images_dir, m.group(10).strip('/'))
57 | pattern = re.compile("^(.*?)%s(.*?)$" % self.image_pattern.pattern, re.DOTALL | re.UNICODE)
58 | match = re.match(pattern, ' ' + image + ' ')
59 | el = self.image_pattern.handleMatch(match)
60 | # each markdown image should have default style
61 | el.set('style', self.unescape('max-width: 100%;'))
62 | return el
63 |
64 |
65 | class NoTrackingLinkPattern(LinkPattern):
66 | def __init__(self, *args):
67 | super().__init__(*args)
68 |
69 | def handleMatch(self, m):
70 | el = super().handleMatch(m)
71 | if el.get('href') and el.get('href').startswith('!'):
72 | el.set('href', el.get('href')[1:])
73 | el.set('clicktracking', 'off')
74 | return el
75 |
76 |
77 | class InlineTextExtension(Extension):
78 | def extendMarkdown(self, md, md_globals):
79 | md.parser.blockprocessors.add('inline_text', InlineBlockProcessor(md.parser), '\n\t\t$img\n\t""")
103 | img = string.Template("""
""")
104 | mapping = dict(self._opt_attr)
105 | optional = ""
106 | div_style = "vertical-align: middle;text-align: center;"
107 | constraints = ""
108 | if self.alt:
109 | optional += " alt=\"{}\"".format(self.alt)
110 | for style_tag in ['max-width', 'max-height']:
111 | if style_tag in mapping:
112 | constraints += "{}: {};".format(style_tag, mapping[style_tag])
113 | mapping.update({
114 | 'style': div_style + constraints,
115 | 'id': self.id,
116 | 'optional': optional,
117 | 'src': self.src,
118 | 'img_style': constraints
119 | })
120 | mapping['img'] = img.substitute(mapping)
121 | final_wrapper = wrapper.substitute(mapping)
122 | return final_wrapper
123 |
124 | def set_attr(self, attr):
125 | self._opt_attr = attr
126 |
127 |
128 | class MissingPatternParamError(Exception):
129 | pass
130 |
131 |
132 | class MissingSubjectError(Exception):
133 | pass
134 |
135 |
136 | class MissingTemplatePlaceholderError(Exception):
137 | pass
138 |
139 |
140 | class RenderingError(Exception):
141 | pass
142 |
--------------------------------------------------------------------------------
/email_parser/cmd.py:
--------------------------------------------------------------------------------
1 | """
2 | Handles command line and calls the email parser with corrent options.
3 | """
4 |
5 | import argparse
6 | import logging
7 | import sys
8 | import os
9 | import shutil
10 | import asyncio
11 | import concurrent.futures
12 | from itertools import islice
13 | from functools import reduce
14 | from multiprocessing import Manager
15 |
16 | from . import const, Parser, config, fs
17 |
18 | logger = logging.getLogger(__name__)
19 |
20 |
21 | class ProgressConsoleHandler(logging.StreamHandler):
22 | store_msg_loglevels = (logging.ERROR, logging.WARN)
23 |
24 | on_same_line = False
25 | flush_errors = False
26 |
27 | def __init__(self, err_queue, warn_queue, *args, **kwargs):
28 | self.err_msgs_queue = err_queue
29 | self.warn_msgs_queue = warn_queue
30 | super(ProgressConsoleHandler, self).__init__(*args, **kwargs)
31 |
32 | def _store_msg(self, msg, loglevel):
33 | if loglevel == logging.ERROR:
34 | self.err_msgs_queue.put(msg)
35 | if loglevel == logging.WARN:
36 | self.warn_msgs_queue.put(msg)
37 |
38 | def error_msgs(self):
39 | while not self.err_msgs_queue.empty():
40 | yield self.err_msgs_queue.get()
41 |
42 | def warning_msgs(self):
43 | while not self.warn_msgs_queue.empty():
44 | yield self.warn_msgs_queue.get()
45 |
46 | def _print_msg(self, stream, msg, record):
47 | same_line = hasattr(record, 'same_line')
48 | if self.on_same_line and not same_line:
49 | stream.write(self.terminator)
50 | stream.write(msg)
51 | if same_line:
52 | self.on_same_line = True
53 | else:
54 | stream.write(self.terminator)
55 | self.on_same_line = False
56 | self.flush()
57 |
58 | def _flush_store(self, stream, msgs, header):
59 | stream.write(self.terminator)
60 | stream.write(header)
61 | stream.write(self.terminator)
62 | for idx, msg in enumerate(msgs):
63 | stream.write('%s. %s' % (idx + 1, msg))
64 | stream.write(self.terminator)
65 |
66 | def _flush_errors(self, stream):
67 | if not self.err_msgs_queue.empty():
68 | self._flush_store(stream, self.error_msgs(), 'ERRORS:')
69 | if not self.warn_msgs_queue.empty():
70 | self._flush_store(stream, self.warning_msgs(), 'WARNINGS:')
71 |
72 | def _write_msg(self, stream, msg, record):
73 | flush_errors = hasattr(record, 'flush_errors')
74 | if flush_errors:
75 | self._flush_errors(stream)
76 | self._print_msg(stream, msg, record)
77 |
78 | def emit(self, record):
79 | try:
80 | msg = self.format(record)
81 | stream = self.stream
82 | if record.levelno in self.store_msg_loglevels:
83 | self._store_msg(msg, record.levelno)
84 | else:
85 | self._write_msg(stream, msg, record)
86 | except (KeyboardInterrupt, SystemExit):
87 | raise
88 | except Exception:
89 | self.handleError(record)
90 |
91 |
92 | def read_args(argsargs=argparse.ArgumentParser):
93 | logger.debug('reading arguments list')
94 | args = argsargs(epilog='Brought to you by KeepSafe - www.getkeepsafe.com')
95 |
96 | args.add_argument('-i', '--images', help='Images base directory')
97 | args.add_argument('-vv', '--verbose', help='Generate emails despite errors', action='store_true')
98 | args.add_argument('-v', '--version', help='Show version', action='store_true')
99 |
100 | subparsers = args.add_subparsers(help='Parser additional commands', dest='command')
101 |
102 | config_parser = subparsers.add_parser('config')
103 | config_parser.add_argument('config_name', help='Name of config to generate. Available: `placeholders`')
104 |
105 | return args.parse_args()
106 |
107 |
108 | def _parse_and_save(email, parser):
109 | result = parser.render_email(email)
110 | if result:
111 | subject, text, html = result
112 | fs.save_parsed_email(parser.root_path, email, subject, text, html)
113 | return True
114 | else:
115 | return False
116 |
117 |
118 | def _parse_emails_batch(emails, parser):
119 | results = []
120 | for email in emails:
121 | try:
122 | results.append(_parse_and_save(email, parser))
123 | except Exception as ex:
124 | logger.exception('Cannot _parse_and_save email %s', email, exc_info=ex)
125 |
126 | result = reduce(lambda acc, res: acc and res, results)
127 | return result
128 |
129 |
130 | def _parse_emails(loop, root_path):
131 | shutil.rmtree(os.path.join(root_path, config.paths.destination), ignore_errors=True)
132 | emails = fs.emails(root_path)
133 | executor = concurrent.futures.ProcessPoolExecutor(max_workers=const.DEFAULT_WORKER_POOL)
134 | tasks = []
135 | parser = Parser(root_path)
136 |
137 | emails_batch = list(islice(emails, const.DEFAULT_WORKER_POOL))
138 | while emails_batch:
139 | task = loop.run_in_executor(executor, _parse_emails_batch, emails_batch, parser)
140 | tasks.append(task)
141 | emails_batch = list(islice(emails, const.DEFAULT_WORKER_POOL))
142 | results = yield from asyncio.gather(*tasks)
143 | result = reduce(lambda acc, result: True if acc and result else False, results)
144 | return result
145 |
146 |
147 | def parse_emails(root_path):
148 | loop = init_loop()
149 | result = loop.run_until_complete(_parse_emails(loop, root_path))
150 | return result
151 |
152 |
153 | def print_version():
154 | import pkg_resources
155 | version = pkg_resources.require('ks-email-parser')[0].version
156 | print(version)
157 | return True
158 |
159 |
160 | def generate_config(root_path):
161 | logger.info('generating config for placeholders')
162 | Parser(root_path).refresh_email_placeholders_config()
163 | return True
164 |
165 |
166 | def execute_command(args):
167 | if args.command == 'config' and args.config_name == 'placeholders':
168 | return generate_config(args)
169 | return False
170 |
171 |
172 | def init_log(verbose):
173 | log_level = logging.DEBUG if verbose else logging.INFO
174 | error_msgs_queue = Manager().Queue()
175 | warning_msgs_queue = Manager().Queue()
176 | handler = ProgressConsoleHandler(error_msgs_queue, warning_msgs_queue, stream=sys.stdout)
177 | logger.setLevel(log_level)
178 | logger.addHandler(handler)
179 |
180 |
181 | def init_loop():
182 | loop = asyncio.get_event_loop()
183 | loop.set_debug(False)
184 | return loop
185 |
186 |
187 | def main():
188 | root_path = os.getcwd()
189 | args = read_args()
190 | init_log(args.verbose)
191 | if args.images:
192 | config.base_img_path = args.images
193 | if args.version:
194 | result = print_version()
195 | elif args.command:
196 | result = execute_command(args)
197 | else:
198 | result = parse_emails(root_path)
199 | logger.info('\nAll done', extra={'flush_errors': True})
200 | sys.exit(0) if result else sys.exit(1)
201 |
202 |
203 | if __name__ == '__main__':
204 | main()
205 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # ks-email-parser [](https://travis-ci.org/KeepSafe/ks-email-parser)
2 |
3 | A command line tool name `ks-email-parser` to render HTML and text emails of markdown content.
4 |
5 | ## Goal
6 | The goal is to store emails in a unified format that is easy to translate and to generate HTML and text emails off. It should be easy for translators to maintain content formatting accords different languages.
7 |
8 | ## Requirements
9 |
10 | 1. Python 3.+
11 | 2. libxml - on OSX install with `xcode-select --install`
12 |
13 | ## Installation
14 |
15 | `make install`
16 |
17 | ## Usage
18 |
19 | `ks-email-parser` in root folder to generate all emails.
20 |
21 |
22 | ### Options
23 |
24 | Run `ks-email-parser --help` to see available options.
25 |
26 |
27 | ## Format
28 | Emails are defined as plain text or markdown for simple translation. The folder structure makes it easy to plug into an existing translation tool.
29 | The content of each email is stored in a XML file that contains all content, the subject and the assosiated HTML template.
30 |
31 | ### Syntax
32 |
33 | ```
34 |
35 |
36 | text_string
40 |
41 | ```
42 |
43 | #### Inline text
44 |
45 | By default any text simple text you put in a `` tag will be wrapped in a `` tag, so `simple text` would become `
simple text
`.
46 | This is standard markdown behaviour. In case you want to get raw text output, for example if you want to use it in a link tag,
47 | wrap the entire block in `[[text]]`, for example `[[www.google.pl]]` would become `www.google.pl`.
48 | This is true only for entire blocks of text (paragraphs separated by blanck lines), `[[www.google.pl]] hello`
49 | would be rendered as `[[www.google.pl]] hello`
50 |
51 | #### Link locale
52 |
53 | `{link_locale}` is special placeholder, which is resolved by mapping located in `src/link_locale_mappings.json` during rendering.
54 | If email locale couldn't be mapped to link locale, it's `en` by default.
55 |
56 | ```
57 | {
58 | "email locale": "link locale",
59 | "zh-TW-Hant": "zh"
60 | }
61 | ```
62 | If email locale will be `zh-TW-Hant`, then `[something](https://getkeepsafe.com/?locale={link_locale})` will be rendered as `[something](https://getkeepsafe.com/?locale=zh)`.
63 |
64 |
65 | #### Base url for images
66 |
67 | The parser will automatically add base_url to any image tag in markdown, so `` and base url `base_url`
68 | will produce `
`
69 |
70 | #### No tracking for links in Sendgrid
71 |
72 | Click tracking interferes with our deep links by augmenting the link. To disable click tracking for specific links add `!` in front of a link like so `[Alt text](!keepsafe://)`
73 |
74 | ### Elements
75 |
76 | #### `resource`
77 | Resource attributes:
78 |
79 | - **template** - the name of the corresponding HTML template to render
80 | - **style** - (optional) comma separated value of CSS to be used for HTML templates. Those will be applied in order of the list.
81 |
82 | #### `string`
83 | Content formatting
84 |
85 | - Plain text
86 | - Markdown wrapped in `![CDATA[`
87 |
88 | String attributes:
89 |
90 | - **name** - Name of the matching place holder `[name_value]` in the HTML template
91 | - **order** - (optional) in case of multiple string elements, they get rendered in the right order into text emails.
92 |
93 | #### Example
94 |
95 | ```
96 |
97 |
98 | Verify your email address with KeepSafe.
99 |
107 |
108 | ```
109 |
110 | ## Templates
111 |
112 | HTML templates use [Mustache](http://mustache.github.io/) to parse the template. You can use `{{name}}` inside a template and it will be replace by `name` string element from the email XML or you can use `{{global_name}}` and it will be replaced by `name` string elemenet from `/global.xml` (this file has exactly the same structure as usual email XML and behaviour, except it won't be parsed). You can find example of the templates in `templates_html` folder in this repo.
113 |
114 |
115 | ## Folder structure
116 |
117 | ```
118 | src/
119 | en/
120 | email_template.xml
121 | global.xml
122 | target/
123 | templates_html/
124 | html_template.html
125 | html_template.css
126 | ```
127 |
128 | - `src/` - all email content by local. e.g. `src/en/` for english content
129 | - `target/` - Output folder generated. Contains the same local subfolders as `src/`. Each email generates 3 files with the self explained file extensions `.txt`, `.html` and `.subject`
130 | - `templates_html/` - all HTML templates and CSS styles. A HTML template can have a corresponding CSS file with the same name.
131 |
132 | This structure is configurable. By changing `source`, `destination`, `templates` and `pattern` you can use a structure you like. The `pattern` parameter is especially useful as it controls directory layout and email names. the default is `{locale}/{name}.xml` but you can use `{name}.{locale}.xml` if you don't want to have nested directories. Keep in mind both `name` and `locale` are required in the pattern.
133 |
134 | ## Rendering
135 | *ks-email-parser* renders email content into HTML in 2 steps.
136 |
137 | 1. Render markdown files into simple HTML
138 | 2. Inserting CSS style definitions from `html_template.css` inline into the HTML. The goal is to support email clients that don't support inline CSS formatting.
139 |
140 | ### Strict mode
141 |
142 | You can use `--strict` option to make sure all placeholders are filled. If there are leftover placeholders the parsing will fail with an error.
143 |
144 | ### isText
145 |
146 | In case you want to put some non-text values in emails, like colors, you can use placeholders which will be ignored in text emails:
147 |
148 | `[[#C0D9D9]]`
149 |
150 | The only valid false value for isText is `false`, everything else counts as true including omitting the attribute.
151 |
152 | ## Placeholders validation
153 |
154 | To make sure the placeholders are consistent between languages and every language has all needed placeholders you can create configuration file to hold needed placeholders.
155 |
156 | ### Config file
157 |
158 | The file is a mapping of name to required placeholders and the number of times they appear. It's a json file with structure as below:
159 |
160 | ```
161 | {
162 | "email name" : {"placeholder1":1, "placeholder2":2}
163 | }
164 | ```
165 |
166 | You can generate the file in the provided source directory from existing emails with
167 |
168 | ```
169 | $ ks-email-parser config placeholders
170 | ```
171 |
172 | It will go through your email and extract placeholders.
173 |
174 | ### Validation
175 |
176 | If the config file is present in the source directory each email will be validated for having placeholders specified in the file. The parsing will fail with an error if any email is missing one of required placeholders.
177 |
178 | ## 3rd party support
179 | Some 3rd party services have custom formats to represent emails in multiple languages. This is a list of supported providers.
180 |
181 | ### Customer.io
182 | Customer.io defines their own multi language email format. More: [http://customer.io/docs/localization-i18n.html](http://customer.io/docs/localization-i18n.html)
183 |
184 | #### Usage
185 | `ks-email-parser customerio [email_name]`
186 | Generates a valid customer.io multi language email format into the `target/` folder.
187 |
--------------------------------------------------------------------------------
/email_parser/fs.py:
--------------------------------------------------------------------------------
1 | """
2 | All filesystem interaction.
3 | """
4 |
5 | import logging
6 | import os
7 | import parse
8 | from pathlib import Path
9 | from string import Formatter
10 |
11 | from . import const, config
12 | from .model import *
13 |
14 | logger = logging.getLogger(__name__)
15 |
16 |
17 | def _parse_params(pattern):
18 | params = [p for p in map(lambda e: e[1], Formatter().parse(pattern)) if p]
19 | if 'name' not in params:
20 | raise MissingPatternParamError(
21 | '{{name}} is a required parameter in the pattern but it is not present in {}'.format(pattern))
22 | if 'locale' not in params:
23 | raise MissingPatternParamError(
24 | '{{locale}} is a required parameter in the pattern but it is not present in {}'.format(pattern))
25 | return params
26 |
27 |
28 | def _has_correct_ext(path, pattern):
29 | return os.path.splitext(str(path))[1] == os.path.splitext(pattern)[1]
30 |
31 |
32 | # TODO extract globals
33 | def _emails(root_path, pattern, params):
34 | source_path = os.path.join(root_path, config.paths.source)
35 | wildcard_params = {k: '*' for k in params}
36 | wildcard_pattern = pattern.format(**wildcard_params)
37 | parser = parse.compile(pattern)
38 | glob_path = Path(source_path).glob(wildcard_pattern)
39 | for path in sorted(glob_path, key=lambda path: str(path)):
40 | if not path.is_dir() and _has_correct_ext(path, pattern):
41 | str_path = str(path.relative_to(source_path))
42 | result = parser.parse(str_path)
43 | if result: # HACK: result can be empty when pattern doesn't contain any placeholder
44 | result.named['path'] = str(path.resolve())
45 | if not str_path.endswith(const.GLOBALS_EMAIL_NAME + const.SOURCE_EXTENSION):
46 | logger.debug('loading email %s', result.named['path'])
47 | yield result
48 |
49 |
50 | def get_email_filepath(email_name, locale):
51 | pattern = config.pattern.replace('{name}', email_name)
52 | pattern = pattern.replace('{locale}', locale)
53 | filepath = os.path.join(config.paths.source, pattern)
54 | return filepath
55 |
56 |
57 | def get_template_filepath(root_path, template_name, template_type):
58 | return Path(root_path, config.paths.templates, template_type, template_name)
59 |
60 |
61 | def get_template_resources_filepaths(root_path, template):
62 | """
63 | :return: list of file paths of resources (css & html) related with given email
64 | """
65 | paths = [Path(root_path, config.paths.templates, f) or ' ' for f in template.styles_names]
66 | paths.append(get_template_filepath(root_path, template.name, template.type))
67 | return paths
68 |
69 |
70 | def emails(root_path, email_name=None, locale=None):
71 | """
72 | Resolves a pattern to a collection of emails.
73 |
74 | :param src_dir: base dir for the search
75 | :param pattern: search pattern
76 | :exclusive_path: single email path, glob path for emails subset or None to not affect emails set
77 |
78 | :returns: generator for the emails matching the pattern
79 | """
80 | params = _parse_params(config.pattern)
81 | pattern = config.pattern
82 | if email_name:
83 | pattern = pattern.replace('{name}', email_name)
84 | if locale:
85 | pattern = pattern.replace('{locale}', locale)
86 | for result in _emails(root_path, pattern, params):
87 | if email_name:
88 | result.named['name'] = email_name
89 | if locale:
90 | result.named['locale'] = locale
91 | yield Email(**result.named)
92 |
93 |
94 | def email(root_path, email_name, locale):
95 | """
96 | Gets an email by name and locale
97 |
98 | :param src_dir: base dir for the search
99 | :param pattern: search pattern
100 | :param email_name: email name
101 | :param locale: locale name or None for all locales
102 |
103 | :returns: generator for the emails with email_name
104 | """
105 | params = _parse_params(config.pattern)
106 | pattern = config.pattern.replace('{name}', email_name)
107 | pattern = pattern.replace('{locale}', locale)
108 | for result in _emails(root_path, pattern, params):
109 | result.named['name'] = email_name
110 | result.named['locale'] = locale
111 | return Email(**result.named)
112 | return None
113 |
114 |
115 | def global_email(root_path, locale):
116 | path = os.path.join(root_path, config.paths.source, locale, const.GLOBALS_EMAIL_NAME + const.SOURCE_EXTENSION)
117 | return Email(const.GLOBALS_EMAIL_NAME, locale, path)
118 |
119 |
120 | def read_file(*path_parts):
121 | """
122 | Helper for reading files
123 | """
124 | path = os.path.join(*path_parts)
125 | logger.debug('reading file from %s', path)
126 | with open(path) as fp:
127 | return fp.read()
128 |
129 |
130 | def save_file(content, *path_parts):
131 | """
132 | Helper for saving files
133 | """
134 | path = os.path.join(*path_parts)
135 | logger.debug('saving file to %s', path)
136 | with open(path, 'w') as fp:
137 | return fp.write(content)
138 |
139 |
140 | def delete_file(*path_parts):
141 | """
142 | Helper for deleting files
143 | """
144 | path = os.path.join(*path_parts)
145 | logger.debug('deleting file to %s', path)
146 | os.remove(path)
147 |
148 |
149 | def save_email(root_path, content, email_name, locale):
150 | pattern = config.pattern.replace('{locale}', locale)
151 | pattern = pattern.replace('{name}', email_name)
152 | path = os.path.join(root_path, config.paths.source, pattern)
153 | save_file(content, path)
154 | return path
155 |
156 |
157 | def save_template(root_path, template_filename, template_type, template_content):
158 | path = os.path.join(root_path, config.paths.templates, template_type.value, template_filename)
159 | written = save_file(template_content, path)
160 | return path, written
161 |
162 |
163 | def save_parsed_email(root_path, email, subject, text, html):
164 | """
165 | Saves an email. The locale and name are taken from email tuple.
166 |
167 | :param email: Email tuple
168 | :param subject: email's subject
169 | :param text: email's body as text
170 | :param html: email's body as html
171 | :param dest_dir: root destination directory
172 | """
173 | locale = email.locale or const.DEFAULT_LOCALE
174 | folder = os.path.join(root_path, config.paths.destination, locale)
175 | os.makedirs(folder, exist_ok=True)
176 | save_file(subject, folder, email.name + const.SUBJECT_EXTENSION)
177 | save_file(text, folder, email.name + const.TEXT_EXTENSION)
178 | save_file(html, folder, email.name + const.HTML_EXTENSION)
179 |
180 |
181 | def resources(root_path):
182 | """
183 | TODO: separate styles and templates
184 | Returns a tuple of lists: html templates list and css styles list
185 | :param root_path:
186 | :return:
187 | """
188 | templates = {}
189 | styles = []
190 | templates_path = os.path.join(root_path, config.paths.templates)
191 | css_glob = Path(templates_path).glob('*' + const.CSS_EXTENSION)
192 | css_files = sorted(css_glob, key=lambda p: str(p))
193 | styles.extend(map(lambda p: p.name, css_files))
194 | html_glob = Path(templates_path).glob('**/*' + const.HTML_EXTENSION)
195 | html_files = sorted(html_glob, key=lambda p: str(p))
196 | for html_file in html_files:
197 | parent = html_file.relative_to(templates_path).parent
198 | try:
199 | template_type = EmailType(str(parent)).value
200 | templates_list_by_type = templates.setdefault(template_type, [])
201 | templates_list_by_type.append(html_file.name)
202 | except ValueError:
203 | continue
204 | return templates, styles
205 |
206 |
207 | def get_html_sections_map(root_path):
208 | html_sections = {}
209 | html_sections_path = os.path.join(root_path, config.paths.sections)
210 | html_sections_glob = Path(html_sections_path).glob('*' + const.HTML_EXTENSION)
211 | html_sections_files = sorted(html_sections_glob, key=lambda p: str(p))
212 | for html_sections_path in html_sections_files:
213 | html_sections[html_sections_path.name] = read_file(*html_sections_path.parts)
214 | return html_sections
215 |
--------------------------------------------------------------------------------
/email_parser/renderer.py:
--------------------------------------------------------------------------------
1 | """
2 | Different ways of rendering emails.
3 | """
4 |
5 | import logging
6 | import re
7 | import xml.etree.ElementTree as ET
8 |
9 | import bs4
10 | import inlinestyler.utils as inline_styler
11 | import markdown
12 | import pystache
13 |
14 | from . import markdown_ext, const, utils, config
15 | from .model import *
16 | from .reader import parse_placeholder
17 |
18 | logger = logging.getLogger(__name__)
19 |
20 |
21 | def _md_to_html(text, base_url=None):
22 | extensions = [markdown_ext.inline_text(), markdown_ext.no_tracking()]
23 | if base_url:
24 | extensions.append(markdown_ext.base_url(base_url))
25 | return markdown.markdown(text, extensions=extensions)
26 |
27 |
28 | def _split_subject(placeholders):
29 | return (placeholders.get(const.SUBJECT_PLACEHOLDER),
30 | dict((k, v) for k, v in placeholders.items() if k != const.SUBJECT_PLACEHOLDER))
31 |
32 |
33 | def _transform_extended_tags(content):
34 | regex = r"{{(.*):(.*):(.*)}}"
35 | return re.sub(regex, lambda match: '{{%s}}' % match.group(2), content)
36 |
37 |
38 | class HtmlRenderer(object):
39 | """
40 | Renders email' body as html.
41 | """
42 |
43 | def __init__(self, template, email_locale):
44 | self.template = template
45 | self.locale = utils.normalize_locale(email_locale)
46 |
47 | def _inline_css(self, html, css):
48 | # an empty style will cause an error in inline_styler so we use a space instead
49 | css = css or ' '
50 | html_with_css = inline_styler.inline_css(css + html)
51 |
52 | # inline_styler will return a complete html filling missing html and body tags which we don't want
53 | if html.startswith('<'):
54 | body = ET.fromstring(html_with_css).find('.//body')
55 | body = ''.join(ET.tostring(e, encoding='unicode') for e in body)
56 | else:
57 | body = ET.fromstring(html_with_css).find('.//body/p')
58 | if body is None:
59 | raise ValueError()
60 | body = body.text
61 |
62 | return body.strip()
63 |
64 | def _wrap_with_text_direction(self, html):
65 | if self.locale in config.rtl_locales:
66 | soup = bs4.BeautifulSoup(html, 'html.parser')
67 | for element in soup.contents:
68 | try:
69 | element['dir'] = 'rtl'
70 | break
71 | except TypeError:
72 | continue
73 | return soup.prettify()
74 | else:
75 | return html
76 |
77 | def _wrap_with_highlight(self, html, highlight):
78 | attr_id = highlight.get('id', '')
79 | attr_style = highlight.get('style', '')
80 |
81 | soup = bs4.BeautifulSoup(html, 'html.parser')
82 | tag = soup.new_tag('div', id=attr_id, style=attr_style)
83 | tag.insert(0, soup)
84 | return tag.prettify()
85 |
86 | def _render_placeholder(self, placeholder, variant=None, highlight=None):
87 | content = placeholder.get_content(variant)
88 | if not content.strip():
89 | return content
90 | content = content.replace(const.LOCALE_PLACEHOLDER, self.locale)
91 | if placeholder.type == PlaceholderType.raw:
92 | return content
93 | else:
94 | html = _md_to_html(content, config.base_img_path)
95 | html = self._inline_css(html, self.template.styles)
96 | if highlight and highlight.get('placeholder') == placeholder.name and highlight.get('variant') == variant:
97 | html = self._wrap_with_highlight(html, highlight)
98 | return html
99 |
100 | def _concat_parts(self, subject, parts, variant):
101 | subject = subject.get_content(variant) if subject is not None else ''
102 | placeholders = dict(parts.items() | {'subject': subject, 'base_url': config.base_img_path}.items())
103 | try:
104 | # pystache escapes html by default, we pass escape option to disable this
105 | renderer = pystache.Renderer(escape=lambda u: u, missing_tags='strict')
106 | # since pystache tags parsing cant be easily extended: transform all tags extended with types to names only
107 | content = _transform_extended_tags(self.template.content)
108 | return renderer.render(content, placeholders)
109 | except pystache.context.KeyNotFoundError as e:
110 | message = 'template %s for locale %s has missing placeholders: %s' % (self.template.name, self.locale, e)
111 | raise MissingTemplatePlaceholderError(message) from e
112 |
113 | def render(self, placeholders, variant=None, highlight=None):
114 | subject, contents = _split_subject(placeholders)
115 | parts = {k: self._render_placeholder(v, variant, highlight) for k, v in contents.items()}
116 | html = self._concat_parts(subject, parts, variant)
117 | html = self._wrap_with_text_direction(html)
118 | return html
119 |
120 |
121 | class TextRenderer(object):
122 | """
123 | Renders email's body as text.
124 | """
125 |
126 | def __init__(self, template, email_locale):
127 | # self.shortener = link_shortener.shortener(settings.shortener)
128 | self.template = template
129 | self.locale = utils.normalize_locale(email_locale)
130 |
131 | def _html_to_text(self, html):
132 | soup = bs4.BeautifulSoup(html, const.HTML_PARSER)
133 |
134 | # replace the value in with the href because soup.get_text() takes the value inside instead or href
135 | anchors = soup.find_all('a')
136 | for anchor in anchors:
137 | text = anchor.string or ''
138 | href = anchor.get('href') or text
139 | # href = self.shortener.shorten(href)
140 | if href != text:
141 | anchor.replace_with('{} ({})'.format(text, href))
142 | elif href:
143 | anchor.replace_with(href)
144 |
145 | # add prefix to lists, it wont be added automatically
146 | unordered_lists = soup('ul')
147 | for unordered_list in unordered_lists:
148 | for element in unordered_list('li'):
149 | if element.string:
150 | element.replace_with('- ' + element.string)
151 | ordered_lists = soup('ol')
152 | for ordered_list in ordered_lists:
153 | for idx, element in enumerate(ordered_list('li')):
154 | element.replace_with('%s. %s' % (idx + 1, element.string))
155 |
156 | return soup.get_text()
157 |
158 | def _md_to_text(self, text, base_url=None):
159 | html = _md_to_html(text, base_url)
160 | return self._html_to_text(html)
161 |
162 | def render(self, placeholders, variant=None):
163 | _, contents = _split_subject(placeholders)
164 | parts = [
165 | self._md_to_text(contents[p].get_content(variant).replace(const.LOCALE_PLACEHOLDER, self.locale))
166 | for p in self.template.placeholders if p in contents if contents[p].type != PlaceholderType.attribute]
167 | return const.TEXT_EMAIL_PLACEHOLDER_SEPARATOR.join(v for v in filter(bool, parts))
168 |
169 |
170 | class SubjectRenderer(object):
171 | """
172 | Renders email's subject as text.
173 | """
174 |
175 | def render(self, placeholders, variant=None):
176 | subject, _ = _split_subject(placeholders)
177 | if subject is None:
178 | raise MissingSubjectError('Subject is required for every email')
179 | return subject.get_content(variant)
180 |
181 |
182 | def render(email_locale, template, placeholders, variant=None, highlight=None):
183 | subject_renderer = SubjectRenderer()
184 | subject = subject_renderer.render(placeholders, variant)
185 |
186 | text_renderer = TextRenderer(template, email_locale)
187 | text = text_renderer.render(placeholders, variant)
188 |
189 | html_renderer = HtmlRenderer(template, email_locale)
190 | try:
191 | html = html_renderer.render(placeholders, variant, highlight)
192 | except MissingTemplatePlaceholderError as e:
193 | message = 'failed to generate html content for locale: {} with message: {}'.format(email_locale, e)
194 | raise RenderingError(message) from e
195 |
196 | return subject, text, html
197 |
--------------------------------------------------------------------------------
/tests/test_reader.py:
--------------------------------------------------------------------------------
1 | import os.path
2 | from unittest import TestCase
3 | from unittest.mock import patch
4 |
5 | from lxml import etree
6 |
7 | from email_parser import reader
8 | from email_parser.model import *
9 |
10 |
11 | def read_fixture(filename, decoder=None):
12 | with open(os.path.join('tests/fixtures', filename)) as fp:
13 | content = fp.read()
14 | if decoder:
15 | return decoder(content)
16 | return content
17 |
18 |
19 | class TestReader(TestCase):
20 | def setUp(self):
21 | super().setUp()
22 | self.maxDiff = None
23 | self.email = Email(name='dummy', locale='dummy', path='dummy')
24 | self.email_content = """
25 |
26 | dummy subject
27 | dummy content
28 |
29 | - hello
30 | - bye
31 |
32 |
33 | """
34 | self.globals_xml = etree.fromstring("""
35 |
36 | dummy global
37 | asc
38 |
39 | """)
40 | self.template_str = '{{content}}{{global_content}}'
41 |
42 | self.patch_parse = patch('email_parser.reader.etree.parse')
43 | self.mock_parse = self.patch_parse.start()
44 | self.mock_parse.return_value = etree.ElementTree(self.globals_xml).getroot()
45 |
46 | self.patch_fs = patch('email_parser.reader.fs')
47 | self.mock_fs = self.patch_fs.start()
48 | self.mock_fs.read_file.return_value = 'test'
49 |
50 | def tearDown(self):
51 | super().tearDown()
52 | self.patch_fs.stop()
53 | self.patch_parse.stop()
54 |
55 | def test_template(self):
56 | self.mock_fs.read_file.side_effect = iter([self.email_content, self.template_str, 'test'])
57 | expected_placeholders = {
58 | 'content': MetaPlaceholder('content'),
59 | 'global_content': MetaPlaceholder('global_content')
60 | }
61 | expected_template = Template('dummy_template.html', ['dummy_template.css'], '',
62 | self.template_str, expected_placeholders, EmailType.transactional.value)
63 | template, _ = reader.read('.', self.email)
64 | self.assertEqual(expected_template, template)
65 |
66 | def test_get_template_parts_no_email_type(self):
67 | self.mock_fs.read_file.side_effect = iter([self.template_str])
68 | content, placeholders = reader.get_template_parts('root-path', 'basic_template.html', None)
69 | self.assertEqual(content, self.template_str)
70 |
71 | def test_placeholders(self):
72 | self.mock_fs.read_file.side_effect = iter([self.email_content, self.template_str, 'test'])
73 | expected = {
74 | 'subject': Placeholder('subject', 'dummy subject'),
75 | 'content': Placeholder('content', 'dummy content'),
76 | 'global_content': Placeholder('global_content', 'dummy global', True),
77 | 'block': Placeholder('block', 'hello', False, PlaceholderType.text, {'B': 'bye'})
78 | }
79 | _, placeholders = reader.read('.', self.email)
80 | self.assertEqual(expected.keys(), placeholders.keys())
81 |
82 | def test_template_with_multiple_styles(self):
83 | email_content = """
84 |
86 |
87 |
88 |
89 | """
90 | self.mock_fs.read_file.return_value = 'test'
91 | self.mock_fs.global_email().path = 'test'
92 | template, _ = reader.read_from_content('.', email_content, 'en')
93 | self.assertEqual('', template.styles)
94 |
95 | def test_template_with_bitmap_placeholder(self):
96 | email_content = """
97 |
98 |
99 | - XXX
100 | - YYY
101 |
102 |
103 | """
104 | template_str = '{{image:MYIMAGE:max_width=160}}'
105 | self.mock_fs.read_file.side_effect = iter([email_content, template_str, 'test'])
106 | template, placeholders = reader.read_from_content('.', email_content, 'en')
107 | self.assertTrue('MY_IMAGE' in placeholders.keys())
108 |
109 | def test_read_by_content(self):
110 | self.mock_fs.read_file.return_value = self.template_str
111 | template, _ = reader.read_from_content('.', self.email_content, 'en')
112 | self.assertEqual(template.name, 'dummy_template.html')
113 |
114 | def test_on_missing_content_return_fallback(self):
115 | self.mock_fs.read_file.return_value = None
116 | template, _ = reader.read('.', self.email)
117 | self.assertEqual(self.mock_fs.read_file.call_count, 2)
118 |
119 | def test_on_malformed_content_return_fallback(self):
120 | malformed_email_content = """
121 |
123 |
124 |
125 | """
126 | self.mock_fs.read_file.side_effect = iter([malformed_email_content,
127 | self.email_content,
128 | self.template_str,
129 | 'test'])
130 | template, _ = reader.read('.', self.email)
131 | self.assertEqual(self.mock_fs.read_file.call_count, 4)
132 |
133 |
134 | class TestWriter(TestCase):
135 | def setUp(self):
136 | self.maxDiff = None
137 |
138 | self.patch_fs = patch('email_parser.reader.fs')
139 | self.mock_fs = self.patch_fs.start()
140 | self.mock_fs.read_file.return_value = 'test'
141 |
142 | self.template_str = '{{subject}}{{content}}{{global_content}}'
143 | self.mock_fs.read_file.side_effect = iter([self.template_str])
144 |
145 | def elements_equal(self, e1, e2):
146 | self.assertEqual(e1.tag, e2.tag)
147 | self.assertEqual(e1.text, e2.text)
148 | self.assertEqual(e1.tail, e2.tail)
149 | self.assertEqual(e1.attrib, e2.attrib)
150 | self.assertEqual(len(e1), len(e2))
151 | return all(self.elements_equal(c1, c2) for c1, c2 in zip(e1, e2))
152 |
153 | def tearDown(self):
154 | super().tearDown()
155 | self.patch_fs.stop()
156 |
157 | def test_create_email_content(self):
158 | expected = read_fixture('email.xml').strip()
159 | placeholders = [
160 | Placeholder('content', 'dummy content'),
161 | Placeholder('subject', 'dummy subject', False, PlaceholderType.text, {'B': 'better subject'}),
162 | ]
163 |
164 | result = reader.create_email_content('dummy_root', 'basic_template.html', ['style1.css'], placeholders,
165 | EmailType.transactional)
166 | self.assertMultiLineEqual(expected, result.strip())
167 |
168 | def test_create_content_with_type(self):
169 | expected = read_fixture('email_with_type.xml').strip()
170 | placeholders = [
171 | Placeholder('content', 'dummy content'),
172 | Placeholder('subject', 'dummy subject', False, PlaceholderType.text, {'B': 'better subject'}),
173 | ]
174 |
175 | result = reader.create_email_content('dummy_root', 'dummy_template_name.html', ['style1.css'], placeholders,
176 | EmailType.transactional)
177 | self.assertMultiLineEqual(expected, result.strip())
178 |
179 | def test_create_content_with_metaplaceholder(self):
180 | expected_xml = read_fixture('email_inference.xml', etree.XML)
181 | tpl = read_fixture('email_inference_raw.html').strip()
182 | bitmap_attr = {
183 | 'max-width': '160px',
184 | 'max-height': '160px'
185 | }
186 | placeholders = [
187 | BitmapPlaceholder('MY_BITMAP', 'ID', 'SRC_LINK', 'ALT_TEXT', None, None, **bitmap_attr),
188 | ]
189 | self.mock_fs.read_file.side_effect = iter([tpl])
190 | result = reader.create_email_content('dummy_root', 'basic_template.html', ['style1.css'], placeholders,
191 | EmailType.transactional)
192 | xml_result = etree.fromstring(result)
193 | self.elements_equal(expected_xml, xml_result)
194 |
195 |
196 | class TestParsing(TestCase):
197 | def test_parsing_meta_complex(self):
198 | placeholder_str = 'text:name:arg1=0;arg2=abcd'
199 | expected = MetaPlaceholder('name', PlaceholderType.text, {'arg1': '0', 'arg2': 'abcd'})
200 | result = reader.parse_placeholder(placeholder_str)
201 | self.assertEqual(result, expected)
202 |
203 | def test_parsing_meta_simple(self):
204 | placeholder_str = 'name'
205 | expected = MetaPlaceholder('name')
206 | result = reader.parse_placeholder(placeholder_str)
207 | self.assertEqual(result, expected)
208 |
--------------------------------------------------------------------------------
/tests/test_parser.py:
--------------------------------------------------------------------------------
1 | import os
2 | from unittest import TestCase
3 | from unittest.mock import patch
4 |
5 | import email_parser
6 | from email_parser import config
7 | from email_parser.model import EmailType
8 |
9 |
10 | def read_fixture(filename):
11 | with open(os.path.join('tests/fixtures', filename)) as fp:
12 | return fp.read()
13 |
14 |
15 | class TestParser(TestCase):
16 | def setUp(self):
17 | self.parser = email_parser.Parser('./tests')
18 | self.maxDiff = None
19 |
20 | def tearDown(self):
21 | config.init()
22 |
23 | def test_get_template_for_email(self):
24 | email = self.parser.get_template_for_email('email', 'en')
25 | self.assertEqual(email, read_fixture('email.raw.html'))
26 |
27 | def test_parse_email(self):
28 | subject, text, html = self.parser.render('email', 'en')
29 | self.assertEqual(subject, read_fixture('email.subject').strip())
30 | self.assertEqual(html, read_fixture('email.html'))
31 | self.assertEqual(text, read_fixture('email.text').strip())
32 |
33 | def test_parse_email_variant(self):
34 | subject, text, html = self.parser.render('email', 'en', 'B')
35 | self.assertEqual(subject, read_fixture('email.b.subject').strip())
36 | self.assertEqual(html, read_fixture('email.b.html'))
37 | self.assertEqual(text, read_fixture('email.b.text').strip())
38 |
39 | def test_get_email_names(self):
40 | names = self.parser.get_email_names()
41 | self.assertListEqual(
42 | list(names), [
43 | 'email', 'email_globale', 'email_order', 'email_render_with_inference',
44 | 'email_subject_resend', 'email_subjects_ab', 'fallback', 'missing_placeholder', 'placeholder'
45 | ])
46 |
47 | def test_get_email_placeholders(self):
48 | placeholders = self.parser.get_email_placeholders()
49 | self.assertEqual(len(placeholders.keys()), 8)
50 | self.assertCountEqual(placeholders['placeholder'], ['unsubscribe_link', 'placeholder'])
51 |
52 | def test_render(self):
53 | subject, text, html = self.parser.render('placeholder', 'en')
54 | self.assertEqual(text, 'Dummy content {{placeholder}}\n\nDummy inline')
55 |
56 | def test_create_email(self):
57 | placeholders = {
58 | 'subject': {
59 | 'content': "dummy subject",
60 | 'is_text': True,
61 | 'is_global': False,
62 | 'type': 'text',
63 | 'is_global': False,
64 | 'variants': {
65 | 'B': 'better subject'
66 | }
67 | },
68 | 'content': {
69 | 'content': "dummy content",
70 | 'type': 'text',
71 | 'is_global': False
72 | },
73 | 'global_content': {
74 | 'content': "global dummy content",
75 | 'type': 'text',
76 | 'is_global': True
77 | },
78 | }
79 | expected = read_fixture('email.xml').strip()
80 | content = self.parser.create_email_content('basic_template.html', ['style1.css'], placeholders, 'transactional')
81 | self.assertMultiLineEqual(content.strip(), expected.strip())
82 |
83 | def test_create_email_without_emailtype(self):
84 | placeholders = {
85 | 'subject': {
86 | 'content': "dummy subject",
87 | 'is_text': True,
88 | 'is_global': False,
89 | 'type': 'text',
90 | 'is_global': False,
91 | 'variants': {
92 | 'B': 'better subject'
93 | }
94 | },
95 | 'content': {
96 | 'content': "dummy content",
97 | 'type': 'text',
98 | 'is_global': False
99 | },
100 | 'global_content': {
101 | 'content': "global dummy content",
102 | 'type': 'text',
103 | 'is_global': True
104 | },
105 | }
106 | self.parser.create_email_content('basic_template.html', ['style1.css'], placeholders)
107 |
108 | def test_get_template(self):
109 | expected = ['subject', 'color', 'content', 'inline', 'image', 'image_absolute']
110 | _, actual = self.parser.get_template('basic_template.html', 'transactional')
111 | self.assertEqual(set(actual), set(expected))
112 |
113 | def test_get_template_without_type(self):
114 | expected = ['subject', 'color', 'content', 'inline', 'image', 'image_absolute']
115 | _, actual = self.parser.get_template('basic_template.html')
116 | self.assertEqual(set(actual), set(expected))
117 |
118 | def test_get_resources(self):
119 | basic_template_placeholders = ['subject', 'color', 'content', 'inline', 'image', 'image_absolute']
120 | globale_template_placeholders = ['subject', 'color', 'content', 'inline', 'image', 'image_absolute',
121 | 'global_unsubscribe']
122 | actual_templates, styles, sections = self.parser.get_resources()
123 | # are templates separated by type?
124 | self.assertIn('marketing', actual_templates.keys())
125 | self.assertIn('transactional', actual_templates.keys())
126 | # are templates assigned to correct type?
127 | self.assertIn('basic_marketing_template.html', actual_templates['marketing'].keys())
128 | self.assertIn('basic_template.html', actual_templates['transactional'].keys())
129 | # are placeholders assigned to correct template?
130 | template_html__keys = actual_templates['transactional']['basic_template.html'].keys()
131 | self.assertEqual(basic_template_placeholders, list(template_html__keys))
132 | template_html__keys = actual_templates['marketing']['globale_template.html'].keys()
133 | self.assertEqual(globale_template_placeholders, list(template_html__keys))
134 | self.assertIn('header-with-background.html', sections.keys())
135 | self.assertIn('basic_template.css', styles)
136 |
137 | def test_get_email_filepaths_all_locale(self):
138 | expected = ['src/ar/email.xml', 'src/en/email.xml', 'src/fr/email.xml']
139 | actual = self.parser.get_email_filepaths('email')
140 | self.assertEqual(actual, expected)
141 |
142 | def test_get_email_filepaths_single_locale(self):
143 | expected = ['src/ar/email.xml']
144 | actual = self.parser.get_email_filepaths('email', 'ar')
145 | self.assertEqual(actual, expected)
146 |
147 | def test_equality(self):
148 | parserA = email_parser.Parser('./tests')
149 | parserB = email_parser.Parser('./tests')
150 | self.assertEqual(parserA, parserB)
151 |
152 | def test_get_email_components(self):
153 | expected = ('basic_template.html', EmailType.transactional.value, ['basic_template.css'],
154 | {
155 | 'color': {
156 | 'content': '[[#C0D9D9]]',
157 | 'is_global': False,
158 | 'name': 'color',
159 | 'type': 'attribute',
160 | 'variants': {}
161 | },
162 | 'content': {
163 | 'content': 'Dummy content',
164 | 'is_global': False,
165 | 'name': 'content',
166 | 'type': 'text',
167 | 'variants': {
168 | 'B': 'Awesome content'
169 | }
170 | },
171 | 'image': {
172 | 'content': '',
173 | 'is_global': False,
174 | 'name': 'image',
175 | 'type': 'text',
176 | 'variants': {}
177 | },
178 | 'image_absolute': {
179 | 'content': '',
180 | 'is_global': False,
181 | 'name': 'image_absolute',
182 | 'type': 'text',
183 | 'variants': {}
184 | },
185 | 'inline': {
186 | 'content': 'Dummy inline',
187 | 'is_global': False,
188 | 'name': 'inline',
189 | 'type': 'raw',
190 | 'variants': {}
191 | },
192 | 'subject': {
193 | 'content': 'Dummy subject',
194 | 'is_global': False,
195 | 'name': 'subject',
196 | 'type': 'text',
197 | 'variants': {'B': 'Awesome subject'}
198 | }
199 | })
200 | actual = self.parser.get_email_components('email', 'en')
201 | self.assertEqual(actual, expected)
202 |
203 | def test_get_email_variants(self):
204 | actual = self.parser.get_email_variants('email')
205 | self.assertEqual(actual, ['B'])
206 |
207 | @patch('email_parser.fs.save_file')
208 | def test_save_email_variant_default_content(self, mock_save):
209 | expected = read_fixture('email_en_default.xml')
210 | self.parser.save_email_variant_as_default('email', ['en'], None)
211 | content, _ = mock_save.call_args[0]
212 | self.assertMultiLineEqual(content.strip(), expected.strip())
213 |
214 | @patch('email_parser.fs.save_file')
215 | def test_save_email_variant_b_content(self, mock_save):
216 | expected = read_fixture('email_en_b.xml')
217 | self.parser.save_email_variant_as_default('email', ['en'], 'B')
218 | content, _ = mock_save.call_args[0]
219 | self.assertMultiLineEqual(content.strip(), expected.strip())
220 |
221 | def test_original(self):
222 | actual = self.parser.original('email_order', 'en')
223 | self.assertEqual(actual, read_fixture('original.txt').strip())
224 |
225 | def test_get_emails_resources_paths(self):
226 | expected = ['templates_html/basic_template.css',
227 | 'templates_html/transactional/basic_template.html']
228 | actual = self.parser.get_email_resources_filepaths('email')
229 | self.assertEqual(set(expected), set(actual))
230 |
231 | def test_parse_email_with_inference(self):
232 | subject, text, html = self.parser.render('email_render_with_inference', 'en')
233 | self.assertEqual(html, read_fixture('email_render_with_inference.html'))
234 |
--------------------------------------------------------------------------------
/email_parser/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | email_parser
3 | ~~~~~~~~~~
4 |
5 | Parses emails from a source directory and outputs text and html format.
6 |
7 | :copyright: (c) 2014 by KeepSafe.
8 | :license: Apache, see LICENSE for more details.
9 | """
10 | import json
11 | import os
12 |
13 | from . import placeholder, fs, reader, renderer, const, config
14 | from .model import *
15 |
16 |
17 | class Parser:
18 | def __init__(self, root_path, **kwargs):
19 | self.root_path = root_path
20 | config.init(**kwargs)
21 |
22 | def __hash__(self):
23 | return hash(self.root_path)
24 |
25 | def __eq__(self, other):
26 | """
27 | since parser is just wrapper it means essentially that parsers are equal if they work on the same root_path
28 | sorting this out is needed for caching
29 | """
30 | return self.__hash__() == hash(other)
31 |
32 | def get_template_for_email(self, email_name, locale):
33 | email = fs.email(self.root_path, email_name, locale)
34 | if not email:
35 | return None
36 | template, _ = reader.read(self.root_path, email)
37 | return template.content
38 |
39 | def get_email_type(self, email_name, locale):
40 | email = fs.email(self.root_path, email_name, locale)
41 | return reader.get_email_type(self.root_path, email)
42 |
43 | def render(self, email_name, locale, variant=None):
44 | email = fs.email(self.root_path, email_name, locale)
45 | return self.render_email(email, variant)
46 |
47 | def render_email(self, email, variant=None):
48 | if not email:
49 | return None
50 | template, persisted_placeholders = reader.read(self.root_path, email)
51 | if template:
52 | return renderer.render(email.locale, template, persisted_placeholders, variant)
53 |
54 | def render_email_content(self, content, locale=const.DEFAULT_LOCALE, variant=None, highlight=None):
55 | template, persisted_placeholders = reader.read_from_content(self.root_path, content, locale)
56 | return renderer.render(locale, template, persisted_placeholders, variant=variant, highlight=highlight)
57 |
58 | def get_email(self, email_name, locale):
59 | email = fs.email(self.root_path, email_name, locale)
60 | return fs.read_file(email.path)
61 |
62 | def original(self, email_name, locale, variant=None):
63 | email = fs.email(self.root_path, email_name, locale)
64 | if not email:
65 | return None
66 | template, placeholders = reader.read(self.root_path, email)
67 | return '\n\n'.join([placeholders[name].get_content() for name in template.placeholders if
68 | placeholders[name].type == PlaceholderType.text])
69 |
70 | def get_email_components(self, email_name, locale):
71 | email = fs.email(self.root_path, email_name, locale)
72 | template, placeholders = reader.read(self.root_path, email)
73 | serialized_placeholders = {name: dict(placeholder) for name, placeholder in placeholders.items()}
74 | return template.name, template.type, template.styles_names, serialized_placeholders
75 |
76 | def get_email_variants(self, email_name):
77 | email = fs.email(self.root_path, email_name, const.DEFAULT_LOCALE)
78 | _, placeholders = reader.read(self.root_path, email)
79 | variants = set([name for _, p in placeholders.items() for name in p.variants.keys()])
80 | return list(variants)
81 |
82 | def delete_email(self, email_name):
83 | emails = fs.emails(self.root_path, email_name=email_name)
84 | files = []
85 | for email in emails:
86 | files.append(email.path)
87 | fs.delete_file(email.path)
88 | self.refresh_email_placeholders_config()
89 | return files
90 |
91 | def save_email(self, email_name, locale, content):
92 | saved_path = fs.save_email(self.root_path, content, email_name, locale)
93 | self.refresh_email_placeholders_config()
94 | return saved_path
95 |
96 | def save_email_variant_as_default(self, email_name, locales, variant, email_type=None):
97 | paths = []
98 | for locale in locales:
99 | email = fs.email(self.root_path, email_name, locale)
100 | template, placeholders = reader.read(self.root_path, email)
101 | placeholders_list = [p.pick_variant(variant) for _, p in placeholders.items() if not p.is_global]
102 | if email_type:
103 | email_type = EmailType(email_type)
104 | elif template.type:
105 | email_type = EmailType(template.type)
106 | content = reader.create_email_content(self.root_path, template.name, template.styles_names,
107 | placeholders_list, email_type)
108 | email_path = fs.save_email(self.root_path, content, email_name, locale)
109 | paths.append(email_path)
110 | return paths
111 |
112 | def create_email_content(self, template_name, styles_names, placeholders, email_type=None):
113 | placeholder_list = []
114 | non_globals = filter(lambda key: not placeholders[key].get('is_global', False), placeholders.keys())
115 | for placeholder_name in non_globals:
116 | placeholder_props = placeholders[placeholder_name]
117 | is_global = placeholder_props.get('is_global', False)
118 | variants = placeholder_props.get('variants', {})
119 | pt = placeholder_props.get('type', PlaceholderType.text.value)
120 | pt = PlaceholderType[pt]
121 | if pt != PlaceholderType.bitmap:
122 | content = placeholder_props['content']
123 | p = Placeholder(placeholder_name, content, is_global, pt, variants)
124 | else:
125 | bitmap_id = placeholder_props['id']
126 | bitmap_src = placeholder_props['src']
127 | bitmap_alt = placeholder_props.get('alt')
128 | bitmap_attr = placeholder_props.get('attributes', {})
129 | p = BitmapPlaceholder(placeholder_name, bitmap_id, bitmap_src, bitmap_alt, is_global, variants,
130 | **bitmap_attr)
131 | placeholder_list.append(p)
132 | if email_type:
133 | email_type = EmailType(email_type)
134 | return reader.create_email_content(self.root_path, template_name, styles_names, placeholder_list, email_type)
135 |
136 | def render_template_content(self, template_content, styles_names, placeholders, locale=const.DEFAULT_LOCALE):
137 | styles = reader.get_inline_style(self.root_path, styles_names)
138 | placeholders_objs = {name: Placeholder(name, content) for name, content in placeholders.items()}
139 | template = Template('preview', styles_names, styles, template_content, placeholders_objs, None)
140 | return renderer.render(locale, template, placeholders_objs)
141 |
142 | def get_email_names(self):
143 | return (email.name for email in fs.emails(self.root_path, locale=const.DEFAULT_LOCALE))
144 |
145 | def get_emails(self, locale=const.DEFAULT_LOCALE):
146 | return (email._asdict() for email in fs.emails(self.root_path, locale=locale))
147 |
148 | def get_email_placeholders(self):
149 | expected_placeholders = placeholder.expected_placeholders_file(self.root_path)
150 | return {k: list(v) for k, v in expected_placeholders.items()}
151 |
152 | def get_template(self, template_filename, template_type=None):
153 | try:
154 | template_type = EmailType(template_type)
155 | except ValueError:
156 | template_type = None
157 | content, placeholders = reader.get_template_parts(self.root_path, template_filename, template_type)
158 | return content, placeholders
159 |
160 | def save_template(self, template_filename, template_type, template_content):
161 | template_type = EmailType(template_type)
162 | return fs.save_template(self.root_path, template_filename, template_type, template_content)
163 |
164 | def refresh_email_placeholders_config(self):
165 | placeholders_config = placeholder.generate_config(self.root_path)
166 | if placeholders_config:
167 | fs.save_file(
168 | json.dumps(placeholders_config, sort_keys=True, indent=const.JSON_INDENT),
169 | self.get_placeholders_filepath())
170 | placeholder.expected_placeholders_file.cache_clear()
171 |
172 | def get_placeholders_filepath(self):
173 | return os.path.join(self.root_path, const.REPO_SRC_PATH, const.PLACEHOLDERS_FILENAME)
174 |
175 | def get_templates_directory_filepath(self):
176 | return os.path.join(self.root_path, config.paths.templates)
177 |
178 | def get_email_filepaths(self, email_name, locale=None):
179 | """
180 | return list of file paths for single email or collection of emails if locale = None
181 | :param email_name:
182 | :param locale:
183 | :return:
184 | """
185 | emails = fs.emails(self.root_path, email_name, locale)
186 | abs_paths = map(lambda email: fs.get_email_filepath(email.name, email.locale), emails)
187 | return list(abs_paths)
188 |
189 | def get_email_resources_filepaths(self, email_name):
190 | email = fs.email(self.root_path, email_name, 'en')
191 | if not email:
192 | return None
193 | template, _ = reader.read(self.root_path, email)
194 | file_paths = fs.get_template_resources_filepaths(self.root_path, template)
195 | return list(map(lambda p: str(p.relative_to(self.root_path)), file_paths))
196 |
197 | def get_email_placeholders_validation_errors(self, email_name, locale):
198 | email = fs.email(self.root_path, email_name, locale)
199 | return placeholder.get_email_validation(self.root_path, email)['errors']
200 |
201 | def get_resources(self):
202 | templates_view = {}
203 | templates, styles = fs.resources(self.root_path)
204 | sections_map = fs.get_html_sections_map(self.root_path)
205 | for template_type in templates:
206 | types_templates = templates[template_type]
207 | templates_view_type = templates_view.setdefault(template_type, {})
208 | for template_name in types_templates:
209 | tpl_content, tpl_placeholders = self.get_template(template_name, template_type)
210 | templates_view_type[template_name] = tpl_placeholders
211 | return templates_view, styles, sections_map
212 |
213 | def get_global_placeholders_map(self, locale=const.DEFAULT_LOCALE):
214 | global_placeholders = reader.get_global_placeholders(self.root_path, locale)
215 | return {name: placeholder.get_content() for name, placeholder in global_placeholders.items()}
216 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "{}"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright {yyyy} {name of copyright owner}
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/email_parser/reader.py:
--------------------------------------------------------------------------------
1 | """
2 | Extracts email information from an email file.
3 | """
4 |
5 | import logging
6 | import re
7 | from collections import OrderedDict
8 |
9 | from lxml import etree
10 |
11 | from . import fs, const, config
12 | from .model import *
13 |
14 | logger = logging.getLogger(__name__)
15 |
16 |
17 | def parse_placeholder(placeholder_str):
18 | args = {}
19 | regex = r"(.*):(.*):(.*)"
20 | try:
21 | match = re.match(regex, placeholder_str)
22 | placeholder_type = match.group(1)
23 | name = match.group(2)
24 | args_str = match.group(3)
25 | except AttributeError:
26 | return MetaPlaceholder(placeholder_str)
27 | try:
28 | placeholder_type = PlaceholderType(placeholder_type)
29 | except ValueError:
30 | raise ValueError('Placeholder definition %s uses invalid PlaceholderType' % placeholder_str)
31 | for attr_str in args_str.split(';'):
32 | try:
33 | attr_name, attr_value = attr_str.split('=')
34 | args[attr_name] = attr_value
35 | except ValueError:
36 | ValueError('Malformed attributes definition: %s'.format(args_str))
37 | return MetaPlaceholder(name, placeholder_type, args)
38 |
39 |
40 | def _placeholders(tree, prefix=''):
41 | if tree is None:
42 | return {}
43 | is_global = (prefix == const.GLOBALS_PLACEHOLDER_PREFIX)
44 | result = OrderedDict()
45 | for element in tree.xpath('./string | ./string-array | bitmap | ./array'):
46 | name = '{0}{1}'.format(prefix, element.get('name'))
47 | placeholder_type = PlaceholderType[element.get('type', PlaceholderType.text.value)]
48 | opt_attrs = dict(element.items())
49 | del opt_attrs['name']
50 | try:
51 | del opt_attrs['type']
52 | except KeyError:
53 | pass
54 | if element.tag == 'string':
55 | content = element.text or ''
56 | result[name] = Placeholder(name, content.strip(), is_global, placeholder_type, None, opt_attrs)
57 | elif element.tag == 'bitmap':
58 | del opt_attrs['src']
59 | del opt_attrs['id']
60 | p_id = element.get('id')
61 | src = element.get('src')
62 | alt = element.get('alt') or None
63 | if alt:
64 | del opt_attrs['alt']
65 | result[name] = BitmapPlaceholder(name, p_id, src, alt, is_global, None, **opt_attrs)
66 | elif element.tag in ['string-array', 'array']:
67 | content = ''
68 | variants = {}
69 | for item in element.findall('./item'):
70 | variant = item.get('variant')
71 | if variant:
72 | variants[variant] = item.text.strip()
73 | else:
74 | content = item.text.strip()
75 | result[name] = Placeholder(name, content, is_global, placeholder_type, variants, opt_attrs)
76 | else:
77 | raise Exception('Unknown tag:\n%s' % element)
78 |
79 | return result
80 |
81 |
82 | def get_template_parts(root_path, template_filename, template_type):
83 | content = None
84 | placeholders = OrderedDict()
85 |
86 | try:
87 | template_type = EmailType(template_type)
88 | except ValueError:
89 | template_type = None
90 |
91 | if template_type:
92 | template_path = str(fs.get_template_filepath(root_path, template_filename, template_type.value))
93 | content = fs.read_file(template_path)
94 | else:
95 | logger.warning('FIXME: no email_type set for: %s, trying all types..', template_filename)
96 | for email_type in EmailType:
97 | try:
98 | template_path = str(fs.get_template_filepath(root_path, template_filename, email_type.value))
99 | content = fs.read_file(template_path)
100 | break
101 | except FileNotFoundError:
102 | continue
103 |
104 | for m in re.finditer(r'{{(.+?)}}', content):
105 | placeholder_def = m.group(1)
106 | placeholder_meta = parse_placeholder(placeholder_def)
107 | placeholders[placeholder_meta.name] = placeholder_meta
108 |
109 | try:
110 | # TODO sad panda, refactor
111 | # base_url placeholder is not a content block
112 | del placeholders['base_url']
113 | except KeyError:
114 | pass
115 |
116 | return content, placeholders
117 |
118 |
119 | def get_inline_style(root_path, styles_names):
120 | if not len(styles_names):
121 | return ''
122 | css = [fs.read_file(root_path, config.paths.templates, f) or ' ' for f in styles_names]
123 | styles = '\n'.join(css)
124 | return '' % styles
125 |
126 |
127 | def _template(root_path, tree):
128 | styles = ''
129 | styles_names = []
130 |
131 | template_filename = tree.getroot().get('template')
132 | email_type = tree.getroot().get('email_type')
133 | content, placeholders = get_template_parts(root_path, template_filename, email_type)
134 | style_element = tree.getroot().get('style')
135 |
136 | if style_element:
137 | styles_names = style_element.split(',')
138 | styles = get_inline_style(root_path, styles_names)
139 |
140 | # TODO either read all or leave just names for content and styles
141 | return Template(template_filename, styles_names, styles, content, placeholders, email_type)
142 |
143 |
144 | def _handle_xml_parse_error(file_path, exception):
145 | pos = exception.position
146 | with open(file_path) as f:
147 | lines = f.read().splitlines()
148 | error_line = lines[pos[0] - 1]
149 | node_matches = re.findall(const.SEGMENT_REGEX, error_line[:pos[1]])
150 | segment_id = None
151 |
152 | if not len(node_matches):
153 | prev_line = pos[0] - 1
154 | search_part = ''.join(lines[:prev_line])
155 | node_matches = re.findall(const.SEGMENT_REGEX, search_part)
156 |
157 | if len(node_matches):
158 | name_matches = re.findall(const.SEGMENT_NAME_REGEX, node_matches[-1])
159 | if len(name_matches):
160 | segment_id = name_matches[-1]
161 | logger.exception('Unable to read content from %s\n%s\nSegment ID: %s\n_______________\n%s\n%s\n', file_path,
162 | exception, segment_id, error_line.replace('\t', ' '), " " * exception.position[1] + "^")
163 |
164 |
165 | def _read_xml(path):
166 | if not path:
167 | return None
168 | try:
169 | parser = etree.XMLParser(encoding='utf-8')
170 | return etree.parse(path, parser=parser)
171 | except etree.ParseError as e:
172 | _handle_xml_parse_error(path, e)
173 | return None
174 |
175 |
176 | def _read_xml_from_content(content):
177 | if not content:
178 | return None
179 | try:
180 | parser = etree.XMLParser(encoding='utf-8')
181 | root = etree.fromstring(content.encode('utf-8'), parser=parser)
182 | return etree.ElementTree(root)
183 | except etree.ParseError as e:
184 | logger.exception('Unable to parse XML content %s %s', content, e)
185 | return None
186 | except TypeError:
187 | # got None? no results
188 | return None
189 |
190 |
191 | def _sort_from_template(root_path, template_filename, template_type, placeholders):
192 | _, placeholders_ordered = get_template_parts(root_path, template_filename, template_type)
193 | ordered_placeholders_names = list(placeholders_ordered.keys())
194 | placeholders.sort(
195 | key=lambda item: ordered_placeholders_names.index(item.name) if item.name in ordered_placeholders_names else 99)
196 |
197 |
198 | def create_email_content(root_path, template_name, styles, placeholders, email_type):
199 | root = etree.Element('resources')
200 | root.set('template', template_name)
201 | root.set('style', ','.join(styles))
202 | if email_type:
203 | root.set('email_type', email_type.value)
204 | _sort_from_template(root_path, template_name, email_type, placeholders)
205 | for placeholder in placeholders:
206 | if placeholder.variants:
207 | new_content_tag = etree.SubElement(root, 'string-array', {
208 | 'name': placeholder.name,
209 | 'type': placeholder.type.value or PlaceholderType.text.value
210 | })
211 | default_item_tag = etree.SubElement(new_content_tag, 'item')
212 | default_item_tag.text = etree.CDATA(placeholder.get_content())
213 | for variant_name, variant_content in placeholder.variants.items():
214 | new_item_tag = etree.SubElement(new_content_tag, 'item', {'variant': variant_name})
215 | new_item_tag.text = etree.CDATA(variant_content)
216 | elif placeholder.type == PlaceholderType.bitmap:
217 | attr = {
218 | 'name': placeholder.name,
219 | 'type': placeholder.type.value,
220 | 'id': placeholder.id,
221 | 'src': placeholder.src
222 | }
223 | if placeholder.alt:
224 | attr['alt'] = placeholder.alt
225 | try:
226 | etree.SubElement(root, 'bitmap', attr)
227 | except TypeError:
228 | msg = 'Cannot create xml element with attrs: %s' % attr
229 | raise TypeError(msg)
230 | else:
231 | new_content_tag = etree.SubElement(root, 'string', {
232 | 'name': placeholder.name,
233 | 'type': placeholder.type.value or PlaceholderType.text.value
234 | })
235 | new_content_tag.text = etree.CDATA(placeholder.get_content())
236 | xml_as_str = etree.tostring(root, encoding='utf8', pretty_print=True)
237 | return xml_as_str.decode('utf-8')
238 |
239 |
240 | def get_global_placeholders(root_path, locale):
241 | globals_xml = _read_xml(fs.global_email(root_path, locale).path)
242 | return _placeholders(globals_xml, const.GLOBALS_PLACEHOLDER_PREFIX)
243 |
244 |
245 | def get_inferred_placeholders(meta_placeholders, placeholders):
246 | inferred_placeholders = OrderedDict(placeholders)
247 | for placeholder_name in placeholders:
248 | meta_placeholder = meta_placeholders.get(placeholder_name)
249 | if meta_placeholder and meta_placeholder.attributes:
250 | attr = meta_placeholder.attributes
251 | inferred_placeholders[placeholder_name].set_attr(attr)
252 | return inferred_placeholders
253 |
254 |
255 | def read_from_content(root_path, email_content, locale):
256 | email_xml = _read_xml_from_content(email_content)
257 | if not email_xml:
258 | return None, None
259 | template = _template(root_path, email_xml)
260 | if not template.name:
261 | logger.error('no HTML template name defined for given content')
262 | global_placeholders = get_global_placeholders(root_path, locale)
263 | placeholders = OrderedDict({name: content for name, content
264 | in global_placeholders.items()
265 | if name in template.placeholders})
266 | placeholders.update(_placeholders(email_xml).items())
267 | inferred_placeholders = get_inferred_placeholders(template.placeholders, placeholders)
268 |
269 | return template, inferred_placeholders
270 |
271 |
272 | def read(root_path, email):
273 | """
274 | Reads an email from a path.
275 |
276 | :param root_path: root path of repository
277 | :param email: instance of Email namedtuple
278 | :returns: tuple of email template, a collection of placeholders
279 | """
280 | email_content = fs.read_file(email.path)
281 | results = read_from_content(root_path, email_content, email.locale)
282 | if not results[0] and email.locale != const.DEFAULT_LOCALE:
283 | email = fs.email(root_path, email.name, const.DEFAULT_LOCALE)
284 | email_content = fs.read_file(email.path)
285 | results = read_from_content(root_path, email_content, email.locale)
286 |
287 | return results
288 |
289 |
290 | def get_email_type(root_path, email):
291 | email_content = fs.read_file(email.path)
292 | email_xml = _read_xml_from_content(email_content)
293 | return email_xml.getroot().get('email_type')
294 |
--------------------------------------------------------------------------------
/tests/test_renderer.py:
--------------------------------------------------------------------------------
1 | from unittest import TestCase
2 | from unittest.mock import patch
3 |
4 | from email_parser.model import *
5 | from email_parser import renderer, const, config
6 |
7 |
8 | class TestTextRenderer(TestCase):
9 | def setUp(self):
10 | self.email_locale = 'locale'
11 | self.template = Template('dummy', [], '', '{{content1}}',
12 | ['content', 'content1', 'content2'], None)
13 | self.r = renderer.TextRenderer(self.template, self.email_locale)
14 |
15 | def test_happy_path(self):
16 | placeholders = {'content': Placeholder('content', 'dummy content')}
17 | actual = self.r.render(placeholders)
18 | self.assertEqual('dummy content', actual)
19 |
20 | def test_render_variant(self):
21 | placeholders = {'content': Placeholder('content', 'dummy content', variants={'B': 'awesome content'})}
22 | actual = self.r.render(placeholders, variant='B')
23 | self.assertEqual('awesome content', actual)
24 |
25 | def test_concat_multiple_placeholders(self):
26 | placeholders = {
27 | 'content1': Placeholder('content', 'dummy content'),
28 | 'content2': Placeholder('content2', 'dummy content')
29 | }
30 | expected = const.TEXT_EMAIL_PLACEHOLDER_SEPARATOR.join(['dummy content', 'dummy content'])
31 | actual = self.r.render(placeholders)
32 | self.assertEqual(expected, actual)
33 |
34 | def test_concat_multiple_placeholders_with_variants(self):
35 | placeholders = {
36 | 'content1': Placeholder('content', 'dummy content'),
37 | 'content2': Placeholder('content2', 'dummy content', variants={'B': 'awesome content'})
38 | }
39 | expected = const.TEXT_EMAIL_PLACEHOLDER_SEPARATOR.join(['dummy content', 'awesome content'])
40 | actual = self.r.render(placeholders, variant='B')
41 | self.assertEqual(expected, actual)
42 |
43 | def test_ignore_subject(self):
44 | placeholders = {
45 | 'content': Placeholder('content', 'dummy content'),
46 | 'subject': Placeholder('subject', 'dummy subject')
47 | }
48 | actual = self.r.render(placeholders)
49 | self.assertEqual('dummy content', actual)
50 |
51 | def test_ignore_empty_placeholders(self):
52 | placeholders = {'content': Placeholder('content', 'dummy content'), 'empty': Placeholder('empty', '')}
53 | actual = self.r.render(placeholders)
54 | self.assertEqual('dummy content', actual)
55 |
56 | def test_ignored_placeholders(self):
57 | placeholders = {
58 | 'content': Placeholder('content', 'dummy content'),
59 | 'ignore': Placeholder('ignore', 'test', False)
60 | }
61 | r = renderer.TextRenderer(self.template, self.email_locale)
62 | actual = r.render(placeholders)
63 | self.assertEqual('dummy content', actual)
64 |
65 | def test_use_text_and_url_for_links(self):
66 | placeholders = {'content': Placeholder('content', 'dummy [link_text](http://link_url) content')}
67 | actual = self.r.render(placeholders)
68 | self.assertEqual('dummy link_text (http://link_url) content', actual)
69 |
70 | def test_default_link_locale_for_links(self):
71 | placeholders = {
72 | 'content': Placeholder('content', 'dummy [link_text](http://link_url?locale={link_locale}) content')
73 | }
74 | actual = self.r.render(placeholders)
75 | self.assertEqual('dummy link_text (http://link_url?locale=locale) content', actual)
76 |
77 | def test_link_locale_for_links(self):
78 | self.email_locale = 'pt-BR'
79 | placeholders = {
80 | 'content': Placeholder('content', 'dummy [link_text](http://link_url?locale={link_locale}) content')
81 | }
82 | r = renderer.TextRenderer(self.template, self.email_locale)
83 | actual = r.render(placeholders)
84 | self.assertEqual('dummy link_text (http://link_url?locale=pt) content', actual)
85 |
86 | def test_use_text_if_href_is_empty(self):
87 | placeholders = {'content': Placeholder('content', 'dummy [http://link_url]() content')}
88 | actual = self.r.render(placeholders)
89 | self.assertEqual('dummy http://link_url content', actual)
90 |
91 | def test_use_href_if_text_is_same(self):
92 | placeholders = {'content': Placeholder('content', 'dummy [http://link_url](http://link_url) content')}
93 | actual = self.r.render(placeholders)
94 | self.assertEqual('dummy http://link_url content', actual)
95 |
96 | def test_url_with_params(self):
97 | placeholders = {
98 | 'content': Placeholder('content', 'dummy [param_link](https://something.com/thing?id=mooo) content')
99 | }
100 | actual = self.r.render(placeholders)
101 | self.assertEqual('dummy param_link (https://something.com/thing?id=mooo) content', actual)
102 |
103 | def test_unordered_list(self):
104 | placeholders = {'content': Placeholder('content', '- one\n- two\n- three')}
105 | actual = self.r.render(placeholders)
106 | self.assertEqual('- one\n- two\n- three', actual.strip())
107 |
108 | def test_ordered_list(self):
109 | placeholders = {'content': Placeholder('content', '1. one\n2. two\n3. three')}
110 | actual = self.r.render(placeholders)
111 | self.assertEqual('1. one\n2. two\n3. three', actual.strip())
112 |
113 |
114 | class TestSubjectRenderer(TestCase):
115 | def setUp(self):
116 | self.r = renderer.SubjectRenderer()
117 | self.placeholders = {
118 | 'content': Placeholder('content', 'dummy content'),
119 | 'subject': Placeholder('subject', 'dummy subject', variants={'B': 'experiment subject'})
120 | }
121 |
122 | def test_happy_path(self):
123 | actual = self.r.render(self.placeholders)
124 | self.assertEqual('dummy subject', actual)
125 |
126 | def test_variant(self):
127 | actual = self.r.render(self.placeholders, variant='B')
128 | self.assertEqual('experiment subject', actual)
129 |
130 | def test_raise_error_for_missing_subject(self):
131 | placeholders = {'content': 'dummy content'}
132 | with self.assertRaises(MissingSubjectError):
133 | self.r.render(placeholders)
134 |
135 |
136 | class TestHtmlRenderer(TestCase):
137 | def _get_renderer(self, template_html, template_placeholders, **kwargs):
138 | template = Template(
139 | name='template_name',
140 | styles_names=['template_style.css', 'template_style2.css'],
141 | styles='',
142 | content=template_html,
143 | placeholders=template_placeholders,
144 | type=None)
145 | return renderer.HtmlRenderer(template, kwargs.get('email_locale', const.DEFAULT_LOCALE))
146 |
147 | def setUp(self):
148 | self.email_locale = 'locale'
149 | config.init(_base_img_path='images_base')
150 |
151 | def tearDown(self):
152 | config.init()
153 |
154 | def test_happy_path(self):
155 | placeholders = {'content1': Placeholder('content1', 'text1')}
156 | template = Template('dummy', [], '', '{{content1}}', ['content1'], None)
157 | r = renderer.HtmlRenderer(template, self.email_locale)
158 |
159 | actual = r.render(placeholders)
160 | self.assertEqual('text1
', actual)
161 |
162 | def test_variant(self):
163 | placeholders = {'content1': Placeholder('content1', 'text1', variants={'B': 'text2'})}
164 | template = Template('dummy', [], '', '{{content1}}', ['content1'], None)
165 | r = renderer.HtmlRenderer(template, self.email_locale)
166 |
167 | actual = r.render(placeholders, variant='B')
168 | self.assertEqual('text2
', actual)
169 |
170 | def test_empty_style(self):
171 | placeholders = {'content': Placeholder('content', 'dummy_content')}
172 | template = Template('dummy', [], '', '{{content}}', ['content1'], None)
173 | r = renderer.HtmlRenderer(template, self.email_locale)
174 |
175 | actual = r.render(placeholders)
176 | self.assertEqual('dummy_content
', actual)
177 |
178 | def test_include_base_url(self):
179 | template = Template('dummy', [], '', '{{base_url}}', ['base_url'], None)
180 | placeholders = {}
181 | r = renderer.HtmlRenderer(template, self.email_locale)
182 |
183 | actual = r.render(placeholders)
184 | self.assertEqual('images_base', actual)
185 |
186 | def test_fail_on_missing_placeholders(self):
187 | template = Template('dummy', [], '', '{{content}}{{missing}}',
188 | ['content', 'missing'], None)
189 | r = renderer.HtmlRenderer(template, self.email_locale)
190 | placeholders = {'content': Placeholder('content', 'dummy_content')}
191 |
192 | with self.assertRaises(MissingTemplatePlaceholderError):
193 | r.render(placeholders)
194 |
195 | def test_rtl_locale(self):
196 | email_locale = 'he'
197 | template = Template('dummy', [], '', '{{content}}', ['content'], None)
198 | r = renderer.HtmlRenderer(template, email_locale)
199 | placeholders = {'content': Placeholder('content', 'dummy_content')}
200 |
201 | actual = r.render(placeholders)
202 | self.assertEqual('\n \n dummy_content\n
\n', actual)
203 |
204 | def test_rtl_two_placeholders(self):
205 | email_locale = 'ar'
206 | template = Template('dummy', [], '',
207 | '{{content1}}
{{content2}}
', ['content1', 'content2'],
208 | None)
209 | r = renderer.HtmlRenderer(template, email_locale)
210 | placeholders = {
211 | 'content1': Placeholder('content1', 'dummy_content1'),
212 | 'content2': Placeholder('content2', 'dummy_content2')
213 | }
214 |
215 | actual = r.render(placeholders)
216 | expected = '\n \n
\n dummy_content1\n
\n
\n
\n dummy_content2\n
\n
dummy_content
', actual)
228 |
229 | @patch('email_parser.fs.read_file')
230 | def test_no_tracking(self, mock_read):
231 | html = '{{content}}'
232 | html_placeholders = ['content']
233 | placeholders = {'content': Placeholder('content', '[link_title](!http://link.com)', True, False)}
234 | mock_read.side_effect = iter([''])
235 | expected = """
236 | link_title
237 |
"""
238 |
239 | r = self._get_renderer(html, html_placeholders)
240 | actual = r.render(placeholders)
241 |
242 | self.assertEqual(expected, actual)
243 |
244 | def test_empty_placeholders_rendering(self):
245 | template = Template('dummy', [], '', '{{content}}', ['content'], None)
246 | r = renderer.HtmlRenderer(template, self.email_locale)
247 | placeholders = {'content': Placeholder('content', '')}
248 |
249 | actual = r.render(placeholders)
250 | self.assertEqual('', actual)
251 |
252 | def test_transform_extended_tags(self):
253 | content = '{{bitmap:MY_BITMAP:max-width=160;max-height=160;}}'
254 | expected = '{{MY_BITMAP}}'
255 | result = renderer._transform_extended_tags(content)
256 | self.assertEqual(result, expected)
257 |
--------------------------------------------------------------------------------