├── .gitignore
├── .idea
├── .gitignore
├── codeStyles
│ └── codeStyleConfig.xml
└── inspectionProfiles
│ └── Project_Default.xml
├── LICENSE
├── README.md
├── contributors.md
├── examples
└── talkpython
│ └── README.md
├── extensibility.md
├── markdown_subtemplate
├── __init__.py
├── caching
│ ├── __init__.py
│ ├── cache_entry.py
│ ├── memory_cache.py
│ └── subtemplate_cache.py
├── engine.py
├── exceptions.py
├── infrastructure
│ ├── __init__.py
│ ├── markdown_transformer.py
│ └── page.py
├── logging
│ ├── __init__.py
│ ├── log_level.py
│ ├── null_logger.py
│ ├── stdout_logger.py
│ └── subtemplate_logger.py
└── storage
│ ├── __init__.py
│ ├── file_storage.py
│ └── subtemplate_storage.py
├── readme_resources
├── mitlicense.svg
├── workflow_image_layout.png
└── workflow_image_layout.pptx
├── requirements-dev.txt
├── requirements.txt
├── setup.py
├── tests
├── _all_tests.py
├── engine_tests.py
├── logging_tests.py
├── page_tests.py
└── templates
│ ├── _shared
│ ├── basic_import.md
│ ├── nested_import.md
│ ├── replacements.md
│ └── second_import.md
│ └── home
│ ├── basic_markdown.md
│ ├── empty_markdown.md
│ ├── import1.md
│ ├── import_missing.md
│ ├── import_nested.md
│ ├── markdown_with_html.md
│ ├── replacements_case_error.md
│ ├── replacements_import.md
│ ├── two_imports.md
│ └── variables.md
└── tox.ini
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | pip-wheel-metadata/
24 | share/python-wheels/
25 | *.egg-info/
26 | .installed.cfg
27 | *.egg
28 | MANIFEST
29 |
30 | # PyInstaller
31 | # Usually these files are written by a python script from a template
32 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
33 | *.manifest
34 | *.spec
35 |
36 | # Installer logs
37 | pip-log.txt
38 | pip-delete-this-directory.txt
39 |
40 | # Unit test / coverage reports
41 | htmlcov/
42 | .tox/
43 | .nox/
44 | .coverage
45 | .coverage.*
46 | .cache
47 | nosetests.xml
48 | coverage.xml
49 | *.cover
50 | *.py,cover
51 | .hypothesis/
52 | .pytest_cache/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | local_settings.py
61 | db.sqlite3
62 | db.sqlite3-journal
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | target/
76 |
77 | # Jupyter Notebook
78 | .ipynb_checkpoints
79 |
80 | # IPython
81 | profile_default/
82 | ipython_config.py
83 |
84 | # pyenv
85 | .python-version
86 |
87 | # pipenv
88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
91 | # install all needed dependencies.
92 | #Pipfile.lock
93 |
94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
95 | __pypackages__/
96 |
97 | # Celery stuff
98 | celerybeat-schedule
99 | celerybeat.pid
100 |
101 | # SageMath parsed files
102 | *.sage.py
103 |
104 | # Environments
105 | .env
106 | .venv
107 | env/
108 | venv/
109 | ENV/
110 | env.bak/
111 | venv.bak/
112 |
113 | # Spyder project settings
114 | .spyderproject
115 | .spyproject
116 |
117 | # Rope project settings
118 | .ropeproject
119 |
120 | # mkdocs documentation
121 | /site
122 |
123 | # mypy
124 | .mypy_cache/
125 | .dmypy.json
126 | dmypy.json
127 |
128 | # Pyre type checker
129 | .pyre/
130 | .idea/$CACHE_FILE$
131 | .idea/markdown-subtemplate.iml
132 | .idea/misc.xml
133 | .idea/modules.xml
134 | .idea/vcs.xml
135 | .idea/dictionaries/mkennedy.xml
136 | .idea/inspectionProfiles/profiles_settings.xml
137 |
--------------------------------------------------------------------------------
/.idea/.gitignore:
--------------------------------------------------------------------------------
1 | # Default ignored files
2 | /workspace.xml
--------------------------------------------------------------------------------
/.idea/codeStyles/codeStyleConfig.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
--------------------------------------------------------------------------------
/.idea/inspectionProfiles/Project_Default.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 Michael Kennedy
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # markdown-subtemplate
2 | [](https://www.python.org/downloads/)
3 | [](https://github.com/mikeckennedy/markdown-subtemplate/blob/master/LICENSE)
4 | [](https://pypi.org/project/markdown-subtemplate/)
5 |
6 | A template engine to render Markdown with external template imports and basic variable replacements.
7 |
8 | ## Motivation
9 |
10 | We often make a choice between data-driven server apps (typical Flask app), CMSes that let us edit content on the web such as WordPress, and even flat file systems like Pelican.
11 |
12 | These are presented as an either-or. You either get a full database driven app or you get a CMS, but not both. This project is meant to help add CMS like features to your data-driven web apps and even author them as static markdown files.
13 |
14 | Here's how it works:
15 |
16 | 1. You write standard markdown content.
17 | 2. Markdown content can be shared and imported into your top-level markdown.
18 | 3. Fragments of HTML can be used when css classes and other specializations are needed, but generally HTML is avoided.
19 | 4. A dictionary of variables and their values to replace in the merged markdown is processed.
20 | 5. Markdown content is converted to HTML and embedded in your larger site layout (e.g. within a Jinja2 template).
21 | 6. Markdown transforms are cached to achieve very high performance regardless of the complexity of the content.
22 |
23 | ## Standard workflow
24 |
25 | Write markdown content, merge it with other markdown files, deliver it as HTML as part of your larger site.
26 |
27 | 
28 |
29 | ## Usage
30 |
31 | To use the library, simply install it.
32 |
33 | ```bash
34 | pip3 install markdown-subtemplate
35 | ```
36 |
37 | Next, write a markdown template, `page.md`:
38 |
39 | ```markdown
40 | ## This is a sub-title
41 |
42 | * Here's an entry
43 | * And another
44 | ```
45 |
46 | Register the template engine in your web app startup:
47 |
48 | ```python
49 | from markdown_subtemplate import engine
50 | # Set the template folder so that when you ask for page.md
51 | # the system knows where to look.
52 |
53 | engine.set_template_folder(full_path_to_template_folder)
54 | ```
55 |
56 | Then generate the HTML content via:
57 |
58 | ```python
59 | data = {'variable1': 'Value 1', 'variable2': 'Value 2'}
60 | contents = engine.get_page('page.md', data)
61 | ```
62 |
63 | Finally, pass the HTML fragment to be rendered in the larger page context:
64 |
65 | ```python
66 | # A Pyramid view method:
67 |
68 | @view_config(route_name='landing', renderer='landing.pt')
69 | def landing(request):
70 | data = {'variable1': 'Value 1', 'variable2': 'Value 2'}
71 | contents = engine.get_page('page.md', data)
72 |
73 | return {
74 | 'name': 'Project name',
75 | 'contents': contents
76 | }
77 | ```
78 |
79 | And the larger website template grabs the content and renders it, `landing.pt`:
80 |
81 | ```html
82 | ...
83 |
84 | ${structure:contents}
85 |
86 | ...
87 | ```
88 |
89 | ## Beware the danger!
90 |
91 | This library is meant for INTERNAL usage only. It's to help you add CMS features to your app. It is **not** for taking user input and making a forum or something like that.
92 |
93 | To allow for the greatest control, you can embed small fragments of HTML in the markdown (e.g. to add a CSS class or other actions). This means the markdown is processed in **UNSAFE** mode. It would allow for script injection attacks if opened to the public.
94 |
95 | ## Extensibility
96 |
97 | `markdown-subtemplate` has three axis of extensibility:
98 |
99 | * **Storage** - Load markdown content from disk, db, or elsewhere.
100 | * **Caching** - Cache generated markdown and HTML in memory, DB, or you pick!
101 | * **Logging** - If you are using a logging framework, plug in logging messages from the library.
102 |
103 | See the [extensibility doc](https://github.com/mikeckennedy/markdown-subtemplate/blob/master/extensibility.md) for details and examples.
104 |
105 |
106 | ## Nested markdown
107 |
108 | One of the reason's this project exists, rather than just passing markdown text to a markdown library is allowing nesting / importing of markdown files.
109 |
110 | If you have page fragments that need to appear more than once, create a dedicated markdown import file that can be managed and versioned in one place. Here's how:
111 |
112 | ### Created an imported file in TEMPLATES/_shared
113 |
114 | All imported markdown files are located in subpaths of `TEMPLATES/_shared` where `TEMPLATES` is the path you set during startup.
115 |
116 | ```
117 | TEMPLATES
118 | |- _shared
119 | |- contact.md
120 | |- footer.md
121 | |-pages
122 | | - page.md
123 | | - about.md
124 | ```
125 |
126 | Write the imported / shared markdown, `contact.md`:
127 |
128 | ```markdown
129 | Contact us via email [us@us.com](mailto:us.com) or on
130 | Twitter via [@us](https://twitter.com/us)
131 | ```
132 |
133 | Then in your page, e.g. `page.md` you can add an import statement:
134 |
135 | ```markdown
136 | # Our amazing page
137 |
138 | Here is some info **about the page**. It's standard markdown.
139 |
140 | Want to contact us? Here are some options:
141 | [IMPORT CONTACT]
142 |
143 | And a footer:
144 | [IMPORT FOOTER]
145 | ```
146 |
147 | The resulting markdown is just replacing the `IMPORT` statements with the contents of those files, then passing the whole thing through a markdown to HTML processor.
148 |
149 | ## Variables
150 |
151 | `markdown_subtemplate` has some limited support for variable replacements. Given this markdown page:
152 |
153 | ```markdown
154 | # Example: $TITLE$
155 |
156 | Welcome to the $PROJECT$ project. Here are some details
157 | ...
158 | ```
159 |
160 | You can populate the variable values with:
161 |
162 | ```python
163 | data = {'title': 'Markdown Transformers', 'project': 'sub templates'}
164 | contents = engine.get_page('page.md', data)
165 | ```
166 |
167 | Note that the variable names must be all-caps in the template. Missing variable statements in markdown that appear in the data dictionary are ignored.
168 |
169 | ## Variables within nested templates
170 |
171 | One problem you might run into is you have some reusable section, like this:
172 |
173 | ```
174 | // shared_promo.md
175 |
176 | We are running a special! Click here to see the featured item:
177 |
178 | [Our latest project, 50% off](/markdown-editor?utm_source={SOMETHING_FROM_THE_PAGE})
179 | ```
180 |
181 | It's generally reusable, but you want that `SOMETHING_FROM_THE_PAGE` to vary depending on which page they are on.
182 | You could do this in Python code but that would be out of band and actually a little hard to do. So you can
183 | now define hierarchical variables in markdown.
184 |
185 | Define nested variables as follows with `name=VARNAME` and `value=var-value`:
186 |
187 | ```
188 | [VARIABLE VARNAME="var-value"]
189 | ```
190 |
191 | In the following page, we choose a `source` and reuse this shared template:
192 |
193 | ```
194 | // top-level.md
195 |
196 | # Welcome to the page
197 |
198 | Page content here...
199 |
200 | [VARIABLE SOMETHING_FROM_THE_PAGE="editor-youtube-landing-page"]
201 | [IMPORT shared_promo]
202 | ```
203 |
204 | The result would be effective this markdown, then turned into HTML:
205 |
206 | ```markdown
207 | # Welcome to the page
208 |
209 | Page content here...
210 |
211 | We are running a special! Click here to see the featured item:
212 |
213 | [Our latest project, 50% off](/markdown-editor?utm_source=editor-youtube-landing-page)
214 | ```
215 |
216 | Just like regular variables, they are enclosed in `{` and `}` and are upper case where they are used.
217 |
218 |
219 |
220 | ## Requirements
221 |
222 | This library requires **Python 3.6 or higher**. Because, *f-yes*! (f-strings).
223 |
224 | ## Licence
225 |
226 | `markdown-subtemplate` is distributed under the MIT license.
227 |
228 | ## Authors
229 |
230 | `markdown_subtemplate` was written by [Michael Kennedy](https://github.com/mikeckennedy).
231 |
--------------------------------------------------------------------------------
/contributors.md:
--------------------------------------------------------------------------------
1 | # markdown-subtemplate Contributors
2 |
3 | ## Authors
4 |
5 | [Michael Kennedy](https://github.com/mikeckennedy)
6 |
7 | ## Contributors
8 |
9 | [RhinoCodes](https://github.com/RhinoCodes)
10 |
11 |
--------------------------------------------------------------------------------
/examples/talkpython/README.md:
--------------------------------------------------------------------------------
1 | # Talk Python Training Example
2 |
3 | This project was extracted from an internal CMS built to power [Talk Python Training](https://training.talkpython.fm/)'s landing pages, for example, [this one](https://training.talkpython.fm/courses/explore_100days_web/100-days-of-web-in-python).
4 |
5 | We have lots of sections on our landing pages that are repeated. Hence the ability to import them into the landing page is very helpful. If you visit the page:
6 |
7 | [https://training.talkpython.fm/courses/explore_100days_web/100-days-of-web-in-python](https://training.talkpython.fm/courses/explore_100days_web/100-days-of-web-in-python)
8 |
9 | The sections from **What's this course about and how is it different?** to **The time to act is now** are generated with content similar to below.
10 |
11 | ## 100-web.md (top level page)
12 |
13 | ```markdown
14 | ## What's this course about and how is it different?
15 |
16 | 100 days of code isn’t just about the time commitment.
17 | **The true power is in utilising that time effectively
18 | with a tailored set of projects**. That’s why we have 24
19 | practical projects, each paired with 20-60 minute video
20 | lessons at the beginning of the project.
21 |
22 | Just a small sampling of the projects you’ll work on include:
23 |
24 | * Create your very own Static Site
25 | * ...
26 | * Web Scraping with BeautifulSoup4 and newspaper3k
27 | * And 17 more projects!
28 |
29 | View the full [course outline](#course_outline).
30 | ...
31 |
32 | ## Who is this course for?
33 |
34 | This course is for **anyone who knows the basics of Python and
35 | wants to push themselves into the world of Python Web Development
36 | for 100 days with hands-on projects**.
37 | ...
38 |
39 | [IMPORT TRANSCRIPTS]
40 | ## Who we are and why should you take our course?
41 |
42 | **Meet Michael Kennedy:**
43 |
44 | [IMPORT MICHAEL]
45 |
46 | **Meet Bob Belderbos:**
47 |
48 | [IMPORT BOB_B]
49 |
50 | **Meet Julian Sequeira:**
51 |
52 | [IMPORT JULIAN_S]
53 |
54 | [IMPORT OFFICE_HOURS]
55 |
56 | [IMPORT PYTHON_3_STATEMENT]
57 |
58 | ## The time to act is now
59 |
60 | The **#100DaysOfCode** challenge is an epic adventure.
61 | Don't got it alone. Take our course and we'll
62 | be your guide with both lessons and projects.
63 | ```
64 |
65 |
66 | ## transcripts.md (imported section)
67 |
68 | ```markdown
69 | ## Follow along with subtitles and transcripts
70 |
71 | Each course comes with subtitles and full transcripts.
72 | The transcripts are available as a separate searchable page for
73 | each lecture. They also are available in course-wide search results
74 | to help you find just the right lecture.
75 |
76 |
77 |
81 |
82 | ```
83 |
84 | ## Explore course Chameleon template
85 |
86 | The template contents are show in the larger context of the site via:
87 |
88 | ```html
89 | ... explore_course.pt:[]() main contents, nav, etc. ...
90 |
91 | ${structure:contents}
92 |
93 | ```
94 |
95 | These templates are rendered with `markdown_subtemplate` using this code:
96 |
97 | ```python
98 | # contents returned as part of the model/dictionary from Pyramid.
99 | contents = markdown_subtemplate.engine.get_page('100-web.md', {})
100 | ```
101 |
102 | ## Other details
103 |
104 | This site is using two custom extensibility points. First, we are using a wrapper around the logging to integrate with [logbook](https://logbook.readthedocs.io/en/stable/). Secondly, we have a custom cache provider that is backed by MongoDB rather than in-memory. This allows us to reset the cache and have effects across our scaled out web worker processes.
105 |
106 |
--------------------------------------------------------------------------------
/extensibility.md:
--------------------------------------------------------------------------------
1 | # Extensibility
2 |
3 | `markdown-subtemplate` has three axis of extensibility:
4 |
5 | * **Storage** - Load markdown content from disk, db, or elsewhere.
6 | * **Caching** - Cache generated markdown and HTML in memory, DB, or you pick!
7 | * **Logging** - If you are using a logging framework, plug in logging messages from the library.
8 |
9 | ## Storage
10 |
11 | Out of the box, `markdown-subtemplate` will load markdown files from a structure directory:
12 |
13 | ```
14 | TEMPLATE_FOLDER
15 | |
16 | |- shared
17 | | - contact.md
18 | | - social.md
19 | |- home # arbitrary organizing directories
20 | | - index.md # Use template_path: home/index.md
21 | | - about.md
22 | ```
23 |
24 | This is implemented by the `markdown_subtemplate.storage.file_storage.FileStore` class. It must be configured as follows:
25 |
26 | ```python
27 | from markdown_subtemplate.storage.file_storage import FileStore
28 | folder = FULL_PATH_TO-TEMPLATE_FOLDER
29 | FileStore.set_template_folder(folder)
30 | ```
31 |
32 | If you want to change the storage engine, just create a base class of `markdown_subtemplate.storage.SubtemplateStorage`. It's an abstract class so just implement the abstract methods.
33 |
34 | Here is an example from SQLAlchemy. Define a model to read/write data:
35 |
36 | ```python
37 | # SQLAlchemy entity class:
38 | class MarkdownPage(SqlAlchemyBase):
39 | __tablename__ = 'markdown_pages'
40 |
41 | id = sa.Column(sa.String, primary_key=True)
42 | name = sa.Column(sa.String, index=True)
43 | created_date = sa.Column(sa.DateTime, default=datetime.datetime.now)
44 | is_shared = sa.Column(sa.Boolean, index=True, default=False)
45 | text = sqlalchemy.Column(sa.String)
46 | ```
47 |
48 | Then implement the storage engine class:
49 |
50 | ```python
51 | class MarkdownSubTemplateDBStorage(storage.SubtemplateStorage):
52 | def get_markdown_text(self, template_path: str) -> Optional[str]:
53 | if not template_path:
54 | return None
55 |
56 | template_path = template_path.strip().lower()
57 | session = DbSession.create() # Method to generate a SQLAlchemy session.
58 |
59 | mk: MarkdownPage = session.query(MarkdownPage) \
60 | .filter(MarkdownPage.id == template_path) \
61 | .first()
62 |
63 | session.close()
64 |
65 | if not mk:
66 | return None
67 |
68 | return mk.text
69 |
70 | def get_shared_markdown(self, import_name: str) -> Optional[str]:
71 | if not import_name:
72 | return None
73 |
74 | import_name = import_name.strip().lower()
75 | session = DbSession.create()
76 |
77 | mk: MarkdownPage = session.query(MarkdownPage) \
78 | .filter(MarkdownPage.name == import_name, MarkdownPage.is_shared == True) \
79 | .first()
80 |
81 | session.close()
82 |
83 | if not mk:
84 | return None
85 |
86 | return mk.text
87 |
88 | def is_initialized(self) -> bool:
89 | # Check whether connection string, etc set in SQLAlchemy
90 | return True
91 |
92 | def clear_settings(self):
93 | pass
94 | ```
95 |
96 | Of course, you'll need a way to enter these into the DB but that's technically outside of the content of this discussion.
97 |
98 | Finally, you'll need to set this storage engine as the implementation at process startup:
99 |
100 | ```python
101 | from markdown_subtemplate import storage
102 |
103 | store = MarkdownSubTemplateDBStorage()
104 | storage.set_storage(store)
105 | ```
106 |
107 | ## Caching
108 |
109 | By default, `markdown-subtemplate` will cache generated markdown and HTML in memory. This often is fine. If you do nothing, this will happen automatically and your page generation will be much faster if you reuse content or request it more than once.
110 |
111 | But web environments typically have many processes serving their content. For example, at [Talk Python Training](https://training.talkpython.fm/) we currently have 8-10 uWSGI worker processes running in parallel.
112 |
113 | In this situation, caching all the content in memory has a few drawbacks.
114 | * All content is cached in memory 10x what it would normally cost.
115 | * Content that has to be generated, which can be much slower, is done 10x as often.
116 | * Restarting the server for a new version of code requires everything to be regenerated 10x again making startup slow.
117 | * Clearing the cache, if wanted, is effectively impossible (how to you cleanly signal all 10 processes exactly once?)
118 |
119 | In these situations, storing the cache content in a database or Redis would be better. At Talk Python, we use MongoDB as the backing cache store.
120 |
121 | Below are two examples. They follow the pattern:
122 |
123 | 1. Create an entity to store in the DB for cache data
124 | 2. Create a base class of `markdown_subtemplate.caching.SubtemplateCache`
125 | 3. Override the abstract methods
126 | 4. Register your caching engine with `markdown-subtemplate` at startup.
127 |
128 | ### SQLAlchemy Cachine Engine:
129 |
130 | **First, create the entity to store in the DB**.
131 |
132 | ```python
133 | class MarkdownCache(SqlAlchemyBase):
134 | __tablename__ = 'markdown_cache'
135 |
136 | id = sa.Column(sa.Integer, primary_key=True, autoincrement=True)
137 | key = sa.Column(sa.String, index=True)
138 | type = sa.Column(sa.String, index=True)
139 | name = sa.Column(sa.String, index=True)
140 | contents = sa.Column(sa.String)
141 |
142 | created_date = sa.Column(sa.DateTime, default=datetime.datetime.now, index=True)
143 | ```
144 |
145 | **Second, implement the caching engine**:
146 |
147 | ```python
148 | from markdown_subtemplate import caching
149 | class MarkdownSubTemplateDBCache(caching.SubtemplateCache):
150 | def get_html(self, key: str) -> caching.CacheEntry:
151 | session = DbSession.create()
152 |
153 | cache_entry = session.query(MarkdownCache).filter(
154 | MarkdownCache.key == key, MarkdownCache.type == 'html'
155 | ).first()
156 |
157 | session.close()
158 |
159 | return cache_entry
160 |
161 | def add_html(self, key: str, name: str, html_contents: str) -> caching.CacheEntry:
162 | session = DbSession.create()
163 |
164 | item = self.get_html(key)
165 | if not item:
166 | item = MarkdownCache()
167 | session.add(item)
168 |
169 | item.type = 'html'
170 | item.key = key
171 | item.name = name
172 | item.contents = html_contents
173 |
174 | if html_contents:
175 | session.commit()
176 |
177 | session.close()
178 |
179 | # Not technical a base class, but duck-type equivalent.
180 | # noinspection PyTypeChecker
181 | return item
182 |
183 | def get_markdown(self, key: str) -> caching.CacheEntry:
184 | session = DbSession.create()
185 |
186 | cache_entry = session.query(MarkdownCache).filter(
187 | MarkdownCache.key == key, MarkdownCache.type == 'markdown'
188 | ).first()
189 |
190 | session.close()
191 |
192 | return cache_entry
193 |
194 | def add_markdown(self, key: str, name: str, markdown_contents: str) -> caching.CacheEntry:
195 | session = DbSession.create()
196 |
197 | item = self.get_markdown(key)
198 | if not item:
199 | item = MarkdownCache()
200 | session.add(item)
201 |
202 | item.type = 'markdown'
203 | item.key = key
204 | item.name = name
205 | item.contents = markdown_contents
206 |
207 | if markdown_contents:
208 | session.commit()
209 | session.close()
210 |
211 | # Not technical a base class, but duck-type equivalent.
212 | # noinspection PyTypeChecker
213 | return item
214 |
215 | def clear(self):
216 | session = DbSession.create()
217 |
218 | for entry in session.query(MarkdownCache):
219 | session.delete(entry)
220 |
221 | session.commit()
222 |
223 | def count(self) -> int:
224 | session = DbSession.create()
225 | count = session.query(MarkdownCache).count()
226 | session.close()
227 |
228 | return count
229 | ```
230 |
231 | One minor oddity is the return value is `caching.CacheEntry` whereas that's not the real return value. But the SQLAlchemy entity does implement every field that is present in `CacheEntry`, so duck typing and all that.
232 |
233 | **Finally, register the new caching engine at process startup**.
234 |
235 | ```python
236 | from markdown_subtemplate import caching
237 |
238 | cache = MarkdownSubTemplateDBCache()
239 | caching.set_cache(cache)
240 | ```
241 |
242 | ### MongoDB Cachine Engine:
243 |
244 | Using MongoDB as a backing store for the cache is basically the same as SQLAlchemy in principle. We'll be using MongoEngine.
245 |
246 | **First, create the entity to store in the DB**.
247 |
248 | ```python
249 | import mongoengine as me
250 |
251 | class CmsCache(me.Document):
252 | key: str = me.StringField(required=True)
253 | type: str = me.StringField(required=True)
254 | name: str = me.StringField()
255 | contents: str = me.StringField(required=True)
256 | created_date: datetime.datetime = me.DateTimeField(default=datetime.now)
257 |
258 | meta = {
259 | 'db_alias': 'core',
260 | 'collection': 'cms_cache',
261 | 'indexes': [
262 | {'fields': ['key', 'type']},
263 | 'key',
264 | 'type',
265 | 'name',
266 | 'created_date',
267 | ],
268 | 'ordering': ['-created_date']
269 | }
270 | ```
271 |
272 | **Second, implement the caching engine**:
273 |
274 | ```python
275 | class MarkdownSubTemplateMongoDBCache(SubtemplateCache):
276 | def get_html(self, key: str) -> CacheEntry:
277 | return CmsCache.objects(key=key, type='html').first()
278 |
279 | def add_html(self, key: str, name: str, html_contents: str) -> CacheEntry:
280 | item = self.get_html(key)
281 | if item:
282 | return item
283 |
284 | item = CmsCache()
285 | item.type = 'html'
286 | item.key = key
287 | item.name = name
288 | item.contents = html_contents
289 | item.save()
290 |
291 | # Not technical a base class, but duck-type equivalent.
292 | # noinspection PyTypeChecker
293 | return item
294 |
295 | def get_markdown(self, key: str) -> CacheEntry:
296 | return CmsCache.objects(key=key, type='markdown').first()
297 |
298 | def add_markdown(self, key: str, name: str, markdown_contents: str) -> CacheEntry:
299 | item = self.get_markdown(key)
300 | if item:
301 | return item
302 |
303 | item = CmsCache()
304 | item.type = 'markdown'
305 | item.key = key
306 | item.name = name
307 | item.contents = markdown_contents
308 | item.save()
309 |
310 | # Not technical a base class, but duck-type equivalent.
311 | # noinspection PyTypeChecker
312 | return item
313 |
314 | def clear(self):
315 | CmsCache.objects().delete()
316 |
317 | def count(self) -> int:
318 | return CmsCache.objects().count()
319 | ```
320 |
321 | One minor oddity is the return value is `caching.CacheEntry` whereas that's not the real return value. But the MongoEngine entity does implement every field that is present in `CacheEntry`, so duck typing and all that.
322 |
323 | **Finally, register the new caching engine at process startup**.
324 |
325 | ```python
326 | from markdown_subtemplate import caching
327 |
328 | cache = MarkdownSubTemplateMongoDBCache()
329 | caching.set_cache(cache)
330 | ```
331 |
332 |
333 | ## Logging
334 |
335 | By default, `markdown-subtemplate` will log to standard out using `print()` and log level `INFO` from the builtin `StdOutLogger` class.
336 |
337 | You can change the log level by using the `markdown_subtemplate.logging.LogLevel` class:
338 |
339 | ```python
340 | # Change logging level from default LogLevel.info to LogLevel.error
341 |
342 | log = logging.get_log()
343 | log.log_level = LogLevel.error
344 | ```
345 |
346 | You can disable logging by setting its level to `LogLevel.off`.
347 |
348 | If you use a logging framework, you likely want to direct log messages through that framework. So you can, like the above two subsystems, implement a class based on an abstract base class.
349 |
350 | Let's log through a preconfigured [Logbook](https://logbook.readthedocs.io/en/stable/) setup (which goes to stdout in dev and rotating files in prod).
351 |
352 | **First, create a base class**:
353 |
354 | ```python
355 | import logbook
356 | import markdown_subtemplate
357 | from markdown_subtemplate.logging import LogLevel
358 |
359 | class MarkdownLogger(markdown_subtemplate.logging.SubtemplateLogger):
360 | def __init__(self, log_level: int):
361 | super().__init__(log_level)
362 | self.logbook_logger = logbook.Logger("Markdown Templates")
363 |
364 | def verbose(self, text: str):
365 | if not self.should_log(LogLevel.verbose, text):
366 | return
367 |
368 | self.logbook_logger.trace(text)
369 |
370 | def trace(self, text: str):
371 | if not self.should_log(LogLevel.trace, text):
372 | return
373 |
374 | self.logbook_logger.trace(text)
375 |
376 | def info(self, text: str):
377 | if not self.should_log(LogLevel.info, text):
378 | return
379 |
380 | self.logbook_logger.info(text)
381 |
382 | def error(self, text: str):
383 | if not self.should_log(LogLevel.error, text):
384 | return
385 |
386 | self.logbook_logger.error(text)
387 | ```
388 |
389 | That's pretty straightforward. But do be sure to check at each level whether you should log as we do with:
390 |
391 | ```python
392 | if not self.should_log(LEVEL, text):
393 | return
394 | ```
395 |
396 | Finally, register this logging engine at process startup:
397 |
398 | ```python
399 | From markdown_subtemplate import logging
400 |
401 | log = MarkdownLogger(LogLevel.info) # Set the level you want
402 | logging.set_log(log)
403 | ```
404 |
--------------------------------------------------------------------------------
/markdown_subtemplate/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | markdown_subtemplate - A template engine to render
3 | Markdown with external template imports and variable replacements.
4 | """
5 |
6 | __version__ = '0.2.22'
7 | __author__ = 'Michael Kennedy '
8 | __all__ = []
9 |
10 | from . import caching
11 | from . import engine
12 | from . import exceptions
13 | from . import logging
14 | from . import storage
15 | from .infrastructure import markdown_transformer
16 |
--------------------------------------------------------------------------------
/markdown_subtemplate/caching/__init__.py:
--------------------------------------------------------------------------------
1 | from .cache_entry import CacheEntry
2 | from .memory_cache import MemoryCache
3 | from .subtemplate_cache import SubtemplateCache
4 | from ..exceptions import ArgumentExpectedException
5 |
6 | __cache: SubtemplateCache = MemoryCache()
7 |
8 |
9 | def set_cache(cache_instance: SubtemplateCache):
10 | global __cache
11 | if not cache_instance or not isinstance(cache_instance, SubtemplateCache):
12 | raise ArgumentExpectedException('cache_instance')
13 |
14 | __cache = cache_instance
15 |
16 |
17 | def get_cache() -> SubtemplateCache:
18 | return __cache
19 |
--------------------------------------------------------------------------------
/markdown_subtemplate/caching/cache_entry.py:
--------------------------------------------------------------------------------
1 | from collections import namedtuple
2 |
3 | CacheEntry = namedtuple("CacheEntry", "key, name, created, contents")
4 |
--------------------------------------------------------------------------------
/markdown_subtemplate/caching/memory_cache.py:
--------------------------------------------------------------------------------
1 | from datetime import datetime
2 |
3 | from .cache_entry import CacheEntry
4 | from .subtemplate_cache import SubtemplateCache
5 |
6 |
7 | class MemoryCache(SubtemplateCache):
8 | markdown_cache = {}
9 | html_cache = {}
10 |
11 | def get_html(self, key: str) -> CacheEntry:
12 | return self.html_cache.get(key)
13 |
14 | def add_html(self, key: str, name: str, html_contents: str) -> CacheEntry:
15 | entry = CacheEntry(key=key, name=name, created=datetime.now(), contents=html_contents)
16 | self.html_cache[key] = entry
17 |
18 | return entry
19 |
20 | def get_markdown(self, key: str) -> CacheEntry:
21 | return self.markdown_cache.get(key)
22 |
23 | def add_markdown(self, key: str, name: str, markdown_contents: str) -> CacheEntry:
24 | entry = CacheEntry(key=key, name=name, created=datetime.now(), contents=markdown_contents)
25 | self.markdown_cache[key] = entry
26 |
27 | return entry
28 |
29 | def clear(self):
30 | self.markdown_cache.clear()
31 | self.html_cache.clear()
32 |
33 | def count(self) -> int:
34 | return len(self.markdown_cache) + len(self.html_cache)
35 |
--------------------------------------------------------------------------------
/markdown_subtemplate/caching/subtemplate_cache.py:
--------------------------------------------------------------------------------
1 | import abc
2 | from typing import Optional
3 |
4 | from .cache_entry import CacheEntry
5 |
6 |
7 | class SubtemplateCache(abc.ABC):
8 | @abc.abstractmethod
9 | def get_html(self, key: str) -> CacheEntry:
10 | pass
11 |
12 | @abc.abstractmethod
13 | def add_html(self, key: str, name: str, html_contents: str) -> CacheEntry:
14 | pass
15 |
16 | @abc.abstractmethod
17 | def get_markdown(self, key: str) -> CacheEntry:
18 | pass
19 |
20 | @abc.abstractmethod
21 | def add_markdown(self, key: str, name: str, markdown_contents: str) -> CacheEntry:
22 | pass
23 |
24 | @abc.abstractmethod
25 | def clear(self):
26 | pass
27 |
28 | @abc.abstractmethod
29 | def count(self) -> int:
30 | pass
31 |
32 |
33 |
--------------------------------------------------------------------------------
/markdown_subtemplate/engine.py:
--------------------------------------------------------------------------------
1 | from typing import Any, Dict
2 |
3 | from . import caching as __caching, storage
4 | from . import logging as __logging
5 | from .infrastructure import page as __page
6 |
7 |
8 | def get_page(template_path: str, data: Dict[str, Any] = {}) -> str:
9 | from markdown_subtemplate.exceptions import InvalidOperationException
10 | log = __logging.get_log()
11 |
12 | if not storage.is_initialized():
13 | msg = "Storage engine is not initialized."
14 | log.error("engine.get_page: " + msg)
15 | raise InvalidOperationException(msg)
16 |
17 | log.verbose(f"engine.get_page: Getting page content for {template_path}")
18 | return __page.get_page(template_path, data)
19 |
20 |
21 | def clear_cache():
22 | log = __logging.get_log()
23 | cache = __caching.get_cache()
24 |
25 | item_count = cache.count()
26 | cache.clear()
27 |
28 | log.info(f"engine.clear_cache: Cache cleared, reclaimed {item_count:,} items.")
29 |
30 |
--------------------------------------------------------------------------------
/markdown_subtemplate/exceptions.py:
--------------------------------------------------------------------------------
1 | class MarkdownTemplateException(Exception):
2 | pass
3 |
4 |
5 | class ArgumentExpectedException(MarkdownTemplateException):
6 | pass
7 |
8 |
9 | class InvalidOperationException(MarkdownTemplateException):
10 | pass
11 |
12 |
13 | class PathException(MarkdownTemplateException):
14 | pass
15 |
16 |
17 | class TemplateNotFoundException(PathException):
18 | def __init__(self, template_path):
19 | super().__init__(f'Template not found: {template_path}.')
20 |
21 |
22 |
--------------------------------------------------------------------------------
/markdown_subtemplate/infrastructure/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeckennedy/markdown-subtemplate/138f46c433b641ad18882da6d3f232f9736d9a32/markdown_subtemplate/infrastructure/__init__.py
--------------------------------------------------------------------------------
/markdown_subtemplate/infrastructure/markdown_transformer.py:
--------------------------------------------------------------------------------
1 | import hashlib
2 |
3 | import markdown2
4 | from .. import caching as __caching
5 |
6 | # Note: Do NOT enable link-patterns, it causes a crash.
7 | __enabled_markdown_extras = [
8 | "cuddled-lists",
9 | "code-friendly",
10 | "fenced-code-blocks",
11 | "tables"
12 | ]
13 |
14 |
15 | def transform(text, safe_mode=True):
16 | if not text:
17 | return text
18 |
19 | hash_val = get_hash(text)
20 |
21 | cache = __caching.get_cache()
22 | entry = cache.get_html(hash_val)
23 | if entry:
24 | return entry.contents
25 |
26 | html = markdown2.markdown(text, extras=__enabled_markdown_extras, safe_mode=safe_mode)
27 | cache.add_html(hash_val, f"markdown_transformer:{hash_val}", html)
28 |
29 | return html
30 |
31 |
32 | def get_hash(text):
33 | md5 = hashlib.md5()
34 | data = text.encode('utf-8')
35 | md5.update(data)
36 |
37 | return md5.hexdigest()
38 |
--------------------------------------------------------------------------------
/markdown_subtemplate/infrastructure/page.py:
--------------------------------------------------------------------------------
1 | import os
2 | import datetime
3 | from typing import Dict, Optional, Any, List
4 |
5 | from markdown_subtemplate import caching as __caching
6 | from markdown_subtemplate.infrastructure import markdown_transformer
7 | from markdown_subtemplate.exceptions import ArgumentExpectedException, TemplateNotFoundException
8 | from markdown_subtemplate import logging as __logging
9 | import markdown_subtemplate.storage as __storage
10 | from markdown_subtemplate.logging import SubtemplateLogger
11 | from markdown_subtemplate.storage import SubtemplateStorage
12 |
13 |
14 | # noinspection DuplicatedCode
15 | def get_page(template_path: str, data: Dict[str, Any]) -> str:
16 | if not template_path or not template_path.strip():
17 | raise ArgumentExpectedException('template_path')
18 |
19 | template_path = template_path.strip().lower()
20 |
21 | cache = __caching.get_cache()
22 | log = __logging.get_log()
23 |
24 | key = f'html: {template_path}'
25 | entry = cache.get_html(key)
26 | if entry:
27 | log.trace(f"CACHE HIT: Reusing {template_path} from HTML cache.")
28 | contents = entry.contents
29 |
30 | # Is there data that needs to be folded in? Process it.
31 | if data:
32 | contents = process_variables(contents, data)
33 |
34 | # Return the cached data, no need to transform for variables.
35 | return contents
36 |
37 | t0 = datetime.datetime.now()
38 |
39 | # Get the markdown with imports and substitutions
40 | markdown = get_markdown(template_path)
41 | inline_variables = {}
42 | markdown = get_inline_variables(markdown, inline_variables, log)
43 | # Convert markdown to HTML
44 | html = get_html(markdown)
45 |
46 | # Cache inline variables, but not the passed in data as that varies per request (query string, etc).
47 | html = process_variables(html, inline_variables)
48 | cache.add_html(key, key, html)
49 |
50 | # Replace the passed variables each time.
51 | html = process_variables(html, data)
52 |
53 | dt = datetime.datetime.now() - t0
54 |
55 | msg = f"Created contents for {template_path}:{data} in {int(dt.total_seconds() * 1000):,} ms."
56 | log.info(f"GENERATING HTML: {msg}")
57 |
58 | return html
59 |
60 |
61 | def get_html(markdown_text: str, unsafe_data=False) -> str:
62 | html = markdown_transformer.transform(markdown_text, unsafe_data)
63 | return html
64 |
65 |
66 | # noinspection DuplicatedCode
67 | def get_markdown(template_path: str, data: Dict[str, Any] = None) -> str:
68 | if data is None:
69 | data = {}
70 |
71 | cache = __caching.get_cache()
72 | log = __logging.get_log()
73 |
74 | key = f'markdown: {template_path}'
75 | entry = cache.get_markdown(key)
76 | if entry:
77 | log.trace(f"CACHE HIT: Reusing {template_path} from MARKDOWN cache.")
78 | if not data:
79 | return entry.contents
80 | else:
81 | return process_variables(entry.contents, data)
82 |
83 | t0 = datetime.datetime.now()
84 |
85 | text = load_markdown_contents(template_path)
86 | cache.add_markdown(key, key, text)
87 | if data:
88 | text = process_variables(text, data)
89 |
90 | dt = datetime.datetime.now() - t0
91 |
92 | msg = f"Created contents for {template_path} in {int(dt.total_seconds() * 1000):,} ms."
93 | log.trace(f"GENERATING MARKDOWN: {msg}")
94 |
95 | return text
96 |
97 |
98 | def load_markdown_contents(template_path: str) -> Optional[str]:
99 | if not template_path:
100 | return ''
101 |
102 | log = __logging.get_log()
103 | log.verbose(f"Loading markdown template: {template_path}")
104 |
105 | page_md = get_page_markdown(template_path)
106 | if not page_md:
107 | return ''
108 |
109 | lines = page_md.split('\n')
110 | lines = process_imports(lines)
111 |
112 | final_markdown = "\n".join(lines).strip()
113 |
114 | return final_markdown
115 |
116 |
117 | def get_page_markdown(template_path: str) -> Optional[str]:
118 | if not template_path or not template_path.strip():
119 | raise TemplateNotFoundException("No template file specified: template_path=''.")
120 |
121 | store: SubtemplateStorage = __storage.get_storage()
122 | return store.get_markdown_text(template_path)
123 |
124 |
125 | def get_shared_markdown(import_name: str) -> Optional[str]:
126 | if not import_name or not import_name.strip():
127 | raise ArgumentExpectedException('import_name')
128 |
129 | store: SubtemplateStorage = __storage.get_storage()
130 | return store.get_shared_markdown(import_name)
131 |
132 |
133 | def process_imports(lines: List[str]) -> List[str]:
134 | log = __logging.get_log()
135 | line_data = list(lines)
136 |
137 | for idx, line in enumerate(line_data):
138 | if not line.strip().startswith('[IMPORT '):
139 | continue
140 |
141 | import_statement = line.strip()
142 | import_name = import_statement \
143 | .replace('[IMPORT ', '') \
144 | .replace(']', '') \
145 | .strip()
146 |
147 | log.verbose(f"Loading import: {import_name}...")
148 |
149 | markdown = get_shared_markdown(import_name)
150 | if markdown is not None:
151 | markdown_lines = markdown.split('\n')
152 | else:
153 | markdown_lines = ['', f'ERROR: IMPORT {import_name} not found', '']
154 |
155 | line_data = line_data[:idx] + markdown_lines + line_data[idx + 1:]
156 |
157 | return process_imports(line_data)
158 |
159 | return line_data
160 |
161 |
162 | def process_variables(raw_text: str, data: Dict[str, Any]) -> str:
163 | if not raw_text:
164 | return raw_text
165 |
166 | log = __logging.get_log()
167 |
168 | keys = list(data.keys())
169 | key_placeholders = {
170 | key: f"${key.strip().upper()}$"
171 | for key in keys
172 | if key and isinstance(key, str)
173 | }
174 |
175 | transformed_text = raw_text
176 | for key in keys:
177 | if key_placeholders[key] not in transformed_text:
178 | continue
179 |
180 | log.verbose(f"Replacing {key_placeholders[key]}...")
181 | transformed_text = transformed_text.replace(key_placeholders[key], str(data[key]))
182 |
183 | return transformed_text
184 |
185 |
186 | def get_inline_variables(markdown: str, new_vars: Dict[str, str], log: Optional[SubtemplateLogger]) -> str:
187 | pattern = '[VARIABLE '
188 |
189 | if pattern not in markdown and pattern.lower() not in markdown:
190 | return markdown
191 |
192 | lines: List[str] = markdown.split('\n')
193 | final_lines = []
194 |
195 | for l in lines:
196 |
197 | if not( l and l.strip().upper().startswith(pattern)):
198 | final_lines.append(l)
199 | continue
200 |
201 | text = l.strip()
202 | text = text[len(pattern):].strip("]")
203 | parts = text.split('=')
204 | if len(parts) != 2:
205 | if log:
206 | log.error(f"Invalid variable definition in markdown: {l}.")
207 | continue
208 |
209 | name = parts[0].strip().upper()
210 | value = parts[1].strip()
211 | has_quotes = (
212 | (value.startswith('"') or value.startswith("'")) and
213 | (value.endswith('"') or value.endswith("'"))
214 | )
215 |
216 | if not has_quotes:
217 | if log:
218 | log.error(f"Invalid variable definition in markdown, missing quotes surrounding value: {l}.")
219 | continue
220 |
221 | value = value.strip('\'"').strip()
222 |
223 | new_vars[name]=value
224 |
225 | if new_vars:
226 | return "\n".join(final_lines)
227 | else:
228 | return markdown
229 |
--------------------------------------------------------------------------------
/markdown_subtemplate/logging/__init__.py:
--------------------------------------------------------------------------------
1 | from .log_level import LogLevel
2 | from .stdout_logger import StdOutLogger
3 | from .null_logger import NullLogger
4 | from .subtemplate_logger import SubtemplateLogger
5 |
6 | from ..exceptions import MarkdownTemplateException
7 |
8 | __log: SubtemplateLogger = StdOutLogger(LogLevel.info)
9 |
10 |
11 | def get_log() -> SubtemplateLogger:
12 | return __log
13 |
14 |
15 | def set_log(log: SubtemplateLogger):
16 | global __log
17 |
18 | if not log or not isinstance(log, SubtemplateLogger):
19 | raise MarkdownTemplateException('log')
20 |
21 | __log = log
22 |
--------------------------------------------------------------------------------
/markdown_subtemplate/logging/log_level.py:
--------------------------------------------------------------------------------
1 | class LogLevel:
2 | verbose = 0
3 | trace = 1
4 | info = 2
5 | error = 3
6 | off = 100
7 |
8 | names = {
9 | 0: 'verbose',
10 | 1: 'trace',
11 | 2: 'info',
12 | 3: 'error',
13 | 100: 'off',
14 | }
15 |
--------------------------------------------------------------------------------
/markdown_subtemplate/logging/null_logger.py:
--------------------------------------------------------------------------------
1 | from .subtemplate_logger import SubtemplateLogger
2 | from .log_level import LogLevel
3 |
4 |
5 | class NullLogger(SubtemplateLogger):
6 | def __init__(self):
7 | super().__init__(LogLevel.error)
8 |
9 | def verbose(self, text: str):
10 | pass
11 |
12 | def trace(self, text: str):
13 | pass
14 |
15 | def info(self, text: str):
16 | pass
17 |
18 | def error(self, text: str):
19 | pass
20 |
--------------------------------------------------------------------------------
/markdown_subtemplate/logging/stdout_logger.py:
--------------------------------------------------------------------------------
1 | from .log_level import LogLevel
2 | from .subtemplate_logger import SubtemplateLogger
3 |
4 |
5 | class StdOutLogger(SubtemplateLogger):
6 | prefix = 'Markdown Subtemplates'
7 |
8 | def verbose(self, text: str):
9 | if not self.should_log(LogLevel.verbose, text):
10 | return
11 |
12 | self._publish(text, LogLevel.verbose)
13 |
14 | def trace(self, text: str):
15 | if not self.should_log(LogLevel.trace, text):
16 | return
17 |
18 | self._publish(text, LogLevel.trace)
19 |
20 | def info(self, text: str):
21 | if not self.should_log(LogLevel.info, text):
22 | return
23 |
24 | self._publish(text, LogLevel.info)
25 |
26 | def error(self, text: str):
27 | if not self.should_log(LogLevel.error, text):
28 | return
29 |
30 | self._publish(text, LogLevel.error)
31 |
32 | def _publish(self, text: str, level: int):
33 | print(f"[{self.prefix}: {LogLevel.names[level].capitalize()}] {text}")
34 |
--------------------------------------------------------------------------------
/markdown_subtemplate/logging/subtemplate_logger.py:
--------------------------------------------------------------------------------
1 | import abc
2 |
3 |
4 | class SubtemplateLogger(abc.ABC):
5 |
6 | def __init__(self, log_level: int):
7 | self.log_level = log_level
8 |
9 | @abc.abstractmethod
10 | def verbose(self, text: str):
11 | pass
12 |
13 | @abc.abstractmethod
14 | def trace(self, text: str):
15 | pass
16 |
17 | @abc.abstractmethod
18 | def info(self, text: str):
19 | pass
20 |
21 | @abc.abstractmethod
22 | def error(self, text: str):
23 | pass
24 |
25 | def should_log(self, level: int, text: str) -> bool:
26 | return self.log_level <= level and (text and text.strip())
27 |
--------------------------------------------------------------------------------
/markdown_subtemplate/storage/__init__.py:
--------------------------------------------------------------------------------
1 | from markdown_subtemplate.exceptions import ArgumentExpectedException
2 | from markdown_subtemplate.storage.subtemplate_storage import SubtemplateStorage
3 | from markdown_subtemplate.storage import file_storage
4 |
5 | __storage: SubtemplateStorage = None
6 |
7 |
8 | def set_storage(storage_instance: SubtemplateStorage):
9 | global __storage
10 | if not storage_instance or not isinstance(storage_instance, SubtemplateStorage):
11 | raise ArgumentExpectedException('storage_instance')
12 |
13 | __storage = storage_instance
14 |
15 |
16 | def get_storage() -> SubtemplateStorage:
17 | global __storage
18 | if not __storage:
19 | from markdown_subtemplate.storage.file_storage import FileStore
20 | __storage = FileStore()
21 | return __storage
22 |
23 |
24 | def is_initialized() -> bool:
25 | return get_storage().is_initialized()
26 |
--------------------------------------------------------------------------------
/markdown_subtemplate/storage/file_storage.py:
--------------------------------------------------------------------------------
1 | import os
2 | from typing import Optional, List
3 |
4 | from markdown_subtemplate.exceptions import TemplateNotFoundException, ArgumentExpectedException, \
5 | InvalidOperationException
6 | from . import SubtemplateStorage
7 |
8 |
9 | class FileStore(SubtemplateStorage):
10 | __template_folder: Optional[str] = None
11 |
12 | def get_markdown_text(self, template_path) -> str:
13 | if not template_path or not template_path.strip():
14 | raise TemplateNotFoundException("No template file specified: template_path=''.")
15 |
16 | file_name = os.path.basename(template_path)
17 | file_parts = os.path.dirname(template_path).split(os.path.sep)
18 | folder = FileStore.get_folder(file_parts)
19 | full_file = os.path.join(folder, file_name).lower()
20 |
21 | if not os.path.exists(full_file):
22 | raise TemplateNotFoundException(full_file)
23 |
24 | with open(full_file, 'r', encoding='utf-8') as fin:
25 | return fin.read()
26 |
27 | def get_shared_markdown(self, import_name):
28 | folder = FileStore.get_folder(['_shared'])
29 | file = os.path.join(folder, import_name.strip().lower() + '.md')
30 |
31 | if not os.path.exists(file):
32 | raise TemplateNotFoundException(file)
33 |
34 | with open(file, 'r', encoding='utf-8') as fin:
35 | return fin.read()
36 |
37 | def is_initialized(self) -> bool:
38 | return bool(FileStore.__template_folder)
39 |
40 | @staticmethod
41 | def get_folder(path_parts: List[str]) -> str:
42 | if not path_parts:
43 | raise ArgumentExpectedException('path_parts')
44 |
45 | if not FileStore.__template_folder:
46 | raise InvalidOperationException("You must set the template folder before calling this method.")
47 |
48 | parts = [
49 | p.strip().strip('/').strip('\\').lower()
50 | for p in path_parts
51 | ]
52 | parent_folder = os.path.abspath(FileStore.__template_folder)
53 | folder = os.path.join(parent_folder, *parts)
54 |
55 | return folder
56 |
57 | @staticmethod
58 | def set_template_folder(full_path: str):
59 | from ..exceptions import PathException
60 | import markdown_subtemplate.logging as logging
61 | log = logging.get_log()
62 |
63 | test_path = os.path.abspath(full_path)
64 | if test_path != full_path:
65 | msg = f"{full_path} is not an absolute path."
66 | log.error("engine.set_template_folder: " + msg)
67 | raise PathException(msg)
68 |
69 | if not os.path.exists(full_path):
70 | msg = f"{full_path} does not exist."
71 | log.error("engine.set_template_folder: " + msg)
72 | raise PathException(msg)
73 |
74 | if not os.path.isdir(full_path):
75 | msg = f"{full_path} is not a directory."
76 | log.error("engine.set_template_folder: " + msg)
77 | raise PathException(msg)
78 |
79 | log.info(f"Template folder set: {full_path}")
80 |
81 | FileStore.__template_folder = full_path
82 |
83 | def clear_settings(self):
84 | FileStore.__template_folder = None
85 |
--------------------------------------------------------------------------------
/markdown_subtemplate/storage/subtemplate_storage.py:
--------------------------------------------------------------------------------
1 | import abc
2 |
3 |
4 | class SubtemplateStorage(abc.ABC):
5 | @abc.abstractmethod
6 | def get_markdown_text(self, template_path) -> str:
7 | pass
8 |
9 | @abc.abstractmethod
10 | def get_shared_markdown(self, import_name) -> str:
11 | pass
12 |
13 | @abc.abstractmethod
14 | def is_initialized(self) -> bool:
15 | pass
16 |
17 | @abc.abstractmethod
18 | def clear_settings(self):
19 | pass
20 |
--------------------------------------------------------------------------------
/readme_resources/mitlicense.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/readme_resources/workflow_image_layout.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeckennedy/markdown-subtemplate/138f46c433b641ad18882da6d3f232f9736d9a32/readme_resources/workflow_image_layout.png
--------------------------------------------------------------------------------
/readme_resources/workflow_image_layout.pptx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikeckennedy/markdown-subtemplate/138f46c433b641ad18882da6d3f232f9736d9a32/readme_resources/workflow_image_layout.pptx
--------------------------------------------------------------------------------
/requirements-dev.txt:
--------------------------------------------------------------------------------
1 | -r requirements.txt
2 |
3 | pytest
4 | twine
5 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | markdown2
2 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | import io
2 | import os
3 | import re
4 |
5 | from setuptools import find_packages
6 | from setuptools import setup
7 |
8 |
9 | def read(filename):
10 | filename = os.path.join(os.path.dirname(__file__), filename)
11 | text_type = type(u"")
12 | with io.open(filename, mode="r", encoding='utf-8') as fd:
13 | return re.sub(text_type(r':[a-z]+:`~?(.*?)`'), text_type(r'``\1``'), fd.read())
14 |
15 |
16 | def read_version():
17 | filename = os.path.join(os.path.dirname(__file__), 'markdown_subtemplate', '__init__.py')
18 | with open(filename, mode="r", encoding='utf-8') as fin:
19 | for line in fin:
20 | if line and line.strip() and line.startswith('__version__'):
21 | return line.split('=')[1].strip().strip("'")
22 |
23 | return "0.0.0.0"
24 |
25 |
26 | requirements_txt = os.path.join(
27 | os.path.dirname(__file__),
28 | 'requirements.txt'
29 | )
30 |
31 | with open(requirements_txt, 'r', encoding='utf-8') as fin:
32 | requires = [
33 | line.strip()
34 | for line in fin
35 | if line and line.strip() and not line.strip().startswith('#')
36 | ]
37 |
38 |
39 | setup(
40 | name="markdown_subtemplate",
41 | version=read_version(),
42 | url="https://github.com/mikeckennedy/markdown-subtemplate",
43 | license='MIT',
44 |
45 | author="Michael Kennedy",
46 | author_email="michael@talkpython.fm",
47 |
48 | description="A template engine to render Markdown with external template imports and variable replacements.",
49 | long_description=read("README.md"),
50 | long_description_content_type='text/markdown',
51 |
52 | packages=find_packages(exclude=('tests',)),
53 |
54 | install_requires=requires,
55 |
56 | classifiers=[
57 | 'Development Status :: 3 - Alpha',
58 | 'License :: OSI Approved :: MIT License',
59 | 'Programming Language :: Python',
60 | 'Programming Language :: Python :: 3',
61 | 'Programming Language :: Python :: 3.6',
62 | 'Programming Language :: Python :: 3.7',
63 | 'Programming Language :: Python :: 3.8',
64 | ],
65 | )
66 |
--------------------------------------------------------------------------------
/tests/_all_tests.py:
--------------------------------------------------------------------------------
1 | # noinspection PyUnresolvedReferences
2 | from engine_tests import *
3 | # noinspection PyUnresolvedReferences
4 | from page_tests import *
5 | # noinspection PyUnresolvedReferences
6 | from logging_tests import *
7 |
--------------------------------------------------------------------------------
/tests/engine_tests.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | import pytest
4 |
5 | from markdown_subtemplate import engine, caching, storage
6 | from markdown_subtemplate import exceptions
7 |
8 | template_folder = os.path.join(os.path.dirname(__file__), 'templates')
9 |
10 |
11 | def test_init_folder_required():
12 | storage.get_storage().clear_settings()
13 |
14 | with pytest.raises(exceptions.InvalidOperationException):
15 | engine.get_page('abc', {})
16 |
17 |
18 | def test_init_folder_missing():
19 | with pytest.raises(exceptions.PathException):
20 | storage.file_storage.FileStore.set_template_folder('bad/cats/')
21 |
22 |
23 | def test_init_folder_success():
24 | storage.file_storage.FileStore.set_template_folder(template_folder)
25 |
26 |
27 | def test_clear_cache():
28 | caching.get_cache().clear()
29 |
--------------------------------------------------------------------------------
/tests/logging_tests.py:
--------------------------------------------------------------------------------
1 | from markdown_subtemplate import logging
2 | from markdown_subtemplate.logging import LogLevel
3 |
4 |
5 | def test_default_log_level():
6 | log = logging.get_log()
7 | assert log.log_level == LogLevel.info
8 |
9 |
10 | def test_can_change_log_level():
11 | log = logging.get_log()
12 |
13 | level = log.log_level
14 | try:
15 | log.log_level = LogLevel.error
16 | assert log.log_level == LogLevel.error
17 | finally:
18 | log.log_level = level
19 |
20 |
21 | def test_should_log_yes():
22 | log = logging.get_log()
23 | assert log.should_log(LogLevel.info, "MSG")
24 | assert log.should_log(LogLevel.error, "MSG")
25 |
26 |
27 | def test_should_log_no():
28 | log = logging.get_log()
29 | assert not log.should_log(LogLevel.verbose, "MSG")
30 | assert not log.should_log(LogLevel.trace, "MSG")
31 |
32 |
33 | def test_logging_off():
34 | log = logging.get_log()
35 | level = log.log_level
36 | try:
37 | log.log_level = LogLevel.off
38 | assert not log.should_log(LogLevel.error, 'MSG')
39 | assert not log.should_log(LogLevel.verbose, 'MSG')
40 | finally:
41 | log.log_level = level
42 |
--------------------------------------------------------------------------------
/tests/page_tests.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | import pytest
4 |
5 | from markdown_subtemplate import engine
6 | from markdown_subtemplate import exceptions
7 | from markdown_subtemplate.infrastructure import page
8 | from markdown_subtemplate.storage.file_storage import FileStore
9 |
10 | FileStore.set_template_folder(
11 | os.path.join(os.path.dirname(__file__), 'templates'))
12 |
13 |
14 | def test_missing_template_by_file():
15 | with pytest.raises(exceptions.TemplateNotFoundException):
16 | engine.get_page(os.path.join('home', 'hiding.md'), {})
17 |
18 |
19 | def test_missing_template_by_folder():
20 | with pytest.raises(exceptions.TemplateNotFoundException):
21 | engine.get_page(os.path.join('hiding', 'index.md'), {})
22 |
23 |
24 | def test_empty_template():
25 | html = engine.get_page(os.path.join('home', 'empty_markdown.md'), {})
26 | assert html == ''
27 |
28 |
29 | def test_basic_markdown_template():
30 | template = os.path.join('home', 'basic_markdown.md')
31 | md = page.get_markdown(template, {'a': 1, 'b': 2})
32 |
33 | text = '''
34 | # This is the basic title
35 |
36 | We have a paragraph with [a link](https://talkpython.fm).
37 |
38 | * Bullet 1
39 | * Bullet 2
40 | * Bullet 3
41 |
42 | '''.strip()
43 | assert text == md.strip()
44 |
45 |
46 | def test_basic_markdown_html():
47 | template = os.path.join('home', 'basic_markdown.md')
48 | html = engine.get_page(template, {'a': 1, 'b': 2})
49 |
50 | text = '''
51 |
161 | '''.strip()
162 |
163 | assert text == html.strip()
164 |
165 |
166 | def test_variable_definition_html_cached():
167 | template = os.path.join('home', 'variables.md')
168 |
169 | # for some reason, the cached version doesn't include the local variables
170 | # let's call it twice and see if we can catch it and guard against it.
171 | page.get_page(template, {})
172 | html = page.get_page(template, {})
173 |
174 | text = '''
175 |
81 | 