├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── lettersmith ├── __init__.py ├── absolutize.py ├── archive.py ├── blog.py ├── cli │ └── scaffold.py ├── data.py ├── date.py ├── doc.py ├── docs.py ├── edge.py ├── file.py ├── files.py ├── func.py ├── html.py ├── io.py ├── jinjatools.py ├── lens.py ├── markdowntools.py ├── package_data │ ├── scaffold │ │ └── blog │ │ │ ├── build.py │ │ │ ├── data │ │ │ └── .empty │ │ │ ├── page │ │ │ ├── .empty │ │ │ ├── About.md │ │ │ ├── Page A.md │ │ │ ├── Page B.md │ │ │ └── index.md │ │ │ ├── post │ │ │ └── Example Post.md │ │ │ ├── static │ │ │ └── .empty │ │ │ └── template │ │ │ ├── _base.html │ │ │ ├── archive.html │ │ │ ├── index.html │ │ │ ├── page.html │ │ │ └── post.html │ └── template │ │ ├── rss.xml │ │ └── sitemap.xml ├── path.py ├── permalink.py ├── query.py ├── rss.py ├── sitemap.py ├── stringtools.py ├── stub.py ├── taxonomy.py ├── util.py ├── wikidoc.py ├── wikimarkup.py └── write.py ├── setup.py └── test ├── __init__.py ├── package_data └── fixtures │ ├── Bare doc.md │ ├── Doc with meta.md │ ├── doc.json │ └── doc.yaml ├── scripts └── generate_fixtures.py ├── test_doc.py ├── test_func.py ├── test_html.py ├── test_lens.py ├── test_query.py ├── test_stringtools.py ├── test_util.py └── test_wikilink.py /.gitignore: -------------------------------------------------------------------------------- 1 | **.pyc 2 | **__pycache__ 3 | **.egg-info 4 | **.DS_Store -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Gordon Brander 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include lettersmith/package_data * -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Lettersmith 2 | 3 | A set of tools for static site generation. It comes with a static site generator bundled in. You can also use it as a library to build your own custom static site generator. 4 | 5 | I built it for myself, because I found other solutions to be pretty baroque and difficult to customize. Right now, it's a simple set of fairly stable tools for personal use. I might package it up later. 6 | 7 | ## Installing 8 | 9 | Lettersmith requires Python 3.6+, and a version of pip compatible with Python 3. 10 | 11 | ```bash 12 | git clone https://github.com/gordonbrander/lettersmith_py 13 | cd lettersmith_py 14 | pip3 install -e . 15 | ``` 16 | 17 | ## lettersmith_scaffold 18 | 19 | You can easily scaffold a site using `lettersmith_scaffold`. 20 | 21 | ```bash 22 | lettersmith_scaffold ./blog --type blog 23 | ``` 24 | 25 | This will stub out a directory structure and a build script for a typical blogging setup. You can customize the build script from there. 26 | 27 | 28 | ## What it does 29 | 30 | Lettersmith comes bundled with a static site generator, but it's really just a library of tools for transforming text. You can use these tools to create your own custom static site generators, build tools, project scaffolders, ebook generators, or wikis — whatever you like. 31 | 32 | Lettersmith loads text files as Python namedtuples, so a markdown file like this: 33 | 34 | ```markdown 35 | --- 36 | title: "My post" 37 | created: 2018-01-17 38 | --- 39 | 40 | Some content 41 | ``` 42 | 43 | Becomes this: 44 | 45 | ```python 46 | Doc( 47 | id_path='path/to/post.md', 48 | output_path='path/to/post.md', 49 | input_path='path/to/post.md', 50 | created=datetime.datetime(2018, 1, 17, 0, 0), 51 | modified=datetime.datetime(2018, 1, 17, 0, 0), 52 | title='My post', 53 | content='Some content', 54 | meta={ 55 | "title": "My post", 56 | "date": "2018-01-17" 57 | }, 58 | template="" 59 | ) 60 | ``` 61 | 62 | 63 | ## Plugins 64 | 65 | Plugins are just functions that transform doc namedtuples. 66 | 67 | To transform many files, you can load them into an iterable, then use list comprehensions, generator expressions, and map, filter, reduce: 68 | 69 | ```python 70 | # Get all markdown docs under source/ 71 | posts = docs.find("posts/*.md") 72 | # Transform them with your function. 73 | posts = my_plugin(posts) 74 | ``` 75 | 76 | To write a plugin, all you need to do is define a generator function that takes an iterator of docs and yields transformed docs. 77 | 78 | ```python 79 | def my_plugin(docs) 80 | for doc in docs: 81 | yield do_something(doc) 82 | ``` 83 | 84 | You can pipe docs through many transforming functions using `pipe`. 85 | 86 | ```python 87 | posts = pipe( 88 | docs.find("source/*.md"), 89 | markdown.content, 90 | my_plugin, 91 | my_other_plugin 92 | ) 93 | ``` 94 | 95 | Which is equivalent to: 96 | 97 | ```python 98 | posts = my_other_plugin(my_plugin(markdown.content(docs.find("source/*.md")))) 99 | ``` 100 | 101 | When you're done transforming things, you can pass the iterable to `write`, which takes care of writing out the files to an output directory. 102 | 103 | ```python 104 | write(posts, directory="public") 105 | ``` 106 | 107 | That's it! 108 | 109 | Check out [blog/build.py](/lettersmith/package_data/scaffold/blog/build.py) for an example of a build script that uses some of the built-in plugins to create a typical blogging setup. 110 | 111 | Lettersmith comes with a swiss army knife of helpful tools for things like Markdown, templates, drafts, tags, wikilinks, and more — and if you see something missing it's easy to write your own functions. -------------------------------------------------------------------------------- /lettersmith/__init__.py: -------------------------------------------------------------------------------- 1 | from lettersmith import absolutize 2 | from lettersmith import archive 3 | from lettersmith import blog 4 | from lettersmith import data 5 | from lettersmith import docs 6 | from lettersmith import files 7 | from lettersmith import html 8 | from lettersmith import jinjatools 9 | from lettersmith import markdowntools 10 | from lettersmith import permalink 11 | from lettersmith import query 12 | from lettersmith import rss 13 | from lettersmith import sitemap 14 | from lettersmith import stub 15 | from lettersmith import taxonomy 16 | from lettersmith import wikidoc 17 | from lettersmith.write import write 18 | from lettersmith.func import rest, pipe, compose, thrush 19 | from itertools import chain -------------------------------------------------------------------------------- /lettersmith/absolutize.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools for making relative URLs absolute in doc content. 3 | """ 4 | import re 5 | from lettersmith.docs import renderer 6 | from lettersmith.func import composable 7 | from lettersmith import path as pathtools 8 | 9 | 10 | URL_ATTR = r"""(src|href)=["'](.*?)["']""" 11 | 12 | 13 | def absolutize(base_url): 14 | """ 15 | Absolutize URLs in content. Replaces any relative URLs in content 16 | that start with `/` and instead starts them with `base_url`. 17 | 18 | URLS are found by matching against `href=` and `src=`. 19 | """ 20 | def render_inner_match(match): 21 | attr = match.group(1) 22 | value = match.group(2) 23 | url = pathtools.qualify_url(value, base_url) 24 | return '{attr}="{url}"'.format(attr=attr, url=url) 25 | 26 | @renderer 27 | def render(content): 28 | """ 29 | Absolutize URLs in doc content fields. 30 | """ 31 | return re.sub(URL_ATTR, render_inner_match, content) 32 | return render -------------------------------------------------------------------------------- /lettersmith/archive.py: -------------------------------------------------------------------------------- 1 | """ 2 | Generate an archive page. 3 | """ 4 | from lettersmith import doc as Doc 5 | from lettersmith import stub as Stub 6 | from lettersmith.func import composable 7 | 8 | 9 | @composable 10 | def archive( 11 | docs, 12 | output_path, 13 | title="Archive", 14 | template="archive.html" 15 | ): 16 | """ 17 | Generate an archive doc for a list of docs. 18 | """ 19 | archive = tuple(Stub.stubs(docs)) 20 | return Doc.create( 21 | id_path=output_path, 22 | output_path=output_path, 23 | title=title, 24 | content="", 25 | template=template, 26 | meta={"archive": archive} 27 | ) -------------------------------------------------------------------------------- /lettersmith/blog.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools for blogging 3 | """ 4 | from lettersmith.func import compose 5 | from lettersmith import wikidoc 6 | from lettersmith import absolutize 7 | from lettersmith import permalink 8 | from lettersmith import docs as Docs 9 | 10 | 11 | def markdown_doc(base_url): 12 | """ 13 | Handle typical transformations for a generic markdown doc. 14 | 15 | - Markdown 16 | - Wikilinks 17 | - Transclusions 18 | - Absolutizes post links 19 | - Changes file extension to .html 20 | - Sets template in prep for Jinja rendering later 21 | """ 22 | return compose( 23 | absolutize.absolutize(base_url), 24 | wikidoc.content_markdown(base_url), 25 | Docs.autotemplate, 26 | Docs.uplift_frontmatter 27 | ) 28 | 29 | 30 | def markdown_page(base_url, relative_to="."): 31 | """ 32 | Performs typical transformations for a page. 33 | 34 | - Sets nice page permalink 35 | - Markdown 36 | - Wikilinks 37 | - Transclusions 38 | - Absolutizes post links 39 | - Changes file extension to .html 40 | - Sets template in prep for Jinja rendering later 41 | """ 42 | return compose( 43 | markdown_doc(base_url), 44 | permalink.rel_page_permalink(relative_to) 45 | ) 46 | 47 | 48 | def markdown_post(base_url): 49 | """ 50 | Performs typical transformations for a blog post. 51 | 52 | - Sets nice date-based permalink 53 | - Markdown 54 | - Wikilinks 55 | - Transclusions 56 | - Absolutizes post links 57 | - Changes file extension to .html 58 | - Sets template in prep for Jinja rendering later 59 | """ 60 | return compose( 61 | markdown_doc(base_url), 62 | permalink.post_permalink 63 | ) 64 | 65 | 66 | def html_doc(base_url): 67 | """ 68 | Handle typical transformations for a generic html doc. 69 | 70 | - Wrap non-HTML lines with paragraph tags. 71 | - Wikilinks 72 | - Transclusions 73 | - Absolutizes post links 74 | - Changes file extension to .html 75 | - Sets template in prep for Jinja rendering later 76 | """ 77 | return compose( 78 | absolutize.absolutize(base_url), 79 | wikidoc.content_html(base_url), 80 | Docs.autotemplate, 81 | Docs.uplift_frontmatter 82 | ) 83 | 84 | 85 | def html_page(base_url, relative_to="."): 86 | """ 87 | Performs typical transformations for a page. 88 | 89 | - Sets nice page permalink 90 | - Wrap non-HTML lines with paragraph tags. 91 | - Wikilinks 92 | - Transclusions 93 | - Absolutizes post links 94 | - Changes file extension to .html 95 | - Sets template in prep for Jinja rendering later 96 | """ 97 | return compose( 98 | html_doc(base_url), 99 | permalink.rel_page_permalink(relative_to) 100 | ) 101 | 102 | 103 | def html_post(base_url): 104 | """ 105 | Performs typical transformations for a blog post. 106 | 107 | - Sets nice date-based permalink 108 | - Wrap non-HTML lines with paragraph tags. 109 | - Wikilinks 110 | - Transclusions 111 | - Absolutizes post links 112 | - Changes file extension to .html 113 | - Sets template in prep for Jinja rendering later 114 | """ 115 | return compose( 116 | html_doc(base_url), 117 | permalink.post_permalink 118 | ) -------------------------------------------------------------------------------- /lettersmith/cli/scaffold.py: -------------------------------------------------------------------------------- 1 | """ 2 | Command line tool for scaffolding Lettersmith sites. 3 | """ 4 | from pathlib import Path, PurePath 5 | from os import makedirs 6 | import shutil 7 | import argparse 8 | import random 9 | 10 | 11 | parser = argparse.ArgumentParser( 12 | description="""A tool for scaffolding Lettersmith sites""") 13 | parser.add_argument("project_path", 14 | type=Path, 15 | help="Path to your project directory") 16 | parser.add_argument("-t", "--type", 17 | type=str, default='blog', choices=["blog"], 18 | help="The type of project to scaffold") 19 | args = parser.parse_args() 20 | 21 | 22 | def main(): 23 | project_path = Path(args.project_path) 24 | module_path = Path(__file__).parent 25 | scaffold_path = Path( 26 | module_path, "..", "package_data", "scaffold", args.type) 27 | 28 | try: 29 | shutil.copytree(scaffold_path, project_path) 30 | 31 | messages = ( 32 | "Hocus pocus — Your new site is ready!", 33 | "Alakazam — Your new site is ready!", 34 | "Tada — Your new site is ready!", 35 | "A wild website appears!" 36 | ) 37 | 38 | print(random.choice(messages)) 39 | except FileExistsError: 40 | message = ( 41 | f"Error: project_path \"{project_path}\" already exists.\n\n" 42 | "Your project_path should be a path to a directory that " 43 | "doesn't exist yet. Lettersmith will create the directory " 44 | "and write all of the project files into it. " 45 | "That way we won't accidentally overwrite anything important!" 46 | ) 47 | print(message) -------------------------------------------------------------------------------- /lettersmith/data.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import yaml 3 | from lettersmith.path import glob_all 4 | 5 | 6 | YAML_EXT = (".yaml", ".yml") 7 | JSON_EXT = (".json") 8 | 9 | 10 | def _smart_read_data_file(file_path): 11 | """ 12 | Given an open file object, this function will do its best to 13 | interpret structured data. 14 | 15 | Supported types: 16 | 17 | * .json 18 | * .yaml 19 | """ 20 | ext = Path(file_path).suffix 21 | with open(file_path, "r") as f: 22 | if ext in JSON_EXT: 23 | return json.load(f) 24 | elif ext in YAML_EXT: 25 | return yaml.load(f) 26 | else: 27 | raise ValueError("Unsupported file type: {}".format(ext)) 28 | 29 | 30 | def find(dir_path): 31 | """ 32 | Create a data dictionary for the template. Each file in the list 33 | will be loaded (supported types include JSON, YAML). The structured 34 | data will be stored under a key corresponding to the filename 35 | (without extension). 36 | 37 | Returns a dictionary of structured Python data. 38 | """ 39 | data = {} 40 | for file_path in glob_all(dir_path, ("*.yaml", "*.yml", "*.json")): 41 | try: 42 | stem = Path(file_path).stem 43 | data[stem] = _smart_read_data_file(file_path) 44 | except ValueError: 45 | pass 46 | return data -------------------------------------------------------------------------------- /lettersmith/date.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime 2 | from os import path 3 | from functools import singledispatch 4 | 5 | 6 | def read_file_times(pathlike): 7 | """ 8 | Given a pathlike, return a tuple of `(created_time, modified_time)`. 9 | Both return values are datetime objects. 10 | 11 | If no value can be found, will return unix epoch for both. 12 | """ 13 | path_str = str(pathlike) 14 | try: 15 | modified_time = datetime.fromtimestamp(path.getmtime(path_str)) 16 | created_time = datetime.fromtimestamp(path.getctime(path_str)) 17 | return created_time, modified_time 18 | except OSError: 19 | return EPOCH, EPOCH 20 | 21 | 22 | @singledispatch 23 | def to_datetime(x): 24 | """ 25 | Given a date or datetime, return a datetime. 26 | Used to read datetime values from meta fields. 27 | """ 28 | raise TypeError("read function not implemented for type {}".format(type(x))) 29 | 30 | 31 | @to_datetime.register(datetime) 32 | def datetime_to_datetime(dt): 33 | return dt 34 | 35 | 36 | @to_datetime.register(date) 37 | def date_to_datetime(d): 38 | """ 39 | Convert a date to a datetime. 40 | """ 41 | return datetime(d.year, d.month, d.day) 42 | 43 | 44 | @to_datetime.register(str) 45 | def date_to_datetime(s): 46 | """ 47 | Convert an ISO 8601 date string to a datetime. 48 | """ 49 | return parse_isoformat(s) 50 | 51 | 52 | def parse_isoformat(dt_str): 53 | """ 54 | Parse an ISO 8601 date string into a datetime. Supports the following date 55 | styles: 56 | 57 | 2017-01-01 58 | 20170101 59 | 2017 01 01 60 | """ 61 | try: 62 | return datetime.strptime(dt_str, "%Y-%m-%d") 63 | except ValueError: 64 | pass 65 | try: 66 | return datetime.strptime(dt_str, "%Y%m%d") 67 | except ValueError: 68 | pass 69 | return datetime.strptime(dt_str, "%Y %m %d") 70 | 71 | 72 | def format_isoformat(dt): 73 | """ 74 | Format datetime as ISO 8601 https://en.wikipedia.org/wiki/ISO_8601 75 | '%Y-%m-%d' 76 | """ 77 | return dt.date().isoformat() 78 | 79 | 80 | EPOCH = datetime.fromtimestamp(0) 81 | EPOCH_ISO_8601 = format_isoformat(EPOCH) 82 | -------------------------------------------------------------------------------- /lettersmith/doc.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools for working with Doc type. 3 | 4 | Docs are namedtuples that represent a file to be transformed. 5 | The `content` field of a doc contains the file contents, read as a 6 | Python string with UTF-8 encoding. 7 | 8 | Most lettersmith plugins transform Docs or iterables of Docs. 9 | 10 | For working with non-text files, images, binary files, or text files 11 | with other encodings, see `lettersmith.file` which stores the raw bytes 12 | instead of reading them into a Python string. 13 | """ 14 | from pathlib import PurePath, Path 15 | import json 16 | from collections import namedtuple 17 | from functools import wraps 18 | 19 | import frontmatter 20 | import yaml 21 | 22 | from lettersmith.util import mix 23 | from lettersmith.date import read_file_times, EPOCH, to_datetime 24 | from lettersmith import path as pathtools 25 | from lettersmith import lens 26 | from lettersmith.lens import ( 27 | Lens, lens_compose, get, put, key, over_with, update 28 | ) 29 | from lettersmith.func import compose 30 | 31 | 32 | Doc = namedtuple("Doc", ( 33 | "id_path", "output_path", "input_path", "created", "modified", 34 | "title", "content", "meta", "template" 35 | )) 36 | Doc.__doc__ = """ 37 | Docs are namedtuples that represent a document to be transformed, 38 | and eventually written to disk. 39 | 40 | Docs contain a content field. This is a string that typically contains the 41 | contents of the file. 42 | """ 43 | 44 | 45 | def create(id_path, output_path, 46 | input_path=None, created=EPOCH, modified=EPOCH, 47 | title="", content="", meta=None, template=""): 48 | """ 49 | Create a Doc tuple, populating it with sensible defaults 50 | """ 51 | return Doc( 52 | id_path=str(id_path), 53 | output_path=str(output_path), 54 | input_path=str(input_path) if input_path is not None else None, 55 | created=to_datetime(created), 56 | modified=to_datetime(modified), 57 | title=str(title), 58 | content=str(content), 59 | meta=meta if meta is not None else {}, 60 | template=str(template) 61 | ) 62 | 63 | 64 | def load(pathlike): 65 | """ 66 | Loads a doc namedtuple from a file path. 67 | `content` field will contain contents of file. 68 | Typically, you decorate the doc later with meta and other fields. 69 | 70 | Returns a doc. 71 | """ 72 | file_created, file_modified = read_file_times(pathlike) 73 | with open(pathlike, 'r') as f: 74 | content = f.read() 75 | title = pathtools.to_title(pathlike) 76 | return create( 77 | id_path=pathlike, 78 | output_path=pathlike, 79 | input_path=pathlike, 80 | created=file_created, 81 | modified=file_modified, 82 | title=title, 83 | meta={}, 84 | content=content 85 | ) 86 | 87 | 88 | def writeable(doc): 89 | """ 90 | Return a writeable tuple for doc. 91 | 92 | writeable tuple is any 2-tuple of `output_path`, `bytes`. 93 | `lettersmith.write` knows how to write these tuples to disk. 94 | """ 95 | return doc.output_path, doc.content.encode() 96 | 97 | 98 | id_path = Lens( 99 | lambda doc: doc.id_path, 100 | lambda doc, id_path: doc._replace(id_path=id_path) 101 | ) 102 | 103 | 104 | output_path = Lens( 105 | lambda doc: doc.output_path, 106 | lambda doc, output_path: doc._replace(output_path=output_path) 107 | ) 108 | 109 | ext = lens_compose(output_path, pathtools.ext) 110 | 111 | title = Lens( 112 | lambda doc: doc.title, 113 | lambda doc, title: doc._replace(title=title) 114 | ) 115 | 116 | 117 | content = Lens( 118 | lambda doc: doc.content, 119 | lambda doc, content: doc._replace(content=content) 120 | ) 121 | 122 | 123 | created = Lens( 124 | lambda doc: doc.created, 125 | lambda doc, created: doc._replace(created=created) 126 | ) 127 | 128 | 129 | modified = Lens( 130 | lambda doc: doc.modified, 131 | lambda doc, modified: doc._replace(modified=modified) 132 | ) 133 | 134 | 135 | meta = Lens( 136 | lambda doc: doc.meta, 137 | lambda doc, meta: doc._replace(meta=meta) 138 | ) 139 | 140 | 141 | template = Lens( 142 | lambda doc: doc.template, 143 | lambda doc, template: doc._replace(template=template) 144 | ) 145 | 146 | 147 | meta_summary = lens_compose(meta, key("summary", "")) 148 | 149 | 150 | def update_meta(doc, patch): 151 | """ 152 | Mix keys from `patch` into `doc.meta`. 153 | """ 154 | return update(meta, mix, doc, patch) 155 | 156 | 157 | def with_ext_html(doc): 158 | """ 159 | Set doc extension to ".html" 160 | """ 161 | return put(ext, doc, ".html") 162 | 163 | 164 | output_tld = compose(pathtools.tld, output_path.get) 165 | id_tld = compose(pathtools.tld, id_path.get) 166 | 167 | 168 | _infer_template = compose( 169 | pathtools.ext_html, 170 | pathtools.to_slug, 171 | id_tld 172 | ) 173 | 174 | 175 | def autotemplate(doc): 176 | """ 177 | Set template based on top-level directory in doc's id_path. 178 | 179 | E.g. if top-level-directory is "posts", template gets set to "posts.html". 180 | """ 181 | if get(template, doc) != "": 182 | return doc 183 | else: 184 | return put(template, doc, _infer_template(doc)) 185 | 186 | 187 | def with_template(t): 188 | """ 189 | Set template `t`, but only if doc doesn't have one already. 190 | """ 191 | def with_template_on_doc(doc): 192 | if get(template, doc) != "": 193 | return doc 194 | else: 195 | return put(template, doc, t) 196 | return with_template_on_doc 197 | 198 | 199 | def to_json(doc): 200 | """ 201 | Serialize a doc as JSON-serializable data 202 | """ 203 | return { 204 | "@type": "doc", 205 | "id_path": doc.id_path, 206 | "output_path": doc.output_path, 207 | "input_path": doc.input_path, 208 | "created": doc.created.timestamp(), 209 | "modified": doc.modified.timestamp(), 210 | "title": doc.title, 211 | "content": doc.content, 212 | "meta": doc.meta, 213 | "template": doc.template 214 | } 215 | 216 | 217 | def uplift_meta(doc): 218 | """ 219 | Reads "magic" fields in the meta and uplifts their values to doc 220 | properties. 221 | 222 | We use this to uplift... 223 | 224 | - title 225 | - created 226 | - modified 227 | - permalink 228 | - template 229 | 230 | ...in the frontmatter, overriding original or default values on doc. 231 | """ 232 | return doc._replace( 233 | title=doc.meta.get("title", doc.title), 234 | created=to_datetime(doc.meta.get("created", doc.created)), 235 | modified=to_datetime(doc.meta.get("modified", doc.modified)), 236 | output_path=doc.meta.get("permalink", doc.output_path), 237 | template=doc.meta.get("template", "") 238 | ) 239 | 240 | 241 | class DocException(Exception): 242 | pass 243 | 244 | 245 | def annotate_exceptions(func): 246 | """ 247 | Decorates a mapping function for docs, giving it a more useful 248 | exception message. 249 | """ 250 | @wraps(func) 251 | def func_with_annotated_exceptions(doc): 252 | try: 253 | return func(doc) 254 | except Exception as e: 255 | msg = ( 256 | 'Error encountered while mapping doc ' 257 | '"{id_path}" with {module}.{func}.' 258 | ).format( 259 | id_path=doc.id_path, 260 | func=func.__qualname__, 261 | module=func.__module__ 262 | ) 263 | raise DocException(msg) from e 264 | return func_with_annotated_exceptions 265 | 266 | 267 | @annotate_exceptions 268 | def parse_frontmatter(doc): 269 | """ 270 | Parse frontmatter as YAML. Set frontmatter on meta field, and 271 | remaining content on content field. 272 | 273 | If there is no frontmatter, will set an empty object on meta field, 274 | and leave content as-is. 275 | """ 276 | meta, content = frontmatter.parse(doc.content) 277 | return doc._replace( 278 | meta=meta, 279 | content=content 280 | ) 281 | 282 | 283 | uplift_frontmatter = compose(uplift_meta, parse_frontmatter) 284 | 285 | 286 | def renderer(render): 287 | """ 288 | Create a renderer for doc content using a string rendering function. 289 | 290 | Will also annotate any exceptions that happen during rendering, 291 | transforming them into DocExceptions that will record the doc's 292 | id_path and the render function where exception occurred. 293 | 294 | Can be used as a decorator. 295 | """ 296 | return annotate_exceptions(over_with(content, render)) -------------------------------------------------------------------------------- /lettersmith/docs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools for working with collections of docs 3 | """ 4 | from fnmatch import fnmatch 5 | from lettersmith import path as pathtools 6 | from lettersmith import doc as Doc 7 | from lettersmith import query 8 | from lettersmith.func import composable, compose 9 | from lettersmith.lens import get 10 | 11 | 12 | load = query.maps(Doc.load) 13 | 14 | 15 | def find(glob): 16 | """ 17 | Load all docs under input path that match a glob pattern. 18 | 19 | Example: 20 | 21 | docs.find("posts/*.md") 22 | """ 23 | return load(pathtools.glob_files(".", glob)) 24 | 25 | 26 | @composable 27 | def remove_id_path(docs, id_path): 28 | """ 29 | Remove docs with a given id_path. 30 | """ 31 | for doc in docs: 32 | if doc.id_path != id_path: 33 | yield doc 34 | 35 | 36 | @composable 37 | def matching(docs, glob): 38 | """ 39 | Filter an iterator of docs to only those docs whos id_path 40 | matches a unix-style glob pattern. 41 | """ 42 | for doc in docs: 43 | if fnmatch(doc.id_path, glob): 44 | yield doc 45 | 46 | 47 | @composable 48 | def filter_siblings(docs, id_path): 49 | """ 50 | Filter a list of dicts with `id_path`, returning a generator 51 | yielding only those dicts who's id_path is a sibling to 52 | `id_path`. 53 | """ 54 | for doc in docs: 55 | if pathtools.is_sibling(id_path, doc.id_path): 56 | yield doc 57 | 58 | 59 | remove_drafts = query.rejects(compose(pathtools.is_draft, Doc.id_path.get)) 60 | remove_index = query.rejects(compose(pathtools.is_index, Doc.id_path.get)) 61 | dedupe = query.dedupes(Doc.id_path.get) 62 | uplift_frontmatter = query.maps(Doc.uplift_frontmatter) 63 | sort_by_created = query.sorts(Doc.created.get, reverse=True) 64 | sort_by_modified = query.sorts(Doc.modified.get, reverse=True) 65 | sort_by_title = query.sorts(Doc.title.get) 66 | autotemplate = query.maps(Doc.autotemplate) 67 | with_ext_html = query.maps(Doc.with_ext_html) 68 | 69 | 70 | def most_recent(n): 71 | """ 72 | Get most recent `n` docs, ordered by created. 73 | """ 74 | return compose( 75 | query.takes(n), 76 | sort_by_created 77 | ) 78 | 79 | 80 | def with_template(template): 81 | """ 82 | Set template, but only if doc doesn't have one already. 83 | """ 84 | return query.maps(Doc.with_template(template)) 85 | 86 | 87 | def renderer(render): 88 | """ 89 | Create a renderer for docs using a string render function. 90 | 91 | Can be used as a decorator. 92 | """ 93 | return query.maps(Doc.renderer(render)) -------------------------------------------------------------------------------- /lettersmith/edge.py: -------------------------------------------------------------------------------- 1 | """ 2 | Data structure for describing directed graph connections. 3 | """ 4 | from collections import namedtuple 5 | 6 | 7 | Edge = namedtuple("Edge", ("tail", "head")) 8 | Edge.__doc__ = """ 9 | A directed edge that points from one thing to another. 10 | If you imagine an arrow, tail is the base of the arrow, and head is 11 | the pointy arrow head. 12 | """ -------------------------------------------------------------------------------- /lettersmith/file.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools for working with files. 3 | 4 | Files are namedtuples that represent the raw bytes in a file to be 5 | copied or transformed. 6 | """ 7 | from collections import namedtuple 8 | from pathlib import PurePath 9 | from lettersmith.date import read_file_times, EPOCH, to_datetime 10 | from lettersmith import doc as Doc 11 | 12 | 13 | File = namedtuple("File", ( 14 | "id_path", "output_path", "input_path", 15 | "created", "modified", "blob" 16 | )) 17 | File.__doc__ = """ 18 | Files are namedtuples that represent the raw bytes in a file to be 19 | copied or transformed. 20 | 21 | Files contain a `blob` field that contains the bytes of the file. 22 | """ 23 | 24 | 25 | def create(id_path, output_path, blob, 26 | input_path=None, created=EPOCH, modified=EPOCH): 27 | """ 28 | Create a File tuple, populating it with sensible defaults 29 | """ 30 | return File( 31 | id_path=str(id_path), 32 | output_path=str(output_path), 33 | input_path=str(input_path) if input_path is not None else None, 34 | created=to_datetime(created), 35 | modified=to_datetime(modified), 36 | blob=bytes(blob) 37 | ) 38 | 39 | 40 | def load(pathlike): 41 | """ 42 | Loads a File namedtuple from a file path. 43 | `blob` field will contain bytes of file. 44 | Returns a File. 45 | """ 46 | file_created, file_modified = read_file_times(pathlike) 47 | with open(pathlike, 'rb') as f: 48 | blob = f.read() 49 | return create( 50 | id_path=pathlike, 51 | output_path=pathlike, 52 | input_path=pathlike, 53 | created=file_created, 54 | modified=file_modified, 55 | blob=blob 56 | ) 57 | 58 | 59 | def writeable(file): 60 | """ 61 | Return a writeable tuple for file. 62 | 63 | writeable tuple is any 2-tuple of `output_path`, `bytes`. 64 | `lettersmith.write` knows how to write these tuples to disk. 65 | """ 66 | return file.output_path, file.blob 67 | 68 | 69 | def to_doc(file): 70 | """ 71 | Create a Doc from a File. 72 | """ 73 | return Doc.create( 74 | id_path=file.id_path, 75 | output_path=file.output_path, 76 | input_path=file.input_path, 77 | created=file.created, 78 | modified=file.modified, 79 | content=file.blob.decode() 80 | ) 81 | 82 | 83 | def from_doc(file): 84 | """ 85 | Create a File from a Doc. 86 | """ 87 | return create( 88 | id_path=doc.id_path, 89 | output_path=doc.output_path, 90 | input_path=doc.input_path, 91 | created=doc.created, 92 | modified=doc.modified, 93 | blob=doc.content.encode() 94 | ) -------------------------------------------------------------------------------- /lettersmith/files.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools for working with collections of files 3 | """ 4 | from lettersmith.path import glob_files 5 | from lettersmith import file as File 6 | from lettersmith import query 7 | 8 | 9 | load = query.maps(File.load) 10 | 11 | 12 | def find(glob): 13 | """ 14 | Load all docs under input path that match a glob pattern. 15 | 16 | Example: 17 | 18 | docs.find("posts/*.md") 19 | """ 20 | return load(glob_files(".", glob)) 21 | 22 | 23 | to_doc = query.maps(File.to_doc) 24 | from_doc = query.maps(File.from_doc) -------------------------------------------------------------------------------- /lettersmith/func.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools for working with higher-order functions. 3 | """ 4 | from functools import reduce, wraps 5 | 6 | 7 | def id(x): 8 | """ 9 | The id function. 10 | """ 11 | return x 12 | 13 | 14 | def _compose2(b, a): 15 | """Compose 2 functions""" 16 | def composed(x): 17 | """Composed function""" 18 | return b(a(x)) 19 | return composed 20 | 21 | 22 | def compose(*funcs): 23 | """Compose n functions from right to left""" 24 | return reduce(_compose2, funcs, id) 25 | 26 | 27 | def thrush(*funcs): 28 | """ 29 | Compose n functions from left to right. 30 | 31 | This is the same as compose, but using left-to-right application 32 | instead of right-to-left application. 33 | 34 | What's with the name? 35 | 36 | Following on Racket's naming convention 37 | https://docs.racket-lang.org/point-free/index.html 38 | 39 | It's named after 40 | https://en.wikipedia.org/wiki/To_Mock_a_Mockingbird 41 | """ 42 | return reduce(_compose2, reversed(funcs), id) 43 | 44 | 45 | def _apply_to(value, func): 46 | """ 47 | Apply value to a single argument function. 48 | """ 49 | return func(value) 50 | 51 | 52 | def pipe(value, *funcs): 53 | """ 54 | Pipe value through a series of single argument functions. 55 | This is basically a function version of a pipeline operator. 56 | 57 | pipe(value, a, b, c) 58 | 59 | is equivalent to 60 | 61 | c(b(a(value))) 62 | 63 | Returns transformed value. 64 | """ 65 | return reduce(_apply_to, funcs, value) 66 | 67 | 68 | def composable(func): 69 | """ 70 | Decorator to transform a function into a composable function 71 | that consumes all of the "rest" of the arguments (everything 72 | after the first argument), then returns a bound function taking 73 | one argument (the first argument). 74 | """ 75 | @wraps(func) 76 | def composable_func(*args, **kwargs): 77 | return rest(func, *args, **kwargs) 78 | return composable_func 79 | 80 | 81 | def rest(func, *args, **kwargs): 82 | """ 83 | Binds the "rest" of the arguments, then returns a bound 84 | function taking one argument (the first argument). 85 | """ 86 | @wraps(func) 87 | def bound(first): 88 | return func(first, *args, **kwargs) 89 | return bound -------------------------------------------------------------------------------- /lettersmith/html.py: -------------------------------------------------------------------------------- 1 | """ 2 | HTML markup -- a very unfancy markup language that automatically wraps 3 | bare lines with `

` tags, but only if they don't contain a 4 | block-level element. 5 | 6 | Example: 7 | 8 | Bare lines are wrapped in paragraph tags. 9 | 10 | Blank spaces are ignored. 11 | 12 | You can use HTML like bold, or italic in paragraphs. 13 | 14 |

15 | Indented lines, or lines containing block-level elements, will 16 | not be wrapped. 17 |
18 | 19 | That's it! 20 | """ 21 | import re 22 | from collections import namedtuple 23 | from lettersmith.stringtools import first_sentence 24 | from lettersmith import docs as Docs 25 | 26 | 27 | def strip_html(html_str): 28 | """Remove html tags from a string.""" 29 | return re.sub('<[^<]+?>', '', html_str) 30 | 31 | 32 | def get_summary(doc): 33 | """ 34 | Get summary for doc. Uses "summary" meta field if it exists. 35 | Otherwise, generates a summary by truncating doc content. 36 | """ 37 | try: 38 | return strip_html(doc.meta["summary"]) 39 | except KeyError: 40 | return first_sentence(strip_html(doc.content)) 41 | 42 | 43 | class RenderError(Exception): 44 | pass 45 | 46 | 47 | Token = namedtuple("Token", ("type", "body")) 48 | Token.__doc__ = """ 49 | A parsed token from the markup. 50 | """ 51 | 52 | _BLOCK_ELS = r"]*>" 53 | 54 | 55 | def _tokenize(lines): 56 | """ 57 | Turn lines into tagged tokens. 58 | """ 59 | for line in lines: 60 | line_clean = line.strip() 61 | if line_clean is "": 62 | pass 63 | elif line.startswith(" "): 64 | yield Token("html", line_clean) 65 | elif re.match(_BLOCK_ELS, line): 66 | yield Token("html", line_clean) 67 | else: 68 | yield Token("p", line_clean) 69 | 70 | 71 | def _render_token(token): 72 | """ 73 | Render HTML tokens. This markup language is very simple, so we only 74 | have two. 75 | """ 76 | if token.type is "html": 77 | return token.body 78 | elif token.type is "p": 79 | return "

{}

".format(token.body) 80 | else: 81 | raise RenderError("Unknown token type {}".format(token.type)) 82 | 83 | 84 | def render_html(text): 85 | """ 86 | Renders text as HTML. 87 | """ 88 | lines = text.splitlines() 89 | return "\n".join(_render_token(token) for token in _tokenize(lines)) 90 | 91 | 92 | content = Docs.renderer(render_html) -------------------------------------------------------------------------------- /lettersmith/io.py: -------------------------------------------------------------------------------- 1 | """ 2 | File utilities 3 | """ 4 | import shutil 5 | from pathlib import Path, PurePath 6 | 7 | 8 | def write_file_deep(pathlike, content, mode="w"): 9 | """Write a file to filepath, creating directories if necessary""" 10 | file_path = Path(pathlike) 11 | file_path.parent.mkdir(exist_ok=True, parents=True) 12 | with open(file_path, mode) as f: 13 | f.write(content) -------------------------------------------------------------------------------- /lettersmith/jinjatools.py: -------------------------------------------------------------------------------- 1 | import random 2 | import itertools 3 | from datetime import datetime 4 | 5 | from jinja2 import Environment, FileSystemLoader 6 | 7 | from lettersmith import util 8 | from lettersmith import docs as Docs 9 | from lettersmith import doc as Doc 10 | from lettersmith import query 11 | from lettersmith.lens import get, put 12 | from lettersmith import path as pathtools 13 | from lettersmith.markdowntools import markdown 14 | 15 | 16 | def _choice(iterable): 17 | return random.choice(tuple(iterable)) 18 | 19 | 20 | def _shuffle(iterable): 21 | """ 22 | Shuffles the elements in an iterable, returning a new list. 23 | 24 | Will collect any iterable before sampling. 25 | This prevents annoying in-template errors, where collecting 26 | an iterator into a tuple can be non-trivial. 27 | """ 28 | t = tuple(iterable) 29 | return random.sample(t, k=len(t)) 30 | 31 | 32 | def _sample(iterable, k): 33 | """ 34 | Will collect any iterable before sampling. 35 | This prevents annoying in-template errors, where collecting 36 | an iterator into a tuple can be non-trivial. 37 | """ 38 | l = list(iterable) 39 | try: 40 | return random.sample(l, k) 41 | except ValueError: 42 | return l 43 | 44 | 45 | def _permalink(base_url): 46 | def permalink_bound(output_path): 47 | return pathtools.to_url(output_path, base_url) 48 | return permalink_bound 49 | 50 | 51 | class FileSystemEnvironment(Environment): 52 | def __init__(self, templates_path, filters={}, context={}): 53 | loader = FileSystemLoader(templates_path) 54 | super().__init__(loader=loader) 55 | self.filters.update(filters) 56 | self.globals.update(context) 57 | 58 | 59 | TEMPLATE_FUNCTIONS = { 60 | "sorted": sorted, 61 | "len": len, 62 | "islice": itertools.islice, 63 | "choice": _choice, 64 | "sample": _sample, 65 | "shuffle": _shuffle, 66 | "to_url": pathtools.to_url, 67 | "join": util.join, 68 | "tuple": tuple 69 | } 70 | 71 | 72 | class LettersmithEnvironment(FileSystemEnvironment): 73 | """ 74 | Specialized version of default Jinja environment class that 75 | offers additional filters and environment variables. 76 | """ 77 | def __init__(self, templates_path, filters={}, context={}): 78 | loader = FileSystemLoader(templates_path) 79 | super().__init__( 80 | templates_path, 81 | filters=TEMPLATE_FUNCTIONS, 82 | context=TEMPLATE_FUNCTIONS 83 | ) 84 | self.filters.update(filters) 85 | self.globals.update(context) 86 | 87 | 88 | def should_template(doc): 89 | """ 90 | Check if a doc should be templated. Returns a bool. 91 | """ 92 | return get(Doc.template, doc) is not "" 93 | 94 | 95 | def jinja(templates_path, base_url, context={}, filters={}): 96 | """ 97 | Wraps up the gory details of creating a Jinja renderer. 98 | Returns a render function that takes a doc and returns a rendered doc. 99 | Template comes preloaded with Jinja default filters, and 100 | Lettersmith default filters and globals. 101 | """ 102 | now = datetime.now() 103 | env = LettersmithEnvironment( 104 | templates_path, 105 | filters={"permalink": _permalink(base_url), **filters}, 106 | context={"now": now, **context} 107 | ) 108 | 109 | @query.maps 110 | @Doc.annotate_exceptions 111 | def render(doc): 112 | if should_template(doc): 113 | template = env.get_template(doc.template) 114 | rendered = template.render({"doc": doc}) 115 | return put(Doc.content, doc, rendered) 116 | else: 117 | return doc 118 | 119 | return render -------------------------------------------------------------------------------- /lettersmith/lens.py: -------------------------------------------------------------------------------- 1 | """ 2 | A minimal implementation of Haskel-style lenses 3 | 4 | Inspired by Elm's Focus library and Racket's Lenses library. 5 | 6 | Lenses let you create getters and setters for complex data structures. 7 | The combination of a getter and setter is called a lens. 8 | 9 | Lenses can be composed to provide a way to do deep reads and deep writes 10 | to complex data structures. 11 | """ 12 | from collections import namedtuple 13 | from functools import reduce 14 | 15 | 16 | Lens = namedtuple("Lens", ("get", "put")) 17 | Lens.__doc__ = """ 18 | Container type for Lenses. 19 | A lens is any structure with `get` and `put` functions that 20 | follow the lens signature. 21 | """ 22 | 23 | 24 | def _lens_compose2(big_lens, small_lens): 25 | """ 26 | Compose 2 lenses. This allows you to create a lens that can 27 | do a deep get/set. 28 | """ 29 | def get(big): 30 | """ 31 | Lens `get` method (composed) 32 | """ 33 | return small_lens.get(big_lens.get(big)) 34 | 35 | def put(big, small): 36 | """ 37 | Lens `update` method (composed) 38 | """ 39 | return big_lens.put( 40 | big, 41 | small_lens.put(big_lens.get(big), small) 42 | ) 43 | 44 | return Lens(get, put) 45 | 46 | 47 | def lens_compose(big_lens, *smaller_lenses): 48 | """ 49 | Compose many lenses 50 | """ 51 | return reduce(_lens_compose2, smaller_lenses, big_lens) 52 | 53 | 54 | def get(lens, big): 55 | """ 56 | Get a value from `big` using `lens`. 57 | """ 58 | return lens.get(big) 59 | 60 | 61 | def put(lens, big, small): 62 | """ 63 | Set a value in `big`. 64 | """ 65 | return lens.put(big, small) 66 | 67 | 68 | def over(lens, func, big): 69 | """ 70 | Map value(s) in `big` using a `mapping` function. 71 | """ 72 | return put(lens, big, func(get(lens, big))) 73 | 74 | 75 | def over_with(lens, func): 76 | """ 77 | Given a lens and a function, returns a single-argument function 78 | that will map over value in `big` using `func`, and returning 79 | a new instance of `big`. 80 | """ 81 | def over_bound(big): 82 | """ 83 | Map value(s) in `big` using a bound mapping function. 84 | """ 85 | return over(lens, func, big) 86 | return over_bound 87 | 88 | 89 | def update(lens, up, big, msg): 90 | """ 91 | Update `big` through an update function, `up` which takes the 92 | current small, and a `msg`, and returns a new small. 93 | """ 94 | return put(lens, big, up(get(lens, big), msg)) 95 | 96 | 97 | def key(k, default=None): 98 | """ 99 | Lens to get and set a key on a dictionary, with default value. 100 | Because it allows for a default, it technically violates the 101 | lens laws. However, in practice, it's too darn useful not to have. 102 | """ 103 | def get(big): 104 | """ 105 | Get key from dict 106 | """ 107 | return big.get(k, default) 108 | 109 | def put(big, small): 110 | """ 111 | Put value in key from dict, returning new dict. 112 | """ 113 | # Check that we're actually making a change before creating 114 | # a new dict. 115 | if big.get(k, default) == small: 116 | return big 117 | else: 118 | return {**big, k: small} 119 | 120 | return Lens(get, put) 121 | 122 | 123 | def _pick(d, keys): 124 | return {k: d[k] for k in keys} 125 | 126 | 127 | def keys(*keys): 128 | """ 129 | Lens to get and set multiple keys on a dictionary. Note that 130 | no default values are allowed. 131 | """ 132 | def get(big): 133 | """ 134 | Get key from dict 135 | """ 136 | return _pick(big, keys) 137 | 138 | def put(big, small): 139 | """ 140 | Put value in key from dict, returning new dict. 141 | """ 142 | patch = _pick(small, keys) 143 | return {**big, **patch} 144 | 145 | return Lens(get, put) -------------------------------------------------------------------------------- /lettersmith/markdowntools.py: -------------------------------------------------------------------------------- 1 | from commonmark import commonmark 2 | from lettersmith.html import strip_html 3 | from lettersmith import docs as Docs 4 | from lettersmith.func import compose 5 | 6 | 7 | markdown = commonmark 8 | strip_markdown = compose(strip_html, markdown) 9 | content = Docs.renderer(markdown) -------------------------------------------------------------------------------- /lettersmith/package_data/scaffold/blog/build.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | A blog-aware Lettersmith build script. Modify it to your heart's content. 4 | """ 5 | from lettersmith import * 6 | 7 | # Configuration 8 | base_url = "http://yourwebsite.com" 9 | site_title = "My very cool website" 10 | site_description = "A very cool website" 11 | site_author = "A very cool person" 12 | 13 | # Load data directory 14 | template_data = data.find("data") 15 | 16 | # Load static and binary files 17 | static = files.find("static/**/*") 18 | 19 | # Load post docs and pipe through plugins 20 | posts = pipe( 21 | docs.find("post/*.md"), 22 | blog.markdown_post(base_url), 23 | docs.sort_by_created, 24 | tuple 25 | ) 26 | 27 | # Load page docs and pipe through plugins 28 | pages = pipe( 29 | docs.find("page/*.md"), 30 | blog.markdown_page(base_url, relative_to="page") 31 | ) 32 | 33 | posts_rss_doc = pipe(posts, rss.rss( 34 | base_url=base_url, 35 | title=site_title, 36 | description=site_description, 37 | author=site_author, 38 | output_path="posts.xml" 39 | )) 40 | 41 | archive_doc = pipe(posts, archive.archive("archive/index.html")) 42 | recent_posts = pipe(posts, stub.stubs, query.takes(5)) 43 | 44 | posts_and_pages = (*posts, *pages) 45 | 46 | sitemap_doc = pipe(posts_and_pages, sitemap.sitemap(base_url)) 47 | 48 | context = { 49 | "rss_docs": (posts_rss_doc,), 50 | "recent": recent_posts, 51 | "site": { 52 | "title": site_title, 53 | "description": site_description, 54 | "author": site_author 55 | }, 56 | "data": template_data, 57 | "base_url": base_url 58 | } 59 | 60 | rendered_docs = pipe( 61 | (sitemap_doc, posts_rss_doc, archive_doc, *posts_and_pages), 62 | jinjatools.jinja("template", base_url, context) 63 | ) 64 | 65 | write(chain(static, rendered_docs), directory="public") 66 | 67 | print("Done!") -------------------------------------------------------------------------------- /lettersmith/package_data/scaffold/blog/data/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordonbrander/lettersmith_py/96ddaf1268ac53062b2e7b1d05d06dc092865666/lettersmith/package_data/scaffold/blog/data/.empty -------------------------------------------------------------------------------- /lettersmith/package_data/scaffold/blog/page/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordonbrander/lettersmith_py/96ddaf1268ac53062b2e7b1d05d06dc092865666/lettersmith/package_data/scaffold/blog/page/.empty -------------------------------------------------------------------------------- /lettersmith/package_data/scaffold/blog/page/About.md: -------------------------------------------------------------------------------- 1 | A bit about this site. -------------------------------------------------------------------------------- /lettersmith/package_data/scaffold/blog/page/Page A.md: -------------------------------------------------------------------------------- 1 | Example link to [[page B]]. -------------------------------------------------------------------------------- /lettersmith/package_data/scaffold/blog/page/Page B.md: -------------------------------------------------------------------------------- 1 | Example link to [[page A]]. -------------------------------------------------------------------------------- /lettersmith/package_data/scaffold/blog/page/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Home 3 | template: "index.html" 4 | --- 5 | 6 | This is your home page. -------------------------------------------------------------------------------- /lettersmith/package_data/scaffold/blog/post/Example Post.md: -------------------------------------------------------------------------------- 1 | --- 2 | created: 2019-11-24 3 | --- 4 | 5 | Content -------------------------------------------------------------------------------- /lettersmith/package_data/scaffold/blog/static/.empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordonbrander/lettersmith_py/96ddaf1268ac53062b2e7b1d05d06dc092865666/lettersmith/package_data/scaffold/blog/static/.empty -------------------------------------------------------------------------------- /lettersmith/package_data/scaffold/blog/template/_base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {{site.title}} 6 | 7 | 8 | {% block body %}{% endblock %} 9 | 10 | -------------------------------------------------------------------------------- /lettersmith/package_data/scaffold/blog/template/archive.html: -------------------------------------------------------------------------------- 1 | {% extends "_base.html" %} 2 | 3 | {% block body %} 4 | {% for stub in doc.meta.archive %} 5 |
6 |

{{stub.title}}

7 |
{{stub.summary}}
8 |
9 | {% endfor %} 10 | {% endblock %} -------------------------------------------------------------------------------- /lettersmith/package_data/scaffold/blog/template/index.html: -------------------------------------------------------------------------------- 1 | {% extends "_base.html" %} 2 | 3 | {% block body %} 4 |
5 |

{{doc.title}}

6 |
{{doc.content}}
7 | 8 |

Recent

9 | {% for stub in recent %} 10 |
11 |

{{stub.title}}

12 |
{{stub.summary}}
13 |
14 | {% endfor %} 15 |
16 | {% endblock %} -------------------------------------------------------------------------------- /lettersmith/package_data/scaffold/blog/template/page.html: -------------------------------------------------------------------------------- 1 | {% extends "_base.html" %} 2 | 3 | {% block body %} 4 |
5 |

{{doc.title}}

6 |
{{doc.content}}
7 |
8 | {% endblock %} -------------------------------------------------------------------------------- /lettersmith/package_data/scaffold/blog/template/post.html: -------------------------------------------------------------------------------- 1 | {% extends "_base.html" %} 2 | 3 | {% block body %} 4 |
5 |

{{doc.title}}

6 |
{{doc.content}}
7 |
8 | {% endblock %} -------------------------------------------------------------------------------- /lettersmith/package_data/template/rss.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | {{title | escape}} 4 | {{base_url}} 5 | {{description | escape}} 6 | {{generator}} 7 | 8 | {{last_build_date.strftime("%a, %d %b %Y %H:%M:%S %Z")}} 9 | 10 | {% for doc in docs %} 11 | 12 | {{doc.title}} 13 | {{doc.output_path | to_url(base_url)}} 14 | {{doc.output_path | to_url(base_url)}} 15 | {{doc | get_summary | escape}} 16 | 19 | {{doc.created.strftime("%a, %d %b %Y %H:%M:%S %Z")}} 20 | {% if doc.meta.author %} 21 | {{doc.meta.author | escape}} 22 | {% elif author %} 23 | {{author | escape}} 24 | {% endif %} 25 | 26 | {% endfor %} 27 | 28 | -------------------------------------------------------------------------------- /lettersmith/package_data/template/sitemap.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | {% for doc in docs %} 4 | 5 | {{ doc.output_path | to_url(base_url) }} 6 | {{ doc.modified.isoformat() }} 7 | 8 | {% endfor %} 9 | -------------------------------------------------------------------------------- /lettersmith/path.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import urlparse, urljoin 2 | from pathlib import Path, PurePath 3 | import re 4 | from lettersmith.func import compose 5 | from lettersmith.lens import Lens, put 6 | from lettersmith import query 7 | 8 | 9 | _STRANGE_CHARS = "[](){}<>:^&%$#@!'\"|*~`," 10 | _STRANGE_CHAR_PATTERN = "[{}]".format(re.escape(_STRANGE_CHARS)) 11 | 12 | 13 | def _space_to_dash(text): 14 | """Replace spaces with dashes.""" 15 | return re.sub(r"\s+", "-", text) 16 | 17 | 18 | def _remove_strange_chars(text): 19 | """Remove funky characters that don't belong in a URL.""" 20 | return re.sub(_STRANGE_CHAR_PATTERN, "", text) 21 | 22 | 23 | def _lower(s): 24 | return s.lower() 25 | 26 | 27 | def _strip(s): 28 | return s.strip() 29 | 30 | 31 | to_slug = compose( 32 | _space_to_dash, 33 | _remove_strange_chars, 34 | _lower, 35 | _strip, 36 | str 37 | ) 38 | 39 | 40 | def to_title(pathlike): 41 | """ 42 | Read a pathlike as a title. This takes the stem and removes any 43 | leading "_". 44 | """ 45 | stem = PurePath(pathlike).stem 46 | return stem[1:] if stem.startswith("_") else stem 47 | 48 | 49 | def is_file_like(pathlike): 50 | """Check if path is file-like, that is, ends with an `xxx.xxx`""" 51 | return len(PurePath(pathlike).suffix) > 0 52 | 53 | 54 | def ensure_trailing_slash(pathlike): 55 | """Append a trailing slash to a path if missing.""" 56 | path_str = str(pathlike) 57 | if is_file_like(path_str) or path_str.endswith("/"): 58 | return path_str 59 | else: 60 | return path_str + "/" 61 | 62 | 63 | def is_local_url(url): 64 | """Does the URL have a scheme?""" 65 | o = urlparse(url) 66 | return not o.scheme 67 | 68 | 69 | def qualify_url(pathlike, base="/"): 70 | """ 71 | Qualify a URL with a basepath. Will leave URL if the URL is already 72 | qualified. 73 | """ 74 | path_str = str(pathlike) 75 | if not path_str.startswith(base) and is_local_url(path_str): 76 | return urljoin(base, path_str) 77 | else: 78 | return path_str 79 | 80 | 81 | def remove_base_slash(any_path): 82 | """Remove base slash from a path.""" 83 | return re.sub("^/", "", any_path) 84 | 85 | 86 | def undraft(pathlike): 87 | """ 88 | Remove the leading `_` from a path, if any. 89 | """ 90 | path = PurePath(pathlike) 91 | if path.stem.startswith("_"): 92 | return path.with_name(re.sub(r'^_', "", path.name)) 93 | else: 94 | return path 95 | 96 | 97 | def relative_to(tlds): 98 | """ 99 | Create a mapping function that will transform pathlike so it is 100 | relative to some top-level directories (tlds). 101 | """ 102 | return lambda pathlike: str(PurePath(pathlike).relative_to(tlds)) 103 | 104 | 105 | def to_nice_path(ugly_pathlike): 106 | """ 107 | Makes an ugly path into a "nice path". Nice paths are paths that end with 108 | an index file, so you can reference them `like/this/` instead of 109 | `like/This.html`. 110 | 111 | ugly_path: 112 | some/File.md 113 | 114 | nice_path: 115 | some/file/index.html 116 | """ 117 | purepath = PurePath(ugly_pathlike) 118 | purepath = undraft(purepath) 119 | # Don't touch index pages 120 | if purepath.stem == "index": 121 | return purepath 122 | index_file = "index" + purepath.suffix 123 | index_path = PurePath(purepath.parent, purepath.stem, index_file) 124 | # Slug-ify and then convert slug string to path object. 125 | nice_path = PurePath(to_slug(index_path)) 126 | return nice_path 127 | 128 | 129 | def _get_ext(pathlike): 130 | return PurePath(pathlike).suffix 131 | 132 | 133 | def _put_ext(pathlike, ext): 134 | return str(PurePath(pathlike).with_suffix(ext)) 135 | 136 | 137 | ext = Lens(_get_ext, _put_ext) 138 | 139 | 140 | def ext_html(pathlike): 141 | """ 142 | Set suffix `.html` on a pathlike. 143 | Return a path string. 144 | """ 145 | return put(ext, pathlike, ".html") 146 | 147 | 148 | def to_url(pathlike, base="/"): 149 | """ 150 | Makes a nice path into a url. 151 | Basically gets rid of the trailing `index.html`. 152 | 153 | nice_path: 154 | some/file/index.html 155 | 156 | url: 157 | /some/file/ 158 | """ 159 | slug = to_slug(pathlike) 160 | purepath = PurePath(slug) 161 | if purepath.name == "index.html": 162 | purepath = ensure_trailing_slash(purepath.parent) 163 | qualified = qualify_url(purepath, base=base) 164 | return qualified 165 | 166 | 167 | def is_draft(pathlike): 168 | return PurePath(pathlike).name.startswith("_") 169 | 170 | 171 | def is_index(pathlike): 172 | return PurePath(pathlike).stem == 'index' 173 | 174 | 175 | def tld(pathlike): 176 | """ 177 | Get the name of the top-level directory in this path. 178 | """ 179 | parts = PurePath(pathlike).parts 180 | return parts[0] if len(parts) > 1 else '' 181 | 182 | 183 | def read_dir(some_path): 184 | """ 185 | Read a path to return the directory portion. 186 | If the path looks like a file, will return the dirname. 187 | Otherwise, will leave the path untouched. 188 | """ 189 | return path.dirname(some_path) if is_file_like(some_path) else some_path 190 | 191 | 192 | def is_sibling(path_a, path_b): 193 | """ 194 | What is a sibling: 195 | 196 | foo/bar/baz.html 197 | foo/bar/bing.html 198 | 199 | What is not a sibling: 200 | 201 | foo/bar/boing/index.html 202 | """ 203 | return ( 204 | PurePath(path_a).parent == PurePath(path_b).parent 205 | and not is_index(path_b)) 206 | 207 | 208 | def filter_files(paths): 209 | """ 210 | Given an iterable of paths, filter paths to just those which are 211 | file paths. 212 | """ 213 | for pathlike in paths: 214 | path = Path(pathlike) 215 | if path.is_file(): 216 | yield path 217 | 218 | 219 | def glob_files(directory, glob): 220 | """ 221 | Return files matching glob 222 | """ 223 | return filter_files(Path(directory).glob(glob)) 224 | 225 | 226 | def glob_all(pathlike, globs): 227 | """ 228 | Given a pathlike and an iterable of glob patterns, will glob 229 | all of them under the path. 230 | Returns a generator of all results. 231 | """ 232 | realpath = Path(pathlike) 233 | for glob_pattern in globs: 234 | for p in realpath.glob(glob_pattern): 235 | yield p -------------------------------------------------------------------------------- /lettersmith/permalink.py: -------------------------------------------------------------------------------- 1 | from pathlib import PurePath 2 | from lettersmith.docs import with_ext_html 3 | from lettersmith.func import composable, compose 4 | from lettersmith import path as pathtools 5 | from lettersmith.lens import over_with, put 6 | from lettersmith import doc as Doc 7 | from lettersmith import query 8 | 9 | 10 | def read_doc_permalink(doc): 11 | """ 12 | Read doc, producing a flat dictionary of permalink template token values. 13 | """ 14 | id_path = PurePath(doc.id_path) 15 | return { 16 | "name": id_path.name, 17 | "stem": id_path.stem, 18 | "suffix": id_path.suffix, 19 | "parents": str(id_path.parent), 20 | "parent": id_path.parent.stem, 21 | "tld": Doc.id_tld(doc), 22 | "yy": doc.created.strftime("%y"), 23 | "yyyy": doc.created.strftime("%Y"), 24 | "mm": doc.created.strftime("%m"), 25 | "dd": doc.created.strftime("%d") 26 | } 27 | 28 | 29 | @composable 30 | def doc_permalink(doc, permalink_template): 31 | """ 32 | Given a doc dict and a permalink template, render 33 | the output_path field of the doc. 34 | """ 35 | output_path = permalink_template.format(**read_doc_permalink(doc)) 36 | return put(Doc.output_path, doc, output_path) 37 | 38 | 39 | def relative_to(tlds): 40 | """ 41 | Create a function that maps doc output path to be relative 42 | to some top-level path. 43 | """ 44 | rel_to_tlds = pathtools.relative_to(tlds) 45 | return query.maps(over_with(Doc.output_path, rel_to_tlds)) 46 | 47 | 48 | nice_path = query.maps( 49 | over_with(Doc.output_path, pathtools.to_nice_path) 50 | ) 51 | 52 | 53 | def permalink(permalink_template): 54 | """ 55 | Set output_path on docs using a python string template. 56 | 57 | For example, here's a typical blog year-based permalink: 58 | 59 | permalink("{yyyy}/{mm}/{dd}/{stem}/index.html") 60 | 61 | Available tokens: 62 | 63 | - name: the doc's file name, including extension (e.g. `name.html`) 64 | - stem: the doc's file name, sans extension (e.g. `name`) 65 | - suffix: the doc's file extension (e.g. `.html`) 66 | - parents: full directory path to the doc, sans file name. 67 | - parent: the immediate parent directory 68 | - tld: the top-level directory 69 | - yy: the 2-digit year 70 | - yyyy: the 4-digit year 71 | - mm: the 2-digit month 72 | - dd: the 2-digit day 73 | """ 74 | return query.maps(doc_permalink(permalink_template)) 75 | 76 | 77 | post_permalink = permalink("{yyyy}/{mm}/{dd}/{stem}/index.html") 78 | post_permalink.__doc__ = """ 79 | Sets typical blog date-based output_path on docs: 80 | 81 | 2019/12/01/my-post/index.html 82 | """ 83 | 84 | page_permalink = compose(with_ext_html, nice_path) 85 | page_permalink.__doc__ = """ 86 | Sets nice path on doc, retaining original directory path, but 87 | giving it a nice URL, and an .html extension. 88 | 89 | path/to/some/file.md 90 | 91 | Becomes: 92 | 93 | path/to/some/file/index.html 94 | 95 | """ 96 | 97 | 98 | def rel_page_permalink(tlds): 99 | """ 100 | Sets nice path that keeps original directory structure, relative to 101 | some top-level path. 102 | 103 | path/to/some/file.md 104 | 105 | Where `tlds` is "path/to", becomes: 106 | 107 | some/file/index.html 108 | """ 109 | return compose(with_ext_html, nice_path, relative_to(tlds)) -------------------------------------------------------------------------------- /lettersmith/query.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools for querying data structures. Kind of a lightweight LINQ. 3 | """ 4 | from itertools import islice 5 | from random import sample 6 | 7 | 8 | def filters(predicate): 9 | """ 10 | Keep items if they pass predicate function test. 11 | """ 12 | def filter_bound(iterable): 13 | """ 14 | Filter iterable with bound predicate function. 15 | """ 16 | return filter(predicate, iterable) 17 | return filter_bound 18 | 19 | 20 | def rejects(predicate): 21 | """ 22 | Reject items if they pass predicate function test. 23 | Inverse of filter. 24 | """ 25 | def reject_bound(iterable): 26 | """ 27 | Reject items with bound predicate function. 28 | """ 29 | for item in iterable: 30 | if not predicate(item): 31 | yield item 32 | return reject_bound 33 | 34 | 35 | def maps(a2b): 36 | """ 37 | Map `iterable` with function `a2b`. 38 | """ 39 | def map_bound(iterable): 40 | """ 41 | Map iterable using bound function. 42 | """ 43 | return map(a2b, iterable) 44 | return map_bound 45 | 46 | 47 | def sorts(key=None, reverse=False): 48 | """ 49 | Sort `iterable` by key. 50 | """ 51 | def sort_bound(iterable): 52 | """ 53 | Sort iterable using bound arguments. 54 | """ 55 | return sorted(iterable, key=key, reverse=reverse) 56 | return sort_bound 57 | 58 | 59 | def takes(n): 60 | """ 61 | Take `n` elements from `iterable`. 62 | """ 63 | def take_bound(iterable): 64 | """ 65 | Return first n elements of iterable. 66 | """ 67 | return islice(iterable, n) 68 | return take_bound 69 | 70 | 71 | def samples(k): 72 | """ 73 | Sample `k` elements at random from `iterable`. 74 | """ 75 | def sample_bound(iterable): 76 | """ 77 | Sample `k` elements at random from `iterable`. 78 | """ 79 | return sample(iterable, k) 80 | return sample_bound 81 | 82 | 83 | def dedupes(key): 84 | """ 85 | De-duplicate items in an iterable by key, retaining order. 86 | """ 87 | def dedupe(iterable): 88 | """ 89 | De-duplicate items in an iterable using bound key function, 90 | retaining order. 91 | """ 92 | seen = set() 93 | for item in iterable: 94 | k = key(item) 95 | if k not in seen: 96 | seen.add(k) 97 | yield item 98 | return dedupe -------------------------------------------------------------------------------- /lettersmith/rss.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from datetime import datetime 3 | from lettersmith.jinjatools import FileSystemEnvironment 4 | from lettersmith.path import to_url 5 | from lettersmith import doc as Doc 6 | from lettersmith.docs import most_recent 7 | from lettersmith.html import get_summary 8 | from lettersmith.stringtools import first_sentence 9 | from lettersmith.func import composable 10 | 11 | 12 | MODULE_PATH = Path(__file__).parent 13 | TEMPLATE_PATH = Path(MODULE_PATH, "package_data", "template") 14 | 15 | 16 | FILTERS = { 17 | "get_summary": get_summary, 18 | "to_url": to_url 19 | } 20 | 21 | def render_rss( 22 | docs, 23 | base_url, 24 | last_build_date, 25 | title, 26 | description, 27 | author 28 | ): 29 | context = { 30 | "generator": "Lettersmith", 31 | "base_url": base_url, 32 | "title": title, 33 | "description": description, 34 | "author": author, 35 | "last_build_date": last_build_date 36 | } 37 | env = FileSystemEnvironment( 38 | str(TEMPLATE_PATH), 39 | context=context, 40 | filters=FILTERS 41 | ) 42 | rss_template = env.get_template("rss.xml") 43 | return rss_template.render({ 44 | "docs": docs 45 | }) 46 | 47 | 48 | _most_recent_24 = most_recent(24) 49 | 50 | 51 | @composable 52 | def rss( 53 | docs, 54 | base_url, 55 | title, 56 | description, 57 | author, 58 | output_path="rss.xml", 59 | last_build_date=None 60 | ): 61 | """ 62 | Given an iterable of docs and some details, returns an 63 | RSS doc. 64 | """ 65 | last_build_date = ( 66 | last_build_date 67 | if last_build_date is not None 68 | else datetime.now() 69 | ) 70 | recent = _most_recent_24(docs) 71 | content = render_rss( 72 | recent, 73 | base_url=base_url, 74 | last_build_date=last_build_date, 75 | title=title, 76 | description=description, 77 | author=author 78 | ) 79 | return Doc.create( 80 | id_path=output_path, 81 | output_path=output_path, 82 | created=last_build_date, 83 | modified=last_build_date, 84 | title=title, 85 | content=content 86 | ) 87 | -------------------------------------------------------------------------------- /lettersmith/sitemap.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from datetime import datetime 3 | from itertools import islice 4 | from lettersmith import doc as Doc 5 | from lettersmith.jinjatools import FileSystemEnvironment 6 | from lettersmith.path import to_url 7 | from lettersmith.func import composable 8 | 9 | MODULE_PATH = Path(__file__).parent 10 | TEMPLATE_PATH = Path(MODULE_PATH, "package_data", "template") 11 | 12 | FILTERS = { 13 | "to_url": to_url 14 | } 15 | 16 | 17 | def render_sitemap(docs, 18 | base_url="/", last_build_date=None, 19 | title="Feed", description="", author=""): 20 | context = {"base_url": base_url} 21 | env = FileSystemEnvironment( 22 | str(TEMPLATE_PATH), 23 | context=context, 24 | filters=FILTERS 25 | ) 26 | template = env.get_template("sitemap.xml") 27 | return template.render({"docs": docs}) 28 | 29 | 30 | @composable 31 | def sitemap(docs, base_url): 32 | """ 33 | Returns a sitemap doc 34 | """ 35 | # The sitemap spec limits each sitemap to 50k entries. 36 | # https://www.sitemaps.org/protocol.html 37 | docs_50k = islice(docs, 50000) 38 | output_path = "sitemap.xml" 39 | now = datetime.now() 40 | content = render_sitemap(docs_50k, base_url=base_url) 41 | return Doc.create( 42 | id_path=output_path, 43 | output_path=output_path, 44 | created=now, 45 | modified=now, 46 | content=content 47 | ) -------------------------------------------------------------------------------- /lettersmith/stringtools.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | _FIRST_SENTENCE = r'^[^.]+' 5 | 6 | 7 | def first_sentence(plain_text): 8 | """ 9 | Get the first sentence in a block of plain text. 10 | Does not include period at end of sentence. 11 | """ 12 | match = re.match(_FIRST_SENTENCE, plain_text) 13 | if match: 14 | return match.group(0) 15 | else: 16 | return "" 17 | 18 | 19 | def truncate(text, max_len=250, suffix="..."): 20 | """ 21 | Truncate a text string to a certain number of characters, 22 | trimming to the nearest word boundary. 23 | """ 24 | stripped = text.strip() 25 | if len(stripped) <= max_len: 26 | return stripped 27 | substr = stripped[0:max_len + 1] 28 | words = " ".join(re.split(r"\s+", substr)[0:-1]) 29 | return words + suffix -------------------------------------------------------------------------------- /lettersmith/stub.py: -------------------------------------------------------------------------------- 1 | """ 2 | Stubs are summary details for a document. 3 | """ 4 | from collections import namedtuple 5 | from lettersmith import doc as Doc 6 | from lettersmith.lens import get 7 | from lettersmith import query 8 | 9 | 10 | Stub = namedtuple("Stub", ( 11 | "id_path", 12 | "output_path", 13 | "created", 14 | "modified", 15 | "title", 16 | "summary" 17 | )) 18 | Stub.__doc__ = """ 19 | A namedtuple for representing a stub. A stub is just a container for 20 | the summary details of a document. No content, no meta, no template. 21 | 22 | Only hashable properties, so stubs can be used in sets. 23 | (Note that datetime objects are immutable and hashable.) 24 | """ 25 | 26 | def from_doc(doc): 27 | """ 28 | Read stub from doc 29 | """ 30 | return Stub( 31 | get(Doc.id_path, doc), 32 | get(Doc.output_path, doc), 33 | get(Doc.created, doc), 34 | get(Doc.modified, doc), 35 | get(Doc.title, doc), 36 | get(Doc.meta_summary, doc) 37 | ) 38 | 39 | 40 | stubs = query.maps(from_doc) -------------------------------------------------------------------------------- /lettersmith/taxonomy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools for indexing docs by tag (taxonomy). 3 | """ 4 | from datetime import datetime 5 | from lettersmith.func import composable, pipe 6 | from lettersmith import path as pathtools 7 | from lettersmith import stub as Stub 8 | from lettersmith import doc as Doc 9 | from lettersmith import docs as Docs 10 | from lettersmith.lens import lens_compose, key, get, put 11 | 12 | 13 | _empty = tuple() 14 | 15 | 16 | meta_related = lens_compose(Doc.meta, key("related", _empty)) 17 | 18 | 19 | def meta_taxonomy(tax): 20 | """ 21 | Create a lens for taxonomy `tax`. 22 | """ 23 | return lens_compose(Doc.meta, key(tax, _empty)) 24 | 25 | 26 | meta_tags = meta_taxonomy("tags") 27 | 28 | 29 | @composable 30 | def taxonomy_archives( 31 | docs, 32 | key, 33 | template="taxonomy.html", 34 | output_path_template="{taxonomy}/{term}/index.html" 35 | ): 36 | """ 37 | Creates an archive page for each taxonomy term. One page per term. 38 | """ 39 | tax_index = index_taxonomy(docs, key) 40 | for term, docs in tax_index.items(): 41 | output_path = output_path_template.format( 42 | taxonomy=pathtools.to_slug(key), 43 | term=pathtools.to_slug(term) 44 | ) 45 | meta = {"docs": docs} 46 | now = datetime.now() 47 | yield Doc.create( 48 | id_path=output_path, 49 | output_path=output_path, 50 | created=now, 51 | modified=now, 52 | title=term, 53 | template=template, 54 | meta=meta 55 | ) 56 | 57 | 58 | tag_archives = taxonomy_archives("tags") 59 | 60 | 61 | def _get_indexes(index, keys): 62 | for key in keys: 63 | for item in index[key]: 64 | yield item 65 | 66 | 67 | @composable 68 | def index_taxonomy(docs, key): 69 | """ 70 | Create a new index for a taxonomy. 71 | `key` is a whitelisted meta keys that should 72 | be treated as a taxonomy field. 73 | 74 | Returns a dict that looks like: 75 | 76 | { 77 | "term_a": [stub, ...], 78 | "term_b": [stub, ...] 79 | } 80 | """ 81 | tax_index = {} 82 | for doc in docs: 83 | if key in doc.meta: 84 | for term in doc.meta[key]: 85 | if term not in tax_index: 86 | tax_index[term] = [] 87 | tax_index[term].append(Stub.from_doc(doc)) 88 | return tax_index 89 | 90 | 91 | index_tags = index_taxonomy("tags") 92 | 93 | 94 | def related(tax): 95 | """ 96 | Annotate doc meta with a list of related doc stubs. 97 | 98 | A doc is related if it shares any of the same tags in the 99 | same taxonomy. 100 | """ 101 | taxonomy = meta_taxonomy(tax) 102 | build_index = index_taxonomy(tax) 103 | def add_related(docs): 104 | docs = tuple(docs) 105 | index = build_index(docs) 106 | for doc in docs: 107 | tags = get(taxonomy, doc) 108 | related = pipe( 109 | _get_indexes(index, tags), 110 | Docs.dedupe, 111 | Docs.remove_id_path(doc.id_path), 112 | tuple 113 | ) 114 | yield put(meta_related, doc, related) 115 | return add_related 116 | 117 | 118 | related_by_tag = related("tags") -------------------------------------------------------------------------------- /lettersmith/util.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions. 3 | Mostly tools for working with dictionaries and iterables. 4 | """ 5 | from functools import wraps 6 | from fnmatch import fnmatch 7 | from collections import OrderedDict 8 | 9 | 10 | def chunk(iterable, n): 11 | """ 12 | Split an iterable into chunks of size n. 13 | Returns an iterator of sequences. 14 | """ 15 | chunk = [] 16 | for x in iterable: 17 | chunk.append(x) 18 | if len(chunk) == n: 19 | yield chunk 20 | chunk = [] 21 | if len(chunk) > 0: 22 | yield chunk 23 | 24 | 25 | def mix(d, e): 26 | """ 27 | Combine two dicts. 28 | Just a function version of the spread operator for dicts. 29 | """ 30 | return {**d, **e} 31 | 32 | 33 | def _first(pair): 34 | return pair[0] 35 | 36 | 37 | def order_dict_by_keys(d): 38 | """ 39 | Create an OrderedDict, ordered by key asc. 40 | """ 41 | return OrderedDict(sorted( 42 | d.items(), 43 | key=_first 44 | )) 45 | 46 | 47 | def join(words, sep="", template="{word}"): 48 | """ 49 | Join an iterable of strings, with optional template string defining 50 | how each word is to be templated before joining. 51 | """ 52 | return sep.join(template.format(word=word) for word in words) 53 | 54 | 55 | def expand(f, iter, *args, **kwargs): 56 | """ 57 | Expand each item in `iter` using function `f`. 58 | `f` is expected to return an iterator itself... it "expands" 59 | each item. 60 | """ 61 | for x in iter: 62 | for y in f(x, *args, **kwargs): 63 | yield y 64 | 65 | 66 | def index_sets(items): 67 | """ 68 | Create a dictionary of sets from an iterable of `(key, value)` pairs. 69 | 70 | Each item is stored in a set at `key`. More than one item with same key 71 | means items get appended to same list. 72 | 73 | This means items in indices are unique, but they must be hashable. 74 | """ 75 | index = {} 76 | for key, value in items: 77 | try: 78 | index[key].add(value) 79 | except KeyError: 80 | index[key] = set((value,)) 81 | return index 82 | 83 | 84 | def index_many(items): 85 | """ 86 | Create a dictionary of lists from an iterable of `(key, value)` pairs. 87 | 88 | Each item is stored in a list at `key`. More than one item with same key 89 | means items get appended to same list. 90 | """ 91 | index = {} 92 | for key, value in items: 93 | try: 94 | index[key].append(value) 95 | except KeyError: 96 | index[key] = [value] 97 | return index 98 | -------------------------------------------------------------------------------- /lettersmith/wikidoc.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools for rendering wikilinks in content. 3 | """ 4 | import re 5 | from collections import namedtuple 6 | from lettersmith import doc as Doc 7 | from lettersmith import docs as Docs 8 | from lettersmith import stub as Stub 9 | from lettersmith import edge as Edge 10 | from lettersmith import html 11 | from lettersmith import wikimarkup 12 | from lettersmith import markdowntools 13 | from lettersmith.path import to_slug, to_url 14 | from lettersmith.util import index_sets, expand 15 | from lettersmith.lens import lens_compose, key, get, put, over 16 | from lettersmith.func import compose, composable 17 | from lettersmith.stringtools import first_sentence 18 | 19 | 20 | # Read a summary from an HTML text blob 21 | read_summary_html = compose( 22 | first_sentence, 23 | wikimarkup.strip_wikilinks, 24 | html.strip_html 25 | ) 26 | 27 | # Read a summary from a markdown text blob 28 | read_summary_markdown = compose( 29 | first_sentence, 30 | wikimarkup.strip_wikilinks, 31 | markdowntools.strip_markdown 32 | ) 33 | 34 | 35 | def _summary(read_summary): 36 | """ 37 | Render a summary from content using `read_summary` and set it on 38 | `doc.meta["summary"]`. 39 | 40 | If doc already has a `doc.meta["summary"]` it will leave it alone. 41 | """ 42 | def summary(docs): 43 | for doc in docs: 44 | if get(Doc.meta_summary, doc): 45 | yield doc 46 | else: 47 | yield put(Doc.meta_summary, doc, read_summary(doc.content)) 48 | return summary 49 | 50 | 51 | summary_html = _summary(read_summary_html) 52 | summary_markdown = _summary(read_summary_markdown) 53 | 54 | 55 | def _index_by_slug(docs): 56 | return { 57 | to_slug(doc.title): Stub.from_doc(doc) 58 | for doc in docs 59 | } 60 | 61 | 62 | def _extract_links(content, slug_to_stub): 63 | wikilinks = frozenset(wikimarkup.find_wikilinks(content)) 64 | for slug, title in wikilinks: 65 | try: 66 | yield slug_to_stub[slug] 67 | except KeyError: 68 | pass 69 | 70 | 71 | def _expand_edges(doc, slug_to_stub): 72 | tail = Stub.from_doc(doc) 73 | for head in _extract_links(doc.content, slug_to_stub): 74 | yield Edge.Edge(tail, head) 75 | 76 | 77 | def _collect_edges(docs): 78 | docs = tuple(docs) 79 | slug_to_stub = _index_by_slug(docs) 80 | return expand(_expand_edges, docs, slug_to_stub) 81 | 82 | 83 | def _index_by_link(edge): 84 | return edge.tail.id_path, edge.head 85 | 86 | 87 | def _index_by_backlink(edge): 88 | return edge.head.id_path, edge.tail 89 | 90 | 91 | _empty = tuple() 92 | meta_links = lens_compose(Doc.meta, key("links", _empty)) 93 | meta_backlinks = lens_compose(Doc.meta, key("backlinks", _empty)) 94 | 95 | 96 | def has_links(doc): 97 | return len(get(meta_links, doc)) > 0 98 | 99 | 100 | def has_backlinks(doc): 101 | return len(get(meta_backlinks, doc)) > 0 102 | 103 | 104 | def annotate_links(docs): 105 | """ 106 | Annotate docs with links and backlinks. 107 | 108 | Returns an iterator for docs with 2 new meta fields: links and backlinks. 109 | Each contains a tuple of `Stub`s. 110 | """ 111 | docs = tuple(docs) 112 | edges = tuple(_collect_edges(docs)) 113 | link_index = index_sets(_index_by_link(edge) for edge in edges) 114 | backlink_index = index_sets(_index_by_backlink(edge) for edge in edges) 115 | empty = tuple() 116 | for doc in docs: 117 | backlinks = frozenset(backlink_index.get(doc.id_path, empty)) 118 | links = frozenset(link_index.get(doc.id_path, empty)) 119 | yield Doc.update_meta(doc, { 120 | "links": links, 121 | "backlinks": backlinks, 122 | }) 123 | 124 | 125 | _LINK_TEMPLATE = '{title}' 126 | _NOLINK_TEMPLATE = '{title}' 127 | _TRANSCLUDE_TEMPLATE = '''''' 133 | 134 | 135 | @composable 136 | def content_wikilinks( 137 | docs, 138 | base_url, 139 | link_template=_LINK_TEMPLATE, 140 | nolink_template=_NOLINK_TEMPLATE, 141 | transclude_template=_TRANSCLUDE_TEMPLATE 142 | ): 143 | """ 144 | `[[wikilink]]` is replaced with a link to a doc with the same title 145 | (case insensitive), using the `link_template`. 146 | 147 | If no doc exists with that title it will be rendered 148 | using `nolink_template`. 149 | """ 150 | docs = tuple(docs) 151 | slug_to_stub = _index_by_slug(docs) 152 | 153 | def render_wikilink(slug, title, type): 154 | if type is "transclude": 155 | try: 156 | link = slug_to_stub[slug] 157 | url = to_url(link.output_path, base=base_url) 158 | return transclude_template.format( 159 | url=url, 160 | title=link.title, 161 | summary=link.summary 162 | ) 163 | except KeyError: 164 | return "" 165 | else: 166 | try: 167 | link = slug_to_stub[slug] 168 | url = to_url(link.output_path, base=base_url) 169 | return link_template.format(url=url, title=title) 170 | except KeyError: 171 | return nolink_template.format(title=title) 172 | 173 | render_wikilinks = wikimarkup.renderer(render_wikilink) 174 | 175 | for doc in docs: 176 | yield over(Doc.content, render_wikilinks, doc) 177 | 178 | 179 | def content_markdown( 180 | base_url, 181 | link_template=_LINK_TEMPLATE, 182 | nolink_template=_NOLINK_TEMPLATE, 183 | transclude_template=_TRANSCLUDE_TEMPLATE 184 | ): 185 | """ 186 | Render markdown and wikilinks. 187 | 188 | Also annotates doc meta with: 189 | 190 | - A summary 191 | - A list of links and backlinks. 192 | 193 | Example: 194 | 195 | Write _markdown_ like normal. 196 | 197 | - List item 198 | - List item 199 | - List item 200 | 201 | [[Wikilinks]] also work. They will be rendered as Wikilinks. 202 | 203 | [[Transclusion wikilink]] 204 | 205 | If you put a wikilink on it's own line, as above, it will be rendered as a rich snippet (transclude). 206 | """ 207 | return compose( 208 | markdowntools.content, 209 | content_wikilinks( 210 | base_url, 211 | link_template, 212 | nolink_template, 213 | transclude_template 214 | ), 215 | annotate_links, 216 | summary_markdown 217 | ) 218 | 219 | 220 | def content_html( 221 | base_url, 222 | link_template=_LINK_TEMPLATE, 223 | nolink_template=_NOLINK_TEMPLATE, 224 | transclude_template=_TRANSCLUDE_TEMPLATE 225 | ): 226 | """ 227 | Render html (wrap bare lines with paragraphs) and wikilinks. 228 | 229 | Also annotates doc meta with: 230 | 231 | - A summary 232 | - A list of links and backlinks. 233 | 234 | Example: 235 | 236 | Bare lines are wrapped in paragraphs. 237 | 238 | Blank spaces are ignored. 239 | 240 | You can use HTML like bold, or italic text. 241 | 242 | Lines with
block els
will not get wrapped. 243 | 244 |
245 | If you indent a line, it will not get wrapped 246 | in a paragraph 247 |
248 | 249 | [[Wikilinks]] also work. They will be rendered as Wikilinks. 250 | 251 | [[Transclusion wikilink]] 252 | 253 | If you put a wikilink on it's own line, as above, it will be rendered as a rich snippet (transclude). 254 | 255 | """ 256 | return compose( 257 | html.content, 258 | content_wikilinks( 259 | base_url, 260 | link_template, 261 | nolink_template, 262 | transclude_template 263 | ), 264 | annotate_links, 265 | summary_html 266 | ) -------------------------------------------------------------------------------- /lettersmith/wikimarkup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Render wikilinks in text. 3 | """ 4 | import re 5 | from lettersmith.path import to_slug 6 | 7 | 8 | _WIKILINK = r'\[\[([^\]]+)\]\]' 9 | _TRANSCLUDE = r'^\[\[([^\]]+)\]\]$' 10 | 11 | 12 | def _sub_wikilink_title(match): 13 | slug, title = _parse_wikilink(match.group(0)) 14 | return title 15 | 16 | 17 | def strip_wikilinks(text): 18 | """ 19 | Strip markup from text 20 | """ 21 | # Remove transcludes completely 22 | text = re.sub(_TRANSCLUDE, "", text, flags=re.MULTILINE) 23 | # Remove inline wikilinks, but leaves bare text 24 | text = re.sub(_WIKILINK, _sub_wikilink_title, text) 25 | return text 26 | 27 | 28 | def _parse_wikilink(wikilink_str): 29 | """ 30 | Given a `[[WikiLink]]` or a `[[wikilink | Title]]`, return a 31 | tuple of `(wikilink, Title)`. 32 | 33 | Supports both piped and non-piped forms. 34 | """ 35 | inner = wikilink_str.strip('[] ') 36 | try: 37 | _slug, _text = inner.split("|") 38 | slug = to_slug(_slug.strip()) 39 | text = _text.strip() 40 | except ValueError: 41 | text = inner.strip() 42 | slug = to_slug(text) 43 | return slug, text 44 | 45 | 46 | def find_wikilinks(s): 47 | """ 48 | Find all wikilinks in a string (if any) 49 | Returns an iterator of 2-tuples for slug, title. 50 | """ 51 | for match in re.finditer(_WIKILINK, s): 52 | yield _parse_wikilink(match.group(0)) 53 | 54 | 55 | def renderer(render_wikilink): 56 | """ 57 | Creates a renderer function 58 | """ 59 | def _render_wikilink(match): 60 | slug, title = _parse_wikilink(match.group(0)) 61 | return render_wikilink(slug, title, "inline") 62 | 63 | def _render_transclude(match): 64 | slug, title = _parse_wikilink(match.group(0)) 65 | return render_wikilink(slug, title, "transclude") 66 | 67 | def render_text(text): 68 | text = re.sub(_TRANSCLUDE, _render_transclude, text, flags=re.MULTILINE) 69 | text = re.sub(_WIKILINK, _render_wikilink, text) 70 | return text 71 | 72 | return render_text -------------------------------------------------------------------------------- /lettersmith/write.py: -------------------------------------------------------------------------------- 1 | from pathlib import PurePath 2 | import shutil 3 | from lettersmith import doc as Doc 4 | from lettersmith import file as File 5 | from lettersmith.io import write_file_deep 6 | 7 | 8 | def writer(writeable): 9 | """ 10 | Lift a `writeable` function that reads a data object and returns 11 | a 2-tuple of `(pathlike, bytes)`. 12 | 13 | Returns a `write` function that knows how to take these 2-tuples 14 | and write them to disk. 15 | """ 16 | def write(things, directory): 17 | """ 18 | Write files to `directory`. 19 | """ 20 | dir_path = PurePath(directory) 21 | shutil.rmtree(dir_path, ignore_errors=True) 22 | written = 0 23 | for thing in things: 24 | written = written + 1 25 | output_path, blob = writeable(thing) 26 | write_file_deep( 27 | dir_path.joinpath(output_path), 28 | blob, 29 | mode="wb" 30 | ) 31 | return {"written": written} 32 | return write 33 | 34 | 35 | def writeable(thing): 36 | """ 37 | Write a doc or file to `output_path`. 38 | """ 39 | if isinstance(thing, Doc.Doc): 40 | return Doc.writeable(thing) 41 | elif isinstance(thing, File.File): 42 | return File.writeable(thing) 43 | else: 44 | msg = ( 45 | "Don't know how to convert {type} to (path, bytes). " 46 | "If you are using a custom document type, you can create " 47 | "your own function that knows how to return (path, bytes). " 48 | "Then you can pass that function to writer() to create " 49 | "a custom write function." 50 | ) 51 | raise ValueError(msg.format(type=type(thing))) 52 | 53 | 54 | write = writer(writeable) -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | from os import path 3 | 4 | readme_path = path.join(path.dirname(__file__), "README.md") 5 | with open(readme_path) as f: 6 | readme = f.read() 7 | 8 | setup( 9 | name='lettersmith', 10 | version='0.0.1-alpha.1', 11 | author='Gordon Brander', 12 | description='Tools for static site generation', 13 | long_description=readme, 14 | license="MIT", 15 | url="", 16 | classifiers=[ 17 | "Development Status :: 3 - Alpha", 18 | "Intended Audience :: Developers", 19 | "Programming Language :: Python :: 3.6", 20 | ], 21 | packages=find_packages(exclude=("tests", "tests.*")), 22 | install_requires=[ 23 | "PyYAML>=3.13", 24 | "commonmark>=0.9.1", 25 | "python-frontmatter>=0.3.1", 26 | "Jinja2>=2.7" 27 | # TODO 28 | # "watchdog>=0.6.0" 29 | ], 30 | extras_require={}, 31 | include_package_data=True, 32 | entry_points={ 33 | "console_scripts": [ 34 | "lettersmith_scaffold=lettersmith.cli.scaffold:main", 35 | ] 36 | } 37 | ) 38 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordonbrander/lettersmith_py/96ddaf1268ac53062b2e7b1d05d06dc092865666/test/__init__.py -------------------------------------------------------------------------------- /test/package_data/fixtures/Bare doc.md: -------------------------------------------------------------------------------- 1 | Lorem ipsum -------------------------------------------------------------------------------- /test/package_data/fixtures/Doc with meta.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Doc title" 3 | created: 2018-01-28 4 | summary: "The summary" 5 | --- 6 | Lorem ipsum -------------------------------------------------------------------------------- /test/package_data/fixtures/doc.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Doc title", 3 | "date": "2018-01-28", 4 | "summary": "The summary" 5 | } -------------------------------------------------------------------------------- /test/package_data/fixtures/doc.yaml: -------------------------------------------------------------------------------- 1 | title: "Doc title" 2 | date: 2018-01-28 3 | summary: "The summary" -------------------------------------------------------------------------------- /test/scripts/generate_fixtures.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import random 3 | import argparse 4 | from lettersmith import doc as Doc 5 | 6 | PARA_1 = """Lorem ipsum dolor sit amet, consectetur adipiscing elit. 7 | Donec cursus tristique nisi et aliquet. Curabitur posuere 8 | auctor metus at facilisis. Proin ultrices dictum eros non 9 | pharetra. Etiam ultricies pharetra nisl id aliquet. Sed aliquet 10 | efficitur cursus. Nunc aliquet varius tortor, et sollicitudin 11 | libero fermentum sit amet. Phasellus mattis semper magna, congue 12 | laoreet dolor pharetra ac. Proin vitae ornare lacus. 13 | Suspendisse fermentum facilisis congue.""" 14 | 15 | PARA_2 = """Pellentesque varius rhoncus lacus, sed scelerisque nisl viverra nec. 16 | Mauris sit amet pretium dui. Integer non sapien orci. 17 | In lorem tortor, posuere fringilla efficitur ut, consectetur eu 18 | velit. Nam ornare sapien eu tellus mollis efficitur. Fusce ut 19 | venenatis quam. Quisque ligula ante, iaculis id sollicitudin at, 20 | molestie ac enim. Lorem ipsum dolor sit amet, consectetur 21 | adipiscing elit. Morbi sollicitudin dictum lectus, sodales 22 | rutrum nulla efficitur ut. Nullam congue viverra arcu, a 23 | porttitor nisl dictum quis.""" 24 | 25 | PARA_3 = """Aliquam erat volutpat. Nulla luctus interdum dui, nec aliquam 26 | lectus aliquet sed. Duis sed purus dictum, posuere leo non, 27 | sollicitudin diam. Praesent malesuada quis sem id euismod. 28 | Maecenas condimentum dictum augue, vitae interdum nibh 29 | vestibulum at. Maecenas arcu massa, scelerisque vitae auctor non, 30 | tempus nec purus. Quisque placerat risus nec tortor luctus, nec 31 | lobortis nisi tristique. Donec non commodo tortor, non tempus enim. 32 | Proin eget iaculis tellus. Curabitur purus ante, fringilla vitae 33 | suscipit a, condimentum sollicitudin ex. In euismod enim sit amet 34 | purus rutrum molestie.""" 35 | 36 | PARA_4 = """Morbi feugiat felis tellus, nec commodo dui dictum at. Aenean 37 | tincidunt tortor sed tempus placerat. Sed mattis id tellus non 38 | pulvinar. Pellentesque pulvinar semper ultricies. Pellentesque 39 | auctor, lectus sed commodo volutpat, arcu enim luctus odio, at 40 | sodales justo tortor non est. In libero velit, sodales sed elit 41 | ut, elementum scelerisque velit. Morbi eu lacus bibendum, ultrices 42 | purus in, volutpat nisl. Suspendisse posuere dictum auctor. 43 | Vivamus id rutrum nunc. Sed maximus metus nec erat imperdiet 44 | dignissim. Duis sit amet semper urna. Integer non mi tortor.""" 45 | 46 | 47 | PARAS = (PARA_1, PARA_2, PARA_3, PARA_4) 48 | 49 | 50 | HEADINGS = ( 51 | "Sed maximus metus nec", 52 | "Volutpat nisil", 53 | "Elementum velit", 54 | "Semper urna integer non mi", 55 | "Vivamus id rutrum nunc" 56 | ) 57 | 58 | 59 | TEMPLATE = """{para_1} 60 | 61 | ## {heading_2} 62 | 63 | {para_2} 64 | 65 | {para_3} 66 | 67 | - {bullet_1} 68 | - {bullet_2} 69 | - {bullet_3} 70 | 71 | ## {heading_2} 72 | 73 | {para_4} 74 | 75 | {para_5}""" 76 | 77 | 78 | def gen_text(): 79 | """ 80 | Generate some filler markdown text from template 81 | """ 82 | return TEMPLATE.format( 83 | para_1=random.choice(PARAS), 84 | para_2=random.choice(PARAS), 85 | para_3=random.choice(PARAS), 86 | para_4=random.choice(PARAS), 87 | para_5=random.choice(PARAS), 88 | heading_1=random.choice(HEADINGS), 89 | heading_2=random.choice(HEADINGS), 90 | bullet_1=random.choice(HEADINGS), 91 | bullet_2=random.choice(HEADINGS), 92 | bullet_3=random.choice(HEADINGS) 93 | ) 94 | 95 | def gen_doc(i): 96 | id_path = "Test Doc {}.md".format(i) 97 | return Doc.create( 98 | id_path=id_path, 99 | output_path=id_path, 100 | title="Test Doc {}".format(i), 101 | content=gen_text() 102 | ) 103 | 104 | 105 | def gen_docs(n): 106 | for i in range(0, n): 107 | yield gen_doc(i) 108 | 109 | 110 | parser = argparse.ArgumentParser( 111 | description="Generate markdown fixtures" 112 | ) 113 | parser.add_argument( 114 | 'n', 115 | help="Number of documents to generate", 116 | type=int 117 | ) 118 | parser.add_argument( 119 | 'output_path', 120 | help="Where to write documents", 121 | type=str, 122 | default="." 123 | ) 124 | 125 | 126 | def main(): 127 | args= parser.parse_args() 128 | docs = gen_docs(args.n) 129 | for doc in docs: 130 | Doc.write(doc, output_dir=args.output_path) 131 | 132 | if __name__ == '__main__': 133 | main() 134 | -------------------------------------------------------------------------------- /test/test_doc.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit tests for Doc 3 | """ 4 | 5 | import unittest 6 | from pathlib import Path 7 | from lettersmith import doc as Doc 8 | 9 | module_path = Path(__file__).parent 10 | fixtures_path = Path(module_path, "package_data", "fixtures") 11 | 12 | class test_load_bare_doc(unittest.TestCase): 13 | def setUp(self): 14 | doc_path = fixtures_path.joinpath("Bare doc.md") 15 | doc = Doc.load(doc_path, relative_to=fixtures_path) 16 | self.doc = doc 17 | 18 | def test_content(self): 19 | self.assertEqual(self.doc.content, "Lorem ipsum") 20 | 21 | def test_meta(self): 22 | """ 23 | Meta should always be a dict 24 | """ 25 | self.assertIsInstance(self.doc.meta, dict) 26 | 27 | def test_id_path(self): 28 | self.assertEqual(str(self.doc.id_path), "Bare doc.md") 29 | 30 | 31 | class test_parse_frontmatter(unittest.TestCase): 32 | def setUp(self): 33 | doc_path = fixtures_path.joinpath("Doc with meta.md") 34 | doc = Doc.load(doc_path, relative_to=fixtures_path) 35 | doc = Doc.parse_frontmatter(doc) 36 | self.doc = doc 37 | 38 | def test_content(self): 39 | self.assertEqual(self.doc.content, "Lorem ipsum") 40 | 41 | def test_meta(self): 42 | """ 43 | Meta should always be a dict 44 | """ 45 | self.assertIsInstance(self.doc.meta, dict) 46 | 47 | def test_items(self): 48 | self.assertEqual(self.doc.meta["title"], "Doc title") 49 | 50 | 51 | if __name__ == '__main__': 52 | unittest.main() -------------------------------------------------------------------------------- /test/test_func.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from lettersmith.func import compose, thrush, pipe, rest, composable 3 | 4 | 5 | class test_compose(unittest.TestCase): 6 | def test_1(self): 7 | def a(s): 8 | return s + "a" 9 | 10 | def b(s): 11 | return s + "b" 12 | 13 | def c(s): 14 | return s + "c" 15 | 16 | abc = compose(c, b, a) 17 | s = abc("_") 18 | self.assertEqual(s, "_abc") 19 | 20 | 21 | class test_thrush(unittest.TestCase): 22 | def test_1(self): 23 | def a(s): 24 | return s + "a" 25 | 26 | def b(s): 27 | return s + "b" 28 | 29 | def c(s): 30 | return s + "c" 31 | 32 | abc = thrush(a, b, c) 33 | s = abc("_") 34 | self.assertEqual(s, "_abc") 35 | 36 | 37 | class test_pipe(unittest.TestCase): 38 | def test_1(self): 39 | def a(s): 40 | return s + "a" 41 | 42 | def b(s): 43 | return s + "b" 44 | 45 | def c(s): 46 | return s + "c" 47 | 48 | s = pipe("_", a, b, c) 49 | self.assertEqual(s, "_abc") 50 | 51 | 52 | class test_rest(unittest.TestCase): 53 | def test_1(self): 54 | def f(a, b, c): 55 | return (a, b, c) 56 | 57 | fx = rest(f, 2, 3) 58 | v = fx(1) 59 | self.assertEqual(v, (1, 2, 3)) 60 | 61 | 62 | class test_composable(unittest.TestCase): 63 | def test_1(self): 64 | @composable 65 | def f(a, b, c): 66 | return (a, b, c) 67 | 68 | fx = f(2, 3) 69 | v = fx(1) 70 | self.assertEqual(v, (1, 2, 3)) 71 | 72 | 73 | if __name__ == '__main__': 74 | unittest.main() -------------------------------------------------------------------------------- /test/test_html.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit tests for html 3 | """ 4 | import unittest 5 | from lettersmith import html 6 | 7 | 8 | class test_strip_html(unittest.TestCase): 9 | def test_1(self): 10 | text = """

foo

""" 11 | s = html.strip_html(text) 12 | self.assertEqual(s, 'foo') 13 | 14 | def test_2(self): 15 | text = """

foo

""" 16 | s = html.strip_html(text) 17 | self.assertEqual(s, 'foo') 18 | 19 | def test_3(self): 20 | text = """""" 21 | s = html.strip_html(text) 22 | self.assertEqual(s, '') 23 | 24 | def test_4(self): 25 | text = """""" 26 | s = html.strip_html(text) 27 | self.assertEqual(s, '') -------------------------------------------------------------------------------- /test/test_lens.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from experimental.lens2 import key, get, put, over, compose 3 | 4 | 5 | class TestKey(unittest.TestCase): 6 | def test_1(self): 7 | data = { 8 | "a": 10, 9 | "b": 11 10 | } 11 | 12 | a = key("a", 0) 13 | self.assertEqual(get(a, data), 10, "Getter gets the value at key") 14 | 15 | def test_2(self): 16 | data = { 17 | "b": 11 18 | } 19 | 20 | a = key("a", 0) 21 | self.assertEqual( 22 | get(a, data), 23 | 0, 24 | "Getter gets the default if no value is present" 25 | ) 26 | 27 | def test_3(self): 28 | data = { 29 | "b": 11 30 | } 31 | 32 | a = key("a") 33 | self.assertEqual( 34 | get(a, data), 35 | None, 36 | "Getter returns None if no default value provided" 37 | ) 38 | 39 | def test_4(self): 40 | data = { 41 | "b": 11 42 | } 43 | 44 | a = key("a", 0) 45 | data = put(a, data, 10) 46 | self.assertIsInstance(data, dict, "Update returns container type") 47 | self.assertEqual(data["a"], 10, "Update sets the value at key") 48 | 49 | 50 | class TestCompose(unittest.TestCase): 51 | def test_get(self): 52 | data = { 53 | "a": { 54 | "b": 10 55 | } 56 | } 57 | 58 | a = key("a", {}) 59 | b = key("b", 0) 60 | ab = compose(a, b) 61 | self.assertEqual(get(ab, data), 10, "Composed getter gets the value") 62 | 63 | def test_deep_get(self): 64 | data = { 65 | "a": { 66 | "b": { 67 | "c": { 68 | "d": 10 69 | } 70 | } 71 | } 72 | } 73 | 74 | abcd = compose(key("a"), key("b"), key("c"), key("d")) 75 | self.assertEqual(get(abcd, data), 10, "Composed getter gets the value") 76 | 77 | def test_put(self): 78 | data = { 79 | "a": { 80 | "b": 10 81 | } 82 | } 83 | 84 | a = key("a", {}) 85 | b = key("b", 0) 86 | ab = compose(a, b) 87 | data = put(ab, data, 11) 88 | self.assertIsInstance(data, dict, "Setter returns container type") 89 | self.assertEqual(data["a"]["b"], 11, "Setter sets the value at key") 90 | 91 | def test_set_with_defaults(self): 92 | data = {} 93 | 94 | a = key("a", {}) 95 | b = key("b", 0) 96 | ab = compose(a, b) 97 | data = put(ab, data, 11) 98 | self.assertIsInstance(data, dict, "Setter returns container type") 99 | self.assertEqual(data["a"]["b"], 11, "Setter sets the value at key") 100 | 101 | 102 | class TestOver(unittest.TestCase): 103 | def test_1(self): 104 | data = { 105 | "a": 0 106 | } 107 | 108 | a = key("a") 109 | x = over(a, lambda a: a + 1, data) 110 | self.assertEqual(x["a"], 1, "over sets the value using functor") 111 | 112 | 113 | if __name__ == '__main__': 114 | unittest.main() -------------------------------------------------------------------------------- /test/test_query.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from lettersmith import query 3 | 4 | 5 | class test_filters(unittest.TestCase): 6 | def test_1(self): 7 | 8 | @query.filters 9 | def filter_1(x): 10 | return x is 1 11 | 12 | data = (1, 2, 3) 13 | value = tuple(filter_1(data)) 14 | 15 | self.assertEqual(value, (1,)) 16 | 17 | 18 | class test_rejects(unittest.TestCase): 19 | def test_1(self): 20 | 21 | @query.rejects 22 | def reject_1(x): 23 | return x is 1 24 | 25 | data = (1, 2, 3) 26 | value = tuple(reject_1(data)) 27 | 28 | self.assertEqual(value, (2, 3)) 29 | 30 | 31 | class test_maps(unittest.TestCase): 32 | def test_1(self): 33 | 34 | @query.maps 35 | def double(x): 36 | return x * 2 37 | 38 | data = (1, 2, 3) 39 | value = tuple(double(data)) 40 | 41 | self.assertEqual(value, (2, 4, 6)) 42 | 43 | 44 | if __name__ == '__main__': 45 | unittest.main() -------------------------------------------------------------------------------- /test/test_stringtools.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit tests for stringtools 3 | """ 4 | 5 | import unittest 6 | from lettersmith import stringtools 7 | 8 | 9 | class test_truncate(unittest.TestCase): 10 | def test_text(self): 11 | s = """ 12 | Shall I compare thee to a summer’s day? 13 | Thou art more lovely and more temperate. 14 | Rough winds do shake the darling buds of May, 15 | And summer’s lease hath all too short a date. 16 | Sometime too hot the eye of heaven shines, 17 | And often is his gold complexion dimmed; 18 | And every fair from fair sometime declines, 19 | By chance, or nature’s changing course, untrimmed; 20 | But thy eternal summer shall not fade, 21 | Nor lose possession of that fair thou ow’st, 22 | Nor shall death brag thou wand’rest in his shade, 23 | When in eternal lines to Time thou grow’st. 24 | So long as men can breathe, or eyes can see, 25 | So long lives this, and this gives life to thee. 26 | """ 27 | truncated = stringtools.truncate(s, max_len=50) 28 | self.assertEqual( 29 | truncated, 30 | "Shall I compare thee to a summer’s day? Thou art..." 31 | ) 32 | 33 | 34 | if __name__ == '__main__': 35 | unittest.main() -------------------------------------------------------------------------------- /test/test_util.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | if __name__ == '__main__': 5 | unittest.main() -------------------------------------------------------------------------------- /test/test_wikilink.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from lettersmith import wikilink 3 | 4 | 5 | class test_parse_wikilink(unittest.TestCase): 6 | def test_returns_tuple(self): 7 | s = "[[WikiLink]]" 8 | result = wikilink.parse_wikilink(s) 9 | self.assertIsInstance(result, tuple) 10 | self.assertEqual(len(result), 2) 11 | 12 | def test_well_formed(self): 13 | slug, title = wikilink.parse_wikilink("[[WikiLink]]") 14 | self.assertEqual(slug, "wikilink") 15 | self.assertEqual(title, "WikiLink") 16 | 17 | def test_piped(self): 18 | slug, title = wikilink.parse_wikilink("[[dog | Dogs]]") 19 | self.assertEqual(slug, "dog") 20 | self.assertEqual(title, "Dogs") 21 | 22 | def test_piped_snug(self): 23 | slug, title = wikilink.parse_wikilink("[[dog|Dogs]]") 24 | self.assertEqual(slug, "dog") 25 | self.assertEqual(title, "Dogs") 26 | 27 | def test_piped_space(self): 28 | slug, title = wikilink.parse_wikilink("[[dog | Dogs]]") 29 | self.assertEqual(slug, "dog") 30 | self.assertEqual(title, "Dogs") 31 | 32 | 33 | class test_strip_wikilinks(unittest.TestCase): 34 | def test_returns_tuple(self): 35 | s = "lorem ipsum [[WikiLink]] dolar [[wiki| Link]]" 36 | result = wikilink.strip_wikilinks(s) 37 | self.assertEqual( 38 | result, 39 | "lorem ipsum WikiLink dolar Link" 40 | ) 41 | 42 | 43 | if __name__ == '__main__': 44 | unittest.main() --------------------------------------------------------------------------------