13 |
14 | Index
15 | About
16 | About: Story
17 | About: Team
18 | Contact
19 |
20 | {% block body %}{% endblock %}
21 |
22 |
23 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | The MIT License (MIT)
2 |
3 | Copyright (c) 2019 Nick Janetakis
4 | Copyright (c) 2023 Matthew Swabey
5 |
6 | Permission is hereby granted, free of charge, to any person obtaining
7 | a copy of this software and associated documentation files (the
8 | 'Software'), to deal in the Software without restriction, including
9 | without limitation the rights to use, copy, modify, merge, publish,
10 | distribute, sublicense, and/or sell copies of the Software, and to
11 | permit persons to whom the Software is furnished to do so, subject to
12 | the following conditions:
13 |
14 | The above copyright notice and this permission notice shall be
15 | included in all copies or substantial portions of the Software.
16 |
17 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
18 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
21 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
22 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
23 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup
2 |
3 |
4 | setup(
5 | name="Flask-Static-Digest",
6 | version="0.4.1",
7 | author="Nick Janetakis",
8 | author_email="nick.janetakis@gmail.com",
9 | url="https://github.com/nickjj/flask-static-digest",
10 | description=(
11 | "Flask extension for md5 tagging and compressing "
12 | "(gzip / brotli) static files."
13 | ),
14 | license="MIT",
15 | package_data={"Flask-Static-Digest": ["VERSION"]},
16 | packages=["flask_static_digest"],
17 | platforms="any",
18 | python_requires=">=3.6",
19 | zip_safe=False,
20 | install_requires=["Flask>=1.0"],
21 | extras_require={"brotli": ["Brotli>=1.0.9"]},
22 | entry_points={
23 | "flask.commands": ["digest=flask_static_digest.cli:digest"],
24 | },
25 | classifiers=[
26 | "Environment :: Web Environment",
27 | "Framework :: Flask",
28 | "Intended Audience :: Developers",
29 | "License :: OSI Approved :: MIT License",
30 | "Operating System :: OS Independent",
31 | "Programming Language :: Python",
32 | "Programming Language :: Python :: 3",
33 | "Topic :: Internet :: WWW/HTTP :: Dynamic Content",
34 | "Topic :: Software Development :: Libraries :: Python Modules",
35 | "Topic :: System :: Archiving :: Compression",
36 | ],
37 | )
38 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Created by https://www.gitignore.io/api/python,osx
2 |
3 | ### OSX ###
4 | *.DS_Store
5 | .AppleDouble
6 | .LSOverride
7 |
8 | # Icon must end with two \r
9 | Icon
10 |
11 | # Thumbnails
12 | ._*
13 |
14 | # Files that might appear in the root of a volume
15 | .DocumentRevisions-V100
16 | .fseventsd
17 | .Spotlight-V100
18 | .TemporaryItems
19 | .Trashes
20 | .VolumeIcon.icns
21 | .com.apple.timemachine.donotpresent
22 |
23 | # Directories potentially created on remote AFP share
24 | .AppleDB
25 | .AppleDesktop
26 | Network Trash Folder
27 | Temporary Items
28 | .apdisk
29 |
30 | ### Python ###
31 | # Byte-compiled / optimized / DLL files
32 | __pycache__/
33 | *.py[cod]
34 | *$py.class
35 |
36 | # C extensions
37 | *.so
38 |
39 | # Distribution / packaging
40 | .Python
41 | build/
42 | develop-eggs/
43 | dist/
44 | downloads/
45 | eggs/
46 | .eggs/
47 | lib/
48 | lib64/
49 | parts/
50 | sdist/
51 | var/
52 | wheels/
53 | *.egg-info/
54 | .installed.cfg
55 | *.egg
56 |
57 | # PyInstaller
58 | # Usually these files are written by a python script from a template
59 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
60 | *.manifest
61 | *.spec
62 |
63 | # Installer logs
64 | pip-log.txt
65 | pip-delete-this-directory.txt
66 |
67 | # Unit test / coverage reports
68 | htmlcov/
69 | .tox/
70 | .coverage
71 | .coverage.*
72 | .cache
73 | .pytest_cache/
74 | nosetests.xml
75 | coverage.xml
76 | *.cover
77 | .hypothesis/
78 |
79 | # Translations
80 | *.mo
81 | *.pot
82 |
83 | # End of https://www.gitignore.io/api/python,osx
84 |
85 | # Exclude local .venv
86 | .venv/
87 |
--------------------------------------------------------------------------------
/flask_static_digest/cli.py:
--------------------------------------------------------------------------------
1 | import sys
2 |
3 | import click
4 | from flask import current_app
5 | from flask.cli import with_appcontext
6 |
7 | from flask_static_digest.digester import clean as _clean
8 | from flask_static_digest.digester import compile as _compile
9 |
10 |
11 | @click.group()
12 | @click.pass_context
13 | @with_appcontext
14 | def digest(ctx):
15 | """md5 tag and compress static files."""
16 |
17 | ctx.ensure_object(dict)
18 |
19 | ctx.obj["gzip"] = False
20 | ctx.obj["brotli"] = False
21 | ctx.obj["blacklist_filter"] = current_app.config.get(
22 | "FLASK_STATIC_DIGEST_BLACKLIST_FILTER"
23 | )
24 |
25 | compression = current_app.config.get("FLASK_STATIC_DIGEST_COMPRESSION")
26 |
27 | for algo in compression:
28 | if algo == "gzip":
29 | ctx.obj["gzip"] = True
30 | elif algo == "brotli":
31 | try:
32 | import brotli # noqa: F401
33 | except ModuleNotFoundError:
34 | click.echo("Error: Python package 'brotli' not installed.")
35 | sys.exit(78) # sysexits.h 78 EX_CONFIG configuration error
36 | else:
37 | ctx.obj["brotli"] = True
38 | else:
39 | click.echo(
40 | f"{algo} is not a supported compression value, it must be"
41 | " 'gzip' or 'brotli'"
42 | )
43 | sys.exit(78)
44 |
45 |
46 | @digest.command()
47 | @click.pass_context
48 | @with_appcontext
49 | def compile(ctx):
50 | """Generate optimized static files and a cache manifest."""
51 | for blueprint in [current_app, *current_app.blueprints.values()]:
52 | if not blueprint.static_folder:
53 | continue
54 |
55 | _compile(
56 | blueprint.static_folder,
57 | blueprint.static_folder,
58 | ctx.obj["blacklist_filter"],
59 | ctx.obj["gzip"],
60 | ctx.obj["brotli"],
61 | )
62 |
63 |
64 | @digest.command()
65 | @click.pass_context
66 | @with_appcontext
67 | def clean(ctx):
68 | """Remove generated static files and cache manifest."""
69 | for blueprint in [current_app, *current_app.blueprints.values()]:
70 | if not blueprint.static_folder:
71 | continue
72 |
73 | _clean(
74 | blueprint.static_folder,
75 | ctx.obj["blacklist_filter"],
76 | ctx.obj["gzip"],
77 | ctx.obj["brotli"],
78 | )
79 |
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | # Changelog
2 |
3 | All notable changes to this project will be documented in this file.
4 |
5 | The format is based on [Keep a
6 | Changelog](https://keepachangelog.com/en/1.0.0/).
7 |
8 | ## [Unreleased]
9 |
10 | - Nothing yet!
11 |
12 | ## [0.4.1] - 2024-05-17
13 |
14 | - Fix `FLASK_STATIC_DIGEST_HOST_URL` to support URL paths that have multiple `/` segments
15 |
16 | ## [0.4.0] - 2023-05-14
17 |
18 | - Brotli support added by [Dr. Matthew Swabey](https://github.com/mattaw)
19 | - Remove `FLASK_STATIC_DIGEST_GZIP_FILES` gzip config option
20 | - Add `FLASK_STATIC_DIGEST_COMPRESSION` (defaults to `["gzip"]` and replaces the old gzip config option)
21 |
22 | ## [0.3.0] - 2022-03-16
23 |
24 | - Add support for digesting static files in blueprints and nested blueprints
25 | - Spruce up the code base by using Black for code formatting
26 |
27 | ## [0.2.1] - 2020-12-23
28 |
29 | ### Fixed
30 |
31 | - Ensure Flask's `static_url_path` is used if you have a host URL set
32 |
33 | ## [0.2.0] - 2020-12-23
34 |
35 | ### Added
36 |
37 | - Support to prepend your static paths with a host / CDN URL via `FLASK_STATIC_DIGEST_HOST_URL`
38 |
39 | ## [0.1.4] - 2020-11-24
40 |
41 | ### Fixed
42 |
43 | - `static_url_for` will now throw a 404 instead of a 500 if you have an invalid `filename`
44 |
45 | ## [0.1.3] - 2020-01-23
46 |
47 | ### Added
48 |
49 | - Windows support
50 |
51 | ## [0.1.2] - 2019-10-30
52 |
53 | ### Fixed
54 |
55 | - Really fix the version being read dynamically in `setup.py`
56 |
57 | ## [0.1.1] - 2019-10-30
58 |
59 | ### Fixed
60 |
61 | - Attempt to fix the version being read dynamically in `setup.py`
62 |
63 | ## [0.1.0] - 2019-10-30
64 |
65 | ### Added
66 |
67 | - Everything!
68 |
69 | [Unreleased]: https://github.com/nickjj/flask-static-digest/compare/v0.4.1...HEAD
70 | [0.4.1]: https://github.com/nickjj/flask-static-digest/compare/v0.4.0...v0.4.1
71 | [0.4.0]: https://github.com/nickjj/flask-static-digest/compare/v0.3.0...v0.4.0
72 | [0.3.0]: https://github.com/nickjj/flask-static-digest/compare/v0.2.1...v0.3.0
73 | [0.2.1]: https://github.com/nickjj/flask-static-digest/compare/v0.2.0...v0.2.1
74 | [0.2.0]: https://github.com/nickjj/flask-static-digest/compare/v0.1.4...v0.2.0
75 | [0.1.4]: https://github.com/nickjj/flask-static-digest/compare/v0.1.3...v0.1.4
76 | [0.1.3]: https://github.com/nickjj/flask-static-digest/compare/v0.1.2...v0.1.3
77 | [0.1.2]: https://github.com/nickjj/flask-static-digest/compare/v0.1.1...v0.1.2
78 | [0.1.1]: https://github.com/nickjj/flask-static-digest/compare/v0.1.0...v0.1.1
79 | [0.1.0]: https://github.com/nickjj/flask-static-digest/releases/tag/v0.1.0
80 |
--------------------------------------------------------------------------------
/flask_static_digest/__init__.py:
--------------------------------------------------------------------------------
1 | import json
2 | import logging
3 | import os
4 |
5 | from flask import request
6 | from flask import url_for as flask_url_for
7 |
8 |
9 | class FlaskStaticDigest(object):
10 | def __init__(self, app=None):
11 | self.app = app
12 |
13 | if app is not None:
14 | self.init_app(app)
15 |
16 | def init_app(self, app):
17 | """
18 | Mutate the application passed in as explained here:
19 | https://flask.palletsprojects.com/en/1.1.x/extensiondev/
20 | :param app: Flask application
21 | :return: None
22 | """
23 | app.config.setdefault("FLASK_STATIC_DIGEST_BLACKLIST_FILTER", [])
24 | app.config.setdefault("FLASK_STATIC_DIGEST_HOST_URL", None)
25 | app.config.setdefault("FLASK_STATIC_DIGEST_COMPRESSION", ["gzip"])
26 |
27 | compression = set(app.config["FLASK_STATIC_DIGEST_COMPRESSION"])
28 | app.config["FLASK_STATIC_DIGEST_COMPRESSION"] = list(compression)
29 |
30 | self.host_url = app.config.get("FLASK_STATIC_DIGEST_HOST_URL")
31 |
32 | self.manifests = {}
33 |
34 | self._load_manifest("static", app)
35 | for endpoint, blueprint in app.blueprints.items():
36 | self._load_manifest(f"{endpoint}.static", blueprint)
37 |
38 | app.add_template_global(self.static_url_for)
39 |
40 | def _load_manifest(self, endpoint, scaffold):
41 | if not scaffold.has_static_folder:
42 | return
43 |
44 | manifest_path = os.path.join(
45 | scaffold._static_folder, "cache_manifest.json"
46 | )
47 | try:
48 | with scaffold.open_resource(manifest_path, "r") as f:
49 | self.manifests[endpoint] = json.load(f)
50 | except json.JSONDecodeError:
51 | logging.warning(f"Couldn't decode file: {manifest_path}")
52 | except PermissionError:
53 | logging.warning(f"Couldn't access file: {manifest_path}")
54 | except (FileNotFoundError, Exception):
55 | pass
56 |
57 | def _custom_url_join(self, url_path):
58 | """
59 | Join the host URL with another URL path.
60 | :param url_path: The URL returned by url_for
61 | :type url_path: str
62 | :return: Joined URL
63 | """
64 | if self.host_url is None:
65 | return url_path
66 |
67 | return "/".join([self.host_url.rstrip("/"), url_path.lstrip("/")])
68 |
69 | def static_url_for(self, endpoint, **values):
70 | """
71 | This function uses Flask's url_for under the hood and accepts the
72 | same arguments. The only differences are it will prefix a host URL if
73 | one exists and if a manifest is available it will look up the filename
74 | from the manifest.
75 | :param endpoint: The endpoint of the URL
76 | :type endpoint: str
77 | :param values: Arguments of the URL rule
78 | :return: Static file path.
79 | """
80 | if request is not None:
81 | blueprint_name = request.blueprint
82 | if endpoint[:1] == ".":
83 | if blueprint_name is not None:
84 | endpoint = f"{blueprint_name}{endpoint}"
85 | else:
86 | endpoint = endpoint[1:]
87 |
88 | manifest = self.manifests.get(endpoint, {})
89 | filename = values.get("filename", None)
90 | values["filename"] = manifest.get(filename, filename)
91 |
92 | return self._custom_url_join(flask_url_for(endpoint, **values))
93 |
--------------------------------------------------------------------------------
/flask_static_digest/digester.py:
--------------------------------------------------------------------------------
1 | import functools
2 | import glob
3 | import gzip
4 | import hashlib
5 | import json
6 | import os.path
7 | import re
8 | import shutil
9 |
10 | DIGESTED_FILE_REGEX = r"-[a-f\d]{32}"
11 | CHUNK_SIZE = 1024 * 1024
12 |
13 |
14 | def compile(
15 | input_path, output_path, digest_blacklist_filter, gzip_files, brotli_files
16 | ):
17 | """
18 | Generate md5 tagged static files compressed with gzip and brotli.
19 |
20 | :param input_path: The source path of your static files
21 | :type input_path: str
22 | :param output_path: The destination path of your static files
23 | :type output_path: str
24 | :param digest_blacklist_filter: Ignore compiling these file types
25 | :type digest_blacklist_filter: list
26 | :param gzip_files: Whether or not gzipped files will be generated
27 | :type gzip_files: bool
28 | :param brotli_files: Whether or not brotli files will be generated
29 | :type brotli_files: bool
30 | :return: None
31 | """
32 | if not os.path.exists(input_path):
33 | print(f"The input path '{input_path}' does not exist")
34 | return None
35 |
36 | if not os.path.exists(output_path):
37 | print(f"The output path '{output_path}' does not exist")
38 | return None
39 |
40 | files = _filter_files(input_path, digest_blacklist_filter)
41 | manifest = _generate_manifest(files, gzip_files, brotli_files, output_path)
42 | _save_manifest(manifest, output_path)
43 |
44 | print(f"Check your digested files at '{output_path}'")
45 | return None
46 |
47 |
48 | def clean(output_path, digest_blacklist_filter, gzip_files, brotli_files):
49 | """
50 | Delete the generated md5 tagged and gzipped static files.
51 |
52 | :param input_path: The source path of your static files
53 | :type input_path: str
54 | :param output_path: The destination path of your static files
55 | :type output_path: str
56 | :param digest_blacklist_filter: Ignore cleaning these file types
57 | :type digest_blacklist_filter: list
58 | :param gzip_files: Whether or not gzipped files will be cleaned
59 | :type gzip_files: bool
60 | :param brotli_files: Whether or not brotli files will be cleaned
61 | :type brotli_files: bool
62 | :return: None
63 | """
64 | for item in glob.iglob(output_path + "**/**", recursive=True):
65 | if os.path.isfile(item):
66 | _, file_extension = os.path.splitext(item)
67 | basename = os.path.basename(item)
68 |
69 | if (
70 | re.search(DIGESTED_FILE_REGEX, basename)
71 | and file_extension not in digest_blacklist_filter
72 | ):
73 | if os.path.exists(item):
74 | os.remove(item)
75 |
76 | if gzip_files and file_extension == ".gz":
77 | if os.path.exists(item):
78 | os.remove(item)
79 |
80 | if brotli_files and file_extension == ".br":
81 | if os.path.exists(item):
82 | os.remove(item)
83 |
84 | manifest_path = os.path.join(output_path, "cache_manifest.json")
85 |
86 | if os.path.exists(manifest_path):
87 | os.remove(manifest_path)
88 |
89 | print(f"Check your cleaned files at '{output_path}'")
90 | return None
91 |
92 |
93 | def _filter_files(input_path, digest_blacklist_filter):
94 | filtered_files = []
95 |
96 | for item in glob.iglob(input_path + "**/**", recursive=True):
97 | if os.path.isfile(item):
98 | if not _is_compiled_file(item, digest_blacklist_filter):
99 | filtered_files.append(item)
100 |
101 | return filtered_files
102 |
103 |
104 | def _is_compiled_file(file_path, digest_blacklist_filter):
105 | file_name, file_extension = os.path.splitext(file_path)
106 | basename = os.path.basename(file_path)
107 |
108 | return (
109 | re.search(DIGESTED_FILE_REGEX, basename)
110 | or file_extension in digest_blacklist_filter
111 | or file_extension == ".gz"
112 | or file_extension == ".br"
113 | or basename == "cache_manifest.json"
114 | )
115 |
116 |
117 | def _generate_manifest(files, gzip_files, brotli_files, output_path):
118 | manifest = {}
119 |
120 | for file in files:
121 | rel_file_path = os.path.relpath(file, output_path).replace("\\", "/")
122 |
123 | file_name, file_extension = os.path.splitext(rel_file_path)
124 |
125 | digest = _generate_digest(file)
126 | digested_file_path = f"{file_name}-{digest}{file_extension}"
127 |
128 | manifest[rel_file_path] = digested_file_path
129 |
130 | _write_to_disk(
131 | file, digested_file_path, gzip_files, brotli_files, output_path
132 | )
133 |
134 | return manifest
135 |
136 |
137 | def _generate_digest(file):
138 | digest = None
139 |
140 | with open(file, "rb") as f:
141 | digest = hashlib.md5(f.read()).hexdigest()
142 |
143 | return digest
144 |
145 |
146 | def _save_manifest(manifest, output_path):
147 | manifest_content = json.dumps(manifest)
148 | manifest_path = os.path.join(output_path, "cache_manifest.json")
149 |
150 | with open(manifest_path, "w") as f:
151 | f.write(manifest_content)
152 |
153 | return None
154 |
155 |
156 | def _write_to_disk(
157 | file, digested_file_path, gzip_files, brotli_files, input_path
158 | ):
159 | full_digested_file_path = os.path.join(input_path, digested_file_path)
160 |
161 | # Copy file while preserving permissions and meta data if supported.
162 | shutil.copy2(file, full_digested_file_path)
163 |
164 | if gzip_files:
165 | with open(file, "rb") as f_in:
166 | with gzip.open(f"{file}.gz", "wb") as f_out:
167 | shutil.copyfileobj(f_in, f_out)
168 |
169 | shutil.copy2(f"{file}.gz", f"{full_digested_file_path}.gz")
170 |
171 | if brotli_files:
172 | import brotli
173 |
174 | compressor = brotli.Compressor(quality=11)
175 |
176 | with open(file, "rb") as f_in:
177 | with open(f"{file}.br", "wb") as f_out:
178 | read_chunk = functools.partial(f_in.read, CHUNK_SIZE)
179 |
180 | for data in iter(read_chunk, b""):
181 | f_out.write(compressor.process(data))
182 |
183 | f_out.write(compressor.finish())
184 |
185 | shutil.copy2(f"{file}.br", f"{full_digested_file_path}.br")
186 |
187 | return None
188 |
--------------------------------------------------------------------------------
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
1 | name: "CI"
2 |
3 | on:
4 | pull_request:
5 | branches:
6 | - "*"
7 | push:
8 | branches:
9 | - "main"
10 | - "master"
11 |
12 | jobs:
13 | test:
14 | runs-on: "ubuntu-22.04"
15 |
16 | steps:
17 | - uses: "actions/checkout@v2"
18 |
19 | - name: "Set up Python 3.10"
20 | uses: "actions/setup-python@v1"
21 | with:
22 | python-version: "3.10"
23 |
24 | - name: "Install dependencies"
25 | run: |
26 | pip3 install flake8 black Flask==3.0.3
27 | pip3 install -e .[brotli]
28 | - name: "Lint code base"
29 | run: |
30 | flake8 .
31 | black --check .
32 | - name: "Run test suite for gzip and brotli"
33 | run: |
34 | cd tests/example_app
35 | export FLASK_APP=example.app
36 | ls -laR example/static/ | wc -l \
37 | | grep -q "26" \
38 | && (echo "File count before compiling test (index): pass" && exit 0) \
39 | || (echo "File count before compiling test (index): fail" && exit 1)
40 | ls -laR example/about/static/ | wc -l \
41 | | grep -q "5" \
42 | && (echo "File count before compiling test (about): pass" && exit 0) \
43 | || (echo "File count before compiling test (about): fail" && exit 1)
44 | ls -laR example/contact/static/ | wc -l \
45 | | grep -q "5" \
46 | && (echo "File count before compiling test (contact): pass" && exit 0) \
47 | || (echo "File count before compiling test (contact): fail" && exit 1)
48 | ls -laR example/about/team/static/ | wc -l \
49 | | grep -q "5" \
50 | && (echo "File count before compiling test (team): pass" && exit 0) \
51 | || (echo "File count before compiling test (team): fail" && exit 1)
52 | flask digest compile \
53 | && ls -laR example/static/ | wc -l \
54 | | grep -q "42" \
55 | && (echo "File count after compiling test (index): pass" && exit 0) \
56 | || (echo "File count after compiling test (index): fail" && exit 1)
57 | ls -laR example/about/static/ | wc -l \
58 | | grep -q "11" \
59 | && (echo "File count after compiling test (about): pass" && exit 0) \
60 | || (echo "File count after compiling test (about): fail" && exit 1)
61 | ls -laR example/contact/static/ | wc -l \
62 | | grep -q "11" \
63 | && (echo "File count after compiling test (contact): pass" && exit 0) \
64 | || (echo "File count after compiling test (contact): fail" && exit 1)
65 | ls -laR example/about/team/static/ | wc -l \
66 | | grep -q "11" \
67 | && (echo "File count after compiling test (team): pass" && exit 0) \
68 | || (echo "File count after compiling test (team): fail" && exit 1)
69 | find . -type f | grep ".gz$" | wc -l \
70 | | grep -q "14" \
71 | && (echo "Check for gzip asset creation: pass" && exit 0) \
72 | || (echo "Check for gzip asset creation: pass: fail" && exit 1)
73 | find . -type f | grep ".br$" | wc -l \
74 | | grep -q "14" \
75 | && (echo "Check for brotli asset creation: pass" && exit 0) \
76 | || (echo "Check for brotli asset creation: pass: fail" && exit 1)
77 | grep -q "js/modules/hello.js" example/static/cache_manifest.json \
78 | && (echo "Cache manifest has nested file test: pass" && exit 0) \
79 | || (echo "Cache manifest has nested file test: fail" && exit 1)
80 | grep -q "js/modules/hello-d41d8cd98f00b204e9800998ecf8427e.js" example/static/cache_manifest.json \
81 | && (echo "Cache manifest has digested file test: pass" && exit 0) \
82 | || (echo "Cache manifest has digested file test: fail" && exit 1)
83 | grep -q "js/modules/hello.js" example/static/cache_manifest.json \
84 | && (echo "Cache manifest has nested file test: pass" && exit 0) \
85 | || (echo "Cache manifest has nested file test: fail" && exit 1)
86 | flask run &
87 | sleep 5 \
88 | && curl http://localhost:5000 \
89 | | grep -q 'https://cdn.example.com/static/css/app-d41d8cd98f00b204e9800998ecf8427e.css' \
90 | && (echo 'Stylesheet is md5 tagged test (index): pass' && exit 0) \
91 | || (echo 'Stylesheet is md5 tagged test (index): fail' && exit 1)
92 | curl http://localhost:5000/about/ \
93 | | grep -q 'https://cdn.example.com/about/static/app-297c671953f6c24f1354214b3f706e0b.css' \
94 | && (echo 'Stylesheet is md5 tagged test (about): pass' && exit 0) \
95 | || (echo 'Stylesheet is md5 tagged test (about): fail' && exit 1)
96 | curl http://localhost:5000/contact \
97 | | grep -q 'https://cdn.example.com/contact/static/app-b71c3f4f5d026dd5198163b655228776.css' \
98 | && (echo 'Stylesheet is md5 tagged test (contact): pass' && exit 0) \
99 | || (echo 'Stylesheet is md5 tagged test (contact): fail' && exit 1)
100 | curl http://localhost:5000/about/team \
101 | | grep -q 'https://cdn.example.com/about/static/team/app-023f768653c556fc2fcaf338b408c464.css' \
102 | && (echo 'Stylesheet is md5 tagged test (team): pass' && exit 0) \
103 | || (echo 'Stylesheet is md5 tagged test (team): fail' && exit 1)
104 | kill $!
105 | flask digest clean \
106 | && ls -laR example/static/ | wc -l \
107 | | grep -q "26" \
108 | && (echo "File count after cleaning test (index): pass" && exit 0) \
109 | || (echo "File count after cleaning test (index): fail" && exit 1)
110 | ls -laR example/about/static/ | wc -l \
111 | | grep -q "5" \
112 | && (echo "File count after cleaning test (about): pass" && exit 0) \
113 | || (echo "File count after cleaning test (about): fail" && exit 1)
114 | ls -laR example/contact/static/ | wc -l \
115 | | grep -q "5" \
116 | && (echo "File count after cleaning test (contact): pass" && exit 0) \
117 | || (echo "File count after cleaning test (contact): fail" && exit 1)
118 | ls -laR example/about/team/static/ | wc -l \
119 | | grep -q "5" \
120 | && (echo "File count after cleaning test (team): pass" && exit 0) \
121 | || (echo "File count after cleaning test (team): fail" && exit 1)
122 | flask run &
123 | sleep 5 \
124 | && curl http://localhost:5000 \
125 | | grep -q 'https://cdn.example.com/static/js/app.js' \
126 | && (echo 'Javascript is not md5 tagged test: pass' && exit 0) \
127 | || (echo 'Javascript is not md5 tagged test: fail' && exit 1)
128 | flask digest clean
129 | kill $!
130 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # What is Flask-Static-Digest? 
2 |
3 | It is a Flask extension that will help make your static files production ready
4 | with very minimal effort on your part. It does this by creating md5 tagged
5 | versions and gzip and / or brotli compressed versions of your static files by
6 | running a `flask digest compile` command that this extension adds to your Flask
7 | app.
8 |
9 | It should be the last thing you do to your static files before uploading them
10 | to your server or CDN. Speaking of which, if you're using a CDN this extension
11 | optionally lets you configure a host URL that will get prepended to your static
12 | file paths. If you're not using a CDN, no problem everything will work as you
13 | would expect by default.
14 |
15 | Other web frameworks like Django, Ruby on Rails and Phoenix all have this
16 | feature built into their framework, and now with this extension Flask does too.
17 |
18 | **This extension will work if you're not using any asset build tools but at the
19 | same time it also works with esbuild, Webpack, Grunt, Gulp or any other build
20 | tool you can think of. This tool does not depend on or compete with existing
21 | asset build tools.**
22 |
23 | If you're already using Webpack or a similar tool, that's great. Webpack takes
24 | care of bundling your assets and helps convert things like SASS to CSS and ES6+
25 | JS to browser compatible JS. That is solving a completely different problem
26 | than what this extension solves. This extension will further optimize your
27 | static files after your build tool produces its output files.
28 |
29 | This extension does things that Webpack alone cannot do because in order for
30 | things like md5 tagging to work Flask needs to be aware of how to map those
31 | hashed file names back to regular file names you would reference in your Jinja
32 | 2 templates.
33 |
34 | ## How does it work?
35 |
36 | There's 3 pieces to this extension:
37 |
38 | 1. It adds a custom Flask CLI command to your project. When you run this
39 | command it looks at your static files and then generates an md5 tagged
40 | version of each file along with optionally compressing them with gzip
41 | and / or brotli.
42 |
43 | 2. When the above command finishes it creates a `cache_manifest.json` file in
44 | your static folder which maps the regular file names, such as
45 | `images/flask.png` to `images/flask-f86b271a51b3cfad5faa9299dacd987f.png`.
46 |
47 | 3. It adds a new template helper called `static_url_for` which uses Flask's
48 | `url_for` under the hood but is aware of the `cache_manifest.json` file so
49 | it knows how to resolve `images/flask.png` to the md5 tagged file name.
50 |
51 | ### Demo video
52 |
53 | This 25 minute video goes over using this extension but it also spends a lot
54 | of time on the "why" where we cover topics like cache busting and why IMO you
55 | might want to use this extension in all of your Flask projects.
56 |
57 | If you prefer reading instead of video, this README file covers installing,
58 | configuring and using this extension too.
59 |
60 | [](https://www.youtube.com/watch?v=-Xd84hlIjkI)
62 |
63 | #### Changes since this video
64 |
65 | - `FLASK_STATIC_DIGEST_HOST_URL` has been added to configure an optional external host, aka. CDN ([explained here](#configuring-this-extension))
66 | - If your blueprints have static files they will get digested now too (including nested blueprints!)
67 | - Optional Brotli support has been added
68 | - `FLASK_STATIC_DIGEST_COMPRESSION` has been added to control compression ([explained here](#configuring-this-extension))
69 |
70 | ## Table of Contents
71 |
72 | - [Installation](#installation)
73 | - [Using the newly added Flask CLI command](#using-the-newly-added-flask-cli-command)
74 | - [Going over the Flask CLI commands](#going-over-the-flask-cli-commands)
75 | - [Configuring this extension](#configuring-this-extension)
76 | - [Modifying your templates to use static_url_for instead of url_for](#modifying-your-templates-to-use-static_url_for-instead-of-url_for)
77 | - [Potentially updating your .gitignore file](#potentially-updating-your-gitignore-file)
78 | - [FAQ](#faq)
79 | - [What about development vs production and performance implications?](#what-about-development-vs-production-and-performance-implications)
80 | - [Why bother compressing your static files here instead of with nginx?](#why-bother-compressing-your-static-files-here-instead-of-with-nginx)
81 | - [How do you use this extension with Webpack or another build tool?](#how-do-you-use-this-extension-with-webpack-or-another-build-tool)
82 | - [Migrating from Flask-Webpack](#migrating-from-flask-webpack)
83 | - [How do you use this extension with Docker?](#how-do-you-use-this-extension-with-docker)
84 | - [How do you use this extension with Heroku?](#how-do-you-use-this-extension-with-heroku)
85 | - [What about user uploaded files?](#what-about-user-uploaded-files)
86 | - [About the author](#about-the-author)
87 |
88 | ## Installation
89 |
90 | *You'll need to be running Python 3.6+ and using Flask 1.0 or greater.*
91 |
92 | `pip install Flask-Static-Digest`
93 |
94 | To install with Brotli support:
95 |
96 | `pip install Flask-Static-Digest[brotli]`
97 |
98 | ### Example directory structure for a 'hello' app
99 |
100 | ```
101 | ├── hello
102 | │ ├── __init__.py
103 | │ ├── app.py
104 | │ └── static
105 | │ └── css
106 | │ ├── app.css
107 | └── requirements.txt
108 | ```
109 |
110 | ### Flask app factory example using this extension
111 |
112 | ```py
113 | from flask import Flask
114 | from flask_static_digest import FlaskStaticDigest
115 |
116 | flask_static_digest = FlaskStaticDigest()
117 |
118 |
119 | def create_app():
120 | app = Flask(__name__)
121 |
122 | flask_static_digest.init_app(app)
123 |
124 | @app.route("/")
125 | def index():
126 | return "Hello, World!"
127 |
128 | return app
129 | ```
130 |
131 | *A more complete example app can be found in the [tests/
132 | directory](https://github.com/nickjj/flask-static-digest/tree/master/tests/example_app).*
133 |
134 | ## Using the newly added Flask CLI command
135 |
136 | You'll want to make sure to at least set the `FLASK_APP` environment variable:
137 |
138 | ```sh
139 | export FLASK_APP=hello.app
140 | export FLASK_ENV=development
141 | ```
142 |
143 | Then run the `flask` binary to see its help menu:
144 |
145 | ```sh
146 | Usage: flask [OPTIONS] COMMAND [ARGS]...
147 |
148 | ...
149 |
150 | Options:
151 | --version Show the flask version
152 | --help Show this message and exit.
153 |
154 | Commands:
155 | digest md5 tag and compress static files.
156 | routes Show the routes for the app.
157 | run Run a development server.
158 | shell Run a shell in the app context.
159 | ```
160 |
161 | If all went as planned you should see the new `digest` command added to the
162 | list of commands.
163 |
164 | ## Going over the Flask CLI commands
165 |
166 | Running `flask digest` will produce this help menu:
167 |
168 | ```sh
169 | Usage: flask digest [OPTIONS] COMMAND [ARGS]...
170 |
171 | md5 tag and compress static files.
172 |
173 | Options:
174 | --help Show this message and exit.
175 |
176 | Commands:
177 | clean Remove generated static files and cache manifest.
178 | compile Generate optimized static files and a cache manifest.
179 | ```
180 |
181 | Each command is labeled, but here's a bit more information on what they do.
182 |
183 | ### compile
184 |
185 | Inspects your Flask app's and blueprint's `static_folder` and uses that as both
186 | the input and output path of where to look for and create the newly digested
187 | and compressed files.
188 |
189 | At a high level it recursively loops over all of the files it finds in that
190 | directory and then generates the md5 tagged and compressed versions of each
191 | file. It also creates a `cache_manifest.json` file in the root of your
192 | `static_folder`.
193 |
194 | That manifest file is machine generated meaning you should not edit it unless
195 | you really know what you're doing.
196 |
197 | This file maps the human readable file name of let's say `images/flask.png` to
198 | the digested file name. It's a simple key / value set up. It's basically a
199 | Python dictionary in JSON format.
200 |
201 | In the end it means if your static folder looked like this originally:
202 |
203 | - `css/app.css`
204 | - `js/app.js`
205 | - `images/flask.png`
206 |
207 | And you decided to run the compile command, it would now look like this:
208 |
209 | - `css/app.css`
210 | - `css/app.css.gz`
211 | - `css/app-5d41402abc4b2a76b9719d911017c592.css`
212 | - `css/app-5d41402abc4b2a76b9719d911017c592.css.gz`
213 | - `js/app.js`
214 | - `js/app.js.gz`
215 | - `js/app-098f6bcd4621d373cade4e832627b4f6.js`
216 | - `js/app-098f6bcd4621d373cade4e832627b4f6.js.gz`
217 | - `images/flask.png`
218 | - `images/flask.png.gz`
219 | - `images/flask-f86b271a51b3cfad5faa9299dacd987f.png`
220 | - `images/flask-f86b271a51b3cfad5faa9299dacd987f.png.gz`
221 | - `cache_manifest.json`
222 |
223 | *Your md5 hashes will be different because it depends on what the contents of
224 | the file are.*
225 |
226 | ### clean
227 |
228 | Inspects your Flask app's and blueprint's `static_folder` and uses that as the
229 | input path of where to look for digested and compressed files.
230 |
231 | It will recursively delete files that have a file extension of `.gz` or `.br`
232 | and files that have been digested. It determines if a file has been digested
233 | based on its file name. In other words, it will delete files that match this
234 | regexp `r"-[a-f\d]{32}"`.
235 |
236 | In the end that means if you had these 6 files in your static folder:
237 |
238 | - `images/flask.png`
239 | - `images/flask.png.gz`
240 | - `images/flask.png.br`
241 | - `images/flask-f86b271a51b3cfad5faa9299dacd987f.png`
242 | - `images/flask-f86b271a51b3cfad5faa9299dacd987f.png.gz`
243 | - `images/flask-f86b271a51b3cfad5faa9299dacd987f.png.br`
244 |
245 | And you decided to run the clean command, the last 5 files would be deleted
246 | leaving you with the original `images/flask.png`.
247 |
248 | ## Configuring this extension
249 |
250 | By default this extension will create md5 tagged versions of all files it finds
251 | in your configured `static_folder`. It will also create gzip'ed versions of each
252 | file and it won't prefix your static files with an external host.
253 |
254 | If you don't like any of this behavior or you wish to enable brotli you can
255 | optionally configure:
256 |
257 | ```py
258 | FLASK_STATIC_DIGEST_BLACKLIST_FILTER = []
259 | # If you want specific extensions to not get md5 tagged you can add them to
260 | # the list, such as: [".htm", ".html", ".txt"]. Make sure to include the ".".
261 |
262 | FLASK_STATIC_DIGEST_COMPRESSION = ["gzip"]
263 | # Optionally compress your static files, supported values are:
264 | # [] avoids any compression
265 | # ["gzip"] uses gzip
266 | # ["brotli"] uses brotli (prefer either gzip or both)
267 | # ["gzip", "brotli"] uses both
268 |
269 | FLASK_STATIC_DIGEST_HOST_URL = None
270 | # When set to a value such as https://cdn.example.com and you use static_url_for
271 | # it will prefix your static path with this URL. This would be useful if you
272 | # host your files from a CDN. Make sure to include the protocol (aka. https://).
273 | ```
274 |
275 | You can override these defaults in your Flask app's config file.
276 |
277 | ## Modifying your templates to use static_url_for instead of url_for
278 |
279 | We're all familiar with this code right?
280 |
281 | ```html
282 |
284 | ```
285 |
286 | When you put the above code into a Flask powered Jinja 2 template, it turns
287 | into this:
288 |
289 | ```html
290 |
292 | ```
293 |
294 | The path might vary depending on how you configured your Flask app's
295 | `static_folder` but you get the idea.
296 |
297 | #### Using static_url_for instead of url_for
298 |
299 | Let's use the same example as above:
300 |
301 | ```html
302 |
304 | ```
305 |
306 | But now take a look at the output this produces:
307 |
308 | ```html
309 |
311 | ```
312 |
313 | Or if you set `FLASK_STATIC_DIGEST_HOST_URL = "https://cdn.example.com"` it
314 | would produce:
315 |
316 | ```html
317 |
319 | ```
320 |
321 | Instead of using `url_for` you would use `static_url_for`. This uses Flask's
322 | `url_for` under the hood so things like `_external=True` and everything else
323 | `url_for` supports is available to use with `static_url_for`.
324 |
325 | That means to use this extension you don't have to do anything other than
326 | install it, optionally run the CLI command to generate the manifest and then
327 | rename your static file references to use `static_url_for` instead of
328 | `url_for`.
329 |
330 | If your editor supports performing a find / replace across multiple files you
331 | can quickly make the change by finding `url_for('static'` and replacing that
332 | with `static_url_for('static'`. If you happen to use double quotes instead of
333 | single quotes you'll want to adjust for that too.
334 |
335 | ## Potentially updating your .gitignore file
336 |
337 | If you're using something like Webpack then chances are you're already git
338 | ignoring the static files it produces as output. It's a common pattern to
339 | commit your Webpack source static files but ignore the compiled static files
340 | it produces.
341 |
342 | But if you're not using Webpack or another asset build tool then the static
343 | files that are a part of your project might have the same source and
344 | destination directory. If that's the case, chances are you'll want to git
345 | ignore the md5 tagged files as well as the compressed and `cache_manifest.json`
346 | files from version control.
347 |
348 | For clarity, you want to ignore them because you'll be generating them on your
349 | server at deploy time or within a Docker image if you're using Docker. They
350 | don't need to be tracked in version control.
351 |
352 | Add this to your `.gitignore` file to ignore certain files this extension
353 | creates:
354 |
355 | ```
356 | *-[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f].*
357 | *.gz
358 | cache_manifest.json
359 | ```
360 |
361 | This allows your original static files but ignores everything else this
362 | extension creates. I am aware at how ridiculous that ignore rule is for the md5
363 | hash but using `[0-9a-f]{32}` does not work. If you know of a better way,
364 | please open a PR!
365 |
366 | ## FAQ
367 |
368 | ### What about development vs production and performance implications?
369 |
370 | You would typically only run the CLI command to prepare your static files for
371 | production. Running `flask digest compile` would become a part of your build
372 | process -- typically after you pip install your dependencies.
373 |
374 | In development when the `cache_manifest.json` likely doesn't exist
375 | `static_url_for` calls `url_for` directly. This allows the `static_url_for`
376 | helper to work in both development and production without any fuss.
377 |
378 | It's also worth pointing out the CLI command is expected to be run before you
379 | even start your Flask server (or gunicorn / etc.), so there's no perceivable
380 | run time performance hit. It only involves doing 1 extra dictionary lookup at
381 | run time which is many orders of magnitude faster than even the most simple
382 | database query.
383 |
384 | **In other words, this extension is not going to negatively impact the
385 | performance of your web application. If anything it's going to speed it up and
386 | save you money on hosting**.
387 |
388 | That's because compressed files can be upwards of 5-10x smaller so there's less
389 | bytes to transfer over the network.
390 |
391 | Also with md5 tagging each file it means you can configure your web server such
392 | as nginx to cache each file forever. That means if a user visits your site a
393 | second time in the future, nginx will be smart enough to load it from their
394 | local browser's cache without even contacting your server. It's a 100% local
395 | look up.
396 |
397 | This is as efficient as it gets. You can't do this normally without md5 tagging
398 | each file because if the file changes in the future, nginx will continue
399 | serving the old file until the cache expires so users will never see your
400 | updates. But due to how md5 hashing works, if the contents of a file changes it
401 | will get generated with a new name and nginx will serve the uncached new file.
402 |
403 | This tactic is commonly referred to as "cache busting" and it's a very good
404 | idea to do this in production. You can even go 1 step further and serve your
405 | static files using a CDN. Using this cache busting strategy makes configuring
406 | your CDN a piece of cake since you don't need to worry about ever expiring your
407 | cache manually.
408 |
409 | ### Why bother compressing your static files here instead of with nginx?
410 |
411 | You would still be using nginx's gzip / brotli features, but now instead of
412 | nginx having to compress your files on the fly at run time you can configure
413 | nginx to use the pre-made compressed files that this extension creates.
414 |
415 | This way you can benefit from having maximum compression without having nginx
416 | waste precious CPU cycles compressing files on the fly. This gives you the best
417 | of both worlds -- the highest compression ratio with no noticeable run time
418 | performance penalty.
419 |
420 | ### How do you use this extension with Webpack or another build tool?
421 |
422 | It works out of the box with no extra configuration or plugins needed for
423 | Webpack or your build tool of choice.
424 |
425 | Typically the Webpack (or another build tool) work flow would look like this:
426 |
427 | - You configure Webpack with your source static files directory
428 | - You configure Webpack with your destination static files directory
429 | - Webpack processes your files in the source directory and copies them to the
430 | destination directory
431 | - Flask is configured to serve static files from that destination directory
432 |
433 | For example, your source directory might be `assets/` inside of your project
434 | and the destination might be `myapp/static`.
435 |
436 | This extension will look at your Flask configuration for the `static_folder`
437 | and determine it's set to `myapp/static` so it will md5 tag and compress those
438 | files. Your Webpack source files will not get digested and compressed.
439 |
440 | ### Migrating from Flask-Webpack
441 |
442 | [Flask-Webpack](https://github.com/nickjj/flask-webpack) is another extension I
443 | wrote a long time ago which was specific to Webpack but had a similar idea to
444 | this extension. Flask-Webpack is now deprecated in favor of
445 | Flask-Static-Digest. Migrating is fairly painless. There are a number of
446 | changes but on the bright side you get to delete more code than you add!
447 |
448 | #### Dependency / Flask app changes
449 |
450 | - Remove `Flask-Webpack` from `requirements.txt`
451 | - Remove all references to Flask-Webpack from your Flask app and config
452 | - Remove `manifest-revision-webpack-plugin` from `package.json`
453 | - Remove all references to this webpack plugin from your webpack config
454 | - Add `Flask-Static-Digest` to `requirements.txt`
455 | - Add the Flask-Static-Digest extension to your Flask app
456 |
457 | #### Jinja 2 template changes
458 |
459 | - Replace `stylesheet_tag('main_css') | safe` with `static_url_for('static', filename='css/main.css')`
460 | - Replace `javascript_tag('main_js') | safe` with `static_url_for('static', filename='js/main.js')`
461 | - Replace any occurrences of `asset_url_for('foo.png')` with `static_url_for('static', filename='images/foo.png')`
462 |
463 | ### How do you use this extension with Docker?
464 |
465 | It's really no different than without Docker, but instead of running `flask
466 | digest compile` on your server directly at deploy time you would run it inside
467 | of your Docker image at build time. This way your static files are already set
468 | up and ready to go by the time you pull and use your Docker image in
469 | production.
470 |
471 | You can see a fully working example of this in the open source version of my
472 | [Build a SAAS App with
473 | Flask](https://github.com/nickjj/build-a-saas-app-with-flask) course. It
474 | leverages Docker's build arguments to only compile the static files when
475 | `FLASK_ENV` is set to `production`. The key files to look at are the
476 | `Dockerfile`, `docker-compose.yml` and `.env` files. That wires up the build
477 | arguments and env variables to make it work.
478 |
479 | ### How do you use this extension with Heroku?
480 |
481 | If you're deploying to Heroku using the Python buildpack you can follow these 2 steps:
482 |
483 | 1. Create a `bin/post_compile` file in your project's source code
484 | 2. Copy the lines below into the `bin/post_compile` file, save it and commit the changes
485 |
486 | ```sh
487 | #!/usr/bin/env bash
488 |
489 | set -e
490 |
491 | echo "-----> Digesting static files"
492 | cd "${1}" && flask digest compile
493 | ```
494 |
495 | The next time you push your code this script will run after your pip
496 | dependencies are installed. It will run before your slug is compiled which
497 | ensures that the digested files are available before any traffic is served to
498 | your Dyno.
499 |
500 | You can view how this file gets executed by Heroku in their [Python buildpack's
501 | source
502 | code](https://github.com/heroku/heroku-buildpack-python/blob/main/bin/steps/hooks/post_compile).
503 |
504 | ### What about user uploaded files?
505 |
506 | Let's say that besides having static files like your logo and CSS / JavaScript
507 | bundles you also have files uploaded by users. This could be things like a user
508 | avatar, blog post images or anything else.
509 |
510 | **You would still want to md5 tag and compress these files but now we've run
511 | into a situation**. The `flask digest compile` command is meant to be run at
512 | deploy time and it could potentially be run from your dev box, inside of a
513 | Docker image, on a CI server or your production server. In these cases you
514 | wouldn't have access to the user uploaded files.
515 |
516 | But at the same time you have users uploading files at run time. They are
517 | changing all the time.
518 |
519 | **Needless to say you can't use the `flask digest compile` command to digest
520 | user uploaded files**. The `cache_manifest.json` file should be reserved for
521 | files that exist in your code repo (such as your CSS / JS bundles, maybe a
522 | logo, fonts, etc.).
523 |
524 | The above files do not change at run time and align well with running the
525 | `flask digest compile` command at deploy time.
526 |
527 | For user uploaded content you wouldn't ever write these entries to the manifest
528 | JSON file. Instead, you would typically upload your files to disk, S3 or
529 | somewhere else and then save the file name of the file you uploaded into your
530 | local database.
531 |
532 | So now when you reference a user uploaded file (let's say an avatar), you would
533 | loop over your users from the database and reference the file name from the DB.
534 |
535 | There's no need for a manifest file to store the user uploaded files because
536 | the database has a reference to the real name and then you are dynamically
537 | referencing that in your template helper (`static_url_for`), so it's never a
538 | hard coded thing that changes at the template level.
539 |
540 | What's cool about this is you already did the database query to retrieve the
541 | record(s) from the database, so there's no extra database work to do. All you
542 | have to do is reference the file name field that's a part of your model.
543 |
544 | But that doesn't fully solve the problem. You'll still want to md5 tag and
545 | compress your user uploaded content at run time and you would want to do this
546 | before you save the uploaded file into its final destination (local file system,
547 | S3, etc.).
548 |
549 | This can be done completely separate from this extension and it's really going
550 | to vary depending on where you host your user uploaded content. For example some
551 | CDNs will automatically create compressed files for you and they use things like
552 | an ETag header in the response to include a unique file name (and this is what
553 | you can store in your DB).
554 |
555 | So maybe md5 hashing and maybe compressing your user uploaded content becomes an
556 | app specific responsibility, although I'm not opposed to maybe creating helper
557 | functions you can use but that would need to be thought out carefully.
558 |
559 | However the implementation is not bad. It's really only about 5 lines of code
560 | to do both things. Feel free to `CTRL + F` around the [code
561 | base](https://github.com/nickjj/flask-static-digest/blob/master/flask_static_digest/digester.py)
562 | for `hashlib`, `gzip` and `brotli` and you'll find the related code.
563 |
564 | **So with that said, here's a work flow you can do to deal with this today:**
565 |
566 | - User uploads file
567 | - Your Flask app potentially md5 tags / gzips the file if necessary
568 | - Your Flask app saves the file name + compressed file to its final destination (local file system, S3, etc.)
569 | - Your Flask app saves the final unique file name to your database
570 |
571 | That final unique file name would be the md5 tagged version of the file that
572 | you created or the unique file name that your CDN returned back to you. I hope
573 | that clears up how to deal with user uploaded files and efficiently serving
574 | them!
575 |
576 | ## About the author
577 |
578 | - Nick Janetakis | | [@nickjanetakis](https://twitter.com/nickjanetakis)
579 |
580 | If you're interested in learning Flask I have a 17+ hour video course called
581 | [Build a SAAS App with
582 | Flask](https://buildasaasappwithflask.com/?utm_source=github&utm_medium=staticdigest&utm_campaign=readme).
583 | It's a course where we build a real world SAAS app. Everything about the course
584 | and demo videos of what we build is on the site linked above.
585 |
--------------------------------------------------------------------------------
 }})
284 | ```
285 |
286 | When you put the above code into a Flask powered Jinja 2 template, it turns
287 | into this:
288 |
289 | ```html
290 | 
292 | ```
293 |
294 | The path might vary depending on how you configured your Flask app's
295 | `static_folder` but you get the idea.
296 |
297 | #### Using static_url_for instead of url_for
298 |
299 | Let's use the same example as above:
300 |
301 | ```html
302 |  }})
304 | ```
305 |
306 | But now take a look at the output this produces:
307 |
308 | ```html
309 | 
311 | ```
312 |
313 | Or if you set `FLASK_STATIC_DIGEST_HOST_URL = "https://cdn.example.com"` it
314 | would produce:
315 |
316 | ```html
317 | 
319 | ```
320 |
321 | Instead of using `url_for` you would use `static_url_for`. This uses Flask's
322 | `url_for` under the hood so things like `_external=True` and everything else
323 | `url_for` supports is available to use with `static_url_for`.
324 |
325 | That means to use this extension you don't have to do anything other than
326 | install it, optionally run the CLI command to generate the manifest and then
327 | rename your static file references to use `static_url_for` instead of
328 | `url_for`.
329 |
330 | If your editor supports performing a find / replace across multiple files you
331 | can quickly make the change by finding `url_for('static'` and replacing that
332 | with `static_url_for('static'`. If you happen to use double quotes instead of
333 | single quotes you'll want to adjust for that too.
334 |
335 | ## Potentially updating your .gitignore file
336 |
337 | If you're using something like Webpack then chances are you're already git
338 | ignoring the static files it produces as output. It's a common pattern to
339 | commit your Webpack source static files but ignore the compiled static files
340 | it produces.
341 |
342 | But if you're not using Webpack or another asset build tool then the static
343 | files that are a part of your project might have the same source and
344 | destination directory. If that's the case, chances are you'll want to git
345 | ignore the md5 tagged files as well as the compressed and `cache_manifest.json`
346 | files from version control.
347 |
348 | For clarity, you want to ignore them because you'll be generating them on your
349 | server at deploy time or within a Docker image if you're using Docker. They
350 | don't need to be tracked in version control.
351 |
352 | Add this to your `.gitignore` file to ignore certain files this extension
353 | creates:
354 |
355 | ```
356 | *-[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f].*
357 | *.gz
358 | cache_manifest.json
359 | ```
360 |
361 | This allows your original static files but ignores everything else this
362 | extension creates. I am aware at how ridiculous that ignore rule is for the md5
363 | hash but using `[0-9a-f]{32}` does not work. If you know of a better way,
364 | please open a PR!
365 |
366 | ## FAQ
367 |
368 | ### What about development vs production and performance implications?
369 |
370 | You would typically only run the CLI command to prepare your static files for
371 | production. Running `flask digest compile` would become a part of your build
372 | process -- typically after you pip install your dependencies.
373 |
374 | In development when the `cache_manifest.json` likely doesn't exist
375 | `static_url_for` calls `url_for` directly. This allows the `static_url_for`
376 | helper to work in both development and production without any fuss.
377 |
378 | It's also worth pointing out the CLI command is expected to be run before you
379 | even start your Flask server (or gunicorn / etc.), so there's no perceivable
380 | run time performance hit. It only involves doing 1 extra dictionary lookup at
381 | run time which is many orders of magnitude faster than even the most simple
382 | database query.
383 |
384 | **In other words, this extension is not going to negatively impact the
385 | performance of your web application. If anything it's going to speed it up and
386 | save you money on hosting**.
387 |
388 | That's because compressed files can be upwards of 5-10x smaller so there's less
389 | bytes to transfer over the network.
390 |
391 | Also with md5 tagging each file it means you can configure your web server such
392 | as nginx to cache each file forever. That means if a user visits your site a
393 | second time in the future, nginx will be smart enough to load it from their
394 | local browser's cache without even contacting your server. It's a 100% local
395 | look up.
396 |
397 | This is as efficient as it gets. You can't do this normally without md5 tagging
398 | each file because if the file changes in the future, nginx will continue
399 | serving the old file until the cache expires so users will never see your
400 | updates. But due to how md5 hashing works, if the contents of a file changes it
401 | will get generated with a new name and nginx will serve the uncached new file.
402 |
403 | This tactic is commonly referred to as "cache busting" and it's a very good
404 | idea to do this in production. You can even go 1 step further and serve your
405 | static files using a CDN. Using this cache busting strategy makes configuring
406 | your CDN a piece of cake since you don't need to worry about ever expiring your
407 | cache manually.
408 |
409 | ### Why bother compressing your static files here instead of with nginx?
410 |
411 | You would still be using nginx's gzip / brotli features, but now instead of
412 | nginx having to compress your files on the fly at run time you can configure
413 | nginx to use the pre-made compressed files that this extension creates.
414 |
415 | This way you can benefit from having maximum compression without having nginx
416 | waste precious CPU cycles compressing files on the fly. This gives you the best
417 | of both worlds -- the highest compression ratio with no noticeable run time
418 | performance penalty.
419 |
420 | ### How do you use this extension with Webpack or another build tool?
421 |
422 | It works out of the box with no extra configuration or plugins needed for
423 | Webpack or your build tool of choice.
424 |
425 | Typically the Webpack (or another build tool) work flow would look like this:
426 |
427 | - You configure Webpack with your source static files directory
428 | - You configure Webpack with your destination static files directory
429 | - Webpack processes your files in the source directory and copies them to the
430 | destination directory
431 | - Flask is configured to serve static files from that destination directory
432 |
433 | For example, your source directory might be `assets/` inside of your project
434 | and the destination might be `myapp/static`.
435 |
436 | This extension will look at your Flask configuration for the `static_folder`
437 | and determine it's set to `myapp/static` so it will md5 tag and compress those
438 | files. Your Webpack source files will not get digested and compressed.
439 |
440 | ### Migrating from Flask-Webpack
441 |
442 | [Flask-Webpack](https://github.com/nickjj/flask-webpack) is another extension I
443 | wrote a long time ago which was specific to Webpack but had a similar idea to
444 | this extension. Flask-Webpack is now deprecated in favor of
445 | Flask-Static-Digest. Migrating is fairly painless. There are a number of
446 | changes but on the bright side you get to delete more code than you add!
447 |
448 | #### Dependency / Flask app changes
449 |
450 | - Remove `Flask-Webpack` from `requirements.txt`
451 | - Remove all references to Flask-Webpack from your Flask app and config
452 | - Remove `manifest-revision-webpack-plugin` from `package.json`
453 | - Remove all references to this webpack plugin from your webpack config
454 | - Add `Flask-Static-Digest` to `requirements.txt`
455 | - Add the Flask-Static-Digest extension to your Flask app
456 |
457 | #### Jinja 2 template changes
458 |
459 | - Replace `stylesheet_tag('main_css') | safe` with `static_url_for('static', filename='css/main.css')`
460 | - Replace `javascript_tag('main_js') | safe` with `static_url_for('static', filename='js/main.js')`
461 | - Replace any occurrences of `asset_url_for('foo.png')` with `static_url_for('static', filename='images/foo.png')`
462 |
463 | ### How do you use this extension with Docker?
464 |
465 | It's really no different than without Docker, but instead of running `flask
466 | digest compile` on your server directly at deploy time you would run it inside
467 | of your Docker image at build time. This way your static files are already set
468 | up and ready to go by the time you pull and use your Docker image in
469 | production.
470 |
471 | You can see a fully working example of this in the open source version of my
472 | [Build a SAAS App with
473 | Flask](https://github.com/nickjj/build-a-saas-app-with-flask) course. It
474 | leverages Docker's build arguments to only compile the static files when
475 | `FLASK_ENV` is set to `production`. The key files to look at are the
476 | `Dockerfile`, `docker-compose.yml` and `.env` files. That wires up the build
477 | arguments and env variables to make it work.
478 |
479 | ### How do you use this extension with Heroku?
480 |
481 | If you're deploying to Heroku using the Python buildpack you can follow these 2 steps:
482 |
483 | 1. Create a `bin/post_compile` file in your project's source code
484 | 2. Copy the lines below into the `bin/post_compile` file, save it and commit the changes
485 |
486 | ```sh
487 | #!/usr/bin/env bash
488 |
489 | set -e
490 |
491 | echo "-----> Digesting static files"
492 | cd "${1}" && flask digest compile
493 | ```
494 |
495 | The next time you push your code this script will run after your pip
496 | dependencies are installed. It will run before your slug is compiled which
497 | ensures that the digested files are available before any traffic is served to
498 | your Dyno.
499 |
500 | You can view how this file gets executed by Heroku in their [Python buildpack's
501 | source
502 | code](https://github.com/heroku/heroku-buildpack-python/blob/main/bin/steps/hooks/post_compile).
503 |
504 | ### What about user uploaded files?
505 |
506 | Let's say that besides having static files like your logo and CSS / JavaScript
507 | bundles you also have files uploaded by users. This could be things like a user
508 | avatar, blog post images or anything else.
509 |
510 | **You would still want to md5 tag and compress these files but now we've run
511 | into a situation**. The `flask digest compile` command is meant to be run at
512 | deploy time and it could potentially be run from your dev box, inside of a
513 | Docker image, on a CI server or your production server. In these cases you
514 | wouldn't have access to the user uploaded files.
515 |
516 | But at the same time you have users uploading files at run time. They are
517 | changing all the time.
518 |
519 | **Needless to say you can't use the `flask digest compile` command to digest
520 | user uploaded files**. The `cache_manifest.json` file should be reserved for
521 | files that exist in your code repo (such as your CSS / JS bundles, maybe a
522 | logo, fonts, etc.).
523 |
524 | The above files do not change at run time and align well with running the
525 | `flask digest compile` command at deploy time.
526 |
527 | For user uploaded content you wouldn't ever write these entries to the manifest
528 | JSON file. Instead, you would typically upload your files to disk, S3 or
529 | somewhere else and then save the file name of the file you uploaded into your
530 | local database.
531 |
532 | So now when you reference a user uploaded file (let's say an avatar), you would
533 | loop over your users from the database and reference the file name from the DB.
534 |
535 | There's no need for a manifest file to store the user uploaded files because
536 | the database has a reference to the real name and then you are dynamically
537 | referencing that in your template helper (`static_url_for`), so it's never a
538 | hard coded thing that changes at the template level.
539 |
540 | What's cool about this is you already did the database query to retrieve the
541 | record(s) from the database, so there's no extra database work to do. All you
542 | have to do is reference the file name field that's a part of your model.
543 |
544 | But that doesn't fully solve the problem. You'll still want to md5 tag and
545 | compress your user uploaded content at run time and you would want to do this
546 | before you save the uploaded file into its final destination (local file system,
547 | S3, etc.).
548 |
549 | This can be done completely separate from this extension and it's really going
550 | to vary depending on where you host your user uploaded content. For example some
551 | CDNs will automatically create compressed files for you and they use things like
552 | an ETag header in the response to include a unique file name (and this is what
553 | you can store in your DB).
554 |
555 | So maybe md5 hashing and maybe compressing your user uploaded content becomes an
556 | app specific responsibility, although I'm not opposed to maybe creating helper
557 | functions you can use but that would need to be thought out carefully.
558 |
559 | However the implementation is not bad. It's really only about 5 lines of code
560 | to do both things. Feel free to `CTRL + F` around the [code
561 | base](https://github.com/nickjj/flask-static-digest/blob/master/flask_static_digest/digester.py)
562 | for `hashlib`, `gzip` and `brotli` and you'll find the related code.
563 |
564 | **So with that said, here's a work flow you can do to deal with this today:**
565 |
566 | - User uploads file
567 | - Your Flask app potentially md5 tags / gzips the file if necessary
568 | - Your Flask app saves the file name + compressed file to its final destination (local file system, S3, etc.)
569 | - Your Flask app saves the final unique file name to your database
570 |
571 | That final unique file name would be the md5 tagged version of the file that
572 | you created or the unique file name that your CDN returned back to you. I hope
573 | that clears up how to deal with user uploaded files and efficiently serving
574 | them!
575 |
576 | ## About the author
577 |
578 | - Nick Janetakis |