93 | {{content}}
94 |
95 | ├── .gitignore ├── .editorconfig ├── vimhelp ├── static │ ├── favicon-vim.ico │ ├── favicon-neovim.ico │ ├── noscript.css │ ├── theme-native-dark.svg │ ├── theme-native-light.svg │ ├── theme-dark-dark.svg │ ├── theme-dark-light.svg │ ├── theme-light-dark.svg │ ├── theme-light-light.svg │ ├── tom-select.min.css │ └── tom-select.base.min.js ├── templates │ ├── prelude.html │ ├── page.html │ ├── vimhelp.js │ └── vimhelp.css ├── secret.py ├── robots.py ├── http.py ├── cache.py ├── tagsearch.py ├── vimhelp.py ├── dbmodel.py ├── webapp.py ├── assets.py ├── vimh2h.py └── update.py ├── requirements.txt ├── .gcloudignore ├── gunicorn.conf.dev.py ├── cron.yaml ├── app.yaml ├── pyproject.toml ├── LICENSE ├── README.md ├── tasks.py └── scripts └── h2h.py /.gitignore: -------------------------------------------------------------------------------- 1 | TODO 2 | /.venv/ 3 | __pycache__/ 4 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*.py] 4 | max_line_length = 88 5 | -------------------------------------------------------------------------------- /vimhelp/static/favicon-vim.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/c4rlo/vimhelp/HEAD/vimhelp/static/favicon-vim.ico -------------------------------------------------------------------------------- /vimhelp/static/favicon-neovim.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/c4rlo/vimhelp/HEAD/vimhelp/static/favicon-neovim.ico -------------------------------------------------------------------------------- /vimhelp/static/noscript.css: -------------------------------------------------------------------------------- 1 | .need-js { display: none !important; } 2 | .site.srch ::placeholder { 3 | opacity: 1 !important; 4 | } 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Flask ~= 3.1 2 | gevent ~= 25.8 3 | geventhttpclient ~= 2.3 4 | google-cloud-ndb ~= 2.3 5 | google-cloud-secret-manager ~= 2.24 6 | google-cloud-tasks ~= 2.19 7 | gunicorn ~= 23.0 8 | -------------------------------------------------------------------------------- /.gcloudignore: -------------------------------------------------------------------------------- 1 | /.editorconfig 2 | /.git 3 | /.gitignore 4 | /.gcloudignore 5 | /.venv 6 | /.ruff_cache 7 | /scripts 8 | /README.md 9 | /LICENSE 10 | /TODO 11 | /tasks.py 12 | /gunicorn.conf.dev.py 13 | __pycache__/ 14 | html/ 15 | -------------------------------------------------------------------------------- /gunicorn.conf.dev.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | 3 | 4 | loglevel = "debug" 5 | reload = True 6 | timeout = 15 7 | worker_class = "gevent" 8 | wsgi_app = "vimhelp.webapp:create_app()" 9 | 10 | 11 | def worker_abort(worker): 12 | traceback.print_stack() 13 | -------------------------------------------------------------------------------- /vimhelp/static/theme-native-dark.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /vimhelp/static/theme-native-light.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /cron.yaml: -------------------------------------------------------------------------------- 1 | cron: 2 | - description: fetch and process vim docs 3 | url: /enqueue_update?project=vim 4 | schedule: every 31 minutes 5 | - description: fetch and process neovim docs 6 | url: /enqueue_update?project=neovim 7 | schedule: every 32 minutes 8 | - description: clean up old static asset versions 9 | url: /enqueue_clean_assets 10 | schedule: every 33 minutes 11 | -------------------------------------------------------------------------------- /app.yaml: -------------------------------------------------------------------------------- 1 | runtime: python313 2 | 3 | automatic_scaling: 4 | max_instances: 2 5 | max_idle_instances: 1 6 | target_cpu_utilization: 0.9 7 | target_throughput_utilization: 0.9 8 | max_concurrent_requests: 50 9 | min_pending_latency: 500ms 10 | 11 | entrypoint: gunicorn -b :$PORT -k gevent -w 1 'vimhelp.webapp:create_app()' 12 | 13 | inbound_services: 14 | - warmup 15 | 16 | handlers: 17 | - url: /.* 18 | script: auto 19 | secure: always 20 | -------------------------------------------------------------------------------- /vimhelp/static/theme-dark-dark.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /vimhelp/static/theme-dark-light.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /vimhelp/templates/prelude.html: -------------------------------------------------------------------------------- 1 | {# This "prelude" is the initial portion of each page. It gets rendered 2 | dynamically in response to each request, based on user preferences (currently 3 | only the color theme). #} 4 | 5 | {% if theme %} 6 | 7 | {% else %} 8 | 9 | {% endif %} 10 |
11 | 12 | {% if theme %} 13 | 14 | {% else %} 15 | 16 | {% endif %} 17 | {# rest of the page is in page.html #} 18 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "vimhelp" 3 | version = "0.1" 4 | dynamic = ["dependencies"] 5 | 6 | [build-system] 7 | requires = ["setuptools", "wheel"] 8 | build-backend = "setuptools.build_meta" 9 | 10 | [tool.setuptools.dynamic] 11 | # Note that Google App Engine requires a requirements.txt file, 12 | # otherwise we'd just specify them directly in here. 13 | dependencies = { file = "requirements.txt" } 14 | 15 | [tool.ruff.lint] 16 | select = ["E", "F", "W", "I002", "N", "UP", "S", "B", "A", "C4", "DTZ", "SIM", "PTH", "PLE", "PLW", "RUF"] 17 | ignore = ["DTZ003", "PLW0603", "SIM102", "SIM108", "UP007"] 18 | 19 | [tool.ruff.lint.per-file-ignores] 20 | "vimhelp/vimh2h.py" = ["E501", "PLW2901", "S704"] 21 | 22 | [tool.ruff.lint.flake8-builtins] 23 | builtins-allowed-modules = ["http"] 24 | -------------------------------------------------------------------------------- /vimhelp/secret.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import threading 4 | 5 | import google.cloud.secretmanager 6 | 7 | 8 | g_secrets = None 9 | g_lock = threading.Lock() 10 | 11 | 12 | def github_token(): 13 | return _get_secrets()["github_token"] 14 | 15 | 16 | def admin_password(): 17 | return _get_secrets()["admin_password"] 18 | 19 | 20 | def _get_secrets(): 21 | with g_lock: 22 | global g_secrets 23 | if g_secrets is None: 24 | client = google.cloud.secretmanager.SecretManagerServiceClient() 25 | cloud_project = os.environ["GOOGLE_CLOUD_PROJECT"] 26 | secret_name = f"projects/{cloud_project}/secrets/secrets/versions/latest" 27 | resp = client.access_secret_version(name=secret_name) 28 | g_secrets = json.loads(resp.payload.data) 29 | return g_secrets 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Carlo Teubner 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # vimhelp.org 2 | 3 | This is the code behind the https://vimhelp.org website. It runs on 4 | [Google App Engine](https://cloud.google.com/appengine/). 5 | 6 | To make testing and deploying easier, a `tasks.py` file exists for use 7 | with the [_Invoke_](https://www.pyinvoke.org/) tool (which is similar in 8 | spirit to _Make_). 9 | 10 | ## Generating static pages 11 | 12 | To generate static HTML pages instead of running on Google App Engine: 13 | 14 | - Create a virtualenv. If you have _Invoke_ installed, this is as easy as 15 | `inv venv`. Alternatively: 16 | ``` 17 | python3 -m venv --upgrade-deps .venv 18 | .venv/bin/pip install -r requirements.txt 19 | ``` 20 | - Run the following (replace the `-i` parameter with the Vim documentation 21 | location on your computer): 22 | ``` 23 | scripts/h2h.py -i /usr/share/vim/vim90/doc/ -o html/ 24 | ``` 25 | The script offers a few options; run with `-h` to see what is available. 26 | 27 | ## License 28 | 29 | This code is made freely available under the MIT License (see file LICENSE). 30 | -------------------------------------------------------------------------------- /vimhelp/robots.py: -------------------------------------------------------------------------------- 1 | # Generate 'robots.txt' and 'sitemap.txt' on-the-fly. 2 | 3 | import itertools 4 | 5 | import flask 6 | 7 | from . import dbmodel 8 | 9 | BASE_URLS = { 10 | "vim": "https://vimhelp.org/", 11 | "neovim": "https://neo.vimhelp.org/", 12 | } 13 | 14 | 15 | def handle_robots_txt(): 16 | return flask.Response( 17 | f"Sitemap: {BASE_URLS[flask.g.project]}/sitemap.txt\n", mimetype="text/plain" 18 | ) 19 | 20 | 21 | def handle_sitemap_txt(): 22 | project = flask.g.project 23 | base_url = BASE_URLS[project] 24 | 25 | with dbmodel.ndb_context(): 26 | query = dbmodel.ProcessedFileHead.query( 27 | dbmodel.ProcessedFileHead.project == project 28 | ) 29 | names = set(query.map(lambda key: key.id().split(":")[-1], keys_only=True)) 30 | names.discard("help.txt") 31 | 32 | return flask.Response( 33 | "".join( 34 | itertools.chain( 35 | (f"{base_url}\n",), 36 | (f"{base_url}{name}.html\n" for name in sorted(names)), 37 | ) 38 | ), 39 | mimetype="text/plain", 40 | ) 41 | -------------------------------------------------------------------------------- /vimhelp/static/theme-light-dark.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /vimhelp/static/theme-light-light.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /vimhelp/http.py: -------------------------------------------------------------------------------- 1 | import json as json_module 2 | import logging 3 | 4 | import geventhttpclient 5 | import geventhttpclient.client 6 | import gevent.ssl 7 | 8 | 9 | class HttpClient: 10 | def __init__(self, concurrency): 11 | self._pool = geventhttpclient.client.HTTPClientPool( 12 | ssl_context_factory=gevent.ssl.create_default_context, 13 | concurrency=concurrency, 14 | ) 15 | 16 | def get(self, url, headers): 17 | try: 18 | url = geventhttpclient.URL(url) 19 | client = self._pool.get_client(url) 20 | response = client.get(url.request_uri, headers=headers) 21 | except Exception as e: 22 | logging.error(e) 23 | raise HttpError(url) from e 24 | return HttpResponse(response, url) 25 | 26 | def post(self, url, json, headers): 27 | try: 28 | url = geventhttpclient.URL(url) 29 | client = self._pool.get_client(url) 30 | response = client.post( 31 | url.request_uri, body=json_module.dumps(json), headers=headers 32 | ) 33 | except Exception as e: 34 | logging.error(e) 35 | raise HttpError(url) from e 36 | return HttpResponse(response, url) 37 | 38 | def close(self): 39 | self._pool.close() 40 | 41 | 42 | class HttpResponse: 43 | def __init__(self, response, url): 44 | self.url = url 45 | self.body = bytes(response.read()) 46 | response.release() 47 | self._response = response 48 | 49 | @property 50 | def status_code(self): 51 | return self._response.status_code 52 | 53 | def header(self, name): 54 | return self._response.get(name) 55 | 56 | 57 | class HttpError(RuntimeError): 58 | def __init__(self, url): 59 | self._url = url 60 | 61 | def __str__(self): 62 | return f"Failed HTTP request for {self._url}" 63 | -------------------------------------------------------------------------------- /vimhelp/cache.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import threading 3 | 4 | import gevent 5 | 6 | from .dbmodel import GlobalInfo, ndb_context 7 | 8 | 9 | _REFRESH_INTERVAL_SEC = 120 10 | 11 | 12 | class Cache: 13 | def __init__(self): 14 | self._cache = {} 15 | self._lock = threading.Lock() 16 | 17 | def get(self, project, key): 18 | with self._lock: 19 | return self._cache.get(project, {}).get(key) 20 | 21 | def put(self, project, key, value): 22 | with self._lock: 23 | logging.info("writing %s:%s to inproc cache", project, key) 24 | self._cache.setdefault(project, {})[key] = value 25 | 26 | def clear(self, project): 27 | with self._lock: 28 | if c := self._cache.get(project): 29 | c.clear() 30 | 31 | def start_refresh_loop(self, refresh_callback): 32 | update_times = Cache._get_update_times() 33 | gevent.spawn_later( 34 | _REFRESH_INTERVAL_SEC, self._refresh, update_times, refresh_callback 35 | ) 36 | 37 | def _refresh(self, old_update_times, refresh_callback): 38 | update_times = Cache._get_update_times() 39 | for project, update_time in update_times.items(): 40 | old_update_time = old_update_times.get(project) 41 | if old_update_time is None or update_time > old_update_time: 42 | logging.info( 43 | "project %s was updated (%s < %s), refreshing cache", 44 | project, 45 | old_update_time, 46 | update_time, 47 | ) 48 | self.clear(project) 49 | refresh_callback(project) 50 | else: 51 | logging.info( 52 | "project %s was not updated, not refreshing cache", project 53 | ) 54 | gevent.spawn_later( 55 | _REFRESH_INTERVAL_SEC, self._refresh, update_times, refresh_callback 56 | ) 57 | 58 | @staticmethod 59 | def _get_update_times(): 60 | with ndb_context(): 61 | return {g.key.id(): g.last_update_time for g in GlobalInfo.query()} 62 | -------------------------------------------------------------------------------- /vimhelp/tagsearch.py: -------------------------------------------------------------------------------- 1 | import bisect 2 | 3 | import flask 4 | import werkzeug.exceptions 5 | 6 | from . import dbmodel 7 | 8 | 9 | # There are about 10k tags. To optimize performance, consider: 10 | # - Dealing with 'bytes' not 'str' 11 | # - Giving this module the Cython treatment 12 | 13 | 14 | MAX_RESULTS = 30 15 | CACHE_KEY_ID = "api/tag-items" 16 | 17 | 18 | class TagItem: 19 | def __init__(self, tag, href): 20 | self.tag = tag 21 | self.tag_lower = tag.casefold() 22 | self.href = href 23 | 24 | def __lt__(self, query): 25 | # This is enough for bisect_left to work... 26 | return self.tag < query 27 | 28 | 29 | def handle_tagsearch(cache): 30 | project = flask.g.project 31 | query = flask.request.args.get("q", "") 32 | items = cache.get(project, CACHE_KEY_ID) 33 | if not items: 34 | with dbmodel.ndb_context(): 35 | entity = dbmodel.TagsInfo.get_by_id(project) 36 | if entity is None: 37 | raise werkzeug.exceptions.NotFound() 38 | items = [TagItem(*tag) for tag in entity.tags] 39 | cache.put(project, CACHE_KEY_ID, items) 40 | 41 | results = do_handle_tagsearch(items, query) 42 | return flask.jsonify({"results": results}) 43 | 44 | 45 | def do_handle_tagsearch(items, query): 46 | results = [] 47 | result_set = set() 48 | 49 | is_lower = query == query.casefold() 50 | 51 | def add_result(item): 52 | if item.tag in result_set: 53 | return False 54 | results.append({"id": item.tag, "text": item.tag, "href": item.href}) 55 | result_set.add(item.tag) 56 | return len(results) == MAX_RESULTS 57 | 58 | # Find all tags beginning with query. 59 | i = bisect.bisect_left(items, query) 60 | for item in items[i:]: 61 | if item.tag.startswith(query): 62 | if add_result(item): 63 | return results 64 | else: 65 | break 66 | 67 | # If we didn't find enough, and the query is all-lowercase, add all case-insensitive 68 | # matches. 69 | if is_lower: 70 | for item in items: 71 | if item.tag_lower.startswith(query): 72 | if add_result(item): 73 | return results 74 | 75 | # If we still didn't find enough, additionally find all tags that contain query as a 76 | # substring. 77 | for item in items: 78 | if query in item.tag: 79 | if add_result(item): 80 | return results 81 | 82 | # If we still didn't find enough, and the query is all-lowercase, additionally find 83 | # all tags that contain query as a substring case-insensitively. 84 | if is_lower: 85 | for item in items: 86 | if query in item.tag_lower: 87 | if add_result(item): 88 | return results 89 | 90 | return results 91 | -------------------------------------------------------------------------------- /vimhelp/vimhelp.py: -------------------------------------------------------------------------------- 1 | # Retrieve a help page from the data store, and present to the user 2 | 3 | import logging 4 | from http import HTTPStatus 5 | 6 | import flask 7 | import werkzeug.exceptions 8 | 9 | from google.cloud import ndb 10 | 11 | from . import dbmodel 12 | from . import vimh2h 13 | 14 | 15 | def handle_vimhelp(filename, cache): 16 | req = flask.request 17 | project = flask.g.project 18 | 19 | if filename in ("help.txt", "help"): 20 | return redirect("./") 21 | 22 | if filename == "": 23 | filename = "help.txt" 24 | 25 | if not filename.endswith(".txt") and filename != "tags": 26 | return redirect(f"{filename}.txt.html") 27 | 28 | theme = req.cookies.get("theme") 29 | if theme not in ("light", "dark"): 30 | theme = None 31 | 32 | if entry := cache.get(project, filename): 33 | logging.info("serving '%s:%s' from inproc cache", project, filename) 34 | head, parts = entry 35 | resp = prepare_response(req, head, theme) 36 | return complete_response(resp, head, parts, theme) 37 | 38 | with dbmodel.ndb_context(): 39 | logging.info("serving '%s:%s' from datastore", project, filename) 40 | head = dbmodel.ProcessedFileHead.get_by_id(f"{project}:{filename}") 41 | if head is None: 42 | logging.warning("%s:%s not found in datastore", project, filename) 43 | raise werkzeug.exceptions.NotFound() 44 | resp = prepare_response(req, head, theme) 45 | parts = [] 46 | if resp.status_code != HTTPStatus.NOT_MODIFIED: 47 | parts = get_parts(head) 48 | complete_response(resp, head, parts, theme) 49 | if head.numparts == 1 or parts: 50 | cache.put(project, filename, (head, parts)) 51 | return resp 52 | 53 | 54 | def prepare_response(req, head, theme): 55 | resp = flask.Response(mimetype="text/html") 56 | resp.last_modified = head.modified 57 | resp.cache_control.max_age = 15 * 60 58 | resp.vary.add("Cookie") 59 | resp.set_etag(head.etag.decode() + (theme or "")) 60 | return resp.make_conditional(req) 61 | 62 | 63 | def complete_response(resp, head, parts, theme): 64 | if resp.status_code != HTTPStatus.NOT_MODIFIED: 65 | logging.info( 66 | "writing %d-part response, modified %s", 67 | 1 + len(parts), 68 | resp.last_modified, 69 | ) 70 | prelude = vimh2h.VimH2H.prelude(theme=theme).encode() 71 | resp.data = b"".join((prelude, head.data0, *(p.data for p in parts))) 72 | return resp 73 | 74 | 75 | def redirect(url): 76 | logging.info("redirecting %s to %s", flask.request.path, url) 77 | return flask.redirect(url, HTTPStatus.MOVED_PERMANENTLY) 78 | 79 | 80 | def get_parts(head): 81 | # We could alternatively achieve this via an ancestor query (retrieving the head and 82 | # its parts simultaneously) to give us strong consistency. 83 | if head.numparts == 1: 84 | return [] 85 | logging.info("retrieving %d extra part(s)", head.numparts - 1) 86 | head_id = head.key.id() 87 | keys = [ 88 | ndb.Key("ProcessedFilePart", f"{head_id}:{i}") for i in range(1, head.numparts) 89 | ] 90 | num_tries = 0 91 | while True: 92 | parts = ndb.get_multi(keys) 93 | if all(p.etag == head.etag for p in parts): 94 | return sorted(parts, key=lambda p: p.key.string_id()) 95 | num_tries += 1 96 | if num_tries >= 10: 97 | logging.error("tried too many times, giving up") 98 | raise werkzeug.exceptions.InternalServerError() 99 | logging.warning("got differing etags, retrying") 100 | -------------------------------------------------------------------------------- /vimhelp/templates/page.html: -------------------------------------------------------------------------------- 1 | {# This is the main content of each page; it gets rendered ahead of time. The 2 | first few lines of HTML that are missing from here are are in prelude.html; 3 | those are the only ones that get rendered dynamically with each request. #} 4 | 5 | 6 |This is an HTML version of the {{project.name}} help pages{% if version %}, current as of {{project.name}} {{version}}{% endif %}. 38 | They are kept up-to-date automatically 39 | from the {{project.name}} source repository. 40 | {% if project.name == "Vim" %} 41 | Also included is the Vim FAQ, kept up to date from its 42 | GitHub repository. 43 | {% endif %} 44 |
45 | 46 |Help pages for {{project.other.contrasted_name}} 47 | are also available.
48 | {% endif %} 49 | 50 | {% set sitenavi %} 51 | Quick links: 52 | help overview · 53 | quick reference · 54 | user manual toc · 55 | reference manual toc 56 | {% if project.name == "Vim" %} 57 | · faq 58 | {% endif %} 59 | {% endset %} 60 | 61 | {% if mode != "offline" %} 62 | 77 | {% else %} 78 |{{sitenavi}}
79 | {% endif %} 80 | 81 |
93 | {{content}}
94 |
95 | {{sitenavi}}
98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- 1 | # Use with https://www.pyinvoke.org/ 2 | 3 | from invoke import call, task 4 | 5 | import os 6 | import pathlib 7 | import sys 8 | 9 | 10 | os.chdir(pathlib.Path(__file__).parent) 11 | 12 | 13 | VENV_DIR = pathlib.Path(".venv") 14 | REQ_TXT = pathlib.Path("requirements.txt") 15 | 16 | PRIV_DIR = pathlib.Path("~/private").expanduser() 17 | STAGING_CREDENTIALS = PRIV_DIR / "gcloud-creds/vimhelp-staging-owner.json" 18 | 19 | PROJECT_STAGING = "vimhelp-staging" 20 | PROJECT_PROD = "vimhelp-hrd" 21 | 22 | DEV_ENV = { 23 | "PYTHONDEVMODE": "1", 24 | "PYTHONWARNINGS": ( 25 | "default," 26 | "ignore:unclosed:ResourceWarning:sys," 27 | "ignore:Type google._upb._message:DeprecationWarning:importlib._bootstrap," 28 | "ignore:This process (pid=:DeprecationWarning:gevent.os" 29 | ), 30 | "VIMHELP_ENV": "dev", 31 | "FLASK_DEBUG": "1", 32 | "GOOGLE_CLOUD_PROJECT": PROJECT_STAGING, 33 | "GOOGLE_APPLICATION_CREDENTIALS": str(STAGING_CREDENTIALS), 34 | } 35 | 36 | 37 | @task(help={"lazy": "Only update venv if out-of-date wrt requirements.txt"}) 38 | def venv(c, lazy=False): 39 | """Populate virtualenv.""" 40 | if not VENV_DIR.exists(): 41 | c.run(f"python -m venv --upgrade-deps {VENV_DIR}") 42 | c.run(f"{VENV_DIR}/bin/pip install -U wheel") 43 | print("Created venv.") 44 | lazy = False 45 | if not lazy or REQ_TXT.stat().st_mtime > VENV_DIR.stat().st_mtime: 46 | c.run(f"{VENV_DIR}/bin/pip install -U --upgrade-strategy eager -r {REQ_TXT}") 47 | c.run(f"touch {VENV_DIR}") 48 | print("Updated venv.") 49 | else: 50 | print("venv was already up-to-date.") 51 | 52 | 53 | venv_lazy = call(venv, lazy=True) 54 | 55 | 56 | @task 57 | def lint(c): 58 | """Run linter/formatter (ruff).""" 59 | c.run("ruff check .") 60 | c.run("ruff format --check") 61 | 62 | 63 | @task( 64 | pre=[venv_lazy], 65 | help={ 66 | "gunicorn": "Run using gunicorn instead of 'flask run'", 67 | "tracemalloc": "Run with tracemalloc enabled", 68 | }, 69 | ) 70 | def run(c, gunicorn=False, tracemalloc=False): 71 | """Run app locally against staging database.""" 72 | _ensure_private_mount(c) 73 | if gunicorn: 74 | cmd = f"{VENV_DIR}/bin/gunicorn -c gunicorn.conf.dev.py" 75 | else: 76 | cmd = f"{VENV_DIR}/bin/flask --app vimhelp.webapp --debug run" 77 | if tracemalloc: 78 | env = DEV_ENV | {"PYTHONTRACEMALLOC": "1"} 79 | else: 80 | env = DEV_ENV 81 | c.run(cmd, env=env) 82 | 83 | 84 | @task(pre=[venv_lazy]) 85 | def show_routes(c): 86 | """Show Flask routes.""" 87 | _ensure_private_mount(c) 88 | c.run(f"{VENV_DIR}/bin/flask --app vimhelp.webapp --debug routes", env=DEV_ENV) 89 | 90 | 91 | @task( 92 | pre=[lint], 93 | help={ 94 | "target": "Target environment: 'staging' (default), 'prod', " 95 | "'all' (= staging + prod)", 96 | "cron": "Deploy cron.yaml instead of main app" 97 | }, 98 | ) # fmt: skip 99 | def deploy(c, target="staging", cron=False): 100 | """Deploy app.""" 101 | _ensure_private_mount(c) 102 | if target == "all": 103 | targets = "staging", "prod" 104 | else: 105 | targets = (target,) 106 | for t in targets: 107 | if t == "staging": 108 | cmd = f"gcloud app deploy --quiet --project={PROJECT_STAGING}" 109 | elif t == "prod": 110 | cmd = f"gcloud app deploy --project={PROJECT_PROD}" 111 | else: 112 | sys.exit(f"Invalid target name: '{t}'") 113 | if cron: 114 | cmd += " cron.yaml" 115 | c.run(cmd, pty=True) 116 | 117 | 118 | @task 119 | def clean(c): 120 | """Clean up build artefacts.""" 121 | for d in VENV_DIR, "__pycache__", "vimhelp/__pycache__", ".ruff_cache": 122 | if pathlib.Path(d).exists(): 123 | c.run(f"rm -rf {d}") 124 | 125 | 126 | @task() 127 | def sh(c): 128 | """Interactive shell with virtualenv and datastore available.""" 129 | _ensure_private_mount(c) 130 | with c.prefix(f". {VENV_DIR}/bin/activate"): 131 | c.run(os.getenv("SHELL", "bash"), env=DEV_ENV, pty=True) 132 | print("Exited vimhelp shell") 133 | 134 | 135 | def _ensure_private_mount(c): 136 | if PRIV_DIR.stat().st_dev == PRIV_DIR.parent.stat().st_dev: 137 | c.run(f"sudo systemctl start {PRIV_DIR}", pty=True) 138 | -------------------------------------------------------------------------------- /vimhelp/templates/vimhelp.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | 4 | // Hide sidebar when it wraps 5 | 6 | const onResize = (e) => { 7 | const sidebar = document.getElementById("vh-sidebar"); 8 | const sidebarTop = sidebar.getBoundingClientRect().top; 9 | const contentBottom = document.getElementById("vh-content").getBoundingClientRect().bottom; 10 | if (sidebarTop >= contentBottom - 4) { 11 | sidebar.style.visibility = "hidden"; 12 | sidebar.style.height = "0px"; 13 | } 14 | else { 15 | sidebar.style.visibility = null; 16 | sidebar.style.height = null; 17 | } 18 | }; 19 | addEventListener("resize", onResize); 20 | onResize(); 21 | 22 | 23 | {% if mode != "offline" %} 24 | 25 | // "Go to keyword" entry 26 | 27 | const tagTS = new TomSelect("#vh-select-tag", { 28 | maxItems: 1, 29 | loadThrottle: 250, 30 | valueField: "href", 31 | placeholder: "Go to keyword (type for autocomplete)", 32 | onFocus: () => { 33 | const ts = document.getElementById("vh-select-tag").tomselect; 34 | ts.clear(); 35 | ts.clearOptions(); 36 | }, 37 | shouldLoad: (query) => query.length >= 1, 38 | load: async (query, callback) => { 39 | let url = "/api/tagsearch?q=" + encodeURIComponent(query); 40 | if (document.location.protocol === "file:") { 41 | url = "http://127.0.0.1:5000" + url; 42 | } 43 | const resp = await fetch(url); 44 | const respJson = await resp.json(); 45 | callback(respJson.results); 46 | }, 47 | onChange: (value) => { 48 | if (value) { 49 | window.location = value; 50 | } 51 | } 52 | }); 53 | 54 | document.querySelector(".tag.srch .placeholder").addEventListener("click", (e) => { 55 | tagTS.focus(); 56 | }); 57 | 58 | 59 | // "Site search" entry 60 | 61 | const srchInput = document.getElementById("vh-srch-input"); 62 | srchInput.placeholder = "Site search (opens new DuckDuckGo tab)"; 63 | srchInput.addEventListener("blur", (e) => { 64 | srchInput.value = ""; 65 | }); 66 | document.querySelector(".site.srch .placeholder").addEventListener("click", (e) => { 67 | srchInput.focus(); 68 | }); 69 | 70 | 71 | // Theme switcher 72 | 73 | for (let theme of ["theme-native", "theme-light", "theme-dark"]) { 74 | document.getElementById(theme).addEventListener("click", (e) => { 75 | const [className, meta] = { 76 | "theme-native": [ "", "light dark" ], 77 | "theme-light": [ "light", "only light" ], 78 | "theme-dark": [ "dark", "only dark" ] 79 | }[theme]; 80 | document.documentElement.className = className; 81 | document.querySelector('meta[name="color-scheme"]').content = meta; 82 | 83 | const cookieDomain = location.hostname.replace(/^neo\./, ""); 84 | const cookieExpiry = theme === "theme-native" 85 | ? "Tue, 01 Jan 1970 00:00:00 GMT" // delete cookie 86 | : "Fri, 31 Dec 9999 23:59:59 GMT"; // set "permanent" cookie 87 | document.cookie = 88 | `theme=${className}; Secure; Domain=${cookieDomain}; SameSite=Lax; Path=/; Expires=${cookieExpiry}`; 89 | }); 90 | } 91 | 92 | document.getElementById("theme-current").addEventListener("click", (e) => { 93 | const themeDropdown = document.getElementById("theme-dropdown"); 94 | if (!themeDropdown.style.display) { 95 | // if currently hidden, show it... 96 | themeDropdown.style.display = "revert"; 97 | // ...and prevent the handler on from running, which would hide it again. 98 | e.stopPropagation(); 99 | } 100 | }); 101 | 102 | document.body.addEventListener("click", (e) => { 103 | // hide theme dropdown (vimhelp.css has it as "display: none") 104 | document.getElementById("theme-dropdown").style.display = null; 105 | }); 106 | 107 | // tweak native theme button tooltip 108 | document.getElementById("theme-native").title = "Switch to native theme" + 109 | (matchMedia("(prefers-color-scheme: dark)").matches ? " (which is dark)" : " (which is light)"); 110 | 111 | 112 | // Keyboard shortcuts 113 | // https://github.com/c4rlo/vimhelp/issues/28 114 | 115 | const onKeyDown = (e) => { 116 | if (e.isComposing || e.keyCode === 229) { 117 | // https://developer.mozilla.org/en-US/docs/Web/API/Element/keydown_event 118 | return; 119 | } 120 | const a = document.activeElement; 121 | if (a && (a.isContentEditable || a.tagName === "INPUT" || a.tagName === "SELECT")) { 122 | return; 123 | } 124 | if (e.key === "k") { 125 | e.preventDefault(); 126 | document.getElementById("vh-select-tag-ts-control").focus(); 127 | } 128 | else if (e.key === "s") { 129 | e.preventDefault(); 130 | document.getElementById("vh-srch-input").focus(); 131 | } 132 | }; 133 | addEventListener("keydown", onKeyDown); 134 | 135 | {% endif %} 136 | -------------------------------------------------------------------------------- /vimhelp/dbmodel.py: -------------------------------------------------------------------------------- 1 | # Definitions of objects stored in Datastore 2 | 3 | from google.cloud import ndb 4 | 5 | 6 | _ndb_client = None 7 | 8 | 9 | def ndb_context(): 10 | global _ndb_client 11 | if _ndb_client is None: 12 | _ndb_client = ndb.Client() 13 | return _ndb_client.context() 14 | 15 | 16 | # There is one of these objects in the datastore, to persist some bits of info that we 17 | # need across update runs; key name is "vim" or "neovim". 18 | class GlobalInfo(ndb.Model): 19 | refs_etag = ndb.BlobProperty() 20 | # HTTP ETag of GraphQL query for latest refs/tags 21 | # (useless in practice since the GitHub GraphQL endpoint seems to not support ETag) 22 | 23 | docdir_etag = ndb.BlobProperty() 24 | # HTTP ETag of the vim repository request for the 'runtime/doc' subdirectory 25 | 26 | master_sha = ndb.TextProperty() 27 | # Git SHA of latest master commit 28 | 29 | vim_version_tag = ndb.TextProperty() 30 | # Git tag of current Vim version 31 | 32 | last_update_time = ndb.DateTimeProperty(indexed=False) 33 | # Time of last changes to generated files 34 | 35 | 36 | # Tags, for use with the "go to tag" feature; key name is "vim" or "neovim". 37 | class TagsInfo(ndb.Model): 38 | tags = ndb.JsonProperty(json_type=list) 39 | # Pairs of vimhelp tag and (site-relative) link. Looks like this: 40 | # [ ["t", "motion.txt.html#t"], ["perl", "if_perl.txt.html#perl"], ... ] 41 | 42 | 43 | # Info related to an unprocessed documentation file from the repository; key name is 44 | # e.g. "vim:help.txt" or "neovim:api.txt" 45 | class RawFileInfo(ndb.Model): 46 | project = ndb.StringProperty(required=True) 47 | # Either "vim" or "neovim", always matches the entity key ID 48 | 49 | git_sha = ndb.BlobProperty() 50 | # 'sha' property returned by GitHub API (not populated for vim_faq.txt) 51 | 52 | etag = ndb.BlobProperty() 53 | # HTTP ETag of the file on github 54 | 55 | 56 | # The actual contents of an unprocessed documentation file from the repository; key name 57 | # is e.g. "vim:faq.txt" or "neovim:help.txt" 58 | class RawFileContent(ndb.Model): 59 | project = ndb.StringProperty(required=True) 60 | # Either "vim" or "neovim", always matches the entity key ID 61 | 62 | data = ndb.BlobProperty(required=True) 63 | # The data 64 | 65 | encoding = ndb.BlobProperty(required=True) 66 | # The encoding, e.g. 'UTF-8' 67 | 68 | 69 | # Info related to a processed (HTMLified) documentation file; key name is e.g. 70 | # "vim:faq.txt" or "neovim:help.txt" 71 | class ProcessedFileHead(ndb.Model): 72 | project = ndb.StringProperty(required=True) 73 | # Either "vim" or "neovim", always matches the entity key ID 74 | 75 | etag = ndb.BlobProperty(required=True) 76 | # HTTP ETag on this server, generated by us as a hash of the contents 77 | 78 | encoding = ndb.BlobProperty(required=True) 79 | # Encoding, always matches the corresponding 'RawFileContent' object 80 | 81 | modified = ndb.DateTimeProperty(indexed=False, auto_now=True) 82 | # Time when this file was generated 83 | 84 | numparts = ndb.IntegerProperty(indexed=False) 85 | # Number of parts; there will be 'numparts - 1' objects of kind 'ProcessedFilePart' 86 | # in the database. Processed files are split up into parts as required by datastore 87 | # blob limitations (currently these can only be up to 1 MiB in size) 88 | 89 | data0 = ndb.BlobProperty(required=True) 90 | # Contents of the first (and possibly only) part 91 | 92 | used_assets = ndb.JsonProperty(json_type=list, indexed=True) 93 | # Names and hashes of assets used by this HTML file. List elements match the 94 | # key names of "Asset" entities. 95 | 96 | 97 | # Part of a processed file; key name is "{project}:{basename}:{partnum}", e.g. 98 | # "neovim:help.txt:1". 99 | # This chunking is necessary because the maximum entity size in the Datastore is 1 MB: 100 | # see https://cloud.google.com/datastore/docs/concepts/limits 101 | # NOTE: vimhelp.py currently relies on the keynames, when sorted lexicographically, 102 | # yielding the correct order; this implies that we must never have a partnum with more 103 | # than one digit. 104 | class ProcessedFilePart(ndb.Model): 105 | data = ndb.BlobProperty(required=True) 106 | # Contents 107 | 108 | etag = ndb.BlobProperty(required=True) 109 | # Same value as corresponding 'ProcessedFileHead.etag'. Used when retrieving the 110 | # 'ProcessedFileHead' and all its 'ProcessedFilePart's to ensure that they were 111 | # retrieved consistently. 112 | 113 | 114 | # Versioned static asset; key name is "{basename}:{hash}", e.g. "vimhelp.js:d34db33f". 115 | class Asset(ndb.Model): 116 | data = ndb.BlobProperty(required=True) 117 | # Contents 118 | 119 | create_time = ndb.DateTimeProperty(required=True, indexed=False, auto_now_add=True) 120 | # Time this entity was created 121 | 122 | unused_time = ndb.DateTimeProperty(indexed=False) 123 | # Time as of which this entity is no longer in use 124 | -------------------------------------------------------------------------------- /scripts/h2h.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env .venv/bin/python3 2 | 3 | # This script is meant to be run from the top-level directory of the 4 | # repository, as 'scripts/h2h.py'. The virtualenv must already exist 5 | # (use "inv venv" to create it). 6 | 7 | import argparse 8 | import os.path 9 | import pathlib 10 | import sys 11 | 12 | import flask 13 | 14 | root_path = pathlib.Path(__file__).parent.parent 15 | 16 | sys.path.append(str(root_path)) 17 | 18 | from vimhelp.vimh2h import VimH2H # noqa: E402 19 | 20 | 21 | def main(): 22 | parser = argparse.ArgumentParser(description="Convert Vim help files to HTML") 23 | parser.add_argument( 24 | "--in-dir", 25 | "-i", 26 | required=True, 27 | type=pathlib.Path, 28 | help="Directory of Vim doc files", 29 | ) 30 | parser.add_argument( 31 | "--out-dir", 32 | "-o", 33 | type=pathlib.Path, 34 | help="Output directory (omit for no output)", 35 | ) 36 | parser.add_argument( 37 | "--project", 38 | "-p", 39 | choices=("vim", "neovim"), 40 | default="vim", 41 | help="Vim flavour (default: vim)", 42 | ) 43 | parser.add_argument( 44 | "--web-version", 45 | "-w", 46 | action="store_true", 47 | help="Generate the web version of the files (default: offline version)", 48 | ) 49 | parser.add_argument( 50 | "--theme", 51 | "-t", 52 | choices=("light", "dark"), 53 | help="Color theme (default: OS-native)", 54 | ) 55 | parser.add_argument( 56 | "--no-tags", 57 | "-T", 58 | action="store_true", 59 | help="Ignore any tags file, always recreate tags from scratch", 60 | ) 61 | parser.add_argument( 62 | "--profile", "-P", action="store_true", help="Profile performance" 63 | ) 64 | parser.add_argument( 65 | "basenames", nargs="*", help="List of files to process (default: all)" 66 | ) 67 | args = parser.parse_args() 68 | 69 | app = flask.Flask( 70 | __name__, 71 | root_path=pathlib.Path(__file__).resolve().parent, 72 | template_folder="../vimhelp/templates", 73 | ) 74 | app.jinja_options["trim_blocks"] = True 75 | app.jinja_options["lstrip_blocks"] = True 76 | app.jinja_env.filters["static_path"] = lambda p: p 77 | 78 | with app.app_context(): 79 | if args.profile: 80 | import cProfile 81 | import pstats 82 | 83 | with cProfile.Profile() as pr: 84 | run(args) 85 | stats = pstats.Stats(pr).sort_stats("cumulative") 86 | stats.print_stats() 87 | else: 88 | run(args) 89 | 90 | 91 | def run(args): 92 | if not args.in_dir.is_dir(): 93 | raise RuntimeError(f"{args.in_dir} is not a directory") 94 | 95 | prelude = VimH2H.prelude(theme=args.theme) 96 | 97 | mode = "hybrid" if args.web_version else "offline" 98 | 99 | if not args.no_tags and (tags_file := args.in_dir / "tags").is_file(): 100 | print("Processing tags file...") 101 | h2h = VimH2H(mode=mode, project=args.project, tags=tags_file.read_text()) 102 | faq = args.in_dir / "vim_faq.txt" 103 | if faq.is_file(): 104 | print("Processing FAQ tags...") 105 | h2h.add_tags(faq.name, faq.read_text()) 106 | else: 107 | print("Initializing tags...") 108 | h2h = VimH2H(mode=mode, project=args.project) 109 | for infile in args.in_dir.iterdir(): 110 | if infile.suffix == ".txt": 111 | h2h.add_tags(infile.name, infile.read_text()) 112 | 113 | if args.out_dir is not None: 114 | args.out_dir.mkdir(exist_ok=True) 115 | 116 | for infile in args.in_dir.iterdir(): 117 | if len(args.basenames) != 0 and infile.name not in args.basenames: 118 | continue 119 | if infile.suffix != ".txt" and infile.name != "tags": 120 | print(f"Ignoring {infile}") 121 | continue 122 | content = infile.read_text() 123 | print(f"Processing {infile}...") 124 | html = h2h.to_html(infile.name, content) 125 | if args.out_dir is not None: 126 | with (args.out_dir / f"{infile.name}.html").open("w") as f: 127 | f.write(prelude) 128 | f.write(html) 129 | 130 | if args.out_dir is not None: 131 | print("Symlinking/creating static files...") 132 | static_dir = root_path / "vimhelp" / "static" 133 | static_dir_rel = os.path.relpath(static_dir, args.out_dir) 134 | for target in static_dir.iterdir(): 135 | target_name = target.name 136 | src = args.out_dir / target_name 137 | src.unlink(missing_ok=True) 138 | src.symlink_to(f"{static_dir_rel}/{target_name}") 139 | for name in "vimhelp.css", "vimhelp.js": 140 | content = flask.render_template(name, mode=mode) 141 | (args.out_dir / name).write_text(content) 142 | 143 | print("Done.") 144 | 145 | 146 | main() 147 | -------------------------------------------------------------------------------- /vimhelp/webapp.py: -------------------------------------------------------------------------------- 1 | import gevent 2 | import gevent.monkey 3 | 4 | # gevent.config.track_greenlet_tree = False 5 | gevent.monkey.patch_all() 6 | 7 | import grpc.experimental.gevent # noqa: E402 8 | 9 | grpc.experimental.gevent.init_gevent() 10 | 11 | from http import HTTPStatus # noqa: E402 12 | import flask # noqa: E402 13 | 14 | import logging # noqa: E402 15 | import os # noqa: E402 16 | import pathlib # noqa: E402 17 | 18 | 19 | _CSP = "default-src 'self'" # Content Security Policy 20 | 21 | _URL_PREFIX_REDIRECTS = ( 22 | ( 23 | # From 24 | ( 25 | "http://www.vimhelp.org/", 26 | "https://www.vimhelp.org/", 27 | "http://vimhelp.appspot.com/", 28 | "https://vimhelp.appspot.com/", 29 | "http://vimhelp.org/", 30 | ), 31 | # To 32 | "https://vimhelp.org", 33 | ), 34 | ( 35 | # From 36 | ("http://neo.vimhelp.org/",), 37 | # To 38 | "https://neo.vimhelp.org", 39 | ), 40 | ) 41 | 42 | _WARMUP_PATH = "/_ah/warmup" 43 | 44 | g_is_dev = False 45 | 46 | 47 | def create_app() -> flask.Flask: 48 | from . import assets 49 | from . import cache 50 | from . import robots 51 | from . import tagsearch 52 | from . import vimhelp 53 | from . import update 54 | 55 | package_path = pathlib.Path(__file__).resolve().parent 56 | 57 | logging.basicConfig(level=logging.INFO) 58 | 59 | cache_ = cache.Cache() 60 | 61 | app = flask.Flask("vimhelp", root_path=package_path, static_folder=None) 62 | 63 | app.jinja_options["trim_blocks"] = True 64 | app.jinja_options["lstrip_blocks"] = True 65 | app.jinja_env.filters["static_path"] = assets.static_path 66 | 67 | global g_is_dev 68 | g_is_dev = os.environ.get("VIMHELP_ENV") == "dev" 69 | if not g_is_dev: 70 | app.config["PREFERRED_URL_SCHEME"] = "https" 71 | 72 | assets.init(app) 73 | 74 | app.add_url_rule( 75 | "/clean_assets", view_func=assets.CleanAssetsHandler.as_view("clean_assets") 76 | ) 77 | app.add_url_rule( 78 | "/enqueue_clean_assets", view_func=assets.handle_enqueue_clean_assets 79 | ) 80 | 81 | bp = flask.Blueprint("bp", "vimhelp", root_path=package_path) 82 | 83 | @bp.route("/