├── .gitignore ├── .readthedocs.yaml ├── LICENSE ├── MANIFEST.in ├── README.md ├── deploy ├── docs ├── Makefile ├── conf.py ├── index.rst └── requirements.txt ├── pyfra ├── __init__.py ├── contrib │ ├── __init__.py │ └── web │ │ ├── .gitignore │ │ ├── LICENSE │ │ ├── README.md │ │ ├── __init__.py │ │ ├── app.py │ │ ├── config.py │ │ ├── emailer.py │ │ ├── forms.py │ │ ├── migrations │ │ ├── README │ │ ├── alembic.ini │ │ ├── env.py │ │ ├── script.py.mako │ │ └── versions │ │ │ ├── 1582bc5883ac_.py │ │ │ └── 1b1532ba9506_add_user_attributes.py │ │ ├── models.py │ │ ├── requirements.txt │ │ ├── server.py │ │ ├── static │ │ ├── .editorconfig │ │ ├── LICENSE.txt │ │ ├── browserconfig.xml │ │ ├── css │ │ │ ├── main.css │ │ │ └── reset.css │ │ ├── favicon.ico │ │ ├── humans.txt │ │ ├── icon.png │ │ ├── img │ │ │ ├── background.jpg │ │ │ ├── eai_logo.png │ │ │ ├── excel.png │ │ │ ├── icon-button-adduser.svg │ │ │ ├── icon-button-admin.svg │ │ │ ├── icon-button-admin │ │ │ │ └── user.svg │ │ │ ├── icon-button-change_password.svg │ │ │ ├── icon-button-demo_form.svg │ │ │ ├── logout.svg │ │ │ └── print_button.jpg │ │ ├── index.html │ │ ├── js │ │ │ ├── main.js │ │ │ ├── plugins.js │ │ │ └── vendor │ │ │ │ ├── jquery-3.3.1.min.js │ │ │ │ ├── modernizr-3.6.0.min.js │ │ │ │ ├── moment-2.23.0.js │ │ │ │ ├── w2ui-1.5.rc1.js │ │ │ │ └── w2ui-1.5.rc1.min.js │ │ ├── robots.txt │ │ └── site.webmanifest │ │ └── templates │ │ ├── 404.html │ │ ├── base.html │ │ ├── email │ │ ├── reset_password.html │ │ └── reset_password.txt │ │ ├── forgot_password.html │ │ ├── form_template.html │ │ ├── index.html │ │ ├── login.html │ │ ├── polls.html │ │ ├── reset_password.html │ │ └── view_template.html ├── delegation.py ├── idempotent.py ├── remote.py ├── setup.py └── shell.py ├── requirements.txt ├── setup.py └── tests ├── Dockerfile └── test_remote.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | build 3 | dist 4 | *.egg-info 5 | env 6 | pyfra/web/state/webapp.db 7 | pyfra/web/state/main.db 8 | state/main.db 9 | state/webapp.db 10 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | # Build documentation in the docs/ directory with Sphinx 4 | sphinx: 5 | configuration: docs/conf.py 6 | 7 | # Optionally build your docs in additional formats such as PDF 8 | formats: 9 | - pdf 10 | 11 | # Optionally set the version of Python and requirements required to build your docs 12 | python: 13 | version: 3.7 14 | install: 15 | - requirements: docs/requirements.txt 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 EleutherAI contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include pyfra/web * 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pyfra 2 | 3 | *The Python Framework for Research Applications.* 4 | 5 | [![Documentation Status](https://readthedocs.org/projects/pyfra/badge/?version=latest)](https://pyfra.readthedocs.io/en/latest/?badge=latest) [![PyPI version](https://badge.fury.io/py/pyfra.svg)](https://badge.fury.io/py/pyfra) 6 | 7 | 8 | ## Design Philosophy 9 | 10 | Research code has some of the fastest shifting requirements of any type of code. It's nearly impossible to plan ahead of time the proper abstractions, because it is exceedingly likely that in the course of the project what you originally thought was your main focus suddenly no longer is. Further, research code (especially in ML) often involves big and complicated pipelines, typically involving many different machines, which are either run by hand or using shell scripts that are far more complicated than any shell script ever should be. 11 | 12 | Therefore, the objective of pyfra is to make it as fast and *low-friction* as possible to write research code involving complex pipelines over many machines. This entails making it as easy as possible to implement a research idea in reality, at the cost of fine-grained control and the long-term maintainability of the system. In other words, pyfra expects that code will either be rapidly obsoleted by newer code, or rewritten using some other framework once it is no longer a research project and requirements have settled down. 13 | 14 | As such, high quality interface design is a top priority of pyfra. The ultimate goal is to make interfaces that feel extremely natural, such that it is intuitively obvious what any piece of pyfra code should do, without requiring consultation of the docs. The core pyfra abstractions should be optimized, as much as possible, for _naturalness_; it should be easy to do the things that are frequently needed, but if making it so would require a sacrifice in generality, the abstraction should be decomposed into a maximally-general core and an application-specific layer on top. As is it expected that pyfra-based code is short lived, when trading off between API stability and cleaner/more intuitive interfaces, pyfra will attempt to choose the latter. That being said, core pyfra should become more and more stable as time progressese. 15 | 16 | The `pyfra.contrib` package is intended for code that is built on top of pyfra and useful in conjunction with pyfra, but has fewer stability guarantees than core pyfra. Especially useful and well designed functions in contrib may be promoted to core pyfra. 17 | 18 | **Pyfra is in its very early stages of development. The interface may change rapidly and without warning.** 19 | 20 | Features: 21 | 22 | - Extremely elegant shell integration—run commands on any server seamlessly. All the best parts of bash and python combined 23 | - Handle files on remote servers with a pathlib-like interface the same way you would local files (WIP, partially implemented) 24 | - Automated remote environment setup, so you never have to worry about provisioning machines by hand again 25 | - Idempotent resumable data and training pipelines with no cognitive overhead 26 | - Spin up an internal webserver complete with a permissions system using only a few lines of code 27 | - (Coming soon) High level API for experiment management/scheduling and resource provisioning 28 | 29 | Want to dive in? See the [documentation](https://pyfra.readthedocs.io/en/latest/). 30 | 31 | ## Example code 32 | 33 | (This example obviously doesn't run as-is, it's just an illustrative example) 34 | 35 | ```python 36 | from pyfra import * 37 | 38 | rem1 = Remote("user@example.com") 39 | rem2 = Remote("goose@8.8.8.8") 40 | 41 | # env creates an environment object, which behaves very similarly to a Remote (in fact Env inherits from Remote), 42 | # but comes with a fresh python environment in a newly created directory (optionally initialized from a git repo) 43 | # also, anything you run in an env will resume where it left off, with semantics similar to dockerfiles. 44 | env1 = rem1.env("tokenization") 45 | env2 = rem2.env("training", "https://github.com/some/repo") 46 | 47 | # path creates a RemotePath object, which behaves similar to a pathlib Path. 48 | raw_data = local.path("training_data.txt") 49 | tokenized_data = env2.path("tokenized_data") 50 | 51 | # tokenize 52 | copy("https://goose.com/files/tokenize_script.py", env1.path("tokenize.py")) # copy can copy from local/remote/url to local/remote 53 | env1.sh(f"python tokenize.py --input {raw_data} --output {tokenized_data}") # implicitly copy files just by using the path object in an f-string 54 | 55 | # start training run 56 | env2.path("config.json").jwrite({...}) 57 | env2.sh("python train.py --input tokenized_data --config config.json") 58 | ``` 59 | 60 | ## Installation 61 | 62 | ```pip3 install pyfra``` 63 | 64 | 65 | ## Webserver screenshots 66 | 67 | ![image](https://user-images.githubusercontent.com/54557097/119907788-4a2f6700-bf0e-11eb-9655-55e3317ba871.png) 68 | ![image](https://user-images.githubusercontent.com/54557097/115158135-fc3f5d80-a049-11eb-8310-a43b7b5c58e0.png) 69 | -------------------------------------------------------------------------------- /deploy: -------------------------------------------------------------------------------- 1 | rm -rf dist 2 | python3 setup.py sdist bdist_wheel 3 | python3 -m twine upload dist/* -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | sys.path.insert(0, os.path.abspath('..')) 16 | 17 | 18 | # -- Project information ----------------------------------------------------- 19 | 20 | project = 'pyfra' 21 | copyright = '2021, Leo Gao' 22 | author = 'Leo Gao' 23 | 24 | # The full version, including alpha/beta/rc tags 25 | release = '0.1.4' 26 | 27 | 28 | # -- General configuration --------------------------------------------------- 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = ['sphinxcontrib.napoleon', 'sphinx.ext.autodoc', # Core library for html generation from docstrings 34 | 'sphinx.ext.autosummary'] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ['_templates'] 38 | 39 | # List of patterns, relative to source directory, that match files and 40 | # directories to ignore when looking for source files. 41 | # This pattern also affects html_static_path and html_extra_path. 42 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 43 | 44 | 45 | # -- Options for HTML output ------------------------------------------------- 46 | 47 | # The theme to use for HTML and HTML Help pages. See the documentation for 48 | # a list of builtin themes. 49 | # 50 | html_theme = 'sphinx_rtd_theme' 51 | 52 | 53 | # Add any paths that contain custom static files (such as style sheets) here, 54 | # relative to this directory. They are copied after the builtin static files, 55 | # so a file named "default.css" will overwrite the builtin "default.css". 56 | html_static_path = ['_static'] 57 | 58 | autoclass_content = 'both' 59 | 60 | autodoc_inherit_docstrings = False 61 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. pyfra documentation master file, created by 2 | sphinx-quickstart on Sat Jun 5 14:55:57 2021. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to pyfra's documentation! 7 | ================================= 8 | 9 | pyfra.remote module 10 | ================================= 11 | 12 | .. automodule:: pyfra.remote 13 | :members: 14 | :undoc-members: 15 | :show-inheritance: 16 | 17 | pyfra.shell module 18 | ================================= 19 | 20 | .. automodule:: pyfra.shell 21 | :members: 22 | :undoc-members: 23 | :show-inheritance: 24 | 25 | Indices and tables 26 | ================== 27 | 28 | * :ref:`genindex` 29 | * :ref:`modindex` 30 | * :ref:`search` 31 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | best_download 2 | ansi2html 3 | sqlitedict 4 | colorama 5 | parse 6 | natsort 7 | deprecation 8 | 9 | # web 10 | flask 11 | flask-login 12 | flask-wtf 13 | flask-sqlalchemy 14 | flask-migrate 15 | flask-admin 16 | flask-bootstrap 17 | pyjwt 18 | sqlalchemy 19 | wtforms[email] 20 | 21 | 22 | sphinx==4.0.2 23 | sphinxcontrib-napoleon==0.7 24 | 25 | imohash 26 | yaspin 27 | -------------------------------------------------------------------------------- /pyfra/__init__.py: -------------------------------------------------------------------------------- 1 | from colorama import init 2 | init() 3 | 4 | from .remote import * 5 | from .shell import * 6 | from .delegation import * 7 | from .idempotent import set_kvstore, cache 8 | 9 | try: 10 | import pyfra.contrib as contrib 11 | except ImportError: 12 | pass 13 | 14 | 15 | # fallback snippet from https://github.com/gruns/icecream 16 | try: 17 | from icecream import ic 18 | except ImportError: # Graceful fallback if IceCream isn't installed. 19 | ic = lambda *a: None if not a else (a[0] if len(a) == 1 else a) # noqa -------------------------------------------------------------------------------- /pyfra/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | from pyfra import * 2 | from pathlib import Path 3 | import pyfra.contrib.web as web 4 | 5 | @always_rerun() 6 | def tpu_vm_create(rem_gcp, tpu_name, zone="europe-west4-a", type="v3-8"): 7 | user = rem_gcp.sh("echo $USER").strip() 8 | 9 | def _get_tpu_ssh(): 10 | ip = rem_gcp.sh(f"gcloud alpha compute tpus tpu-vm describe {tpu_name} --format='get(networkEndpoints[0].accessConfig.externalIp)'".strip()) 11 | return Remote(f"{user}@{ip}") 12 | 13 | try: 14 | r = _get_tpu_ssh() 15 | r.sh("echo hello from tpu") 16 | return r 17 | except ShellException: 18 | pass 19 | 20 | rem_gcp.sh(f""" 21 | echo y | gcloud alpha compute tpus tpu-vm delete {tpu_name} 22 | gcloud alpha compute tpus tpu-vm create {tpu_name} \ 23 | --zone={zone} \ 24 | --accelerator-type={type} \ 25 | --version=v2-alpha 26 | gcloud alpha compute tpus tpu-vm ssh {tpu_name} --zone {zone} --command="echo $(cat {local.path("~/.ssh/id_rsa.pub")}) >> ~/.ssh/authorized_keys" 27 | """) 28 | 29 | time.sleep(10) 30 | 31 | return _get_tpu_ssh() 32 | 33 | def kube_sh(pod, cmd, executable="bash", quiet=False): 34 | """ 35 | Run a command in a kube pod 36 | """ 37 | if executable == "bash": 38 | cmd = f"kubectl exec -it {pod} -- /bin/bash -c {quote(cmd)}" 39 | elif executable == "sh": 40 | cmd = f"kubectl exec -it {pod} -- /bin/sh -c {quote(cmd)}" 41 | elif executable == None: 42 | cmd = f"kubectl exec -it {pod} -- {quote(cmd)}" 43 | else: 44 | raise ValueError(f"executable must be bash or None, not {executable}") 45 | return local.sh(cmd, quiet=quiet) 46 | 47 | 48 | def kube_copy_ssh_key(pod: str, key_path: str = None, quiet: bool = False): 49 | """ 50 | Copy an ssh key to the k8 pod 51 | """ 52 | if key_path is None: 53 | for pubkey in (Path(local.home()) / ".ssh").glob("*.pub"): 54 | kube_copy_ssh_key(pod, pubkey) 55 | return 56 | kube_sh( 57 | pod, 58 | f"echo {quote(local.path(key_path).read().strip())} >> ~/.ssh/authorized_keys", 59 | quiet=quiet, 60 | ) 61 | 62 | 63 | def kube_remote( 64 | pod: str, ssh_key_path: str = None, user=None, service_name=None, quiet=False 65 | ) -> Remote: 66 | """ 67 | Get a remote object for a k8 pod 68 | """ 69 | if service_name is None: 70 | service_name = pod.split("-")[0] + "-service" 71 | get_ip_cmd = f"kubectl get service/{service_name} -o jsonpath='{{.status.loadBalancer.ingress[0].ip}}'" 72 | ip = local.sh(get_ip_cmd, quiet=quiet).strip() 73 | if user is not None: 74 | ip = f"{user}@{ip}" 75 | 76 | # try to connect 77 | try: 78 | r = Remote(ip) 79 | r.sh(f"echo hello from {pod}", quiet=quiet) 80 | return r 81 | except ShellException: 82 | pass 83 | 84 | # copy ssh key 85 | kube_copy_ssh_key(pod, ssh_key_path, quiet=quiet) 86 | 87 | return Remote(ip) 88 | -------------------------------------------------------------------------------- /pyfra/contrib/web/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | 132 | app.db 133 | -------------------------------------------------------------------------------- /pyfra/contrib/web/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 EleutherAI 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /pyfra/contrib/web/README.md: -------------------------------------------------------------------------------- 1 | # Flask Based Polling Website Demo 2 | 3 | Basic implementation of a polling website with authentication and administration backend. 4 | 5 | Currently implements the following: 6 | 7 | Webpage Templating and Endpoints (flask) 8 | Forms (flask-wtf) 9 | Admin Section (flask-admin) 10 | Authentication (flask-login, pyjwt) 11 | Backend sqlite database (flask-sqlalchemy, flask-migrate) 12 | 13 | ## To Do 14 | 15 | Fix the admin section blowing up when you aren't logged in. 16 | Copy over from Votr the polling logic. 17 | Hide logout button when not logged in 18 | 19 | ## Installation 20 | 21 | Clone, copy the config template and set the config items correctly. Secret key 22 | is used to create jwt. Email config is used for sending password reset emails 23 | to users. 24 | 25 | ``` 26 | git clone https://github.com/EleutherAI/poll_website_demo.git 27 | cp config_template.py config.py 28 | vi config.py 29 | ``` 30 | 31 | Create the database: 32 | 33 | ``` 34 | ./recreate_db.sh 35 | ``` 36 | 37 | To run the dev server on your local machines external ip, using port 5000: 38 | ``` 39 | python server.py 40 | ``` 41 | 42 | ## Usage 43 | 44 | Visiting root will show you a login screen where you can register, reset password etc. 45 | 46 | Admin section can be found in /admin which requires an admin user to be logged in. The default admin credentials can be found in users.csv. 47 | 48 | ## Design Details 49 | 50 | The app is wired together inside app.py, which is imported into server.py. 51 | 52 | config.py is imported using app.config.from_object(Config) inside app.py. 53 | 54 | ORM model can be found in models.py 55 | 56 | flask-migrate uses alembic on the backend, which creates database diffs allowing modification of live databases. We have a simple script to generate the initial database in recreate_db.sh, but to make changes to a live site you will need to learn a bit about flask-migrate. Our script calls load_users.py which populates the users table from the users.csv file if you want some users when the application starts (you will need an admin user if you dont want to have to manually setup the user with sqlite). We login with email (not Name). 57 | 58 | Templates are found in the templates directory, with the email subdirectory containing the template used for sending the password reset email. 59 | 60 | All web templates inherit from base.html 61 | 62 | Stylesheets can be found in static/css. We use a reset sheet and a main.css currently. 63 | 64 | static also has a bunch of junk found in the html5 boilerplate in case you want it. You can replace the favicon.ico if wanted. 65 | 66 | jquery is included in the template if you want to use it - just write javascript in the templates you use. To serve react I'd probably build a separate react pipeline and just copy the output of build and serve it up using a flask endpoint (unless there's something better out there I haven't used yet). 67 | 68 | I pulled this application from something else I wrote and left the print and excel buttons in case we ever want to do exporting stuff. 69 | 70 | -------------------------------------------------------------------------------- /pyfra/contrib/web/__init__.py: -------------------------------------------------------------------------------- 1 | from .server import * 2 | from functools import wraps, partial 3 | import inspect 4 | import string 5 | import random 6 | import os 7 | from html import escape 8 | try: 9 | from typing_extensions import Literal 10 | except ModuleNotFoundError: 11 | from typing import Literal 12 | 13 | from flask_migrate import upgrade 14 | from flask_wtf import FlaskForm 15 | from wtforms import StringField, BooleanField, SubmitField, IntegerField, SelectField 16 | from wtforms.validators import DataRequired, Email, EqualTo 17 | from ansi2html import Ansi2HTMLConverter 18 | 19 | 20 | __all__ = [ 21 | 'page', 'webserver', 22 | 'current_user', # so we can know who's making each request inside a @page annotated function 23 | 'User' 24 | ] 25 | 26 | 27 | def dict_replace_if_fn(d): 28 | return { 29 | k: v() if callable(v) else v 30 | for k, v in d.items() 31 | } 32 | 33 | 34 | def page(pretty_name=None, display: Literal["raw", "text", "monospace"]="monospace", field_names={}, dropdowns={}, roles=['everyone']): 35 | def _fn(callback, pretty_name, field_names, roles, display, dropdowns): 36 | 37 | sig = inspect.signature(callback) 38 | 39 | def make_form_class(dropdowns): 40 | dropdowns = dict_replace_if_fn(dropdowns) 41 | class CustomForm(FlaskForm): 42 | pass 43 | 44 | for name in sig.parameters: 45 | type = sig.parameters[name].annotation 46 | is_required = (sig.parameters[name].default == inspect._empty) 47 | 48 | if type == int: 49 | field = IntegerField 50 | elif type == bool: 51 | field = BooleanField 52 | else: 53 | if name in dropdowns: 54 | field = partial(SelectField, choices=dropdowns[name]) 55 | else: 56 | field = StringField 57 | 58 | setattr(CustomForm, name, field( 59 | field_names.get(name, name), 60 | validators=[DataRequired()] if is_required else [], 61 | default = sig.parameters[name].default if not is_required else None 62 | )) 63 | 64 | if len(sig.parameters) > 0: 65 | CustomForm.submit = SubmitField('Submit') 66 | return CustomForm 67 | 68 | form = len(sig.parameters) > 0 69 | 70 | def _callback_wrapper(k): 71 | html = callback(**k) 72 | if display == "raw": 73 | pass 74 | elif display == "text": 75 | html = escape(html) 76 | elif display == "monospace": 77 | converter = Ansi2HTMLConverter() 78 | html = converter.convert(html, full=False) 79 | html = f"{html}" 80 | html += converter.produce_headers() 81 | else: 82 | raise NotImplementedError 83 | 84 | return html 85 | 86 | register_page(callback.__name__, pretty_name, partial(make_form_class, dropdowns), _callback_wrapper, roles, redirect_index=False, has_form=form) 87 | 88 | # used @form and not @form() 89 | if callable(pretty_name): 90 | return _fn(pretty_name, pretty_name=None, field_names=field_names, roles=roles, display=display, dropdowns=dropdowns) 91 | 92 | return partial(_fn, pretty_name=pretty_name, field_names=field_names, roles=roles, display=display, dropdowns=dropdowns) 93 | 94 | 95 | def gen_pass(stringLength=16): 96 | """Generate a random string of letters, digits """ 97 | password_characters = string.ascii_letters + string.digits 98 | return ''.join(random.choice(password_characters) for i in range(stringLength)) 99 | 100 | 101 | @page("Add User", roles=["admin"]) 102 | def adduser(username: str, email: str="example@example.com", roles: str=""): 103 | password = gen_pass() 104 | 105 | add_user(username, email, password, roles) 106 | 107 | return f"Added user {username} with randomly generated password {password}." 108 | 109 | @page("Reset User Password", roles=["admin"]) 110 | def set_password(username: str): 111 | password = gen_pass() 112 | 113 | user = User.query.filter_by(name=username).first() 114 | 115 | user.set_password(password) 116 | db.session.commit() 117 | 118 | return f"Updated user {username} with randomly generated password {password}." 119 | 120 | 121 | def webserver(debug=False): 122 | basedir = os.getcwd() + '/state' 123 | os.makedirs(basedir, exist_ok=True) 124 | with app.app_context(): 125 | migrations_path = os.path.join(os.path.dirname(__file__), "migrations") 126 | upgrade(migrations_path) # equivalent to running "flask db upgrade" every load 127 | 128 | if User.query.count() == 0: 129 | # Add temporary admin user 130 | password = gen_pass() 131 | add_user("root", "example@example.com", password, "admin") 132 | print("=================================================") 133 | print("ADMIN LOGIN CREDENTIALS (WILL ONLY BE SHOWN ONCE)") 134 | print("If you forget you'll need to manually add an") 135 | print("admin user to the database.") 136 | print("Username: root") 137 | print("Password:", password) 138 | print("=================================================") 139 | app.config['TEMPLATES_AUTO_RELOAD'] = True 140 | app.run(host='0.0.0.0', debug=debug) -------------------------------------------------------------------------------- /pyfra/contrib/web/app.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | 3 | from flask_sqlalchemy import SQLAlchemy 4 | from flask_migrate import Migrate 5 | from flask_login import LoginManager 6 | from flask_admin import Admin 7 | from flask_bootstrap import Bootstrap 8 | 9 | from .config import Config 10 | import pathlib 11 | 12 | template_path = pathlib.Path(__file__).parent.absolute() / 'templates' 13 | app = Flask(__name__, template_folder=str(template_path)) 14 | app.config.from_object(Config) 15 | 16 | db = SQLAlchemy(app) 17 | migrate = Migrate(app, db) 18 | bootstrap = Bootstrap(app) 19 | 20 | login = LoginManager(app) 21 | login.login_view = 'login' 22 | 23 | app.config['FLASK_ADMIN_SWATCH'] = 'cerulean' 24 | admin = Admin(app, name='Admin Dashboard', template_mode='bootstrap3') 25 | 26 | import pyfra.contrib.web.models 27 | 28 | -------------------------------------------------------------------------------- /pyfra/contrib/web/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | basedir = os.getcwd() + '/state' 3 | 4 | class Config(object): 5 | 6 | SQLALCHEMY_DATABASE_URI = f'sqlite:///{basedir}/webapp.db' 7 | SQLALCHEMY_TRACK_MODIFICATIONS = False 8 | SECRET_KEY = 'FILL_THIS_IN' 9 | SMTP_SERVER = "smtp.gmail.com" 10 | SMTP_PORT = 587 11 | SMTP_USERNAME = "FILL_THIS_IN" 12 | SMTP_PASSWORD = "FILL_THIS_IN" 13 | 14 | -------------------------------------------------------------------------------- /pyfra/contrib/web/emailer.py: -------------------------------------------------------------------------------- 1 | from smtplib import SMTP 2 | 3 | from email.message import EmailMessage 4 | 5 | from flask import render_template 6 | 7 | from threading import Thread 8 | 9 | def create_email_message(from_address, to_address, subject, body, htmlBody): 10 | msg = EmailMessage() 11 | msg['From'] = from_address 12 | msg['To'] = to_address 13 | msg['Subject'] = subject 14 | msg.set_content(body) 15 | msg.set_content(htmlBody, subtype='html') 16 | return msg 17 | 18 | def send_email(config, to_address, subject, body, htmlBody): 19 | mailserver = SMTP(config["SMTP_SERVER"], config["SMTP_PORT"]) 20 | mailserver.ehlo() 21 | mailserver.starttls() 22 | mailserver.login(config["SMTP_USERNAME"], config["SMTP_PASSWORD"]) 23 | 24 | message = create_email_message(config["SMTP_USERNAME"], to_address, subject, body, htmlBody) 25 | mailserver.send_message(message) 26 | 27 | mailserver.quit() 28 | 29 | def send_password_reset_email(config, targetEmail, targetName, token): 30 | 31 | subject = "[Eleuther Flask App] Reset Your Password" 32 | Thread(target=send_email, 33 | args=(config, targetEmail, subject, 34 | render_template('email/reset_password.txt', 35 | username=targetName, token=token), 36 | render_template('email/reset_password.html', 37 | username=targetName, token=token))).start() -------------------------------------------------------------------------------- /pyfra/contrib/web/forms.py: -------------------------------------------------------------------------------- 1 | from flask_wtf import FlaskForm 2 | from wtforms import StringField, PasswordField, BooleanField, SubmitField 3 | from wtforms.validators import DataRequired, Email, EqualTo 4 | 5 | class LoginForm(FlaskForm): 6 | name = StringField(validators=[DataRequired()]) 7 | password = PasswordField(validators=[DataRequired()]) 8 | remember_me = BooleanField() 9 | submit = SubmitField() 10 | 11 | class ResetPasswordRequestForm(FlaskForm): 12 | email = StringField('Email', validators=[DataRequired(), Email()]) 13 | submit = SubmitField('Request Password Reset') 14 | 15 | class ResetPasswordForm(FlaskForm): 16 | password = PasswordField('New Password', validators=[DataRequired()]) 17 | password2 = PasswordField( 18 | 'Repeat New Password', validators=[DataRequired(), EqualTo('password')]) 19 | submit = SubmitField('Change Password') 20 | -------------------------------------------------------------------------------- /pyfra/contrib/web/migrations/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. -------------------------------------------------------------------------------- /pyfra/contrib/web/migrations/alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | script_location = . 5 | # template used to generate migration files 6 | # file_template = %%(rev)s_%%(slug)s 7 | 8 | # set to 'true' to run the environment during 9 | # the 'revision' command, regardless of autogenerate 10 | # revision_environment = false 11 | 12 | 13 | # Logging configuration 14 | [loggers] 15 | keys = root,sqlalchemy,alembic,flask_migrate 16 | 17 | [handlers] 18 | keys = console 19 | 20 | [formatters] 21 | keys = generic 22 | 23 | [logger_root] 24 | level = WARN 25 | handlers = console 26 | qualname = 27 | 28 | [logger_sqlalchemy] 29 | level = WARN 30 | handlers = 31 | qualname = sqlalchemy.engine 32 | 33 | [logger_alembic] 34 | level = INFO 35 | handlers = 36 | qualname = alembic 37 | 38 | [logger_flask_migrate] 39 | level = INFO 40 | handlers = 41 | qualname = flask_migrate 42 | 43 | [handler_console] 44 | class = StreamHandler 45 | args = (sys.stderr,) 46 | level = NOTSET 47 | formatter = generic 48 | 49 | [formatter_generic] 50 | format = %(levelname)-5.5s [%(name)s] %(message)s 51 | datefmt = %H:%M:%S 52 | -------------------------------------------------------------------------------- /pyfra/contrib/web/migrations/env.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | 3 | import logging 4 | from logging.config import fileConfig 5 | 6 | from flask import current_app 7 | 8 | from alembic import context 9 | 10 | # this is the Alembic Config object, which provides 11 | # access to the values within the .ini file in use. 12 | config = context.config 13 | 14 | # Interpret the config file for Python logging. 15 | # This line sets up loggers basically. 16 | fileConfig(config.config_file_name) 17 | logger = logging.getLogger('alembic.env') 18 | 19 | # add your model's MetaData object here 20 | # for 'autogenerate' support 21 | # from myapp import mymodel 22 | # target_metadata = mymodel.Base.metadata 23 | config.set_main_option( 24 | 'sqlalchemy.url', 25 | str(current_app.extensions['migrate'].db.engine.url).replace('%', '%%')) 26 | target_metadata = current_app.extensions['migrate'].db.metadata 27 | 28 | # other values from the config, defined by the needs of env.py, 29 | # can be acquired: 30 | # my_important_option = config.get_main_option("my_important_option") 31 | # ... etc. 32 | 33 | 34 | def run_migrations_offline(): 35 | """Run migrations in 'offline' mode. 36 | 37 | This configures the context with just a URL 38 | and not an Engine, though an Engine is acceptable 39 | here as well. By skipping the Engine creation 40 | we don't even need a DBAPI to be available. 41 | 42 | Calls to context.execute() here emit the given string to the 43 | script output. 44 | 45 | """ 46 | url = config.get_main_option("sqlalchemy.url") 47 | context.configure( 48 | url=url, target_metadata=target_metadata, literal_binds=True 49 | ) 50 | 51 | with context.begin_transaction(): 52 | context.run_migrations() 53 | 54 | 55 | def run_migrations_online(): 56 | """Run migrations in 'online' mode. 57 | 58 | In this scenario we need to create an Engine 59 | and associate a connection with the context. 60 | 61 | """ 62 | 63 | # this callback is used to prevent an auto-migration from being generated 64 | # when there are no changes to the schema 65 | # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html 66 | def process_revision_directives(context, revision, directives): 67 | if getattr(config.cmd_opts, 'autogenerate', False): 68 | script = directives[0] 69 | if script.upgrade_ops.is_empty(): 70 | directives[:] = [] 71 | logger.info('No changes in schema detected.') 72 | 73 | connectable = current_app.extensions['migrate'].db.engine 74 | 75 | with connectable.connect() as connection: 76 | context.configure( 77 | connection=connection, 78 | target_metadata=target_metadata, 79 | process_revision_directives=process_revision_directives, 80 | **current_app.extensions['migrate'].configure_args 81 | ) 82 | 83 | with context.begin_transaction(): 84 | context.run_migrations() 85 | 86 | 87 | if context.is_offline_mode(): 88 | run_migrations_offline() 89 | else: 90 | run_migrations_online() 91 | -------------------------------------------------------------------------------- /pyfra/contrib/web/migrations/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade(): 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade(): 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /pyfra/contrib/web/migrations/versions/1582bc5883ac_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 1582bc5883ac 4 | Revises: 5 | Create Date: 2021-04-19 15:07:44.848437 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '1582bc5883ac' 14 | down_revision = None 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('user', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('name', sa.String(length=64), nullable=True), 24 | sa.Column('email', sa.String(length=120), nullable=True), 25 | sa.Column('password_hash', sa.String(length=128), nullable=True), 26 | sa.Column('roles', sa.String(length=512), nullable=True), 27 | sa.PrimaryKeyConstraint('id') 28 | ) 29 | op.create_index(op.f('ix_user_email'), 'user', ['email'], unique=False) 30 | op.create_index(op.f('ix_user_name'), 'user', ['name'], unique=True) 31 | # ### end Alembic commands ### 32 | 33 | 34 | def downgrade(): 35 | # ### commands auto generated by Alembic - please adjust! ### 36 | op.drop_index(op.f('ix_user_name'), table_name='user') 37 | op.drop_index(op.f('ix_user_email'), table_name='user') 38 | op.drop_table('user') 39 | # ### end Alembic commands ### 40 | -------------------------------------------------------------------------------- /pyfra/contrib/web/migrations/versions/1b1532ba9506_add_user_attributes.py: -------------------------------------------------------------------------------- 1 | """Add user attributes 2 | 3 | Revision ID: 1b1532ba9506 4 | Revises: 1582bc5883ac 5 | Create Date: 2021-04-22 18:42:10.038438 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '1b1532ba9506' 14 | down_revision = '1582bc5883ac' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('user', sa.Column('attributes', sa.String(length=512), nullable=False, server_default="{}")) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('user', 'attributes') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /pyfra/contrib/web/models.py: -------------------------------------------------------------------------------- 1 | from .app import db 2 | from .app import login 3 | 4 | from werkzeug.security import generate_password_hash, check_password_hash 5 | 6 | from flask_login import UserMixin 7 | 8 | from time import time 9 | import jwt 10 | import json 11 | from .app import app 12 | 13 | class User(db.Model, UserMixin): 14 | __tablename__ = 'user' 15 | id = db.Column(db.Integer, primary_key=True) 16 | name = db.Column(db.String(64), index=True, unique=True) 17 | email = db.Column(db.String(120), index=True) 18 | password_hash = db.Column(db.String(128)) 19 | roles = db.Column(db.String(512), default="") 20 | attributes = db.Column(db.String(512), default="{}", nullable=False) 21 | 22 | def __repr__(self): 23 | return self.name 24 | 25 | def set_password(self, password): 26 | self.password_hash = generate_password_hash(password) 27 | 28 | def check_password(self, password): 29 | return check_password_hash(self.password_hash, password) 30 | 31 | def get_reset_password_token(self, expires_in=600): 32 | return jwt.encode( 33 | {'reset_password': self.id, 'exp': time() + expires_in}, 34 | app.config['SECRET_KEY'], algorithm='HS256') #.decode('utf-8') 35 | 36 | @staticmethod 37 | def verify_reset_password_token(token): 38 | try: 39 | id = jwt.decode(token, app.config['SECRET_KEY'], 40 | algorithms=['HS256'])['reset_password'] 41 | except: 42 | return 43 | return User.query.get(id) 44 | 45 | def get_roles(self): 46 | return list(set(["everyone"] + (self.roles if self.roles is not None else "").lower().split(','))) 47 | 48 | def get_attr(self, k, default=None): 49 | return json.loads(self.attributes).get(k, default) 50 | 51 | def set_attr(self, k, val): 52 | ob = json.loads(self.attributes) 53 | ob[k] = val 54 | self.attributes = json.dumps(ob) 55 | 56 | db.session.commit() 57 | 58 | @staticmethod 59 | def get(username): 60 | return User.query.filter_by(name=username).first() 61 | 62 | @staticmethod 63 | def all(): 64 | return User.query.all() 65 | 66 | @login.user_loader 67 | def load_user(id): 68 | return User.query.get(int(id)) 69 | 70 | -------------------------------------------------------------------------------- /pyfra/contrib/web/requirements.txt: -------------------------------------------------------------------------------- 1 | flask 2 | flask-login 3 | flask-wtf 4 | flask-sqlalchemy 5 | flask-migrate 6 | flask-admin 7 | flask-bootstrap 8 | pyjwt 9 | sqlalchemy 10 | wtforms[email] 11 | -------------------------------------------------------------------------------- /pyfra/contrib/web/server.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, request, render_template, render_template_string, json, jsonify, Response, flash, redirect, url_for, send_file 2 | 3 | from werkzeug.urls import url_parse 4 | 5 | from flask_login import current_user, login_user, login_required, logout_user 6 | 7 | from .forms import LoginForm 8 | from .forms import ResetPasswordRequestForm 9 | from .forms import ResetPasswordForm 10 | 11 | from .emailer import send_password_reset_email 12 | 13 | from .app import app 14 | from .app import db 15 | from .app import admin 16 | 17 | from .models import User 18 | 19 | from flask_admin.contrib.sqla import ModelView 20 | 21 | import collections 22 | import pathlib 23 | 24 | class PageRegistry: 25 | def __init__(self): 26 | self.registry = collections.defaultdict(list) 27 | self.i = 0 28 | 29 | def add_page(self, name, pretty_name, allowed_roles): 30 | for role in allowed_roles: 31 | self.registry[role].append((self.i, name, pretty_name)) 32 | self.i += 1 33 | 34 | def get_pages(self, roles): 35 | ret = [] 36 | for role in roles: 37 | for page in self.registry[role]: 38 | if page not in ret: ret.append(page) 39 | 40 | return [(n,p) for _,n,p in sorted(ret)] 41 | 42 | registry = PageRegistry() 43 | 44 | 45 | def add_user(name, email, password, roles=[]): 46 | newUser = User(email=email, name=name, roles=roles) 47 | 48 | newUser.set_password(password) 49 | 50 | db.session.add(newUser) 51 | db.session.commit() 52 | 53 | 54 | # Index and 404 55 | # =================================================================== 56 | @app.route('/') 57 | @login_required 58 | def index(): 59 | roles = current_user.get_roles() 60 | pages = registry.get_pages(roles) 61 | return render_template('index.html', pages=pages) 62 | 63 | @app.errorhandler(404) 64 | def page_not_found(e): 65 | return render_template('404.html'), 404 66 | 67 | # WTForms stuff 68 | # =================================================================== 69 | template_path = pathlib.Path(__file__).parent.absolute() / 'templates' 70 | 71 | registry.add_page("admin/user", "Admin Dashboard", ["admin"]) 72 | registry.add_page("change_password", "Change Password", ["everyone"]) 73 | 74 | def register_page(name, pretty_name, form_class, callback, allowed_roles, redirect_index, has_form): 75 | if pretty_name is None: pretty_name = name 76 | 77 | registry.add_page(name, pretty_name, allowed_roles) 78 | 79 | @login_required 80 | def _fn(): 81 | 82 | is_authorized = any([ 83 | role in current_user.get_roles() 84 | for role in allowed_roles 85 | ]) 86 | 87 | if not is_authorized: 88 | flash("You are not authorized to view this page.") 89 | return redirect(url_for('index')) 90 | 91 | if has_form: 92 | form = form_class()() 93 | 94 | html = "" 95 | if form.validate_on_submit(): 96 | data = { 97 | field.name: field.data 98 | for field in form 99 | if field.name not in ["submit", "csrf_token"] 100 | } 101 | 102 | ret = callback(data) 103 | html = ret if ret is not None else "" 104 | 105 | if redirect_index: 106 | flash("Form successfully submitted") 107 | return redirect(url_for('index')) 108 | else: 109 | html = callback({}) 110 | form = None 111 | 112 | return render_template_string(open(template_path / 'form_template.html').read(), body=html, form=form, title=pretty_name, has_form=has_form) 113 | 114 | app.add_url_rule(f"/{name}", name, _fn, methods=['GET', 'POST']) 115 | 116 | 117 | # Authentication Stuff 118 | # =================================================================== 119 | @app.route('/change_password/') 120 | @login_required 121 | def change_password(): 122 | token = current_user.get_reset_password_token().decode() 123 | return redirect(f'/reset_password/{token}') 124 | 125 | @app.route('/reset_password/', methods=['GET', 'POST']) 126 | def reset_password(token): 127 | user = User.verify_reset_password_token(token) 128 | if not user: 129 | return redirect(url_for('index')) 130 | form = ResetPasswordForm() 131 | if form.validate_on_submit(): 132 | user.set_password(form.password.data) 133 | db.session.commit() 134 | flash('Your password has been reset.') 135 | return redirect(url_for('login')) 136 | return render_template('reset_password.html', form=form) 137 | 138 | @app.route('/forgot-password', methods=['GET', 'POST']) 139 | def resetPassword(): 140 | if current_user.is_authenticated: 141 | return redirect(url_for('index')) 142 | 143 | form = ResetPasswordRequestForm() 144 | if form.validate_on_submit(): 145 | user = User.query.filter_by(email=form.email.data).first() 146 | if user: 147 | token = user.get_reset_password_token() 148 | send_password_reset_email(app.config, user.email, user.name, token) 149 | flash('Check your email for the instructions to reset your password') 150 | return redirect(url_for('login')) 151 | return render_template('forgot_password.html', 152 | title='Reset Password', form=form) 153 | 154 | @app.route('/login', methods=['GET', 'POST']) 155 | def login(): 156 | if current_user.is_authenticated: 157 | return redirect(url_for('index')) 158 | 159 | form = LoginForm() 160 | if form.validate_on_submit(): 161 | user = User.query.filter_by(name=form.name.data).first() 162 | if user is None or not user.check_password(form.password.data): 163 | flash('Invalid username or password') 164 | return redirect(url_for('login')) 165 | login_user(user, remember=form.remember_me.data) 166 | next_page = request.args.get('next') 167 | if not next_page or url_parse(next_page).netloc != '': 168 | next_page = url_for('index') 169 | return redirect(next_page) 170 | 171 | return render_template('login.html', title='Sign In', form=form) 172 | 173 | @app.route('/logout') 174 | def logout(): 175 | logout_user() 176 | return redirect(url_for('login')) 177 | 178 | 179 | # Admin Section 180 | # =================================================================== 181 | class UserAdminView(ModelView): 182 | create_modal = True 183 | edit_modal = True 184 | can_export = True 185 | 186 | form_columns = ('name', 'email', "roles") 187 | column_exclude_list = ["password_hash"] 188 | column_searchable_list = ['name', 'email', "roles"] 189 | column_filters = ['name', 'email', "roles"] 190 | 191 | def is_accessible(self): 192 | return "admin" in current_user.get_roles() 193 | 194 | def inaccessible_callback(self, name, **kwargs): 195 | flash("You need to be an admin to view this page.") 196 | return redirect(url_for('index')) 197 | 198 | admin.add_view(UserAdminView(User, db.session)) 199 | -------------------------------------------------------------------------------- /pyfra/contrib/web/static/.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | indent_size = 2 8 | indent_style = space 9 | insert_final_newline = true 10 | trim_trailing_whitespace = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /pyfra/contrib/web/static/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) HTML5 Boilerplate 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 7 | of the Software, and to permit persons to whom the Software is furnished to do 8 | so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /pyfra/contrib/web/static/browserconfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /pyfra/contrib/web/static/css/main.css: -------------------------------------------------------------------------------- 1 | @media screen { 2 | @import url("https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,300;0,400;0,700;0,900;1,300;1,700;1,900&display=swap"); 3 | 4 | html, 5 | body { 6 | font-family: "Lato", sans-serif; 7 | width: 100%; 8 | height: 100%; 9 | font-size: 16px; 10 | } 11 | :root { 12 | --white: #ffffff; 13 | --black: #000000; 14 | --green: #699f56; 15 | --blue: #34566f; 16 | --light-blue: #00bbff; 17 | --grey: #959595; 18 | --grey2: #d8d8d8; 19 | --red: #aa4926; 20 | --orange: #cc7832; 21 | } 22 | 23 | /* Main Panel - To Show Full Background*/ 24 | /* ============================================================= */ 25 | .main { 26 | height: 100%; 27 | width: 100%; 28 | } 29 | 30 | .main:after { 31 | content: ""; 32 | display: block; 33 | position: fixed; 34 | top: 0; 35 | left: 0; 36 | background-image: url("/static/img/background.jpg"); 37 | background-size: 100% 100%; 38 | width: 100%; 39 | height: 100%; 40 | opacity: 0.3; 41 | z-index: -1; 42 | background-repeat: no-repeat; 43 | background-position: center; 44 | background-attachment: fixed; 45 | } 46 | 47 | /* Header containing logo and buttons*/ 48 | /* ============================================================= */ 49 | .header-container { 50 | height: 3.75em; 51 | background-color: var(--black); 52 | padding-top: 0.625em; 53 | } 54 | 55 | .header-container .content-wrapper { 56 | flex-direction: row; 57 | } 58 | 59 | header .logo-image { 60 | width: 40px; 61 | height: 40px; 62 | } 63 | 64 | .content-wrapper { 65 | margin: 0 auto; 66 | padding: 0 0.5em; 67 | width: 60em; 68 | overflow-y: hidden; 69 | } 70 | 71 | .row { 72 | display: flex; 73 | } 74 | 75 | .header-left { 76 | display: flex; 77 | } 78 | .header-right { 79 | margin-left: auto; 80 | text-align: center; 81 | } 82 | 83 | h1.logo-text { 84 | font-size: 1.4em; 85 | font-weight: 400; 86 | text-align: left; 87 | margin: 0; 88 | padding: 0; 89 | border: 0px none; 90 | position: relative; 91 | left: 0.75em; 92 | top: 0.55em; 93 | } 94 | 95 | h1.logo-text a { 96 | text-decoration: none; 97 | color: var(--white); 98 | } 99 | 100 | .header-right a { 101 | text-decoration: none; 102 | color: var(--white); 103 | } 104 | 105 | .header-right img { 106 | width: 20px; 107 | height: 20px; 108 | } 109 | 110 | span.logout-link { 111 | font-size: 0.7em; 112 | text-transform: uppercase; 113 | display: block; 114 | position: relative; 115 | bottom: 0px; 116 | } 117 | 118 | header .header-buttons { 119 | display: flex; 120 | } 121 | 122 | .buttonBlock { 123 | display: flex; 124 | flex-direction: column; 125 | align-items: center; 126 | margin-top: 0.375em; 127 | } 128 | 129 | .buttonBlock:last-child { 130 | margin-left: 1.25em; 131 | } 132 | 133 | /* Area to flash messages to user*/ 134 | /* ============================================================= */ 135 | .flashes { 136 | border-top: 1px solid silver; 137 | padding: 10px; 138 | box-shadow: 0px 2px 5px -2px rgba(0, 0, 0, 0.6); 139 | background: white; 140 | display: flex; 141 | flex-direction: row; 142 | justify-content: space-between; 143 | align-items: center; 144 | z-index: 1; 145 | color: green; 146 | } 147 | 148 | /* Full Page With Panel with 20px margin to show pretty background 149 | /* ============================================================= */ 150 | .fullPage { 151 | flex-grow: 1; 152 | flex-shrink: 1; 153 | margin: 20px; 154 | 155 | border: 1px solid silver; 156 | padding: 20px; 157 | box-shadow: 0px 0px 5px rgba(0, 0, 0, 0.3); 158 | background: white; 159 | display: flex; 160 | flex-direction: column; 161 | align-items: stretch; 162 | } 163 | 164 | /* Full Page With Panel with 20px margin to show pretty background 165 | /* ============================================================= */ 166 | .page1024 { 167 | margin: 0 auto; 168 | margin-top: 20px; 169 | border: 1px solid silver; 170 | padding: 20px; 171 | box-shadow: 0px 0px 5px rgba(0, 0, 0, 0.3); 172 | background: var(--white); 173 | width: 100%; 174 | overflow-y: hidden; 175 | } 176 | 177 | /* Global styles*/ 178 | /* ============================================================= */ 179 | h1 { 180 | font-weight: 700; 181 | font-size: 28px; 182 | margin-bottom: 20px; 183 | text-align: center; 184 | border-bottom: 1px solid silver; 185 | padding-bottom: 10px; 186 | } 187 | 188 | /* Login Form */ 189 | /* ============================================================= */ 190 | 191 | .login-form-panel { 192 | width: 350px; 193 | margin: 0 auto; 194 | margin-top: 50px; 195 | 196 | border: 1px solid silver; 197 | padding: 30px; 198 | padding-top: 25px; 199 | padding-bottom: 10px; 200 | box-shadow: 0px 0px 5px rgba(0, 0, 0, 0.3); 201 | background-color: var(--white); 202 | } 203 | 204 | .login-form-panel h1 { 205 | font-weight: 700; 206 | font-size: 28px; 207 | margin-bottom: 20px; 208 | text-align: center; 209 | border-bottom: 1px solid silver; 210 | padding-bottom: 10px; 211 | } 212 | 213 | .login-form-link { 214 | color: var(--blue); 215 | margin: 1em 0em; 216 | display: block; 217 | } 218 | 219 | .login-form-panel label { 220 | display: inline-block; 221 | margin-bottom: 5px; 222 | font-weight: 600; 223 | } 224 | 225 | .login-form-panel .inputBlock { 226 | margin-bottom: 15px; 227 | } 228 | 229 | .login-form-panel .labelLine { 230 | display: flex; 231 | flex-direction: row; 232 | justify-content: space-between; 233 | } 234 | 235 | .login-form-panel input[type="text"], 236 | input[type="password"] { 237 | width: 100%; 238 | font-size: 16px; 239 | padding: 5px; 240 | font-weight: 400; 241 | } 242 | 243 | .login-form-panel input[type="submit"] { 244 | width: 100%; 245 | padding: 10px 0px; 246 | margin: 0px; 247 | margin-left: -1px; 248 | margin-bottom: 10px; 249 | border: 0px none; 250 | border-radius: 10px; 251 | background-color: var(--blue); 252 | font-size: 1em; 253 | cursor: pointer; 254 | color: var(--white); 255 | } 256 | 257 | .login-form-panel input[type="submit"]:hover { 258 | filter: brightness(85%); 259 | } 260 | 261 | .login-form-panel input[type="checkbox"] { 262 | padding: 0px; 263 | margin: 0px; 264 | } 265 | 266 | .remember-me { 267 | display: flex; 268 | flex-direction: row; 269 | align-items: center; 270 | margin-bottom: 15px; 271 | } 272 | 273 | .login-form label[for="remember_me"] { 274 | font-size: 14px; 275 | margin: 0px; 276 | margin-left: 5px; 277 | font-weight: 400; 278 | } 279 | 280 | .login-form-panel a { 281 | display: block; 282 | margin-bottom: 10px; 283 | font-size: 14px; 284 | margin-top: 12px; 285 | } 286 | 287 | form .error { 288 | color: red; 289 | font-size: 14px; 290 | } 291 | 292 | /* Dashboard page */ 293 | /* ============================================================= */ 294 | 295 | .dashboard-list { 296 | width: 21em; 297 | margin: 0 auto; 298 | } 299 | 300 | .dashboard-list li { 301 | margin-bottom: 1.25em; 302 | } 303 | 304 | .dashboard-list li a { 305 | padding: 0.4em 0.875em 0.8em 1.625em; 306 | background-color: var(--blue); 307 | color: var(--white); 308 | display: block; 309 | text-decoration: none; 310 | border-radius: 0.6em; 311 | box-sizing: border-box; 312 | } 313 | 314 | .dashboard-list li a > img { 315 | position: relative; 316 | top: 0.25em; 317 | right: 0.8em; 318 | } 319 | 320 | .dashboard-list li a:hover { 321 | filter: brightness(85%); 322 | } 323 | 324 | .back-button { 325 | margin-bottom: 1em; 326 | } 327 | 328 | .form-section .form-group { 329 | margin-bottom: 1.3em; 330 | } 331 | 332 | .form-section .form-group input { 333 | display: block; 334 | margin: 0.7em 0; 335 | border: 1px solid var(--grey); 336 | padding: 0.5em 1em; 337 | } 338 | 339 | .form-section .checkbox { 340 | margin-bottom: 1.3em; 341 | } 342 | 343 | .form-section .btn.btn-success { 344 | padding: 0.75em 1.375em; 345 | background-color: var(--blue); 346 | color: var(--white); 347 | display: block; 348 | text-decoration: none; 349 | border-radius: 0.6em; 350 | box-shadow: none; 351 | border: 0px none; 352 | cursor: pointer; 353 | font-size: 1em; 354 | } 355 | 356 | .form-body-content { 357 | overflow-x: scroll; 358 | overflow-y: hidden; 359 | } 360 | 361 | .form-section .btn.btn-success:hover { 362 | filter: brightness(85%); 363 | } 364 | 365 | .back-button { 366 | margin-bottom: 1em; 367 | } 368 | 369 | .back-button a { 370 | padding: 8px 0px; 371 | color: var(--blue); 372 | text-decoration: none; 373 | font-size: 1em; 374 | font-style: italic; 375 | } 376 | 377 | .back-button a:hover { 378 | opacity: 0.8; 379 | } 380 | 381 | /* Responsive styles */ 382 | /* ============================================================= */ 383 | 384 | /* Tablet */ 385 | @media (max-width: 960px) { 386 | .content-wrapper { 387 | max-width: 40em; 388 | flex-direction: row; 389 | width: 100%; 390 | } 391 | .dashboard-list { 392 | width: 100%; 393 | } 394 | } 395 | 396 | /* Desktop-small */ 397 | @media (min-width: 1260px) { 398 | .content-wrapper { 399 | max-width: 1260px; 400 | width: 100%; 401 | } 402 | .dashboard-list { 403 | width: 25em; 404 | } 405 | } 406 | 407 | /* Desktop-large */ 408 | @media (min-width: 1600px) { 409 | .content-wrapper { 410 | max-width: 1600px; 411 | width: 100%; 412 | } 413 | .dashboard-list { 414 | width: 28em; 415 | } 416 | } 417 | } 418 | -------------------------------------------------------------------------------- /pyfra/contrib/web/static/css/reset.css: -------------------------------------------------------------------------------- 1 | /* http://meyerweb.com/eric/tools/css/reset/ 2 | v2.0 | 20110126 3 | License: none (public domain) 4 | */ 5 | 6 | html, body, div, span, applet, object, iframe, 7 | h1, h2, h3, h4, h5, h6, p, blockquote, pre, 8 | a, abbr, acronym, address, big, cite, code, 9 | del, dfn, em, img, ins, kbd, q, s, samp, 10 | small, strike, strong, sub, sup, tt, var, 11 | b, u, i, center, 12 | dl, dt, dd, ol, ul, li, 13 | fieldset, form, label, legend, 14 | table, caption, tbody, tfoot, thead, tr, th, td, 15 | article, aside, canvas, details, embed, 16 | figure, figcaption, footer, header, hgroup, 17 | menu, nav, output, ruby, section, summary, 18 | time, mark, audio, video { 19 | margin: 0; 20 | padding: 0; 21 | border: 0; 22 | font-size: 100%; 23 | /* font: inherit; */ 24 | vertical-align: baseline; 25 | } 26 | /* HTML5 display-role reset for older browsers */ 27 | article, aside, details, figcaption, figure, 28 | footer, header, hgroup, menu, nav, section { 29 | display: block; 30 | } 31 | body { 32 | line-height: 1; 33 | } 34 | ol, ul { 35 | list-style: none; 36 | } 37 | blockquote, q { 38 | quotes: none; 39 | } 40 | blockquote:before, blockquote:after, 41 | q:before, q:after { 42 | content: ''; 43 | content: none; 44 | } 45 | table { 46 | border-collapse: collapse; 47 | border-spacing: 0; 48 | } 49 | 50 | *, *:before, *:after { 51 | box-sizing: border-box; 52 | } -------------------------------------------------------------------------------- /pyfra/contrib/web/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/pyfra/ae193a03732b419a7176639cac43d5103f724995/pyfra/contrib/web/static/favicon.ico -------------------------------------------------------------------------------- /pyfra/contrib/web/static/humans.txt: -------------------------------------------------------------------------------- 1 | # humanstxt.org/ 2 | # The humans responsible & technology colophon 3 | 4 | # TEAM 5 | 6 | -- -- 7 | 8 | # THANKS 9 | 10 | 11 | 12 | # TECHNOLOGY COLOPHON 13 | 14 | CSS3, HTML5 15 | Apache Server Configs, jQuery, Modernizr, Normalize.css 16 | -------------------------------------------------------------------------------- /pyfra/contrib/web/static/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/pyfra/ae193a03732b419a7176639cac43d5103f724995/pyfra/contrib/web/static/icon.png -------------------------------------------------------------------------------- /pyfra/contrib/web/static/img/background.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/pyfra/ae193a03732b419a7176639cac43d5103f724995/pyfra/contrib/web/static/img/background.jpg -------------------------------------------------------------------------------- /pyfra/contrib/web/static/img/eai_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/pyfra/ae193a03732b419a7176639cac43d5103f724995/pyfra/contrib/web/static/img/eai_logo.png -------------------------------------------------------------------------------- /pyfra/contrib/web/static/img/excel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/pyfra/ae193a03732b419a7176639cac43d5103f724995/pyfra/contrib/web/static/img/excel.png -------------------------------------------------------------------------------- /pyfra/contrib/web/static/img/icon-button-adduser.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /pyfra/contrib/web/static/img/icon-button-admin.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /pyfra/contrib/web/static/img/icon-button-admin/user.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /pyfra/contrib/web/static/img/icon-button-change_password.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /pyfra/contrib/web/static/img/icon-button-demo_form.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /pyfra/contrib/web/static/img/logout.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /pyfra/contrib/web/static/img/print_button.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/pyfra/ae193a03732b419a7176639cac43d5103f724995/pyfra/contrib/web/static/img/print_button.jpg -------------------------------------------------------------------------------- /pyfra/contrib/web/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 |

Hello world! This is HTML5 Boilerplate.

26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /pyfra/contrib/web/static/js/main.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EleutherAI/pyfra/ae193a03732b419a7176639cac43d5103f724995/pyfra/contrib/web/static/js/main.js -------------------------------------------------------------------------------- /pyfra/contrib/web/static/js/plugins.js: -------------------------------------------------------------------------------- 1 | // Avoid `console` errors in browsers that lack a console. 2 | (function() { 3 | var method; 4 | var noop = function () {}; 5 | var methods = [ 6 | 'assert', 'clear', 'count', 'debug', 'dir', 'dirxml', 'error', 7 | 'exception', 'group', 'groupCollapsed', 'groupEnd', 'info', 'log', 8 | 'markTimeline', 'profile', 'profileEnd', 'table', 'time', 'timeEnd', 9 | 'timeline', 'timelineEnd', 'timeStamp', 'trace', 'warn' 10 | ]; 11 | var length = methods.length; 12 | var console = (window.console = window.console || {}); 13 | 14 | while (length--) { 15 | method = methods[length]; 16 | 17 | // Only stub undefined methods. 18 | if (!console[method]) { 19 | console[method] = noop; 20 | } 21 | } 22 | }()); 23 | 24 | // Place any jQuery/helper plugins in here. 25 | -------------------------------------------------------------------------------- /pyfra/contrib/web/static/js/vendor/modernizr-3.6.0.min.js: -------------------------------------------------------------------------------- 1 | /*! modernizr 3.6.0 (Custom Build) | MIT * 2 | * https://modernizr.com/download/?-cssanimations-csscolumns-customelements-flexbox-history-picture-pointerevents-postmessage-sizes-srcset-webgl-websockets-webworkers-addtest-domprefixes-hasevent-mq-prefixedcssvalue-prefixes-setclasses-testallprops-testprop-teststyles !*/ 3 | !function(e,t,n){function r(e,t){return typeof e===t}function o(){var e,t,n,o,i,s,a;for(var l in C)if(C.hasOwnProperty(l)){if(e=[],t=C[l],t.name&&(e.push(t.name.toLowerCase()),t.options&&t.options.aliases&&t.options.aliases.length))for(n=0;nd;d++)if(h=e[d],v=N.style[h],f(h,"-")&&(h=m(h)),N.style[h]!==n){if(i||r(o,"undefined"))return s(),"pfx"==t?h:!0;try{N.style[h]=o}catch(g){}if(N.style[h]!=v)return s(),"pfx"==t?h:!0}return s(),!1}function v(e,t){return function(){return e.apply(t,arguments)}}function A(e,t,n){var o;for(var i in e)if(e[i]in t)return n===!1?e[i]:(o=t[e[i]],r(o,"function")?v(o,n||t):o);return!1}function g(e,t,n,o,i){var s=e.charAt(0).toUpperCase()+e.slice(1),a=(e+" "+O.join(s+" ")+s).split(" ");return r(t,"string")||r(t,"undefined")?h(a,t,o,i):(a=(e+" "+T.join(s+" ")+s).split(" "),A(a,t,n))}function y(e,t,r){return g(e,n,n,t,r)}var C=[],b={_version:"3.6.0",_config:{classPrefix:"",enableClasses:!0,enableJSClass:!0,usePrefixes:!0},_q:[],on:function(e,t){var n=this;setTimeout(function(){t(n[e])},0)},addTest:function(e,t,n){C.push({name:e,fn:t,options:n})},addAsyncTest:function(e){C.push({name:null,fn:e})}},Modernizr=function(){};Modernizr.prototype=b,Modernizr=new Modernizr;var w=[],S=t.documentElement,x="svg"===S.nodeName.toLowerCase(),_="Moz O ms Webkit",T=b._config.usePrefixes?_.toLowerCase().split(" "):[];b._domPrefixes=T;var E=b._config.usePrefixes?" -webkit- -moz- -o- -ms- ".split(" "):["",""];b._prefixes=E;var P;!function(){var e={}.hasOwnProperty;P=r(e,"undefined")||r(e.call,"undefined")?function(e,t){return t in e&&r(e.constructor.prototype[t],"undefined")}:function(t,n){return e.call(t,n)}}(),b._l={},b.on=function(e,t){this._l[e]||(this._l[e]=[]),this._l[e].push(t),Modernizr.hasOwnProperty(e)&&setTimeout(function(){Modernizr._trigger(e,Modernizr[e])},0)},b._trigger=function(e,t){if(this._l[e]){var n=this._l[e];setTimeout(function(){var e,r;for(e=0;e 6 |

Page Not Found

7 |

Sorry, but the page you were trying to view does not exist.

8 | 9 | 10 | {% endblock %} -------------------------------------------------------------------------------- /pyfra/contrib/web/templates/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | {% if pageTitle %} 8 | {{ pageTitle }} 9 | {% else %} 10 | EleutherAI pyfra internal site 11 | {% endif %} 12 | 13 | 17 | 18 | 19 | 20 | 25 | 26 | 30 | 34 | 38 | 39 | 43 | 47 | 48 | {% block extra_libraries %}{% endblock %} 49 | 50 | 51 | 52 | 53 | 54 | 55 | {% block extra_styles %}{% endblock %} {% block extra_scripts %}{% endblock 56 | %} 57 | 58 | 59 | 60 |
61 | 68 | 69 |
70 |
71 |
72 | 73 | 78 | 79 |

EleutherAI

80 |
81 |
82 |
83 | {% block buttons %}{% endblock %} 84 |
85 | {% if current_user.is_authenticated %} 86 | 87 | 92 | Logout 93 | 94 | {% endif %} 95 |
96 |
97 |
98 |
99 |
100 |
101 | {% with messages = get_flashed_messages() %} {% if messages %} 102 |
    103 | {% for message in messages %} 104 |
  • {{ message }}
    • 105 | {% endfor %} 106 |
107 | {% endif %} {% endwith %} {% block content %}{% endblock %} 108 |
109 |
110 | 111 | 112 | -------------------------------------------------------------------------------- /pyfra/contrib/web/templates/email/reset_password.html: -------------------------------------------------------------------------------- 1 |

Dear {{ username }},

2 |

3 | To reset your password 4 | 5 | click here 6 | . 7 |

8 |

Alternatively, you can paste the following link in your browser's address bar:

9 |

{{ url_for('reset_password', token=token, _external=True) }}

10 |

If you have not requested a password reset simply ignore this message.

11 |

Sincerely,

12 |

Eleuther

-------------------------------------------------------------------------------- /pyfra/contrib/web/templates/email/reset_password.txt: -------------------------------------------------------------------------------- 1 | Dear {{ username }}, 2 | 3 | To reset your password click on the following link: 4 | 5 | {{ url_for('reset_password', token=token, _external=True) }} 6 | 7 | If you have not requested a password reset simply ignore this message. 8 | 9 | Sincerely, 10 | 11 | Eleuther -------------------------------------------------------------------------------- /pyfra/contrib/web/templates/forgot_password.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} {% block extra_libraries %} 2 | 7 | 13 | 14 | {% endblock %} {% block content %} 15 | 33 | 34 | 44 | {% endblock %} 45 | -------------------------------------------------------------------------------- /pyfra/contrib/web/templates/form_template.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} {% import "bootstrap/wtf.html" as wtf %} {% block 2 | extra_libraries %} 3 | 8 | {% endblock %} {% block content %} 9 | 10 |
11 |

{{title}}

12 |
13 | << Return to Dashboard 14 |
15 | {% if (has_form) %} 16 |
17 |
18 | {{ wtf.quick_form(form, button_map={'submit':'success'}) }} 19 |
20 |
21 | {% endif %} 22 |
23 | 24 |

{{ body|safe }}

25 |
26 | 27 | 34 | {% endblock %} 35 | -------------------------------------------------------------------------------- /pyfra/contrib/web/templates/index.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} {% block content %} 2 | 3 |
4 |

Dashboard

5 | 14 |
15 | 16 | {% endblock %} 17 | -------------------------------------------------------------------------------- /pyfra/contrib/web/templates/login.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} {% block extra_libraries %} 2 | 7 | 13 | 14 | {% endblock %} {% block content %} 15 | 44 | 45 | 57 | {% endblock %} 58 | -------------------------------------------------------------------------------- /pyfra/contrib/web/templates/polls.html: -------------------------------------------------------------------------------- 1 | Unused atm -------------------------------------------------------------------------------- /pyfra/contrib/web/templates/reset_password.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} {% block extra_libraries %} 2 | 7 | 13 | 14 | {% endblock %} {% block content %} 15 | 36 | 37 | 50 | 51 | {% endblock %} 52 | -------------------------------------------------------------------------------- /pyfra/contrib/web/templates/view_template.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} {% block extra_libraries %} 2 | 7 | {% endblock %} {% block content %} 8 | 9 |
10 |

{{title}}

11 |
12 | << Return to Dashboard 13 |
14 |

{{ body|safe }}

15 |
16 | 17 | {% endblock %} 18 | -------------------------------------------------------------------------------- /pyfra/delegation.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pathlib 3 | import sys 4 | from shlex import quote 5 | 6 | import pyfra.remote 7 | import pyfra.shell 8 | 9 | __all__ = ["delegate"] 10 | 11 | 12 | def delegate(experiment_name, rem, artifacts=[]): 13 | tmux_name = f"pyfra_delegated_{experiment_name}" 14 | if isinstance(rem, str): rem = pyfra.remote.Remote(rem) 15 | if isinstance(artifacts, str): artifacts = [artifacts] 16 | 17 | if is_delegated(): 18 | return 19 | def _attach_tmux(): 20 | rem.sh(f"tmux a -t {quote(tmux_name)}", maxbuflen=0, forward_keys=True) 21 | 22 | try: 23 | _attach_tmux() 24 | except pyfra.shell.ShellException: 25 | # todo: figure out how to forward ssh keys securely 26 | # the problem with ssh -A is that your remote is screwed if your original client goes offline, which totally 27 | # defeats the purpose of doing this in the first place. also there's something extremely weird going on 28 | # with tmux that makes it so that ssh forward won't carry over to the tmux session, but you can 29 | # fix that with `eval $(tmux show-env -s |grep '^SSH_')` inside the tmux, but that doesn't work with send-keys, it only works with 30 | # actually running it by hand in the tmux for some reason. I've tried running it multiple times using send-keys, 31 | # adding a delay, adding a dummy ssh 127.0.0.1 command in between just to get ssh to use the ssh-agent to auth, etc and it jsut won't work. 32 | # I'm not sure why, and to save my sanity for now I'm just going to require adding the right ssh keys to the delegated server manually. 33 | 34 | env = rem.env(tmux_name) 35 | 36 | ignore = [] if not pathlib.Path(".pyfraignore").exists() else pathlib.Path(".pyfraignore").read_text().strip().splitlines() 37 | env.sh(f"sudo apt install tmux -y; pip install -U git+https://github.com/EleutherAI/pyfra; pip install -r requirements.txt", ignore_errors=True) 38 | 39 | try: 40 | # allow sshing remote into itself 41 | rem_key = rem.path("~/.ssh/id_rsa.pub").read() 42 | if rem_key not in rem.path("~/.ssh/authorized_keys").read(): 43 | rem.path("~/.ssh/authorized_keys").write(rem_key, append=True) 44 | except pyfra.shell.ShellException: 45 | print("WARNING: couldn't add self-key to server") 46 | 47 | pyfra.shell.copy(pyfra.remote.local.path("."), env.path("."), into=False, exclude=ignore) 48 | 49 | with pyfra.remote.force_run(): 50 | env.sh(f"tmux new-session -d -s {quote(tmux_name)}") 51 | 52 | # cmd = f"{cmd} || ( eval $(tmux show-env -s |grep '^SSH_'); {cmd} )" 53 | cmd = f"pyenv shell {env.pyenv_version}" if env.pyenv_version is not None else "" 54 | cmd += f";[ -f env/bin/activate ] && . env/bin/activate; " 55 | cmd += f"PYFRA_DELEGATED=1 PYFRA_DELEGATED_TO={quote(rem.ip)} python "+" ".join([quote(x) for x in sys.argv]) 56 | 57 | with pyfra.remote.force_run(): 58 | env.sh(f"tmux send-keys -t {quote(tmux_name)} {quote(cmd)} Enter") 59 | 60 | _attach_tmux() 61 | 62 | if artifacts: print("Copying artifacts") 63 | for pattern in artifacts: 64 | for path in env.path(".").glob(pattern): 65 | if path.fname.endswith(".pyfra_env_state.json"): continue 66 | pyfra.shell.copy(path, pyfra.remote.local.path("."), exclude=ignore) 67 | 68 | sys.exit(0) 69 | 70 | 71 | def is_delegated(): 72 | return "PYFRA_DELEGATED" in os.environ -------------------------------------------------------------------------------- /pyfra/idempotent.py: -------------------------------------------------------------------------------- 1 | 2 | # EXPERIMENTAL 3 | 4 | from functools import partial, wraps 5 | import types 6 | from typing import Any, Callable, Dict, Type 7 | import pyfra.remote 8 | import abc 9 | import os 10 | import re 11 | import time 12 | import dataclasses 13 | 14 | try: 15 | import blobfile as bf 16 | except ImportError: 17 | pass 18 | 19 | import pickle 20 | import inspect 21 | 22 | 23 | class KVStoreProvider(abc.ABC): 24 | @abc.abstractmethod 25 | def get(self, key: str): 26 | """ 27 | Get the value for a key. 28 | """ 29 | pass 30 | 31 | @abc.abstractmethod 32 | def set(self, key: str, value): 33 | """ 34 | Set the value for a key. 35 | """ 36 | pass 37 | 38 | def cache(self, key=None): 39 | return cache(key, kvstore=self, _callstackoffset=3) 40 | 41 | 42 | class LocalKVStore(KVStoreProvider): 43 | def __init__(self): 44 | self.rem = pyfra.remote.Remote(wd=os.path.expanduser("~")) 45 | 46 | def get(self, key: str): 47 | return self.rem.get_kv(key) 48 | 49 | def set(self, key: str, value): 50 | self.rem.set_kv(key, value) 51 | 52 | 53 | class BlobfileKVStore(KVStoreProvider): 54 | def __init__(self, prefix): 55 | if prefix[-1] == "/": 56 | prefix = prefix[:-1] 57 | self.prefix = prefix 58 | 59 | def get(self, key: str): 60 | try: 61 | return pickle.load(bf.BlobFile(self.prefix + "/" + key, "rb")) 62 | except (FileNotFoundError, EOFError): 63 | raise KeyError(key) 64 | 65 | def set(self, key: str, value): 66 | with bf.BlobFile(self.prefix + "/" + key, "wb") as f: 67 | pickle.dump(value, f) 68 | 69 | 70 | default_kvstore = LocalKVStore() 71 | special_hashing: Dict[Type, Callable[[Any], str]] = {} 72 | 73 | # some pyfra special hashing stuff 74 | special_hashing[pyfra.remote.RemotePath] = lambda x: x.quick_hash() 75 | special_hashing[pyfra.remote.Remote] = lambda x: x.hash 76 | special_hashing[list] = lambda x: list(map(_prepare_for_hash, x)) 77 | special_hashing[dict] = lambda x: {_prepare_for_hash(k): _prepare_for_hash(v) for k, v in x.items()} 78 | special_hashing[tuple] = lambda x: tuple(map(_prepare_for_hash, x)) 79 | special_hashing[types.FunctionType] = lambda x: x.__name__ 80 | special_hashing[type] = lambda x: x.__name__ 81 | 82 | try: 83 | from pandas import DataFrame 84 | special_hashing[DataFrame] = lambda x: x.to_json() 85 | except ImportError: 86 | pass 87 | 88 | def set_kvstore(provider): 89 | global default_kvstore 90 | default_kvstore = provider 91 | 92 | 93 | def _prepare_for_hash(x): 94 | if dataclasses.is_dataclass(x): 95 | return dataclasses.asdict(x) 96 | 97 | for type_, fn in special_hashing.items(): 98 | if isinstance(x, type_): 99 | return fn(x) 100 | 101 | return x 102 | 103 | 104 | def update_source_cache(fname, lineno, new_key): 105 | with open(fname, "r") as f: 106 | file_lines = f.read().split("\n") 107 | 108 | # line numbering is 1-indexed 109 | lineno -= 1 110 | 111 | s = re.match(r"@((?:[^\W0-9]\w*\.)?)cache(\(\))?", file_lines[lineno].lstrip()) 112 | assert s, "@cache can only be used as a decorator!" 113 | leading_whitespace = re.match(r"^\s*", file_lines[lineno]).group(0) 114 | 115 | file_lines[lineno] = f"{leading_whitespace}@{s.group(1)}cache(\"{new_key}\")" 116 | 117 | with open(fname, "w") as f: 118 | f.write("\n".join(file_lines)) 119 | 120 | 121 | def cache(key=None, kvstore=None, *, _callstackoffset=2): 122 | def wrapper(fn, key, kvstore, _callstackoffset): 123 | # execution always gets here, before the function is called 124 | 125 | if key is None: 126 | key = pyfra.remote._hash_obs(fn.__module__, fn.__name__, inspect.getsource(fn))[:8] + "_v0" 127 | # the decorator part of the stack is always the same size because we only get here if key is None 128 | stack_original_function = inspect.stack()[_callstackoffset] 129 | update_source_cache(stack_original_function.filename, stack_original_function.lineno - 1, key) 130 | 131 | if kvstore is None: 132 | global default_kvstore 133 | kvstore = default_kvstore 134 | 135 | @wraps(fn) 136 | def _fn(*args, **kwargs): 137 | # execution gets here only after the function is called 138 | 139 | arg_hash = pyfra.remote._hash_obs( 140 | [_prepare_for_hash(i) for i in args], 141 | [(_prepare_for_hash(k), _prepare_for_hash(v)) for k, v in list(sorted(kwargs.items()))], 142 | ) 143 | 144 | kwargs.pop(kwargs.pop("_pyfra_nonce_kwarg", "v"), None) 145 | 146 | overall_input_hash = key + "_" + arg_hash 147 | 148 | try: 149 | ob = kvstore.get(overall_input_hash) 150 | ret = ob['ret'] 151 | original_awaitable = ob['awaitable'] 152 | original_was_coroutine = ob['iscoroutine'] 153 | current_is_coroutine = inspect.iscoroutinefunction(fn) 154 | 155 | ## ASYNC HANDLING, resume from file 156 | 157 | if original_was_coroutine and current_is_coroutine: 158 | return_awaitable = True # coroutine -> coroutine 159 | elif original_was_coroutine and not current_is_coroutine: 160 | return_awaitable = False # coroutine -> normal 161 | elif not original_was_coroutine and not original_awaitable and current_is_coroutine: 162 | return_awaitable = True # normal -> coroutine 163 | elif not original_was_coroutine and not original_awaitable and not current_is_coroutine: 164 | return_awaitable = False # normal -> normal 165 | elif not original_was_coroutine and original_awaitable and current_is_coroutine: 166 | return_awaitable = True # normal_returning_awaitable -> coroutine 167 | elif not original_was_coroutine and original_awaitable and not current_is_coroutine: 168 | # this case is ambiguous! we can't know if the modifier function returns an awaitable or not 169 | # without actually running the function, so we just assume it's an awaitable, 170 | # since probably nothing changed. 171 | return_awaitable = True # normal_returning_awaitable -> normal/normal_returning_awaitable 172 | else: 173 | return_awaitable = False # fallback - most likely this is a bug 174 | print(f"WARNING: unknown change in async situation for {fn._name__}") 175 | 176 | if return_awaitable: 177 | async def _wrapper(ret): 178 | # wrap ret in a dummy async function 179 | return ret 180 | 181 | return _wrapper(ret) 182 | else: 183 | return ret 184 | except KeyError: 185 | start_time = time.time() 186 | ret = fn(*args, **kwargs) 187 | end_time = time.time() 188 | 189 | ## ASYNC HANDLING, first run 190 | if inspect.isawaitable(ret): 191 | # WARNING: using the same env across multiple async stages is not supported, and not possible to support! 192 | # TODO: detect if two async stages are using the same env and throw an error if so 193 | 194 | async def _wrapper(ret): 195 | # turn the original async function into a synchronous one and return a new async function 196 | ret = await ret 197 | kvstore.set(overall_input_hash, { 198 | "ret": ret, 199 | "awaitable": True, 200 | "iscoroutine": inspect.iscoroutinefunction(fn), 201 | "start_time": start_time, 202 | "end_time": end_time, 203 | }) 204 | return ret 205 | 206 | return _wrapper(ret) 207 | else: 208 | kvstore.set(overall_input_hash, { 209 | "ret": ret, 210 | "awaitable": False, 211 | "iscoroutine": inspect.iscoroutinefunction(fn), 212 | "start_time": start_time, 213 | "end_time": end_time, 214 | }) 215 | return ret 216 | return _fn 217 | 218 | if callable(key): 219 | return wrapper(key, None, kvstore=kvstore, _callstackoffset=_callstackoffset) 220 | 221 | return partial(wrapper, key=key, kvstore=kvstore, _callstackoffset=_callstackoffset) 222 | 223 | 224 | # TODO: make it so Envs and cached Remotes cannot be used in both global and cached fn 225 | # TODO: make sure Envs/Remotes serialize and deserialize properly 226 | -------------------------------------------------------------------------------- /pyfra/remote.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import base64 4 | import csv 5 | import hashlib 6 | import io 7 | import json 8 | import os 9 | import pathlib 10 | import pickle 11 | import random 12 | import uuid 13 | import inspect 14 | import asyncio 15 | from contextlib import contextmanager 16 | from functools import wraps 17 | from typing import * 18 | 19 | import imohash 20 | from colorama import Style 21 | from natsort import natsorted 22 | from yaspin import yaspin 23 | 24 | import pyfra.shell 25 | from pyfra.setup import install_pyenv 26 | 27 | from deprecation import deprecated 28 | 29 | sentinel = object() 30 | 31 | __all__ = [ 32 | "Remote", 33 | "RemotePath", 34 | "Env", 35 | "stage", 36 | "always_rerun", 37 | "local", 38 | ] 39 | 40 | def _normalize_homedir(x): 41 | """ Essentially expanduser(path.join("~", x)) but remote-agnostic """ 42 | if x[:2] == './': 43 | x = x[2:] 44 | if x[-2:] == '/.': 45 | x = x[:-2] 46 | 47 | x = x.replace('/./', '/') 48 | 49 | if '~/' in x: 50 | x = x.split('~/')[-1] 51 | 52 | if x[-2:] == '/~': 53 | return '~' 54 | 55 | # some special cases 56 | if x == '': return '~' 57 | if x == '.': return '~' 58 | if x == '~': return '~' 59 | if x == '/': return '/' 60 | 61 | if x[0] != '/' and x[:2] != '~/': 62 | x = '~/' + x 63 | 64 | if x[-1] == '/' and len(x) > 1: x = x[:-1] 65 | 66 | return x 67 | 68 | 69 | class _ObjectEncoder(json.JSONEncoder): 70 | def default(self, obj): 71 | if isinstance(obj, pyfra.remote.RemotePath): 72 | return obj.sha256sum() 73 | if isinstance(obj, (asyncio.Lock, asyncio.Event, asyncio.Condition, asyncio.Semaphore, asyncio.BoundedSemaphore)): 74 | # don't do anything with asyncio objects 75 | return None 76 | if hasattr(obj, "_to_json"): 77 | return obj._to_json() 78 | 79 | return super().default(obj) 80 | 81 | 82 | def _hash_obs(*args): 83 | jsonobj = json.dumps(args, sort_keys=True, cls=_ObjectEncoder) 84 | arghash = hashlib.sha256(jsonobj.encode()).hexdigest() 85 | return arghash 86 | 87 | 88 | def _print_skip_msg(envname, fn, hash): 89 | print(f"{Style.BRIGHT}[{envname.ljust(15)} {Style.DIM}§{Style.RESET_ALL}{Style.BRIGHT}{fn.rjust(10)}]{Style.RESET_ALL} Skipping {hash}") 90 | 91 | 92 | def _git_hash_key(fn, self, git, branch, python_version): 93 | origin_branch_hash = None 94 | 95 | if git is not None: 96 | # check hash of remote to see if we need to clone 97 | with self.no_hash(): 98 | try: 99 | origin_branch_hash = self.sh(f"git ls-remote https://github.com/gturri/dokuJClient.git refs/heads/{branch} | awk '{{ print $1}}'", quiet=True).strip() 100 | except pyfra.shell.ShellException: 101 | pass 102 | 103 | return (fn.__name__, origin_branch_hash, python_version) 104 | 105 | 106 | def _mutates_state(hash_key=None): 107 | """ 108 | Decorator that marks a function as mutating the state of the underlying environment. 109 | """ 110 | def _f(fn): 111 | @wraps(fn) 112 | def wrapper(self, *args, **kwargs): 113 | if self._no_hash: return fn(self, *args, **kwargs) 114 | new_hash = self.update_hash(fn.__name__, *args, **kwargs) if hash_key is None else self.update_hash(*hash_key(fn, self, *args, **kwargs)) 115 | try: 116 | # if globally we want to ignore hashes, we force a keyerror to run the function again 117 | if global_env_registry.always_rerun: raise KeyError 118 | 119 | # if hash is in the state, then we can just return that 120 | ret = self.get_kv(new_hash) 121 | _print_skip_msg(self.envname, fn.__name__, new_hash) 122 | 123 | return ret 124 | except KeyError: 125 | try: 126 | # otherwise, we need to run the function and save the result 127 | ret = fn(self, *args, **kwargs) 128 | self.set_kv(new_hash, ret) 129 | return ret 130 | except Exception as e: # this prevents the KeyError ending up in the stacktrace 131 | raise e from None 132 | return wrapper 133 | return _f 134 | 135 | 136 | @contextmanager 137 | def always_rerun(): 138 | """ 139 | Use as a context manager to force all Envs to ignore cached results. 140 | Also forces stages to run. 141 | """ 142 | if global_env_registry.always_rerun: 143 | yield 144 | else: 145 | global_env_registry.always_rerun = True 146 | try: 147 | yield 148 | finally: 149 | global_env_registry.always_rerun = False 150 | 151 | # remote stuff 152 | 153 | # global cache 154 | _remotepath_cache = {} 155 | _remotepath_modified_time = {} 156 | def _cache(fn): 157 | """ 158 | Use as an annotation. Caches the response of the function and 159 | check modification time on the file on every call. 160 | """ 161 | @wraps(fn) 162 | def wrapper(self, *args, **kwargs): 163 | modified_time = self.stat().st_mtime 164 | hash = _hash_obs(fn.__name__, args, kwargs) 165 | if hash not in _remotepath_cache or modified_time != _remotepath_modified_time[hash]: 166 | ret = fn(self, *args, **kwargs) 167 | _remotepath_cache[(self.remote.ip, self.fname, hash)] = ret 168 | _remotepath_modified_time[(self.remote.ip, self.fname, hash)] = modified_time 169 | return ret 170 | else: 171 | return _remotepath_cache[hash] 172 | return wrapper 173 | 174 | 175 | class RemotePath: 176 | """ 177 | A RemotePath represents a path somewhere on some Remote. The RemotePath object can be used to manipulate the file. 178 | 179 | Example usage: :: 180 | 181 | # write text 182 | rem.path("goose.txt").write("honk") 183 | 184 | # read text 185 | print(rem.path("goose.txt").read()) 186 | 187 | # write json 188 | rem.path("goose.json").jwrite({"honk": 1}) 189 | 190 | # read json 191 | print(rem.path("goose.json").jread()) 192 | 193 | # write csv 194 | rem.path("goose.csv").csvwrite([{"col1": 1, "col2": "duck"}, {"col1": 2, "col2": "goose"}]) 195 | 196 | # read csv 197 | print(rem.path("goose.csv").csvread()) 198 | 199 | # copy stuff to/from remotes 200 | copy(rem1.path('goose.txt'), 'test1.txt') 201 | copy('test1.txt', rem2.path('goose.txt')) 202 | copy(rem2.path('goose.txt'), rem1.path('testing123.txt')) 203 | """ 204 | def __init__(self, remote, fname): 205 | if remote is None: remote = local 206 | 207 | self.remote = remote 208 | self.fname = fname 209 | 210 | def rsyncstr(self) -> str: 211 | return f"{self.remote.ip}:{self.fname}" if self.remote is not None and self.remote.ip is not None else self.fname 212 | 213 | def _to_json(self) -> Dict[str, str]: 214 | return { 215 | 'remote': self.remote.ip if self.remote is not None else None, 216 | 'fname': self.fname, 217 | } 218 | 219 | def __repr__(self) -> str: 220 | return f"RemotePath({json.dumps(self._to_json())})" 221 | 222 | def _set_cache(self, fn_name, value, *args, **kwargs): 223 | modified_time = self.stat().st_mtime 224 | hash = _hash_obs(fn_name, args, kwargs) 225 | _remotepath_modified_time[(self.remote.ip, self.fname, hash)] = modified_time 226 | _remotepath_cache[(self.remote.ip, self.fname, hash)] = value 227 | 228 | def read(self) -> str: 229 | """ 230 | Read the contents of this file into a string 231 | """ 232 | if self.remote.is_local(): 233 | with open(os.path.expanduser(self.fname)) as fh: 234 | return fh.read() 235 | else: 236 | # TODO: replace with paramiko 237 | nonce = random.randint(0, 99999) 238 | pyfra.shell.copy(self, f".tmp.{nonce}", quiet=True) 239 | with open(f".tmp.{nonce}") as fh: 240 | ret = fh.read() 241 | pyfra.shell.rm(f".tmp.{nonce}") 242 | return ret 243 | 244 | def write(self, content, append=False) -> str: 245 | """ 246 | Write text to this file. 247 | 248 | Args: 249 | content (str): The text to write 250 | append (bool): Whether to append or overwrite the file contents 251 | 252 | """ 253 | self.remote.fwrite(self.fname, content, append) 254 | 255 | def jread(self) -> Dict[str, Any]: 256 | """ 257 | Read the contents of this json file and parses it. Equivalent to :code:`json.loads(self.read())` 258 | """ 259 | return json.loads(self.read()) 260 | 261 | def jwrite(self, content): 262 | """ 263 | Write a json object to this file. Equivalent to :code:`self.write(json.dumps(content))` 264 | 265 | Args: 266 | content (json): The json object to write 267 | 268 | """ 269 | self.write(json.dumps(content)) 270 | 271 | def csvread(self, colnames=None) -> List[dict]: 272 | """ 273 | Read the contents of this csv file and parses it into an array of dictionaries where the keys are column names. 274 | 275 | Args: 276 | colnames (list): Optionally specify the names of the columns for csvs without a header row. 277 | """ 278 | fh = io.StringIO(self.read()) 279 | if self.fname[-4:] == ".tsv": 280 | rdr = csv.reader(fh, delimiter="\t") 281 | else: 282 | rdr = csv.reader(fh) 283 | 284 | if colnames: 285 | cols = colnames 286 | else: 287 | cols = list(next(rdr)) 288 | 289 | for ob in rdr: 290 | yield { 291 | k: v for k, v in zip(cols, [*ob, *[None for _ in range(len(cols) - len(ob))]]) 292 | } 293 | 294 | def csvwrite(self, data, colnames=None): 295 | """ 296 | Write a list of dicts object to this csv file. 297 | 298 | Args: 299 | content (List[dict]): A list of dicts where the keys are column names. Every dicts should have the exact same keys. 300 | 301 | """ 302 | fh = io.StringIO() 303 | if colnames is None: 304 | colnames = data[0].keys() 305 | 306 | wtr = csv.writer(fh) 307 | wtr.writerow(colnames) 308 | 309 | for dat in data: 310 | assert dat.keys() == colnames 311 | 312 | wtr.writerow([dat[k] for k in colnames]) 313 | 314 | fh.seek(0) 315 | self.write(fh.read()) 316 | 317 | def _remote_payload(self, name, *args, **kwargs): 318 | """ 319 | Run an arbitrary Path.* function remotely and return the result. 320 | Restricted to os rather than arbitrary eval for security reasons. 321 | """ 322 | assert all(x in "_.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" for x in name) 323 | 324 | if self.remote.is_local(): 325 | fn = pathlib.Path(self.fname).expanduser() 326 | for k in name.split("."): 327 | fn = getattr(fn, k) 328 | return fn(*args, **kwargs) 329 | else: 330 | payload = f"import pathlib,json; print(json.dumps(pathlib.Path({repr(self.fname)}).expanduser().{name}(*{args}, **{kwargs})))" 331 | ret = self.remote.sh(f"python -c {payload | pyfra.shell.quote}", quiet=True, no_venv=True, pyenv_version=None) 332 | return json.loads(ret) 333 | 334 | def stat(self) -> os.stat_result: 335 | """ 336 | Stat a remote file 337 | """ 338 | with self.remote.no_hash(): 339 | return os.stat_result(self._remote_payload("stat")) 340 | 341 | def exists(self) -> bool: 342 | """ 343 | Check if this file exists 344 | """ 345 | try: 346 | with self.remote.no_hash(): 347 | return self._remote_payload("exists") 348 | except pyfra.shell.ShellException: 349 | # if we can't connect to the remote, the file does not exist 350 | return False 351 | 352 | def is_dir(self) -> bool: 353 | """ 354 | Check if this file exists 355 | """ 356 | with self.remote.no_hash(): 357 | return self._remote_payload("is_dir") 358 | 359 | def unlink(self) -> None: 360 | """ 361 | Delete this file 362 | """ 363 | self._remote_payload("unlink") 364 | 365 | def glob(self, pattern: str) -> List[RemotePath]: 366 | """ 367 | Find all files matching the glob pattern. 368 | """ 369 | def _glob_remote_payload(*args, **kwargs): 370 | if self.remote.is_local(): 371 | return pathlib.Path(self.fname).expanduser().glob(*args, **kwargs) 372 | else: 373 | with self.remote.no_hash(): 374 | payload = f"import pathlib,json; print(json.dumps([str(f) for f in pathlib.Path({repr(self.fname)}).expanduser().glob(*{args}, **{kwargs})]))" 375 | ret = self.remote.sh(f"python -c {payload | pyfra.shell.quote}", quiet=True, no_venv=True, pyenv_version=None) 376 | return json.loads(ret) 377 | 378 | return [RemotePath(self.remote, str(f)) for f in _glob_remote_payload(pattern)] 379 | 380 | def expanduser(self) -> RemotePath: 381 | """ 382 | Return a copy of this path with the home directory expanded. 383 | """ 384 | if self.remote.is_local(): 385 | return RemotePath(None, os.path.expanduser(self.fname)) 386 | else: 387 | homedir = self.remote.home() 388 | 389 | # todo: be more careful with this replace 390 | return RemotePath(self.remote, os.path.expanduser(self.fname).replace("~", homedir)) 391 | 392 | def sh(self, cmd, *args, **kwargs): 393 | try: 394 | return self.remote.sh(f"cd {pyfra.shell.quote(self.expanduser().fname)}; "+cmd, *args, **kwargs) 395 | except pyfra.shell.ShellException as e: # this makes the stacktrace easier to read 396 | raise pyfra.shell.ShellException(e.returncode, rem=not self.remote.is_local()) from e.__cause__ 397 | 398 | def __div__(self, other): 399 | return RemotePath(self.remote, os.path.join(self.fname, other)) 400 | 401 | @_cache 402 | def sha256sum(self) -> str: 403 | """ 404 | Return the sha256sum of this file. 405 | """ 406 | with self.remote.no_hash(): 407 | return self.remote.sh(f"sha256sum {self.fname}", quiet=True).split(" ")[0] 408 | 409 | @_cache 410 | def quick_hash(self) -> str: 411 | """ 412 | Get a hash of this file that catches file changes most of the time 413 | by hashing blocks from the file at th beginning, middle, and end. 414 | Really useful for getting a quick hash of a really big file, but obviously 415 | unsuitable for guaranteeing file integrity. 416 | 417 | Uses imohash under the hood. 418 | """ 419 | if self.remote.is_local(): 420 | return pyfra.shell.quick_hash(self.fname) 421 | else: 422 | # TODO: use paramiko 423 | # TODO: make faster by not trying to install every time 424 | payload = f""" 425 | import json,os,pathlib 426 | import pyfra.shell 427 | 428 | print(pyfra.shell.quick_hash(pathlib.Path(os.path.expanduser({repr(self.fname)})))) 429 | """.strip() 430 | 431 | with self.remote.no_hash(): 432 | ret = self.remote.sh(f"[ -f ~/.pyfra_imohash ] || ( python -m pip --help > /dev/null 2>&1 || sudo apt-get install python3-pip -y > /dev/null 2>&1; python -m pip install imohash 'pyfra>=0.3.0rc5' > /dev/null 2>&1; touch ~/.pyfra_imohash ); python -c {payload | pyfra.shell.quote}", no_venv=True, pyenv_version=None, quiet=True).strip() 433 | 434 | assert all(x in "0123456789abcdef" for x in ret[:32]) 435 | return ret[:32] 436 | 437 | 438 | class Remote: 439 | def __init__(self, ip=None, wd=None, experiment=None, resumable=False, additional_ssh_config=""): 440 | """ 441 | Args: 442 | ip (str): The host to ssh to. This looks something like :code:`12.34.56.78` or :code:`goose.com` or :code:`someuser@12.34.56.78` or :code:`someuser@goose.com`. You must enable passwordless ssh and have your ssh key added to the server first. If None, the Remote represents localhost. 443 | wd (str): The working directory on the server to start out on. 444 | python_version (str): The version of python to use (i.e running :code:`Remote("goose.com", python_version="3.8.10").sh("python --version")` will use python 3.8.10). If this version is not already installed, pyfra will install it. 445 | resumable (bool): If True, this Remote will resume where it left off, with the same semantics as Env. 446 | """ 447 | if ip in ["127.0.0.1", "localhost"]: ip = None 448 | 449 | if "PYFRA_DELEGATED_TO" in os.environ and ip == os.environ["PYFRA_DELEGATED_TO"]: ip = None 450 | 451 | self.ip = ip 452 | self.wd = _normalize_homedir(wd) if wd is not None else "~" 453 | self.experiment = experiment 454 | self.resumable = resumable 455 | 456 | self._home = None 457 | self._no_hash = not resumable 458 | self._kv_cache = None 459 | 460 | self.hash = self._hash(None) 461 | self.envname = "" 462 | 463 | self.additional_ssh_config = additional_ssh_config 464 | 465 | if resumable: 466 | global_env_registry.register(self) 467 | 468 | def env(self, envname, git=None, branch=None, force_rerun=False, python_version="3.9.4") -> Remote: 469 | """ 470 | Arguments are the same as the :class:`pyfra.experiment.Experiment` constructor. 471 | """ 472 | 473 | return Env(ip=self.ip, envname=envname, git=git, branch=branch, force_rerun=force_rerun, python_version=python_version, additional_ssh_config=self.additional_ssh_config) 474 | 475 | @_mutates_state() 476 | def sh(self, x, quiet=False, wrap=True, maxbuflen=1000000000, ignore_errors=False, no_venv=False, pyenv_version=None, forward_keys=False): 477 | """ 478 | Run a series of bash commands on this remote. This command shares the same arguments as :func:`pyfra.shell.sh`. 479 | """ 480 | try: 481 | if self.ip is None: 482 | return pyfra.shell.sh(x, quiet=quiet, wd=self.wd, wrap=wrap, maxbuflen=maxbuflen, ignore_errors=ignore_errors, no_venv=no_venv, pyenv_version=pyenv_version) 483 | else: 484 | return pyfra.shell._rsh(self.ip, x, quiet=quiet, wd=self.wd, wrap=wrap, maxbuflen=maxbuflen, ignore_errors=ignore_errors, no_venv=no_venv, pyenv_version=pyenv_version, forward_keys=forward_keys, additional_ssh_config=self.additional_ssh_config) 485 | except pyfra.shell.ShellException as e: # this makes the stacktrace easier to read 486 | raise pyfra.shell.ShellException(e.returncode, rem=not self.is_local()) from e.__cause__ 487 | 488 | def path(self, fname=None) -> RemotePath: 489 | """ 490 | This is the main way to make a :class:`RemotePath` object; see RemotePath docs for more info on what they're used for. 491 | 492 | If fname is not specified, this command allocates a temporary path 493 | """ 494 | if fname is None: 495 | return self.path(f"pyfra_tmp_{uuid.uuid4().hex}") 496 | 497 | if isinstance(fname, RemotePath): 498 | assert fname.remote == self 499 | return fname 500 | 501 | return RemotePath(self, _normalize_homedir(os.path.join(self.wd, fname) if self.wd is not None else fname)) 502 | 503 | def __repr__(self): 504 | return (self.ip if self.ip is not None else "127.0.0.1") + ":" + self.wd 505 | 506 | def fingerprint(self) -> str: 507 | """ 508 | A unique string for the server that this Remote is pointing to. Useful for detecting 509 | if the server has been yanked under you, or if this different ip actually points 510 | to the same server, etc. 511 | """ 512 | self.sh("if [ ! -f ~/.pyfra.fingerprint ]; then cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1 > ~/.pyfra.fingerprint; fi", quiet=True) 513 | tmpname = ".fingerprint." + str(random.randint(0, 99999)) 514 | pyfra.shell.copy(self.path("~/.pyfra.fingerprint"), tmpname) 515 | ret = pyfra.shell.fread(tmpname) 516 | pyfra.shell.rm(tmpname) 517 | return ret.strip() 518 | 519 | def _to_json(self): 520 | return { 521 | 'ip': self.ip, 522 | 'wd': self.wd, 523 | } 524 | 525 | def ls(self, x='.') -> List[str]: 526 | """ 527 | Lists files, sorted by natsort. 528 | 529 | Args: 530 | x (str): The directory to list. Defaults to current directory. 531 | """ 532 | return list(natsorted(self.sh(f"ls {x} | cat").strip().split("\n"))) 533 | 534 | def rm(self, x, no_exists_ok=True): 535 | """ 536 | Remove a file or directory. 537 | """ 538 | self.sh(f"cd ~; rm -rf {self.path(x).fname}", ignore_errors=no_exists_ok) 539 | 540 | def home(self) -> str: 541 | """ 542 | The home directory on the remote. 543 | """ 544 | if self._home is None: 545 | self._home = self.sh("echo $HOME", quiet=True).strip() 546 | 547 | return self._home 548 | 549 | def glob(self, pattern: str) -> List[RemotePath]: 550 | """ 551 | Find all files matching the glob pattern. 552 | """ 553 | return self.path(".").glob(pattern) 554 | 555 | def _fwrite(self, fname, content, append=False) -> None: 556 | """ 557 | :meta private: 558 | """ 559 | if self.ip is None: 560 | with open(os.path.expanduser(fname), 'a' if append else 'w') as fh: 561 | fh.write(content) 562 | else: 563 | nonce = random.randint(0, 99999) 564 | with open(f".tmp.{nonce}", 'w') as fh: 565 | fh.write(content) 566 | if append: 567 | pyfra.shell.copy(f".tmp.{nonce}", self.path(f".tmp.{nonce}"), quiet=True) 568 | self.sh(f"cat .tmp.{nonce} >> {fname} && rm .tmp.{nonce}", quiet=True) 569 | else: 570 | pyfra.shell.copy(f".tmp.{nonce}", self.path(fname), quiet=True) 571 | pyfra.shell.rm(f".tmp.{nonce}") 572 | 573 | def fwrite(self, fname, content, append=False) -> None: 574 | """ 575 | :meta private: 576 | """ 577 | # wraps fwrite to make it keep track of state hashes 578 | # TODO: replace with paramiko 579 | # TODO: extract this statehash code and the one in shell.copy to a common function or something 580 | needs_set_kv = False 581 | if not self._no_hash: 582 | assert fname.startswith(self.wd) 583 | fname_suffix = fname[len(self.wd):] 584 | new_hash = self.update_hash("fwrite", fname_suffix, content, append) 585 | try: 586 | self.get_kv(new_hash) 587 | _print_skip_msg(self.envname, "fwrite", new_hash) 588 | return 589 | except KeyError: 590 | needs_set_kv = True 591 | 592 | with self.no_hash(): 593 | self._fwrite(fname, content, append) 594 | 595 | if needs_set_kv: 596 | self.set_kv(new_hash, None) 597 | 598 | # key-value store for convienence 599 | 600 | def set_kv(self, key: str, value: Any) -> None: 601 | """ 602 | A key value store to keep track of stuff in this env. The data is stored in the env 603 | on the remote. Current implementation is inefficient (linear time, and the entire json 604 | file is moved each time) but there shouldn't be a lot of data stored in it anyways 605 | (premature optimization is bad), and if we need to store a lot of data in the future 606 | we can always make this more efficient without changing the interface. 607 | 608 | :meta private: 609 | """ 610 | with self.no_hash(): 611 | # TODO: make more efficient 612 | statefile = self.path(".pyfra_env_state.json") 613 | if self._kv_cache is None: 614 | if statefile.exists(): 615 | ob = statefile.jread() 616 | else: 617 | ob = {} 618 | self._kv_cache = ob 619 | pickled_value = base64.b64encode(pickle.dumps(value)).decode() 620 | self._kv_cache[key] = pickled_value 621 | 622 | # for backwards compat if we ever change the encoding format 623 | self._kv_cache[key + "_format"] = "b64" 624 | statefile.jwrite(self._kv_cache) 625 | 626 | def get_kv(self, key: str) -> Any: 627 | """ 628 | Retrieve a value from the state file. 629 | 630 | :meta private: 631 | """ 632 | with self.no_hash(): 633 | # TODO: make more efficient 634 | if self._kv_cache is None: 635 | statefile = self.path(".pyfra_env_state.json") 636 | if statefile.exists(): 637 | ob = statefile.jread() 638 | else: 639 | ob = {} 640 | self._kv_cache = ob 641 | 642 | # check storage format for backwards compat 643 | if key + "_format" not in self._kv_cache: 644 | return pickle.loads(self._kv_cache[key].encode()) 645 | elif self._kv_cache[key + "_format"] == "b64": 646 | return pickle.loads(base64.b64decode(self._kv_cache[key].encode())) 647 | 648 | def update_hash(self, *args, **kwargs) -> str: 649 | """ 650 | :meta private: 651 | """ 652 | self.hash = self._hash(self.hash, *args, **kwargs) 653 | return self.hash 654 | 655 | @contextmanager 656 | def no_hash(self): 657 | """ 658 | Context manager to turn off hashing temporarily. 659 | Example usage: :: 660 | print(env.hash) 661 | with env.no_hash(): 662 | env.sh("echo do something") 663 | print(env.hash) # will be the same as before 664 | """ 665 | if self._no_hash: 666 | yield 667 | return 668 | 669 | self._no_hash = True 670 | try: 671 | yield 672 | finally: 673 | self._no_hash = False 674 | 675 | def is_local(self): 676 | """ 677 | Returns true if this is a local remote/environment. 678 | """ 679 | return self.ip is None 680 | 681 | def __hash__(self): 682 | # this hash is not the Env hash, which represents the state inside the Env, 683 | # but rather is supposed to represent the particular location this 684 | # Remote/Env is pointing to. 685 | return hash((self.ip, self.wd)) 686 | 687 | @classmethod 688 | def _hash(cls, *args, **kwargs) -> str: 689 | return _hash_obs([args, kwargs]) 690 | 691 | # env 692 | class Env(Remote): 693 | """ 694 | An environment is a Remote pointing to a directory that has a virtualenv and a specific version version of python installed, optionally initialized from a git repo. Since environments are also just Remotes, all methods on Remotes work on environments too. 695 | 696 | A typical design pattern sees functions accepting remotes as argument and immediately turning it into an env that's used for the rest of the function. Alternatively, functions can take in already-created envs and perform some task inside the env. 697 | 698 | See :class:`pyfra.remote.Remote` for more information about methods. Envs can be created from an existing Remote using :meth:`pyfra.remote.Remote.env`. 699 | 700 | Example usage: :: 701 | 702 | def train_model(rem, ...): 703 | e = rem.env("neo_experiment", "https://github.com/EleutherAI/gpt-neo", python_version="3.8.10") 704 | e.sh("do something") 705 | e.sh("do something else") 706 | f = some_other_thing(e, ...) 707 | e.path("goose.txt").write(f.jread()["honk"]) 708 | 709 | def some_other_thing(env, ...): 710 | env.sh("do something") 711 | env.sh("do something else") 712 | 713 | return env.path("output.json") 714 | 715 | Args: 716 | ip (str): The host to ssh to. This looks something like :code:`12.34.56.78` or :code:`goose.com` or :code:`someuser@12.34.56.78` or :code:`someuser@goose.com`. You must enable passwordless ssh and have your ssh key added to the server first. If None, the Remote represents localhost. 717 | git (str): The git repo to clone into the fresh env. If None, no git repo is cloned. 718 | branch (str): The git branch to clone. If None, the default branch is used. 719 | force_rerun (bool): If True, all hashing will be disabled and everything will be run every time. Deprecated in favor of `with pyfra.always_rerun()` 720 | python_version (str): The python version to use. 721 | """ 722 | def __init__(self, ip=None, envname=None, git=None, branch=None, force_rerun=False, python_version="3.9.4", additional_ssh_config=""): 723 | self.wd = f"~/pyfra_envs/{envname}" 724 | super().__init__(ip, self.wd, resumable=True, additional_ssh_config=additional_ssh_config) 725 | self.pyenv_version = python_version 726 | 727 | self.envname = envname 728 | 729 | if force_rerun: # deprecated 730 | self.path(".pyfra_env_state.json").unlink() 731 | 732 | self._init_env(git, branch, python_version) 733 | 734 | @_mutates_state(hash_key=_git_hash_key) 735 | def _init_env(self, git, branch, python_version) -> None: 736 | with yaspin(text="Loading", color="white") as spinner, self.no_hash(): 737 | ip = self.ip if self.ip is not None else "localhost" 738 | wd = self.wd 739 | 740 | spinner.text = f"[{ip}:{wd}] Installing python in env" 741 | # install python/pyenv 742 | with spinner.hidden(): 743 | self._install(python_version) 744 | 745 | self.sh(f"mkdir -p {wd}", no_venv=True, quiet=True) 746 | 747 | # pull git 748 | if git is not None: 749 | spinner.text = f"[{ip}:{wd}] Cloning from git repo" 750 | # TODO: make this usable 751 | nonce = str(random.randint(0, 99999)) 752 | 753 | if branch is None: 754 | branch_cmds = "" 755 | else: 756 | branch_cmds = f"git checkout {branch}; git pull origin {branch}; " 757 | 758 | self.sh(f"{{ rm -rf ~/.tmp_git_repo.{nonce} ; git clone {git} ~/.tmp_git_repo.{nonce} ; rsync -ar --delete ~/.tmp_git_repo.{nonce}/ {wd}/ ; rm -rf ~/.tmp_git_repo.{nonce} ; cd {wd}; {branch_cmds} }}", ignore_errors=True, quiet=True) 759 | 760 | # install venv 761 | if wd is not None: 762 | spinner.text = f"[{ip}:{wd}] Creating virtualenv" 763 | pyenv_cmds = f"[ -d env/lib/python{python_version.rsplit('.')[0]} ] || rm -rf env ; python --version ; pyenv shell {python_version} ; python --version;" if python_version is not None else "" 764 | self.sh(f"mkdir -p {wd}; cd {wd}; {pyenv_cmds} [ -f env/bin/activate ] || python -m virtualenv env || ( python -m pip install virtualenv; python -m virtualenv env )", no_venv=True, quiet=True) 765 | spinner.text = f"[{ip}:{wd}] Installing requirements" 766 | self.sh("pip install -e . ; pip install -r requirements.txt", ignore_errors=True, quiet=True) 767 | 768 | spinner.text = f"[{ip}:{wd}] Env created" 769 | spinner.color = "green" 770 | spinner.ok("OK ") 771 | 772 | @_mutates_state() 773 | def sh(self, x, quiet=False, wrap=True, maxbuflen=1000000000, ignore_errors=False, no_venv=False, pyenv_version=sentinel, forward_keys=False): 774 | """ 775 | Run a series of bash commands on this remote. This command shares the same arguments as :func:`pyfra.shell.sh`. 776 | :meta private: 777 | """ 778 | 779 | try: 780 | return super().sh(x, quiet=quiet, wrap=wrap, maxbuflen=maxbuflen, ignore_errors=ignore_errors, no_venv=no_venv, pyenv_version=pyenv_version if pyenv_version is not sentinel else self.pyenv_version, forward_keys=forward_keys) 781 | except pyfra.shell.ShellException as e: # this makes the stacktrace easier to read 782 | raise pyfra.shell.ShellException(e.returncode, rem=not self.is_local()) from e.__cause__ 783 | 784 | def _install(self, python_version) -> None: 785 | # install sudo if it's not installed; this is the case in some docker containers 786 | self.sh("sudo echo hi || { apt-get update; apt-get install sudo; }", pyenv_version=None, ignore_errors=True, quiet=True) 787 | 788 | # set up remote python version 789 | if python_version is not None: install_pyenv(self, python_version) 790 | 791 | # install rsync for copying files 792 | self.sh("rsync --help > /dev/null || ( sudo apt-get update && sudo apt-get install -y rsync )", quiet=True) 793 | 794 | def _to_json(self) -> dict: 795 | return { 796 | 'ip': self.ip, 797 | 'wd': self.wd, 798 | 'pyenv_version': self.pyenv_version, 799 | } 800 | 801 | 802 | @deprecated(details="Will be replaced by pyfra.idempotent eventually") 803 | def stage(fn): 804 | """ 805 | This decorator is used to mark a function as a "stage". 806 | 807 | The purpose of this stage abstraction is for cases where you have some 808 | collection of operations that accomplish some goal and the way this goal 809 | is accomplished is intended to be abstracted away. Some examples would be 810 | tokenization, model training, or evaluation. After a stage runs once, the 811 | return value will be cached and subsequent calls with the same arguments 812 | will return the cached value. 813 | 814 | However, there are several subtleties to the usage of stages. First, you 815 | might be wondering why we need this if Env already resumes where it left 816 | off. The main reason behind this is that since the way a stage accomplises 817 | its goal is meant to be abstracted away, it is possible that the stage will 818 | have changed in implementation, thus invalidating the hash (for example, 819 | the stage is switched to use a more efficient tokenizer that outputs the 820 | same thing). In these cases, just using Env hashing would rerun everything 821 | even when we know we don't need to. Also, any other expensive operations 822 | that are not Env operations will still run every time. Finally, this 823 | decorator correctly handles setting all the env hashes to what they should 824 | be after the stage runs, whereas using some other generic function caching 825 | would not. 826 | 827 | Example usage: :: 828 | 829 | @stage 830 | def train_model(rem, ...): 831 | e = rem.env("neo_experiment", "https://github.com/EleutherAI/gpt-neo", python_version="3.8.10") 832 | e.sh("do something") 833 | e.sh("do something else") 834 | f = some_other_thing(e, ...) 835 | return e.path("checkpoint") 836 | 837 | train_model(rem) 838 | """ 839 | @wraps(fn) 840 | def wrapper(*args, **kwargs): 841 | # get all Envs in args and kwargs 842 | envs = [(i, x) for i, x in enumerate(args) if isinstance(x, Env)] + \ 843 | [(k, kwargs[k]) for k in sorted(kwargs.keys()) if isinstance(kwargs[k], Env)] 844 | inp_hashes = { 845 | k: v.hash for k, v in envs 846 | } 847 | 848 | def _prepare_for_hash(ind): 849 | # we want to handle Envs and RemotePaths specially: 850 | # for Envs, we only care about the Env hash 851 | # for RemotePaths, we only car about the quick_hash 852 | if ind in inp_hashes: 853 | return inp_hashes[ind] 854 | elif isinstance(ind, int) and isinstance(args[ind], RemotePath): 855 | return args[ind].quick_hash() 856 | elif isinstance(ind, str) and isinstance(kwargs[ind], RemotePath): 857 | return kwargs[ind].quick_hash() 858 | elif isinstance(ind, int): # normal arg type 859 | return args[ind] 860 | elif isinstance(ind, str): 861 | return kwargs[ind] 862 | else: 863 | raise Exception(f"Unknown ind type: {type(ind)}") 864 | 865 | # get a hash of all the inputs, except Env objects are counted as their hashes rather than the actual ip and wd 866 | overall_input_hash = _hash_obs( 867 | [_prepare_for_hash(i) for i in range(len(args))], 868 | [_prepare_for_hash(k) for k in sorted(kwargs.keys())], 869 | ) 870 | 871 | global_hashes_before = global_env_registry.hashes_by_env() 872 | 873 | try: 874 | # todo: detect RemotePaths in return value and substitute if broken 875 | # todo: handle Env objects that change the server they're on 876 | 877 | # the following different cases of envs passed are possible: 878 | # - created in block and not returned: 879 | # as long as the Env is never 880 | # independently created again later, we don't need to do anything 881 | # special to track it; if it is created again, we need some kind 882 | # of global tracking to tell it where to resume to 883 | # - created in block and returned 884 | # comes for free because we save the return values. there is 885 | # the problem that if the original Env disappears we might need to 886 | # rerun it, but we can figure that out later 887 | # - passed from outside block and not returned 888 | # we Want to set the hash of the env, because it might be used 889 | # elsewhere by the caller. global tracking would also be useful here 890 | # - passed from outside block and returned 891 | # same as last case, and we can basically ignore the return 892 | # - global from outside block and not returned 893 | # it would be bad if we skip this block but don't update the hash of 894 | # the env, because it might be used elsewhere by the caller, and then 895 | # the hash will be all wrong. 896 | # - global from outside block and returned 897 | # same as last case, except it's slightly easier to detect since we 898 | # can parse the output 899 | 900 | # if globally we want to always rerun, we force a keyerror to run the function again 901 | if global_env_registry.always_rerun: raise KeyError 902 | 903 | # set hashes for envs 904 | r = local.get_kv(overall_input_hash) 905 | # backwards compatibility 906 | if len(r) == 3: 907 | changed_hashes, ret, asynchronous = r 908 | elif len(r) == 2: 909 | changed_hashes, ret = r 910 | asynchronous = False 911 | else: 912 | raise Exception(f"Unknown number of stage kv values: {len(r)}") 913 | 914 | envs_by_ip_envname = global_env_registry.envs_by_ip_envname() 915 | 916 | for ip, envname, orighash, newhash in changed_hashes: 917 | env = envs_by_ip_envname[(ip, envname)] 918 | if env.hash != orighash: 919 | print(f"WARNING: expected env {ip}:{envname} to have hash {orighash} but got {env.hash}! Did the ip change?") 920 | env.hash = newhash 921 | 922 | fmt_args = ", ".join(list(map(str, args)) + [f"{k}={v}" for k, v in kwargs.items()]) 923 | if len(fmt_args) > 100: 924 | fmt_args = fmt_args[:100] + "..." 925 | print(f"Skipping stage {Style.BRIGHT}{fn.__name__}{Style.RESET_ALL}({fmt_args})") 926 | except KeyError: 927 | ret = fn(*args, **kwargs) 928 | 929 | def _write_hashes(ret, asynchronous=False): 930 | # check which env hashes changed, and write that tto local kv store along with ret 931 | 932 | global_hashes_after = global_env_registry.hashes_by_env() 933 | 934 | # get the hashes that changed 935 | changed_hashes = [ 936 | (k.ip, k.envname, global_hashes_before[k], global_hashes_after[k]) for k in global_hashes_after if global_hashes_before[k] != global_hashes_after[k] 937 | ] 938 | 939 | local.set_kv(overall_input_hash, (changed_hashes, ret, asynchronous)) 940 | 941 | ## ASYNC HANDLING, first run 942 | if inspect.isawaitable(ret): 943 | # WARNING: using the same env across multiple async stages is not supported, and not possible to support! 944 | # TODO: detect if two async stages are using the same env and throw an error if so 945 | 946 | async def _wrapper(ret): 947 | # turn the original async function into a synchronous one and return a new async function 948 | ret = await ret 949 | _write_hashes(ret, asynchronous=True) 950 | return ret 951 | 952 | return _wrapper(ret) 953 | else: 954 | _write_hashes(ret) 955 | return ret 956 | 957 | ## ASYNC HANDLING, resume from file 958 | if asynchronous: 959 | async def _wrapper(ret): 960 | # wrap ret in a dummy async function 961 | return ret 962 | 963 | return _wrapper(ret) 964 | else: 965 | return ret 966 | return wrapper 967 | 968 | 969 | class _EnvRegistry: 970 | # everything here is O(n) but there shouldn't be a lot of envs so it's fine 971 | def __init__(self): 972 | self.envs = [] 973 | self.always_rerun = False 974 | 975 | def hashes_by_env(self): 976 | return { 977 | v: v.hash for v in self.envs 978 | } 979 | 980 | def envs_by_ip_envname(self): 981 | return { 982 | (v.ip, v.envname): v for v in self.envs 983 | } 984 | 985 | def register(self, env): 986 | self.envs.append(env) 987 | 988 | 989 | local = Remote(wd=os.getcwd()) 990 | global_env_registry = _EnvRegistry() 991 | 992 | if "PYFRA_ALWAYS_RERUN" in os.environ: global_env_registry = True -------------------------------------------------------------------------------- /pyfra/setup.py: -------------------------------------------------------------------------------- 1 | 2 | import pyfra.remote 3 | import pyfra.shell 4 | 5 | from deprecation import deprecated 6 | 7 | __all__ = [ 8 | 'apt', 9 | 'install_pyenv', 10 | 'ensure_supported', 11 | ] 12 | 13 | def apt(r, packages): 14 | r.sh(f"sudo apt-get update; sudo apt-get install -y {' '.join(packages)}", pyenv_version=None) 15 | 16 | 17 | @deprecated() 18 | def ensure_supported(r): 19 | # todo: wire this up to something 20 | 21 | supported = [ 22 | "Ubuntu 18", "Ubuntu 20", 23 | "stretch" # debian stretch 24 | ] 25 | 26 | @pyfra.remote.block 27 | def _f(r): 28 | print("Checking if", r, "is running a supported distro") 29 | 30 | assert any([ 31 | ver in r.sh("lsb_release -d") 32 | for ver in supported 33 | ]) 34 | 35 | _f(r) 36 | 37 | ## things to install 38 | 39 | 40 | def install_pyenv(r, version="3.9.4"): 41 | if r.sh(f"pyenv shell {version} 2> /dev/null; python --version", no_venv=True, ignore_errors=True, pyenv_version=None, quiet=True).strip().split(" ")[-1] == version: 42 | return 43 | 44 | apt(r, [ 45 | 'build-essential', 46 | 'curl', 47 | 'git', 48 | 'libbz2-dev', 49 | 'libffi-dev', 50 | 'liblzma-dev', 51 | 'libncurses5-dev', 52 | 'libncursesw5-dev', 53 | 'libreadline-dev', 54 | 'libsqlite3-dev', 55 | 'libssl-dev', 56 | 'make', 57 | 'python3-openssl', 58 | 'rsync', 59 | 'tk-dev', 60 | 'wget', 61 | 'xz-utils', 62 | 'zlib1g-dev', 63 | ]) 64 | r.sh("curl https://pyenv.run | bash", ignore_errors=True, pyenv_version=None) 65 | 66 | payload = """ 67 | # pyfra-managed: pyenv stuff 68 | export PYENV_ROOT="$HOME/.pyenv" 69 | export PATH="$PYENV_ROOT/bin:$PATH" 70 | eval "$(pyenv init --path)" 71 | eval "$(pyenv init -)" 72 | eval "$(pyenv virtualenv-init -)" 73 | """ 74 | bashrc = r.sh("cat ~/.bashrc", pyenv_version=None) 75 | 76 | if "# pyfra-managed: pyenv stuff" not in bashrc: 77 | r.sh(f"echo {payload | pyfra.shell.quote} >> ~/.bashrc", pyenv_version=None) 78 | 79 | # install updater 80 | r.sh("git clone https://github.com/pyenv/pyenv-update.git $(pyenv root)/plugins/pyenv-update", ignore_errors=True, pyenv_version=None) 81 | r.sh("pyenv update", ignore_errors=True, pyenv_version=None) 82 | 83 | r.sh(f"pyenv install --verbose -s {version}", pyenv_version=None) 84 | 85 | # make sure the versions all check out 86 | assert r.sh(f"python --version", no_venv=True).strip().split(" ")[-1] == version 87 | assert r.sh(f"python3 --version", no_venv=True).strip().split(" ")[-1] == version 88 | assert version.rsplit('.', 1)[0] in r.sh("pip --version", no_venv=True) 89 | assert version.rsplit('.', 1)[0] in r.sh("pip3 --version", no_venv=True) 90 | 91 | r.sh("pip install virtualenv") 92 | r.sh("virtualenv --version") 93 | -------------------------------------------------------------------------------- /pyfra/shell.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pathlib 3 | import os 4 | import re 5 | import shlex 6 | import shutil 7 | import subprocess 8 | import sys 9 | import time 10 | import urllib 11 | 12 | from best_download import download_file 13 | from colorama import Fore, Style 14 | from natsort import natsorted 15 | from deprecation import deprecated 16 | 17 | import imohash 18 | import pyfra.remote 19 | 20 | class ShellException(Exception): 21 | def __init__(self, code, rem=False): 22 | super().__init__(f"Command exited with non-zero error code {code}" + 23 | (". This could either be because the ssh connection could not be made, or because the command itself failed." if rem and code == 255 else "")) 24 | self.returncode = code 25 | 26 | 27 | __all__ = ['sh', 'copy', 'ls', 'curl', 'quote', 'ShellException'] 28 | 29 | 30 | def _wrap_command(x, no_venv=False, pyenv_version=None): 31 | bashrc_payload = r"""import sys,re; print(re.sub("If not running interactively.{,128}?esac", "", sys.stdin.read(), flags=re.DOTALL).replace('[ -z "$PS1" ] && return', ''))""" 32 | hdr = f"shopt -s expand_aliases; ctrlc() {{ echo Shell wrapper interrupted with C-c, raising error; exit 174; }}; trap ctrlc SIGINT; " 33 | hdr += f"[ -e ~/.bashrc ] && {{ eval \"$(cat ~/.bashrc | python3 -c {bashrc_payload | quote})\" > /dev/null 2>&1; }}; " 34 | hdr += "[ -e ~/.bashrc ] || { [ -e ~/.zshrc ] && . ~/.zshrc; }; " 35 | hdr += "python() { python3 \"$@\"; };" # use python3 by default 36 | if pyenv_version is not None: hdr += f"pyenv shell {pyenv_version} || exit 1 > /dev/null 2>&1; " 37 | if not no_venv: hdr += "[ -f env/bin/activate ] && . env/bin/activate; " 38 | return hdr + x 39 | 40 | 41 | def _process_remotepaths(host, cmd): 42 | candidates = re.findall(r"RemotePath\((.+?)\)", cmd) 43 | 44 | rempaths = [] 45 | 46 | for c in candidates: 47 | ob = json.loads(c) 48 | rem = ob["remote"] if ob["remote"] is not None else "127.0.0.1" 49 | fname = ob["fname"] 50 | 51 | if rem != host: 52 | loc_fname = rem.replace(".", "_").replace("@", "_")+"_"+fname.split("/")[-1] 53 | if loc_fname.startswith("~/"): loc_fname = loc_fname[2:] 54 | if loc_fname.startswith("/"): loc_fname = loc_fname[1:] 55 | loc_fname = "~/.pyfra_remote_files/" + loc_fname 56 | 57 | copyerr = False 58 | try: 59 | frm = pyfra.remote.Remote(rem).path(fname) 60 | 61 | # we want to copy dirs into, but into doesnt work with files 62 | copy(frm, pyfra.remote.Remote(host).path(loc_fname), into=not frm.is_dir()) 63 | except ShellException: 64 | # if this file doesn't exist, it's probably an implicit return 65 | copyerr = True 66 | 67 | rempaths.append((pyfra.remote.Remote(rem).path(fname), pyfra.remote.Remote(host).path(loc_fname), copyerr)) 68 | 69 | cmd = cmd.replace(f"RemotePath({c})", loc_fname) 70 | else: 71 | cmd = cmd.replace(f"RemotePath({c})", fname) 72 | 73 | return cmd, rempaths 74 | 75 | 76 | def _sh(cmd, quiet=False, wd=None, wrap=True, maxbuflen=1000000000, ignore_errors=False, no_venv=False, pyenv_version=None): 77 | if wrap: cmd = _wrap_command(cmd, no_venv=no_venv, pyenv_version=pyenv_version) 78 | 79 | if wd is None: wd = "~" 80 | 81 | cmd = f"cd {wd} > /dev/null 2>&1; {cmd}" 82 | 83 | p = subprocess.Popen(cmd, shell=True, 84 | stdout=subprocess.PIPE, 85 | stderr=subprocess.STDOUT, 86 | executable="/bin/bash") 87 | 88 | ret = bytearray() 89 | while True: 90 | byte = p.stdout.read(1) 91 | 92 | if byte == b'': 93 | break 94 | if not quiet: 95 | sys.stdout.buffer.write(byte) 96 | sys.stdout.flush() 97 | 98 | if maxbuflen is None or len(ret) < maxbuflen: 99 | ret += bytearray(byte) 100 | 101 | p.communicate() 102 | if p.returncode == 174: 103 | raise KeyboardInterrupt() 104 | elif p.returncode != 0 and not ignore_errors: 105 | raise ShellException(p.returncode) 106 | 107 | return ret.decode("utf-8").replace("\r\n", "\n").strip() 108 | 109 | 110 | def sh(cmd, quiet=False, wd=None, wrap=True, maxbuflen=1000000000, ignore_errors=False, no_venv=False, pyenv_version=None): 111 | """ 112 | Runs commands as if it were in a local bash terminal. 113 | 114 | This function patches out the non-interactive detection in bashrc and sources it, activates virtualenvs and sets pyenv shell, handles interrupts correctly, and returns the text printed to stdout. 115 | 116 | Args: 117 | quiet (bool): If turned on, nothing is printed to stdout. 118 | wd (str): Working directory to run in. Defaults to ~ 119 | wrap (bool): Magic for the bashrc, virtualenv, interrupt-handing, and pyenv stuff. Turn off to make this essentially os.system 120 | maxbuflen (int): Max number of bytes to save and return. Useful to prevent memory errors. 121 | ignore_errors (bool): If set, errors will be swallowed. 122 | no_venv (bool): If set, virtualenv will not be activated 123 | pyenv_version (str): Pyenv version to use. Will be silently ignored if not found. 124 | Returns: 125 | The standard output of the command, limited to maxbuflen bytes. 126 | """ 127 | if wd is None: wd = os.getcwd() 128 | 129 | try: 130 | return _rsh("127.0.0.1", cmd, quiet, wd, wrap, maxbuflen, -1, ignore_errors, no_venv, pyenv_version) 131 | except ShellException as e: # this makes the stacktrace easier to read 132 | raise ShellException(e.returncode) from None 133 | 134 | 135 | def _rsh(host, cmd, quiet=False, wd=None, wrap=True, maxbuflen=1000000000, connection_timeout=10, ignore_errors=False, no_venv=False, pyenv_version=None, forward_keys=False, additional_ssh_config=""): 136 | if host is None or host == "localhost": host = "127.0.0.1" 137 | 138 | # implicit-copy files from remote to local 139 | cmd, rempaths = _process_remotepaths(host, cmd) 140 | 141 | if not quiet: 142 | # display colored message 143 | host_style = Fore.GREEN+Style.BRIGHT 144 | sep_style = Style.NORMAL 145 | cmd_style = Fore.WHITE+Style.BRIGHT 146 | dir_style = Fore.BLUE+Style.BRIGHT 147 | hoststr = str(host) 148 | if wd is not None: 149 | wd_display = wd 150 | if not wd.startswith("~/") and wd != '~': 151 | wd_display = os.path.join("~", wd) 152 | else: 153 | wd_display = "~" 154 | hoststr += f"{Style.RESET_ALL}:{dir_style}{wd_display}{Style.RESET_ALL}" 155 | cmd_fmt = cmd.strip().replace('\n', f'\n{ " " * (len(str(host)) + 3 + len(wd_display))}{sep_style}>{Style.RESET_ALL}{cmd_style} ') 156 | print(f"{Style.BRIGHT}{Fore.RED}*{Style.RESET_ALL} {host_style}{hoststr}{Style.RESET_ALL}{sep_style}$ {Style.RESET_ALL}{cmd_style}{cmd_fmt}{Style.RESET_ALL}") 157 | 158 | if host == "127.0.0.1": 159 | return _sh(cmd, quiet, wd, wrap, maxbuflen, ignore_errors, no_venv, pyenv_version) 160 | 161 | if wrap: cmd = _wrap_command(cmd, no_venv=no_venv, pyenv_version=pyenv_version) 162 | if wd: cmd = f"cd {wd} > /dev/null 2>&1; {cmd}" 163 | 164 | ssh_cmd = "eval \"$(ssh-agent -s)\"; ssh-add ~/.ssh/id_rsa; ssh -A" if forward_keys else "ssh" 165 | ret = _sh(f"{ssh_cmd} -q -oConnectTimeout={connection_timeout} -oBatchMode=yes -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null {additional_ssh_config} -t {host} {shlex.quote(cmd)}", quiet=quiet, wrap=False, maxbuflen=maxbuflen, ignore_errors=ignore_errors, no_venv=no_venv) 166 | 167 | # implicit-copy files from local to remote 168 | for remf, locf, copyerr in rempaths: 169 | try: 170 | copy(locf, remf) 171 | except ShellException: 172 | # if it errored both before and after, something is wrong 173 | if copyerr: 174 | raise ShellException(f"implicit-copy file {remf}/{locf} (remote/local) was neither written to nor read from!") 175 | 176 | return ret 177 | 178 | def copy(frm, to, quiet=False, connection_timeout=10, symlink_ok=True, into=True, exclude=[]) -> None: 179 | """ 180 | Copies things from one place to another. 181 | 182 | Args: 183 | frm (str or RemotePath): Can be a string indicating a local path, a :class:`pyfra.remote.RemotePath`, or a URL. 184 | to (str or RemotePath): Can be a string indicating a local path or a :class:`pyfra.remote.RemotePath`. 185 | quiet (bool): Disables logging. 186 | connection_timeout (int): How long in seconds to give up after 187 | symlink_ok (bool): If frm and to are on the same machine, symlinks will be created instead of actually copying. Set to false to force copying. 188 | into (bool): If frm is a file, this has no effect. If frm is a directory, then into=True for frm="src" and to="dst" means "src/a" will get copied to "dst/src/a", whereas into=False means "src/a" will get copied to "dst/a". 189 | """ 190 | 191 | # copy from url 192 | if isinstance(frm, str) and (frm.startswith("http://") or frm.startswith("https://")): 193 | if ":" in to: 194 | to_host, to_path = to.split(":") 195 | _rsh(to_host, f"curl {frm} --create-dirs -o {to_path}") 196 | else: 197 | wget(frm, to) 198 | return 199 | 200 | # get rsync strs and make sure frm and to are RemotePaths 201 | if isinstance(frm, pyfra.remote.RemotePath): 202 | frm_str = frm.rsyncstr() 203 | else: 204 | frm_str = frm 205 | assert ":" not in frm_str 206 | frm = pyfra.remote.local.path(frm) 207 | 208 | if isinstance(to, pyfra.remote.RemotePath): 209 | to_str = to.rsyncstr() 210 | else: 211 | to_str = to 212 | assert ":" not in to_str 213 | to = pyfra.remote.local.path(to) 214 | 215 | # state tracking 216 | needs_set_kv = False 217 | if not to.remote._no_hash: 218 | with to.remote.no_hash(): 219 | checksum = frm.quick_hash() 220 | new_hash = to.remote.update_hash("copy", to.fname, checksum) 221 | try: 222 | to.remote.get_kv(new_hash) 223 | # if already copied, then return 224 | to._set_cache("quick_hash", checksum) # set the checksum of the target file to avoid needing to calculate it again 225 | pyfra.remote._print_skip_msg(to.remote.envname, "copy", new_hash) 226 | 227 | return 228 | except KeyError: 229 | needs_set_kv = True 230 | 231 | # print info 232 | if not quiet: print(f"{Style.BRIGHT}{Fore.RED}*{Style.RESET_ALL} Copying {Style.BRIGHT}{frm_str} {Style.RESET_ALL}to {Style.BRIGHT}{to_str}{Style.RESET_ALL}") 233 | 234 | if frm_str[-1] == '/' and len(frm_str) > 1: frm_str = frm_str[:-1] 235 | if not into: frm_str += '/' 236 | 237 | if quiet: 238 | opts = "-e \"ssh -o StrictHostKeyChecking=no\" -arqL" 239 | else: 240 | opts = "-e \"ssh -o StrictHostKeyChecking=no\" -arL" 241 | 242 | for ex in exclude: 243 | opts += f" --exclude {ex | pyfra.shell.quote}" 244 | 245 | def symlink_frm(frm_str): 246 | # rsync behavior is to_str copy the contents of frm_str into to_str if frm_str ends with a / 247 | if frm_str[-1] == '/': frm_str += '*' 248 | # ln -s can't handle relative paths well! make absolute if not already 249 | if frm_str[0] != '/' and frm_str[0] != '~': frm_str = "$PWD/" + frm_str 250 | 251 | return frm_str 252 | 253 | if ":" in frm_str and ":" in to_str: 254 | frm_host, frm_path = frm_str.split(":") 255 | to_host, to_path = to_str.split(":") 256 | 257 | par_target = to_path.rsplit('/', 1)[0] if "/" in to_path else "" 258 | 259 | if to_host == frm_host: 260 | if symlink_ok: 261 | assert not exclude, "Cannot use exclude symlink" 262 | _rsh(frm_host, f"[ -d {frm_path} ] && mkdir -p {to_path}; ln -sf {symlink_frm(frm_path)} {to_path}", quiet=True) 263 | else: 264 | 265 | if par_target: _rsh(to_host, f"mkdir -p {par_target}", quiet=True) 266 | _rsh(frm_host, f"rsync {opts} {frm_path} {to_path}", quiet=True) 267 | else: 268 | rsync_cmd = f"rsync {opts} {frm_path} {to_str}" 269 | 270 | # make parent dir in terget if not exists 271 | if par_target: _rsh(to_host, f"mkdir -p {par_target}", quiet=True) 272 | 273 | sh(f"eval \"$(ssh-agent -s)\"; ssh-add ~/.ssh/id_rsa; ssh -q -oConnectTimeout={connection_timeout} -oBatchMode=yes -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -A {frm_host} {rsync_cmd | quote}", wrap=False, quiet=True) 274 | else: 275 | # if to_str is host:path, then this gives us path; otherwise, it leaves it unchanged 276 | par_target = to_str.split(":")[-1] 277 | par_target = par_target.rsplit('/', 1)[0] if "/" in par_target else "" 278 | 279 | if symlink_ok and ":" not in frm_str and ":" not in to_str: 280 | assert not exclude, "Cannot use exclude symlink" 281 | sh(f"[ -d {frm_str} ] && mkdir -p {par_target}; ln -sf {symlink_frm(frm_str)} {to_str}", quiet=True) 282 | else: 283 | if ":" in to_str: _rsh(to_str.split(":")[0], f"mkdir -p {par_target}", quiet=True) 284 | sh((f"mkdir -p {par_target}; " if par_target and ":" in frm_str else "") + f"rsync {opts} {frm_str} {to_str}", wrap=False, quiet=True) 285 | 286 | # set value in key value store to flag as done 287 | if needs_set_kv: 288 | to.remote.set_kv(new_hash, None) 289 | to._set_cache("quick_hash", checksum) # set the checksum of the target file to avoid needing to calculate it again 290 | 291 | 292 | def ls(x='.'): 293 | return list(natsorted([x + '/' + fn for fn in os.listdir(x)])) 294 | 295 | def rm(x, no_exists_ok=True): 296 | # from https://stackoverflow.com/a/41789397 297 | if not os.path.exists(x) and no_exists_ok: return 298 | 299 | if os.path.isfile(x) or os.path.islink(x): 300 | os.remove(x) # remove the file 301 | elif os.path.isdir(x): 302 | shutil.rmtree(x) # remove dir and all contains 303 | else: 304 | raise ValueError("file {} is not a file or dir.".format(x)) 305 | 306 | def curl(url, max_tries=10, timeout=30): # TODO: add checksum option 307 | cooldown = 1 308 | for i in range(max_tries): 309 | try: 310 | response = urllib.request.urlopen(url, timeout=timeout) 311 | data = response.read() 312 | except urllib.error.URLError as e: 313 | if e.code == 429: 314 | time.sleep(cooldown) 315 | cooldown *= 1.5 316 | continue 317 | else: 318 | return None 319 | except: 320 | return None 321 | return data 322 | 323 | @deprecated(details="Use best_download directly, or wait for improved file-downloading support in pyfra") 324 | def wget(url, to=None, checksum=None): 325 | # DEPRECATED 326 | # thin wrapper for best_download 327 | 328 | if to is None: 329 | to = os.path.basename(url) 330 | if not to: to = 'index' 331 | 332 | download_file(url, to, checksum) 333 | 334 | def quick_hash(path): 335 | params = { 336 | "hexdigest": True, 337 | "sample_size": 4 * 1024**2, # 4 MB 338 | "sample_threshhold": 16 * 1024**2, # 16 MB 339 | } 340 | path = os.path.expanduser(path) 341 | if pathlib.Path(path).is_dir(): 342 | files = list(sorted(pathlib.Path(path).glob('**/*'))) 343 | res = pyfra.remote._hash_obs(*[(str(f.relative_to(pathlib.Path(path))), imohash.hashfile(str(f.resolve()), **params)) for f in files if f.is_file()])[:32] 344 | return res 345 | return imohash.hashfile(path, **params) 346 | 347 | # convenience function for shlex.quote 348 | class _quote: 349 | def __ror__(self, other): 350 | return shlex.quote(other) 351 | 352 | def __call__(self, other): 353 | return shlex.quote(other) 354 | 355 | quote = _quote() 356 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | 3 | 4 | sphinx==4.0.2 5 | sphinxcontrib-napoleon==0.7 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="pyfra", 8 | version="0.3.1", 9 | author="Leo Gao", 10 | author_email="lg@eleuther.ai", 11 | description="A framework for research code", 12 | long_description=long_description, 13 | long_description_content_type="text/markdown", 14 | url="https://github.com/EleutherAI/pyfra", 15 | packages=setuptools.find_packages(), 16 | include_package_data=True, 17 | classifiers=[ 18 | "Programming Language :: Python :: 3", 19 | "License :: OSI Approved :: MIT License", 20 | "Operating System :: OS Independent", 21 | ], 22 | python_requires='>=3.6', 23 | install_requires=[ 24 | 'best_download', 25 | 'sqlitedict', 26 | 'colorama', 27 | 'parse', 28 | 'natsort', 29 | 'yaspin', 30 | 'imohash', 31 | 'deprecation' 32 | ], 33 | extras_require={ 34 | 'contrib': [ 35 | 'flask', 36 | 'flask-login', 37 | 'flask-wtf', 38 | 'flask-sqlalchemy', 39 | 'flask-migrate', 40 | 'flask-admin', 41 | 'flask-bootstrap', 42 | 'pyjwt', 43 | 'sqlalchemy', 44 | 'wtforms[email]', 45 | 'ansi2html', 46 | ] 47 | } 48 | ) 49 | -------------------------------------------------------------------------------- /tests/Dockerfile: -------------------------------------------------------------------------------- 1 | # modified from https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile 2 | 3 | FROM ubuntu:20.04 4 | 5 | RUN apt-get update 6 | 7 | RUN apt-get install -y openssh-server 8 | RUN mkdir /var/run/sshd 9 | 10 | RUN echo 'root:root' |chpasswd 11 | 12 | RUN sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config 13 | RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config 14 | 15 | RUN mkdir /root/.ssh 16 | 17 | RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 18 | 19 | # makes pyfra installation not take forever 20 | RUN apt-get update && apt-get install -y sudo build-essential libbz2-dev libffi-dev liblzma-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libssl-dev make python3-openssl tk-dev xz-utils zlib1g-dev rsync curl wget git 21 | 22 | COPY id_rsa.pub /root/.ssh/authorized_keys 23 | 24 | EXPOSE 22 25 | 26 | CMD ["/usr/sbin/sshd", "-D"] -------------------------------------------------------------------------------- /tests/test_remote.py: -------------------------------------------------------------------------------- 1 | from pyfra import * 2 | import pyfra.remote as pyr 3 | import os 4 | import random 5 | 6 | 7 | def setup_remote(ind): 8 | """ setup any state specific to the execution of the given module.""" 9 | sh(f""" 10 | # docker kill pyfra_test_remote_{ind} 11 | # docker container rm pyfra_test_remote_{ind} 12 | cd tests 13 | cp ~/.ssh/id_rsa.pub . 14 | docker build -t pyfra_test_remote . 15 | docker run --rm --name pyfra_test_remote_{ind} -d pyfra_test_remote 16 | true 17 | """) 18 | 19 | ip = sh("docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pyfra_test_remote_" + str(ind)) 20 | rem = Remote("root@"+ip) 21 | rem.sh("rm -rf pyfra_envs") 22 | return rem 23 | 24 | 25 | def setup_module(module): 26 | global rem1, rem2 27 | rem1 = setup_remote(1) 28 | rem2 = setup_remote(2) 29 | 30 | def test_workdir_semantics(): 31 | global rem1, rem2 32 | 33 | assert pyr._normalize_homedir("somepath") == "~/somepath" 34 | assert pyr._normalize_homedir("~/somepath") == "~/somepath" 35 | assert pyr._normalize_homedir(".") == "~" 36 | assert pyr._normalize_homedir("~") == "~" 37 | assert pyr._normalize_homedir("/") == "/" 38 | assert pyr._normalize_homedir("somepath/") == "~/somepath" 39 | assert pyr._normalize_homedir("/somepath/") == "/somepath" 40 | assert pyr._normalize_homedir("./somepath") == "~/somepath" 41 | 42 | for rem in [rem1, rem2]: 43 | assert rem.path("somepath").fname == "~/somepath" 44 | assert rem.path("~/somepath").fname == "~/somepath" 45 | assert rem.path(".").fname == "~" 46 | assert rem.path("~").fname == "~" 47 | assert rem.path("/").fname == "/" 48 | assert rem.path("somepath/").fname == "~/somepath" 49 | assert rem.path("/somepath/").fname == "/somepath" 50 | assert rem.path("./somepath").fname == "~/somepath" 51 | 52 | assert local.path("somepath").fname == os.getcwd() + "/somepath" 53 | assert local.path("~/somepath").fname == "~/somepath" 54 | assert local.path(".").fname == os.getcwd() 55 | assert local.path("~").fname == "~" 56 | assert local.path("/").fname == "/" 57 | assert local.path("somepath/").fname == os.getcwd() + "/somepath" 58 | assert local.path("/somepath/").fname == "/somepath" 59 | assert local.path("./somepath").fname == os.getcwd() + "/somepath" 60 | 61 | # sh path 62 | assert sh("echo $PWD") == os.getcwd() 63 | assert local.sh("echo $PWD") == os.getcwd() 64 | assert rem1.sh("echo $PWD") == "/root" 65 | 66 | # env path 67 | env1 = rem1.env("env1") 68 | env2 = rem2.env("env2") 69 | 70 | locenv1 = local.env("locenv1") 71 | locenv2 = local.env("locenv2") 72 | 73 | def copy_path_test(a, b): 74 | payload = random.randint(0, 99999) 75 | a.sh(f"mkdir origin_test_dir; echo hello world {payload} > origin_test_dir/test_pyfra.txt", ignore_errors=True) 76 | b.sh("mkdir test_dir_1", ignore_errors=True) 77 | b.sh("mkdir test_dir_2", ignore_errors=True) 78 | 79 | # check right into=False behavior 80 | copy(a.path("origin_test_dir/test_pyfra.txt"), b.path("test2_pyfra.txt")) 81 | copy(a.path("origin_test_dir"), b.path("test_dir_1")) 82 | copy(a.path("origin_test_dir"), b.path("test_dir_2"), into=False) 83 | ic(b.sh("ls")) 84 | assert b.sh("cat test2_pyfra.txt") == f"hello world {payload}" 85 | assert b.sh("cat test_dir_1/origin_test_dir/test_pyfra.txt") == f"hello world {payload}" 86 | assert b.sh("cat test_dir_2/test_pyfra.txt") == f"hello world {payload}" 87 | a.sh("rm -rf origin_test_dir") 88 | b.sh("rm -rf test2_pyfra.txt test_dir_1 test_dir_2") 89 | 90 | ## env to env 91 | copy_path_test(env1, env2) 92 | 93 | ## env to rem 94 | copy_path_test(env1, rem2) 95 | 96 | ## rem to env 97 | copy_path_test(rem1, env2) 98 | 99 | ## rem to rem 100 | copy_path_test(rem1, rem2) 101 | 102 | ## same rem to rem 103 | copy_path_test(rem1, rem1) 104 | 105 | ## same rem to env 106 | copy_path_test(rem1, env1) 107 | 108 | ## same env to rem 109 | copy_path_test(env1, rem1) 110 | 111 | ## same env to rem 112 | copy_path_test(env1, env1) 113 | 114 | ## local rem to local rem 115 | copy_path_test(local, local) 116 | 117 | ## local rem to local env 118 | copy_path_test(local, locenv2) 119 | 120 | ## local env to local rem 121 | copy_path_test(locenv1, local) 122 | 123 | ## local env to local rem 124 | copy_path_test(locenv1, locenv2) 125 | 126 | # local no rem to rem 127 | sh("echo test 123 > test_pyfra.txt") 128 | copy("test_pyfra.txt", rem1.path("test2_pyfra.txt")) 129 | assert rem1.sh("cat test2_pyfra.txt") == f"test 123" 130 | sh("rm test_pyfra.txt") 131 | rem1.sh("rm test2_pyfra.txt") 132 | 133 | # rem to local no rem 134 | rem1.sh("echo test 1234 > test_pyfra.txt") 135 | copy(rem1.path("test_pyfra.txt"), "test2_pyfra.txt") 136 | assert sh("cat test2_pyfra.txt") == f"test 1234" 137 | rem1.sh("rm test_pyfra.txt") 138 | sh("rm test2_pyfra.txt") 139 | 140 | # todo: test fread/fwrite 141 | 142 | 143 | def test_fns(): 144 | global rem1, rem2 145 | 146 | env1 = rem1.env("env1") 147 | env2 = rem2.env("env2") 148 | 149 | locenv1 = local.env("locenv1") 150 | locenv2 = local.env("locenv2") 151 | 152 | def fns_test(rem): 153 | rem.sh("rm -rf testing_dir_fns; mkdir testing_dir_fns && cd testing_dir_fns && touch a && touch b && touch c && echo $PWD") 154 | assert rem.ls("testing_dir_fns") == ['a', 'b', 'c'] 155 | assert 'testing_dir_fns' in rem.ls() 156 | rem.rm("testing_dir_fns") 157 | 158 | assert 'testing_dir_fns' not in rem.ls() 159 | 160 | # make sure no error when deleting nonexistent 161 | rem.rm("testing_dir_fns") 162 | 163 | # check file read and write 164 | rem.path("testfile.pyfra").write("goose") 165 | assert rem.path("testfile.pyfra").read() == "goose" 166 | rem.path("testfile.pyfra").write("goose", append=True) 167 | assert rem.path("testfile.pyfra").read() == "goosegoose" 168 | rem.rm("testfile.pyfra") 169 | 170 | for rem in [local, rem1, rem2, env1, env2, locenv1, locenv2]: fns_test(rem) 171 | 172 | 173 | def test_remotefile_implicit_copy(): 174 | global rem1, rem2 175 | 176 | rem1.sh("rm *.pyfra", ignore_errors=True) 177 | rem2.sh("rm *.pyfra", ignore_errors=True) 178 | 179 | # implicit-read 180 | f1 = rem1.path("testfile_copy.pyfra") 181 | f1.write("goose") 182 | assert rem2.sh(f"cat {f1}") == "goose" 183 | 184 | # implicit-write 185 | f2 = rem2.path("testfile_copy2.pyfra") 186 | rem1.sh(f"echo geese > {f2}") 187 | assert rem2.sh(f"cat {f2}") == "geese" 188 | 189 | # implicit both read and write 190 | f3 = rem1.path("testfile_copy3.pyfra") 191 | f3.write("canada goose") 192 | f4 = rem1.path("testfile_copy4.pyfra") 193 | assert rem2.sh(f"cat {f3} | tr [a-z] [A-Z] | tee {f4}") == "CANADA GOOSE" 194 | assert rem2.sh(f"cat {f4}") == "CANADA GOOSE" 195 | 196 | 197 | # todo: test env w git 198 | 199 | 200 | def teardown_module(module): 201 | """ teardown any state that was previously setup with a setup_module 202 | method. 203 | """ 204 | # sh("docker kill pyfra_test_remote_1") 205 | # sh("docker kill pyfra_test_remote_2") 206 | 207 | --------------------------------------------------------------------------------