├── doc ├── states.png ├── components.png └── architecture.png ├── bench ├── sample.mp4 ├── bench-host.sh └── sample.ove ├── ssl └── certs │ ├── mkconf.sh │ ├── mksan.sh │ ├── generate_keys.sh │ └── Makefile ├── global_settings.py ├── run_worker_node.py ├── .gitignore ├── examples ├── nfs-example.txt └── ssl │ ├── node.py │ └── master.py ├── project_manager.py ├── ssl_utils.py ├── main.py ├── job_dispatcher.py ├── nfs_mounter.py ├── README.md ├── nfs_exporter.py ├── full_job_dispatcher.py ├── job.py ├── split_job_dispatcher.py ├── worker_node.py └── LICENSE /doc/states.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/morrolinux/olive-distributed/HEAD/doc/states.png -------------------------------------------------------------------------------- /bench/sample.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/morrolinux/olive-distributed/HEAD/bench/sample.mp4 -------------------------------------------------------------------------------- /doc/components.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/morrolinux/olive-distributed/HEAD/doc/components.png -------------------------------------------------------------------------------- /doc/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/morrolinux/olive-distributed/HEAD/doc/architecture.png -------------------------------------------------------------------------------- /ssl/certs/mkconf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cat </dev/null |grep "events per second"|rev|cut -d" " -f 1|rev 4 | # echo 100 5 | # exit 6 | 7 | # a better benchmark? 8 | t=$( (time -p olive-editor bench/sample.ove -e) |& grep real | cut -d" " -f2 | tr "," ".") 9 | 10 | 11 | score=$(echo "scale=8; (1 / $t) * 1000000" | bc) 12 | echo $score 13 | -------------------------------------------------------------------------------- /ssl/certs/mksan.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cat < | " 6 | exit 7 | fi 8 | 9 | # This oneliner is from https://serverfault.com/questions/367141/how-to-get-the-fully-qualified-name-fqn-on-unix-in-a-bash-script 10 | fqn=$(host -TtA $(hostname -s)|grep "has address"|awk '{print $1}') ; if [[ "${fqn}" == "" ]] ; then fqn=$(hostname -s) ; fi 11 | 12 | if [[ $1 == "setup" ]] 13 | then 14 | # Generate the CA: 15 | make rootCA.crt 16 | 17 | # Generate your local certificate needed for NFS Exporter 18 | make DOMAIN=localhost NAME=$(echo -n $(hostname))_local 19 | 20 | # Generate your network-wide certificate 21 | make DOMAIN=$fqn 22 | elif [[ $1 == "add" ]] 23 | then 24 | if [[ $2 != "" ]] 25 | then 26 | make DOMAIN=${2}.$(echo $fqn|cut -d. -f2-) 27 | make DOMAIN=localhost NAME=${2}_local 28 | echo $fqn > whoismaster 29 | scp whoismaster rootCA.crt ${2}* ${2}:olive-distributed/ssl/certs/ 30 | fi 31 | fi 32 | 33 | 34 | -------------------------------------------------------------------------------- /ssl_utils.py: -------------------------------------------------------------------------------- 1 | import Pyro4.core 2 | import socket 3 | import os 4 | from pathlib import Path 5 | 6 | # Doesn't work consistently 7 | # LOCAL_HOSTNAME = socket.gethostbyname_ex(socket.gethostname())[0] 8 | 9 | 10 | # Works consistently 11 | def fqdn(): 12 | full = os.popen('echo -n $(host -TtA $(hostname -s)|grep \"has address\"|awk \'{print $1}\') ; ' 13 | 'if [[ \"${fqn}\" == \"\" ]] ; then fqn=$(hostname -s) ; fi').read() 14 | domain = full[full.find('.'):] 15 | return socket.gethostname() + domain 16 | 17 | 18 | LOCAL_HOSTNAME = fqdn() 19 | SSL_CERTS_DIR = "ssl/certs/" 20 | OD_FOLDER = str(Path.home()) + "/olive-distributed/" 21 | 22 | 23 | class CertCheckingProxy(Pyro4.core.Proxy): 24 | @staticmethod 25 | def verify_cert(cert): 26 | if not cert: 27 | raise Pyro4.errors.CommunicationError("cert missing") 28 | 29 | def _pyroValidateHandshake(self, response): 30 | cert = self._pyroConnection.getpeercert() 31 | self.verify_cert(cert) 32 | 33 | 34 | class CertValidatingDaemon(Pyro4.core.Daemon): 35 | def validateHandshake(self, conn, data): 36 | cert = conn.getpeercert() 37 | if not cert: 38 | raise Pyro4.errors.CommunicationError("node cert missing") 39 | return super(CertValidatingDaemon, self).validateHandshake(conn, data) 40 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import argparse 3 | from project_manager import ProjectManager 4 | from full_job_dispatcher import FullJobDispatcher 5 | from split_job_dispatcher import SplitJobDispatcher 6 | from global_settings import settings 7 | 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument("--folder", dest='folder', help="folder containing projects folders") 10 | parser.add_argument("--project", dest='project', help="project file to be rendered on multiple nodes") 11 | args = parser.parse_args() 12 | 13 | 14 | def get_render_nodes(): 15 | with open('nodes.txt') as fp: 16 | return fp.read().splitlines() 17 | 18 | 19 | if __name__ == '__main__': 20 | 21 | if args.folder is None and args.project is None: 22 | print("usage: --folder | --project ") 23 | exit() 24 | 25 | # initialize the project manager with the render nodes 26 | project_manager = ProjectManager() 27 | 28 | # and feed it the job(s) to be done 29 | job_dispatcher = None 30 | if args.folder is not None: 31 | settings.dispatcher["workflow"] = "full" 32 | project_manager.explore(args.folder) 33 | job_dispatcher = FullJobDispatcher() 34 | job_dispatcher.jobs = project_manager.jobs 35 | elif args.project is not None: 36 | settings.dispatcher["workflow"] = "split" 37 | project_manager.add(args.project, part=True) 38 | job_dispatcher = SplitJobDispatcher() 39 | job_dispatcher.split_job = project_manager.jobs[0] 40 | else: 41 | print("invalid options") 42 | exit() 43 | 44 | job_dispatcher.start() 45 | -------------------------------------------------------------------------------- /examples/ssl/node.py: -------------------------------------------------------------------------------- 1 | import Pyro4.core 2 | import socket 3 | 4 | 5 | class Safe(object): 6 | @Pyro4.expose 7 | def echo(self, message): 8 | print("got message:", message) 9 | return "hi!" 10 | 11 | 12 | Pyro4.config.SSL = True 13 | Pyro4.config.SSL_REQUIRECLIENTCERT = True # enable 2-way ssl 14 | Pyro4.config.SSL_SERVERCERT = "certs/node_cert.pem" 15 | Pyro4.config.SSL_SERVERKEY = "certs/node_key.pem" 16 | Pyro4.config.SSL_CACERTS = "certs/master_cert.pem" # to make ssl accept the self-signed master cert 17 | print("SSL enabled (2-way).") 18 | 19 | 20 | class CertValidatingDaemon(Pyro4.core.Daemon): 21 | def validateHandshake(self, conn, data): 22 | cert = conn.getpeercert() 23 | if not cert: 24 | raise Pyro4.errors.CommunicationError("client cert missing") 25 | ''' 26 | if cert["serialNumber"] != "9BFD9872D96F066C": 27 | raise Pyro4.errors.CommunicationError("cert serial number incorrect") 28 | issuer = dict(p[0] for p in cert["issuer"]) 29 | subject = dict(p[0] for p in cert["subject"]) 30 | if issuer["organizationName"] != "Razorvine.net": 31 | # issuer is not often relevant I guess, but just to show that you have the data 32 | raise Pyro4.errors.CommunicationError("cert not issued by Razorvine.net") 33 | if subject["countryName"] != "NL": 34 | raise Pyro4.errors.CommunicationError("cert not for country NL") 35 | if subject["organizationName"] != "Razorvine.net": 36 | raise Pyro4.errors.CommunicationError("cert not for Razorvine.net") 37 | print("(SSL client cert is ok: serial={ser}, subject={subj})" 38 | .format(ser=cert["serialNumber"], subj=subject["organizationName"])) 39 | ''' 40 | return super(CertValidatingDaemon, self).validateHandshake(conn, data) 41 | 42 | 43 | d = CertValidatingDaemon(host=socket.gethostname(), port=9090) 44 | uri = d.register(Safe, "NodeService") 45 | 46 | print("server uri:", uri) 47 | d.requestLoop() 48 | -------------------------------------------------------------------------------- /examples/ssl/master.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import Pyro4.core 3 | import Pyro4.errors 4 | 5 | 6 | Pyro4.config.SSL = True 7 | Pyro4.config.SSL_CACERTS = "certs/rootCA.crt" # to make ssl accept the self-signed node cert 8 | Pyro4.config.SSL_CLIENTCERT = "certs/t480s.homenet.telecomitalia.it.crt" 9 | Pyro4.config.SSL_CLIENTKEY = "certs/t480s.homenet.telecomitalia.it.key" 10 | print("SSL enabled (2-way).") 11 | 12 | 13 | def verify_cert(cert): 14 | if not cert: 15 | raise Pyro4.errors.CommunicationError("cert missing") 16 | ''' 17 | if cert["serialNumber"] != "D163AB82B8B74DE6": 18 | raise Pyro4.errors.CommunicationError("cert serial number incorrect") 19 | issuer = dict(p[0] for p in cert["issuer"]) 20 | subject = dict(p[0] for p in cert["subject"]) 21 | if issuer["organizationName"] != "Razorvine.net": 22 | # issuer is not often relevant I guess, but just to show that you have the data 23 | raise Pyro4.errors.CommunicationError("cert not issued by Razorvine.net") 24 | if subject["countryName"] != "NL": 25 | raise Pyro4.errors.CommunicationError("cert not for country NL") 26 | if subject["organizationName"] != "Razorvine.net": 27 | raise Pyro4.errors.CommunicationError("cert not for Razorvine.net") 28 | print("(SSL server cert is ok: serial={ser}, subject={subj})" 29 | .format(ser=cert["serialNumber"], subj=subject["organizationName"])) 30 | ''' 31 | 32 | 33 | # to make Pyro verify the certificate on new connections, use the handshake mechanism: 34 | class CertCheckingProxy(Pyro4.core.Proxy): 35 | def _pyroValidateHandshake(self, response): 36 | cert = self._pyroConnection.getpeercert() 37 | verify_cert(cert) 38 | 39 | 40 | node_service_name = ('PYRO:NodeService@' + "x1-yoga.homenet.telecomitalia.it" + ':9090') 41 | # node_service = Pyro4.core.Proxy(node_service_name) 42 | print("node_service: ", node_service_name) 43 | 44 | with CertCheckingProxy(node_service_name) as p: 45 | response = p.echo("client speaking") 46 | print("response:", response) 47 | -------------------------------------------------------------------------------- /ssl/certs/Makefile: -------------------------------------------------------------------------------- 1 | # Deliberately customized from: https://gist.github.com/xenogenesi/1b2137f769aa80b6c99d573071f5d086 2 | 3 | DOMAIN ?= mydomain.com 4 | NAME ?= $(DOMAIN) 5 | 6 | COUNTRY := IT 7 | STATE := IT 8 | COMPANY := Evil Corp. 9 | 10 | # credits to: https://gist.github.com/fntlnz/cf14feb5a46b2eda428e000157447309 11 | 12 | # usage: 13 | # make rootCA.crt # (rootCA.key implicitly created) 14 | # make DOMAIN=somedomain.dev somedomain.dev.csr somedomain.dev.crt or make DOMAIN=somedomain.dev 15 | # make DOMAIN=somedomain.dev verify-csr 16 | # make DOMAIN=somedomain.dev verify-crt 17 | 18 | # import rootCA.crt to the client (chrome) 19 | # upload somedomain.dev.crt and somedomain.dev.key to the host 20 | 21 | all: $(NAME).csr $(NAME).crt 22 | 23 | 24 | rootCA.key: 25 | openssl genrsa -out rootCA.key 4096 26 | 27 | # create and self sign root certificate 28 | rootCA.crt: rootCA.key 29 | echo -ne "$(COUNTRY)\n$(STATE)\n\n$(COMPANY)\n\n\n\n" | openssl req -x509 -new -nodes -key rootCA.key -sha256 -days 1024 -out $@ 30 | 31 | $(NAME).key: 32 | openssl genrsa -out $(NAME).key 2048 33 | 34 | $(NAME).conf: 35 | sh mkconf.sh $(NAME) > $(NAME).conf 36 | 37 | $(NAME).csr: $(NAME).key $(NAME).conf 38 | openssl req -new -sha256 -key $(NAME).key -subj "/C=$(COUNTRY)/ST=$(STATE)/O=$(COMPANY)/CN=$(DOMAIN)" \ 39 | -reqexts SAN \ 40 | -config $(NAME).conf \ 41 | -out $(NAME).csr 42 | 43 | # verify .csr content 44 | .PHONY: verify-csr 45 | verify-csr: 46 | openssl req -in $(NAME).csr -noout -text 47 | 48 | $(NAME).san.conf: 49 | sh mksan.sh $(DOMAIN) $(COUNTRY) $(STATE) "$(COMPANY)" >$@ 50 | 51 | $(NAME).crt: rootCA.key rootCA.crt $(NAME).csr $(NAME).san.conf 52 | openssl x509 -req -in $(NAME).csr -CA ./rootCA.crt -CAkey ./rootCA.key \ 53 | -CAcreateserial -out $@ -days 500 -sha256 \ 54 | -extfile $(NAME).san.conf -extensions req_ext 55 | 56 | # verify the certificate 57 | .PHONY: verify-crt 58 | verify-crt: 59 | openssl x509 -in $(NAME).crt -text -noout 60 | 61 | .PHONY: clean 62 | clean: 63 | -rm -f $(NAME).key $(NAME).csr $(NAME).conf $(NAME).san.conf $(NAME).crt 64 | -------------------------------------------------------------------------------- /job_dispatcher.py: -------------------------------------------------------------------------------- 1 | import Pyro4.core 2 | from worker_node import WorkerNode 3 | from Pyro4.util import SerializerBase 4 | from ssl_utils import CertCheckingProxy, CertValidatingDaemon 5 | from ssl_utils import LOCAL_HOSTNAME, SSL_CERTS_DIR 6 | from global_settings import settings 7 | from job import Job 8 | 9 | 10 | class JobDispatcher: 11 | Pyro4.config.SSL = True 12 | Pyro4.config.SSL_REQUIRECLIENTCERT = True # 2-way ssl 13 | Pyro4.config.SSL_SERVERCERT = SSL_CERTS_DIR + LOCAL_HOSTNAME + ".crt" 14 | Pyro4.config.SSL_SERVERKEY = SSL_CERTS_DIR + LOCAL_HOSTNAME + ".key" 15 | Pyro4.config.SSL_CACERTS = SSL_CERTS_DIR + "rootCA.crt" # to make ssl accept the self-signed master cert 16 | 17 | # For using NFS mounter as a client 18 | Pyro4.config.SSL_CLIENTCERT = Pyro4.config.SSL_SERVERCERT 19 | Pyro4.config.SSL_CLIENTKEY = Pyro4.config.SSL_SERVERKEY 20 | 21 | def __init__(self): 22 | self.daemon = None 23 | self.workers = [] 24 | SerializerBase.register_dict_to_class("worker_node.WorkerNode", WorkerNode.node_dict_to_class) 25 | SerializerBase.register_dict_to_class("job.Job", Job.job_dict_to_class) 26 | self.nfs_exporter = CertCheckingProxy('PYRO:NfsExporter@localhost:9091') 27 | self.first_run = True 28 | 29 | @Pyro4.expose 30 | def test(self): 31 | return "connection ok" 32 | 33 | @Pyro4.expose 34 | def get_worker_options(self): 35 | # options = {"nfs_tuning": ['-o', 'noacl,nocto,noatime,nodiratime']} 36 | options = {"nfs_tuning": ['-o', 'async,soft,timeo=30']} 37 | return options 38 | 39 | @Pyro4.expose 40 | def join_work(self, node): 41 | self.workers.append(node) 42 | 43 | @Pyro4.expose 44 | def report(self, node, job, exit_status, export_range=None): 45 | pass 46 | 47 | @Pyro4.expose 48 | def get_job(self, n): 49 | pass 50 | 51 | def start(self): 52 | print("selected workflow:", settings.dispatcher["workflow"]) 53 | try: 54 | self.nfs_exporter.test() 55 | except Pyro4.errors.CommunicationError as e: 56 | print("Can't connect to local NFS exporter service, make sure it's running.\n", e) 57 | return 58 | 59 | self.daemon = CertValidatingDaemon(host=LOCAL_HOSTNAME, port=9090) 60 | test_uri = self.daemon.register(self, "JobDispatcher") 61 | print("Job dispatcher ready. URI:", test_uri) 62 | self.daemon.requestLoop() 63 | 64 | -------------------------------------------------------------------------------- /nfs_mounter.py: -------------------------------------------------------------------------------- 1 | import Pyro4.core 2 | from ssl_utils import CertValidatingDaemon, SSL_CERTS_DIR 3 | import subprocess 4 | import socket 5 | import os 6 | 7 | 8 | class NfsMounter: 9 | Pyro4.config.SSL = True 10 | Pyro4.config.SSL_REQUIRECLIENTCERT = True # 2-way ssl 11 | Pyro4.config.SSL_SERVERCERT = SSL_CERTS_DIR + socket.gethostname() + "_local.crt" 12 | Pyro4.config.SSL_SERVERKEY = SSL_CERTS_DIR + socket.gethostname() + "_local.key" 13 | Pyro4.config.SSL_CACERTS = SSL_CERTS_DIR + "rootCA.crt" # to make ssl accept the self-signed master cert 14 | 15 | def __init__(self): 16 | self._mounts = set() 17 | 18 | @Pyro4.expose 19 | def test(self): 20 | return "ok" 21 | 22 | @staticmethod 23 | def __nfs4_syntax(path, address): 24 | # get the enclosing folder of the project 25 | if path.find(".ove") > 0: 26 | path = path[:path.rfind("/")] 27 | # add server address prefix 28 | path = address+":"+path 29 | return path 30 | 31 | @Pyro4.expose 32 | def mount(self, path, address, mountpoint, nfs_options): 33 | try: 34 | os.mkdir(mountpoint) 35 | except FileExistsError: 36 | pass 37 | path = self.__nfs4_syntax(path, address) 38 | mount_options = ['mount', path, mountpoint, '-w'] + nfs_options 39 | 40 | # Don't mount twice 41 | if ''.join(mount_options) in self._mounts: 42 | return 0 43 | 44 | print("mounting", path) 45 | mounter = subprocess.run(mount_options, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 46 | 47 | if mounter.returncode != 0: 48 | print("There was an error mounting", path, "- I might not be able to access media.") 49 | print(mounter.stdout, mounter.stderr) 50 | self.umount(mountpoint) 51 | return -1 52 | self._mounts.add(''.join(mount_options)) 53 | return 0 54 | 55 | @Pyro4.expose 56 | def umount(self, mountpoint): 57 | print("umounting", mountpoint) 58 | if subprocess.run(['umount', '-f', mountpoint], stdout=subprocess.PIPE).returncode != 0: 59 | print("There was an error umounting", mountpoint, "- media might still be accessible.") 60 | return -1 61 | self._mounts.clear() 62 | return 0 63 | 64 | def start(self): 65 | d = CertValidatingDaemon(port=9092) 66 | uri = d.register(self, "NfsMounter") 67 | print("NFS Mounter ready. URI:", uri) 68 | d.requestLoop() 69 | 70 | 71 | if __name__ == '__main__': 72 | nfsMounter = NfsMounter() 73 | nfsMounter.start() 74 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # olive-distributed 2 | 3 | ## Dependencies 4 | Software: 5 | 6 | `bind-tools python-pyro` 7 | 8 | Patched olive-editor to support partial export (until it's mainlined): 9 | 10 | `git clone https://github.com/morrolinux/olive.git morrolinux-olive && cd morrolinux-olive && git checkout 0.1.x` 11 | 12 | `cmake . && make -j8 && sudo make install` 13 | 14 | ## Setup 15 | On your "master" node, move to the ssl certs folder: `cd olive-distributed/ssl/certs` 16 | 1) Generate SSL & CA certs for the master node 17 | `./generate_keys.sh setup` 18 | 2) For each worker node you wish to add: 19 | `./generate_keys.sh add ` 20 | 21 | During this setup, SSH service must be running on the worker node in order to copy SSL keys. 22 | 23 | 24 | ## Usage 25 | Move to olive-distributed folder: `cd olive-distributed` 26 | ### On your worker nodes 27 | 1) Start the NFS mounter service: 28 | `sudo python nfs_mounter.py` 29 | 2) Start the worker service: 30 | `./run_worker_node.py` 31 | 32 | ### On the master node 33 | 1) Start the NFS exporter service: 34 | `sudo python nfs_exporter.py` 35 | 2) Submit a job anytime: 36 | `main.py --project /path/to/project.ove` 37 | (to export a single project in a distributed way) or 38 | `main.py --folder /path/to/folder/containing/projects/` 39 | to enqueue multiple projects to be exported in parallel on multiple workers. 40 | 41 | 42 | Note1: Your master node can also be a worker node, just do the steps for worker nodes as well. 43 | 44 | Note2: Once set-up, worker nodes can come and go during a workload, fault tolerance should cope with changes at runtime. 45 | 46 | 47 | ## Logic overview 48 | 49 | A master node is used to dispatch work amongst worker nodes. 50 | The master node has a "NFS Exporter" service running as root and a "Job dispatcher" process running as user. 51 | Each worker node has a "NFS Mounter" service running as root and a "worker" process running as user. 52 | In both cases the user process communicates with the root process via Pyro (RMI) using a 2-Way SSL connection. 53 | The same approach is used for communication between workers and master: 54 | ![Architecture](/doc/architecture.png?raw=true "architecture") 55 | 56 | When a job gets assigned to a worker, it is moved to the "ongoing" queue until a worker reports back on the exit status of the export: 57 | - If it failed, it's moved to the "failed" queue 58 | - If it succeeded, it's moved to the "completed" queue 59 | 60 | When the main job queue becomes empty, "failed" jobs are assigned to free workers (if any). 61 | 62 | When the "failed" queue becomes empty as well, free workers are assigned "ongoing" jobs as they might belong to crashed/unreachable workers who couldn't report back. The first worker to finish an ongoing job gets to push it to the "completed" queue, the others get discarded: 63 | ![States](/doc/states.png?raw=true "states") 64 | 65 | -------------------------------------------------------------------------------- /nfs_exporter.py: -------------------------------------------------------------------------------- 1 | import Pyro4.core 2 | from ssl_utils import CertValidatingDaemon, SSL_CERTS_DIR 3 | import subprocess 4 | from os import stat 5 | import socket 6 | 7 | 8 | class NfsExporter: 9 | Pyro4.config.SSL = True 10 | Pyro4.config.SSL_REQUIRECLIENTCERT = True # 2-way ssl 11 | Pyro4.config.SSL_SERVERCERT = SSL_CERTS_DIR + socket.gethostname() + "_local.crt" 12 | Pyro4.config.SSL_SERVERKEY = SSL_CERTS_DIR + socket.gethostname() + "_local.key" 13 | Pyro4.config.SSL_CACERTS = SSL_CERTS_DIR + "rootCA.crt" # to make ssl accept the self-signed master cert 14 | 15 | @Pyro4.expose 16 | def test(self): 17 | return "ok" 18 | 19 | @staticmethod 20 | def __nfs4_syntax(path, to): 21 | # get the enclosing folder of the project 22 | if path.find(".ove") > 0: 23 | path = path[:path.rfind("/")] 24 | # apply destination 25 | if to is None: 26 | path = "*:"+path 27 | else: 28 | path = to+":"+path 29 | return path 30 | 31 | @Pyro4.expose 32 | def export(self, path, to=None): 33 | folder_uid = str(stat(path).st_uid) 34 | folder_gid = str(stat(path).st_gid) 35 | path = self.__nfs4_syntax(path, to) 36 | print("exporting", path) 37 | if subprocess.run(['exportfs', path, '-o', 'rw,all_squash,anonuid=' + folder_uid + ',anongid=' + folder_gid], 38 | stdout=subprocess.PIPE).returncode != 0: 39 | print("There was an error exporting", path, "- Worker node might not be able to access media.") 40 | 41 | @Pyro4.expose 42 | def unexport(self, path, to=None): 43 | path = self.__nfs4_syntax(path, to) 44 | print("unexporting", path) 45 | if subprocess.run(['exportfs', '-u', path], stdout=subprocess.PIPE).returncode != 0: 46 | print("There was an error unexporting", path, "- media might still be accessible.") 47 | 48 | def start(self): 49 | if subprocess.run(['systemctl', 'start', "nfs-server.service"], stdout=subprocess.PIPE).returncode != 0: 50 | print("There was an error starting nfs-server - make sure NFSv4 is installed.") 51 | 52 | if subprocess.run(['systemctl', 'restart', "rpcbind.service"], stdout=subprocess.PIPE).returncode != 0: 53 | print("There was an error restarting rpcbind - this might be an issue.") 54 | 55 | d = CertValidatingDaemon(port=9091) 56 | uri = d.register(self, "NfsExporter") 57 | print("NFS Exporter ready. URI:", uri) 58 | d.requestLoop() 59 | 60 | def stop(self): 61 | if subprocess.run(['systemctl', 'stop', "nfs-server.service"], stdout=subprocess.PIPE).returncode != 0: 62 | print("There was an error stopping nfs-server.") 63 | 64 | 65 | if __name__ == '__main__': 66 | nfsExporter = NfsExporter() 67 | nfsExporter.start() 68 | nfsExporter.stop() 69 | -------------------------------------------------------------------------------- /full_job_dispatcher.py: -------------------------------------------------------------------------------- 1 | import Pyro4.core 2 | from job_dispatcher import JobDispatcher 3 | from job import Job, abort_job 4 | 5 | 6 | class FullJobDispatcher(JobDispatcher): 7 | def __init__(self): 8 | super().__init__() 9 | self.jobs = None 10 | 11 | @Pyro4.expose 12 | def report(self, node, job, exit_status, export_range=None): 13 | print("\t", node.address, "completed job", job.job_path, "with status", exit_status) 14 | # If the export fails, re-insert it in the job queue 15 | if exit_status != 0: 16 | self.jobs.append(job) 17 | 18 | # In any case, always remove the share after a worker is done 19 | self.nfs_exporter.unexport(job.job_path, to=node.address) 20 | 21 | @Pyro4.expose 22 | def get_job(self, n): 23 | if len(self.jobs) <= 0: 24 | print("PROJECT MANAGER: no more work to do") 25 | return abort_job, None 26 | 27 | if self.first_run: 28 | import time 29 | print("waiting to see if any other workers are joining us...") 30 | time.sleep(5) 31 | self.first_run = False 32 | 33 | # Assign a job based on node benchmark score 34 | max_score = max(node.cpu_score for node in self.workers) 35 | min_score = min(node.cpu_score for node in self.workers) 36 | max_weight = max(job.job_weight for job in self.jobs) 37 | min_weight = min(job.job_weight for job in self.jobs) 38 | fuzzy_job_weight = 0 39 | 40 | if max_score != min_score: 41 | fuzzy_job_weight = min_weight + ((max_weight - min_weight) / (max_score - min_score)) * (n.cpu_score - min_score) 42 | 43 | assigned_job = min(self.jobs, key=lambda x: abs(x.job_weight - fuzzy_job_weight)) 44 | 45 | if assigned_job is None: 46 | return abort_job, None 47 | 48 | # Slow tail fix: 49 | # if there are more nodes than jobs, check weather a faster node is about to finish its work before assignment 50 | # if so, don't assign the current job to the current (slower) node and terminate it. 51 | # TODO: with Pyro, workers in self.render nodes are not updated from remote and cannot provide useful ETA 52 | if len(self.jobs) < len(self.workers): 53 | for worker in self.workers: 54 | w_eta = worker.job_eta() + worker.job_eta(assigned_job) 55 | n_eta = n.job_eta(assigned_job) 56 | if w_eta < n_eta: 57 | return abort_job, None 58 | 59 | # Export the folder via NFS so that the worker node can access it 60 | self.nfs_exporter.export(assigned_job.job_path, to=n.address) 61 | 62 | print(n.address + "\trunning job: ", assigned_job.job_path[assigned_job.job_path.rfind("/")+1:], 63 | "\tWeight: ", assigned_job.job_weight) 64 | self.jobs.remove(assigned_job) 65 | return assigned_job, None 66 | -------------------------------------------------------------------------------- /job.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | 4 | class Job: 5 | def __init__(self, job_path, job_weight, split=False): 6 | self.job_path = job_path 7 | self.job_weight = job_weight 8 | self.len = job_weight 9 | self.split = split 10 | 11 | def __str__(self): 12 | return "" + self.job_path + " : " + str(self.job_weight) 13 | 14 | def __eq__(self, other): 15 | return self.job_weight == other.job_weight 16 | 17 | def __lt__(self, other): 18 | return self.job_weight < other.job_weight 19 | 20 | def __le__(self, other): 21 | return self.job_weight <= other.job_weight 22 | 23 | def __ne__(self, other): 24 | return self.job_weight != other.job_weight 25 | 26 | def __gt__(self, other): 27 | return self.job_weight > other.job_weight 28 | 29 | def __ge__(self, other): 30 | return self.job_weight >= other.job_weight 31 | 32 | @staticmethod 33 | def job_dict_to_class(classname, d): 34 | j = Job(d["job_path"], d["job_weight"]) 35 | j.len = d["len"] 36 | j.split = d["split"] 37 | return j 38 | 39 | 40 | class ExportRange: 41 | def __init__(self, number, start, end, instance_id=None): 42 | self.number = number 43 | self.start = start 44 | self.end = end 45 | self.instance_id = instance_id 46 | if instance_id is None: 47 | self.reset_instance() 48 | 49 | def reset_instance(self): 50 | self.new_instance(-1) 51 | 52 | def new_instance(self, instance_id=None): 53 | if instance_id is None: 54 | self.instance_id = random.randint(1, 99999999) 55 | else: 56 | self.instance_id = instance_id 57 | 58 | @staticmethod 59 | def export_range_dict_to_class(classname, d): 60 | er = ExportRange(d["number"], d["start"], d["end"], d["instance_id"]) 61 | return er 62 | 63 | @staticmethod 64 | def export_range_class_to_dict(obj): 65 | return { 66 | "number": obj.number, 67 | "start": obj.start, 68 | "end": obj.end, 69 | "instance_id": obj.instance_id 70 | } 71 | 72 | def __str__(self): 73 | return str(self.number) + " | (" + str(self.end - self.start) + " frames) | " + str(self.instance_id) 74 | 75 | def __hash__(self): 76 | return hash(self.number) 77 | 78 | def __eq__(self, other): 79 | return self.number == other.number 80 | 81 | def __lt__(self, other): 82 | return self.number < other.number 83 | 84 | def __le__(self, other): 85 | return self.number <= other.number 86 | 87 | def __ne__(self, other): 88 | return self.number != other.number 89 | 90 | def __gt__(self, other): 91 | return self.number > other.number 92 | 93 | def __ge__(self, other): 94 | return self.number >= other.number 95 | 96 | 97 | abort_job = Job("abort", -1) 98 | -------------------------------------------------------------------------------- /split_job_dispatcher.py: -------------------------------------------------------------------------------- 1 | import Pyro4.core 2 | from job_dispatcher import JobDispatcher 3 | import threading 4 | from job import Job, abort_job, ExportRange 5 | from Pyro4.util import SerializerBase 6 | import os 7 | import math 8 | import time 9 | import random 10 | 11 | 12 | class SplitJobDispatcher(JobDispatcher): 13 | def __init__(self): 14 | super().__init__() 15 | self.split_job = None 16 | self.parts_lock = threading.Lock() 17 | self.worker_fails = dict() 18 | self.ongoing_ranges = set() 19 | self.failed_ranges = set() 20 | self.completed_ranges = set() 21 | self.cleanup_queue = set() 22 | self.last_assigned_frame = 0 23 | self.job_parts = 0 24 | SerializerBase.register_dict_to_class("job.ExportRange", ExportRange.export_range_dict_to_class) 25 | SerializerBase.register_class_to_dict(ExportRange, ExportRange.export_range_class_to_dict) 26 | 27 | def set_ongoing_range(self, r): 28 | self.ongoing_ranges.add(r) 29 | try: 30 | self.failed_ranges.remove(r) 31 | except KeyError: 32 | pass 33 | 34 | def fail_range(self, export_range): 35 | self.failed_ranges.add(export_range) 36 | try: 37 | self.ongoing_ranges.remove(export_range) 38 | except KeyError: 39 | pass 40 | self.print_queues() 41 | 42 | def complete_range(self, export_range): 43 | self.completed_ranges.add(export_range) 44 | try: 45 | self.ongoing_ranges.remove(export_range) 46 | self.failed_ranges.remove(export_range) 47 | except KeyError: 48 | pass 49 | self.print_queues() 50 | 51 | def print_queues(self): 52 | print("============================================================") 53 | if (len(self.ongoing_ranges)) > 0: 54 | print("ONGOING:") 55 | for r in self.ongoing_ranges: 56 | print(r) 57 | if len(self.completed_ranges) > 0: 58 | print("COMPLETED:") 59 | for r in self.completed_ranges: 60 | print(r) 61 | if(len(self.failed_ranges)) > 0: 62 | print("FAILED:") 63 | for r in self.failed_ranges: 64 | print(r) 65 | print("============================================================") 66 | 67 | def split_job_finished(self): 68 | if len(self.completed_ranges) == 0: 69 | total_len_covered = False 70 | else: 71 | total_len_covered = self.split_job.len == max(n.end for n in self.completed_ranges) 72 | all_parts_done = self.job_parts == len(self.completed_ranges) 73 | no_failed_ranges = len(self.failed_ranges) == 0 74 | return total_len_covered and all_parts_done and no_failed_ranges 75 | 76 | def write_concat_list(self, list_name): 77 | with open(list_name, "w") as m: 78 | parts = list(self.completed_ranges) 79 | parts.sort() 80 | for p in parts: 81 | m.write("file \'" + str(p.instance_id) + ".mp4\'\n") 82 | 83 | def merge_parts(self, output_name): 84 | os.chdir(self.split_job.job_path[:self.split_job.job_path.rfind("/")]) 85 | list_name = "merge.txt" 86 | self.write_concat_list(list_name) 87 | os.system("ffmpeg -f concat -safe 0 -i " + list_name + " -c copy " + output_name + ".mp4" + " -y") 88 | os.remove(list_name) 89 | 90 | def cleanup_parts(self): 91 | for p in self.cleanup_queue: 92 | try: 93 | print("removing: " + str(p) + ".mp4") 94 | os.remove(str(p) + ".mp4") 95 | except FileNotFoundError: 96 | pass 97 | 98 | def remove_shares(self): 99 | for worker in self.workers: 100 | self.nfs_exporter.unexport(self.split_job.job_path, to=worker.address) 101 | 102 | @Pyro4.expose 103 | def join_work(self, node): 104 | super().join_work(node) 105 | self.worker_fails[node.address] = [] 106 | self.nfs_exporter.export(self.split_job.job_path, to=node.address) 107 | 108 | @Pyro4.expose 109 | def get_job(self, n): 110 | if self.split_job_finished(): 111 | return abort_job, None 112 | 113 | if self.first_run: 114 | print("Welcome, " + str(n.address) + ".\n waiting to see if any other workers are joining us...") 115 | time.sleep(5) 116 | self.first_run = False 117 | 118 | self.parts_lock.acquire() 119 | 120 | # Assign the given worker a chunk size proportional to its rank 121 | tot_workers_score = sum(worker.cpu_score for worker in self.workers) 122 | if len(self.workers) > 1: 123 | s = 900 124 | chunk_size = s + math.ceil((n.cpu_score / tot_workers_score) * s) 125 | else: 126 | chunk_size = 1800 127 | 128 | # TODO: Make sure to ALWAYS match keyframes... (should probably be done in Olive) 129 | # Where to start/end the chunk (+ update seek) 130 | job_start = self.last_assigned_frame 131 | job_end = min(job_start + chunk_size, self.split_job.len) 132 | self.last_assigned_frame = job_end 133 | 134 | # If we're still working but all frames have been assigned, check if there are failed ranges left 135 | if job_end - job_start == 0: 136 | if len(self.failed_ranges) > 0: 137 | r = list(self.failed_ranges)[random.randrange(0, len(self.failed_ranges)) % len(self.failed_ranges)] 138 | # If a worker has already failed this specific range, don't attempt again 139 | if r in self.worker_fails[n.address]: 140 | self.parts_lock.release() 141 | return Job("retry", 1), None 142 | # If there still are ongoing jobs, they *could* belong to crashed/hanged workers. 143 | # We therefore assign them anyways. The first worker to finish will move them to the terminated set 144 | elif len(self.ongoing_ranges) > 0: 145 | r = list(self.ongoing_ranges)[random.randrange(0, len(self.ongoing_ranges)) % len(self.ongoing_ranges)] 146 | # If there are no more failed nor ongoing jobs, we really are finished. 147 | else: 148 | self.parts_lock.release() 149 | return Job("abort", 0), None 150 | else: 151 | # update the current number of job parts 152 | self.job_parts += 1 153 | r = ExportRange(self.job_parts, job_start, job_end) 154 | 155 | r.new_instance(n.address + "-" + str(random.randrange(1, 9999))) 156 | self.cleanup_queue.add(r.instance_id) 157 | self.set_ongoing_range(r) 158 | 159 | print(n.address, "will export part", r) 160 | 161 | self.parts_lock.release() 162 | 163 | # Return the job to the worker node 164 | return self.split_job, r 165 | 166 | @Pyro4.expose 167 | def report(self, node, job, exit_status, export_range): 168 | print(" ", node.address, "completed part", export_range, "with status:", exit_status) 169 | 170 | if isinstance(export_range, dict): 171 | export_range = ExportRange.export_range_dict_to_class("job.ExportRange", export_range) 172 | 173 | # If someone else already completed this job, just discard it 174 | if export_range in self.completed_ranges: 175 | pass 176 | # Otherwise, if export failed, re-insert failed ranges 177 | elif exit_status != 0: 178 | self.fail_range(export_range) 179 | self.worker_fails[node.address].append(export_range) 180 | # If the split job export went fine, move the exported range to the completed ones 181 | else: 182 | self.complete_range(export_range) 183 | 184 | if self.split_job_finished(): 185 | self.remove_shares() 186 | self.merge_parts(self.split_job.job_path[self.split_job.job_path.rfind("/") + 1:]) 187 | print("Export merged. Finished!!!") 188 | self.cleanup_parts() 189 | self.daemon.shutdown() 190 | -------------------------------------------------------------------------------- /worker_node.py: -------------------------------------------------------------------------------- 1 | import time 2 | import Pyro4.core 3 | import Pyro4.errors 4 | import subprocess 5 | from Pyro4.util import SerializerBase 6 | from job import Job, ExportRange 7 | from ssl_utils import CertCheckingProxy, LOCAL_HOSTNAME, SSL_CERTS_DIR, OD_FOLDER 8 | from pathlib import Path 9 | import os 10 | import sys 11 | import shutil 12 | import threading 13 | import signal 14 | 15 | 16 | class WorkerNode: 17 | Pyro4.config.SSL = True 18 | Pyro4.config.SSL_CACERTS = OD_FOLDER + SSL_CERTS_DIR + "rootCA.crt" # to make ssl accept the self-signed node cert 19 | Pyro4.config.SSL_CLIENTCERT = OD_FOLDER + SSL_CERTS_DIR + LOCAL_HOSTNAME + ".crt" 20 | Pyro4.config.SSL_CLIENTKEY = OD_FOLDER + SSL_CERTS_DIR + LOCAL_HOSTNAME + ".key" 21 | sys.excepthook = Pyro4.util.excepthook 22 | 23 | def __init__(self, address): 24 | self.TEMP_DIR = '/tmp/olive' 25 | self.address = address 26 | self.cpu_score = 0 27 | self.net_score = 0 28 | self._job_start_time = None 29 | self._job = None 30 | self.sample_weight = None 31 | self.sample_time = None 32 | self.worker_options = dict() 33 | self.olive_export_process = None 34 | self.terminating = False 35 | self.MASTER_ADDRESS = None 36 | self.MOUNTPOINT_DEFAULT = None 37 | self.job_dispatcher = None 38 | self.nfs_mounter = None 39 | 40 | def setup(self): 41 | signal.signal(signal.SIGINT, self.termination_handler) 42 | with open(OD_FOLDER + SSL_CERTS_DIR + 'whoismaster') as f: 43 | self.MASTER_ADDRESS = f.read().strip() 44 | self.MOUNTPOINT_DEFAULT = str(Path.home())+'/olive-share/' 45 | self.job_dispatcher = CertCheckingProxy('PYRO:JobDispatcher@' + self.MASTER_ADDRESS + ':9090') 46 | self.nfs_mounter = CertCheckingProxy('PYRO:NfsMounter@' + 'localhost' + ':9092') 47 | SerializerBase.register_dict_to_class("job.ExportRange", ExportRange.export_range_dict_to_class) 48 | SerializerBase.register_dict_to_class("job.Job", Job.job_dict_to_class) 49 | 50 | def job_eta(self, j=None): 51 | if self.sample_time is None or self.sample_weight is None: 52 | return 9223372036854775807 53 | 54 | if j is not None: 55 | t = (j.job_weight * self.sample_time) / self.sample_weight 56 | elif self._job is not None: 57 | t = self.job_eta(self._job) - (time.time() - self._job_start_time) 58 | else: 59 | t = 0 60 | return t 61 | 62 | def termination_handler(self, signum, frame): 63 | print("stopping threads and clean termination...") 64 | self.terminating = True 65 | quit(0) 66 | 67 | def __connection_watchdog(self): 68 | while not self.terminating: 69 | time.sleep(5) 70 | try: 71 | self.job_dispatcher.test() 72 | except Pyro4.errors.CommunicationError: 73 | if self.olive_export_process is not None: 74 | print("Lost connection to the master, aborting ongoing exports...") 75 | self.olive_export_process.terminate() 76 | 77 | def run_benchmark(self): 78 | import random 79 | self.cpu_score = random.randrange(1, 10) 80 | self.net_score = random.randrange(1, 10) 81 | self.cpu_score = float(subprocess.run([OD_FOLDER + 'bench/bench-host.sh'], stdout=subprocess.PIPE).stdout) 82 | print("node", self.address, "\t\tCPU:", self.cpu_score) 83 | 84 | def run(self): 85 | if not Path(self.TEMP_DIR).exists(): 86 | os.mkdir(self.TEMP_DIR) 87 | threading.Thread(target=self.__connection_watchdog).start() 88 | 89 | while not self.terminating: 90 | try: 91 | print(self.job_dispatcher.test()) 92 | except Pyro4.errors.CommunicationError as e: 93 | print(e, "\nCan't connect to dispatcher, retrying...") 94 | time.sleep(1) 95 | continue 96 | 97 | if self.cpu_score is None or self.cpu_score == 0: 98 | self.run_benchmark() 99 | self.worker_options.update(self.job_dispatcher.get_worker_options()) 100 | self.job_dispatcher.join_work(self) 101 | self.__run() 102 | time.sleep(1) 103 | 104 | def __run(self): 105 | while True: 106 | try: 107 | j, export_range = self.job_dispatcher.get_job(self) 108 | except Pyro4.errors.CommunicationError: 109 | return 110 | print("got job:", j, (export_range if export_range is not None else "")) 111 | if j.job_path == "abort": 112 | print(self.address, "\tterminating...") 113 | self.nfs_mounter.umount(self.MOUNTPOINT_DEFAULT) 114 | return 115 | if j.job_path == "retry": 116 | time.sleep(j.job_weight) 117 | continue 118 | # mount the NFS share before starting 119 | if self.nfs_mounter.mount(j.job_path, self.MASTER_ADDRESS, self.MOUNTPOINT_DEFAULT, 120 | self.worker_options["nfs_tuning"]) != 0: 121 | self.job_dispatcher.report(self, j, -1, export_range) 122 | return 123 | self.run_job(j, export_range) 124 | 125 | def run_job(self, j, export_range): 126 | self._job_start_time = time.time() 127 | self._job = j 128 | 129 | project_name = j.job_path[j.job_path.rfind("/") + 1:] 130 | olive_args = ['olive-editor', self.MOUNTPOINT_DEFAULT + project_name, '-e'] 131 | 132 | if export_range is not None: 133 | # Here we need to call deserialization manually because of dynamic typing 134 | # ( not all implementations of dispatcher return (Job, ExportRange) ) 135 | if isinstance(export_range, dict): 136 | export_range = ExportRange.export_range_dict_to_class("job.ExportRange", export_range) 137 | olive_args.append(str(export_range.instance_id)) 138 | olive_args.append('--export-start') 139 | olive_args.append(str(export_range.start)) 140 | olive_args.append('--export-end') 141 | olive_args.append(str(export_range.end)) 142 | 143 | # Do the actual export with the given parameters 144 | os.chdir(self.TEMP_DIR) 145 | self.olive_export_process = subprocess.Popen(olive_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 146 | self.olive_export_process.wait() 147 | if export_range is not None: 148 | export_name = export_range.instance_id + ".mp4" 149 | else: 150 | export_name = project_name + ".mp4" 151 | 152 | # Move the exported video to the NFS share 153 | try: 154 | shutil.move(export_name, self.MOUNTPOINT_DEFAULT) 155 | file_moved = True 156 | except OSError: 157 | file_moved = False 158 | 159 | # cleanup partial files 160 | for root, dirs, files in os.walk(self.TEMP_DIR): 161 | for file in files: 162 | os.remove(file) 163 | 164 | # dummy export jobs: 165 | # time.sleep((j.job_weight/self.cpu_score)/100) 166 | # time.sleep(1) 167 | # import random 168 | # if random.randrange(-100, 100) > 0: 169 | # olive_export = subprocess.run(['true'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) # success 170 | # else: 171 | # olive_export = subprocess.run(['false'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) # failure 172 | 173 | if self.olive_export_process.returncode == 0 and file_moved: 174 | print("Job done:", j.job_path, (export_range.number if export_range is not None else "")) 175 | else: 176 | print("Error exporting", j.job_path, (export_range.number if export_range is not None else "")) 177 | 178 | return_code = int(self.olive_export_process.returncode or not file_moved) 179 | self.olive_export_process = None 180 | 181 | # If we completed a with a full job, umount. Otherwise umount on abort 182 | if not j.split: 183 | self.nfs_mounter.umount(self.MOUNTPOINT_DEFAULT) 184 | 185 | try: 186 | self.job_dispatcher.report(self, j, return_code, export_range) 187 | except Pyro4.errors.ConnectionClosedError: 188 | return 189 | except Pyro4.errors.CommunicationError: 190 | return 191 | except ConnectionRefusedError: 192 | return 193 | 194 | self.sample_weight = j.job_weight 195 | self.sample_time = time.time() - self._job_start_time 196 | self._job = None 197 | 198 | @staticmethod 199 | def node_dict_to_class(classname, d): 200 | r = WorkerNode(d["address"]) 201 | r.cpu_score = d["cpu_score"] 202 | r.net_score = d["net_score"] 203 | r._job_start_time = d["_job_start_time"] 204 | r.sample_weight = d["sample_weight"] 205 | r.sample_time = d["sample_time"] 206 | return r 207 | -------------------------------------------------------------------------------- /bench/sample.ove: -------------------------------------------------------------------------------- 1 | 2 | 3 | 190219 4 | bench/sample.ove 5 | 6 | 7 | 8 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | --------------------------------------------------------------------------------