├── .gitignore ├── MANIFEST.in ├── README.md ├── bin ├── bluerun ├── qconvert └── taskrun ├── etc ├── blueprint.cfg ├── env_wrapper.sh ├── postrun.sh └── prerun.sh ├── lib └── blueprint │ ├── __init__.py │ ├── app.py │ ├── archive.py │ ├── backend │ ├── __init__.py │ ├── plowrun.py │ └── test.py │ ├── conf.py │ ├── exception.py │ ├── io.py │ ├── job.py │ ├── layer.py │ ├── modules │ ├── __init__.py │ ├── blender.py │ ├── iconvert.py │ ├── maya.py │ ├── nuke.py │ ├── setup │ │ └── blender_setup.py │ └── shell.py │ └── plugins │ ├── __init__.py │ ├── plow_outputs.py │ └── test_plugin.py ├── setup.py └── tests ├── __init__.py ├── common.py ├── scenes └── sphere_v1.mb ├── scripts ├── maya.bps └── sleep.bps ├── setup.py ├── test_all.py ├── test_app.py ├── test_archive.py ├── test_conf.py ├── test_job.py ├── test_layer.py ├── test_module_blender.py ├── test_module_maya.py └── test_taskrun.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | Blueprint.egg-info 3 | build 4 | lib/Distutils.egg-info 5 | dist 6 | lib/plow_blueprint.egg-info 7 | .idea 8 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | graft etc 2 | graft share 3 | global-include *.sh *.bp *.py *.cfg -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### Blueprint 2 | Blueprint is a job description library for the [Plow Render Farm](http://www.plowrender.com). It simplifies the creation of a Plow job spec structure, allows execution of arbitrary python code on Plow, and provides hooks for integration into your existing pipeline 3 | 4 | ### Application Modules 5 | A Blueprint application module is necessary for wrapping your application/code to run on Plow. Blueprint will eventually include core modules for: 6 | 7 | * Maya 8 | * MotionBuiler 9 | * Nuke 10 | * Katana 11 | * Blender 12 | * Houdini 13 | * Open Image IO 14 | 15 | Also included are modules for executing arbitrary shell commands, python code, file search, and other pipeline related tasks. 16 | 17 | ### Plugins 18 | Blueprint includes an event driven plugin system which can be used to intercept job events and execute arbitrary code. 19 | 20 | ### Dependencies 21 | 22 | ``` 23 | pip install pyyaml 24 | pip install Fileseq 25 | ``` -------------------------------------------------------------------------------- /bin/bluerun: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import argparse 4 | 5 | import blueprint 6 | 7 | class Bluerun(blueprint.Application): 8 | 9 | def __init__(self): 10 | blueprint.app.Application.__init__(self, 11 | "Bluerun - Blueprint script launch tool.") 12 | 13 | group = self._argparser.add_argument_group("Script Options") 14 | group.add_argument("script", metavar="SCRIPT", help="Path to blueprint script.") 15 | group.add_argument("range", metavar="FRAME_RANGE", help="Frame range.") 16 | 17 | job_group = self._argparser.add_argument_group("Job Options") 18 | job_group.add_argument("-pause", action="store_true", help="Launch the job in a paused state.") 19 | job_group.add_argument("-name", metavar="NAME", help="Set the job name.") 20 | job_group.add_argument("-pretend", action="store_true", help="Do everything except launch the job.") 21 | job_group.add_argument("-env", action="append", default=list(), 22 | help="Set an environment variable to be present in every task. (name=value)") 23 | 24 | def handleArgs(self, args): 25 | if args.pause: 26 | self._runner.setArg("pause", args.pause) 27 | if args.name: 28 | self._runner.setArg("name", args.name) 29 | if args.pretend: 30 | self._runner.setArg("pretend", args.pretend) 31 | if args.env: 32 | current_env = self._runner.getArg("env") 33 | for pair in args.env: 34 | key, value = pair.split("=", 1) 35 | current_env[key] = value 36 | self._runner.setArg("env", current_env) 37 | 38 | self._runner.setArg("script", args.script) 39 | self._runner.setArg("range", args.range) 40 | self._runner.launch() 41 | 42 | if __name__ == "__main__": 43 | app = Bluerun() 44 | app.go() 45 | -------------------------------------------------------------------------------- /bin/qconvert: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import argparse 4 | import re 5 | import time 6 | 7 | import blueprint.app 8 | from blueprint.job import Job 9 | from blueprint.modules.iconvert import IConvert 10 | 11 | class QConvert(blueprint.app.Application): 12 | """ A class that converts a commandline tool iconvert into a convert queue """ 13 | def __init__(self): 14 | description = "qconvert - blueprint wrapper of iconvert tool.\n" 15 | description += "Example: qconvert -i /path/input.%04d.png -o /path/output.%04f.jpeg --rot180 --range 1-10 --batch 3" 16 | blueprint.app.Application.__init__(self, description) 17 | 18 | group = self._argparser.add_argument_group("qconvert options") 19 | group.add_argument("-v", "--verbose", action="store_true", help="Verbose status messages.") 20 | group.add_argument("-r", "--range", help="Specify frame range. 1-10") 21 | group.add_argument("-b", "--batch", help="Specify batch numbers. default is 5", default=5) 22 | group.add_argument("-i", help="Specify full path image input. i.e. /tmp/input.%%%%04d.png") 23 | group.add_argument("-o", help="Specify full path image ouput. i.e. /tmp.output.%%%%04d.jpg") 24 | group.add_argument("--jobname", help="Specify job name") 25 | 26 | group.add_argument("--threads", help="Specify number of threads. (default 0=#cores)") 27 | group.add_argument("--d", help="Set the output data format to one of:uint8, sint8, uint10, uint12, uint16, sint16, half, float, double") 28 | group.add_argument("--g", help="Set gamma correction. (default = 1)") 29 | group.add_argument("--tile", help="Output as a tiled image. i.e. 4:4") 30 | group.add_argument("--scanline", action="store_true", help="Output as a scanline image") 31 | group.add_argument("--compression", help="Set the compression method (default = same as input)") 32 | group.add_argument("--quality", help="Set the compression quality, 1-100") 33 | group.add_argument("--no-copy-image", action="store_true", help="Do not use ImageOutput copy_image functionality") 34 | group.add_argument("--adjust-time", action="store_true", help="Adjust file times to match DateTime metadata") 35 | group.add_argument("--caption", help="Set caption (ImageDescription)") 36 | group.add_argument("--keyword", help="Add a keyword") 37 | group.add_argument("--clear-keywords", action="store_true", help="Clear keywords") 38 | group.add_argument("--attrib", help="Set a string attribute, name:value") 39 | group.add_argument("--orientation", help="Set the orientation") 40 | group.add_argument("--rotcw", action="store_true", help="Rotate 90 deg clockwise") 41 | group.add_argument("--rotccw", action="store_true", help="Rotate 90 deg counter-clockwise") 42 | group.add_argument("--rot180", action="store_true", help="Rotate 180 deg") 43 | group.add_argument("--inplace", action="store_true", help="Do operations in place on images") 44 | group.add_argument("--sRGB", action="store_true", help="This file is in sRGB color space") 45 | group.add_argument("--separate", action="store_true", help="Force planarconfig separate") 46 | group.add_argument("--contig", action="store_true", help="Force planarconfig contig") 47 | group.add_argument("--no-clobber", action="store_true", help="Do no overwrite existing files") 48 | 49 | 50 | def batchRange(self, first, last, steps): 51 | """ Given first, last frames and the batch, yield back batched frames """ 52 | size = steps 53 | while first <= last: 54 | next = first + size 55 | yield first, min(next - 1, last) 56 | first = next 57 | 58 | 59 | def handleArgs(self, args): 60 | """ Method to handle args """ 61 | runner = blueprint.app.BlueprintRunner() 62 | 63 | if args.jobname: 64 | runner.setArg("job_name", args.jobname) 65 | else: 66 | jobname = "qconvert_%s" % str(int(time.time())) 67 | 68 | layers = list() 69 | # Create a job instance 70 | j = Job(jobname) 71 | start = end = 0 72 | ranges = args.range 73 | (st, e) = str(args.range).split('-') 74 | 75 | options = self.parseArgs(args) 76 | 77 | frameranges = list(self.batchRange(int(st), int(e), int(args.batch))) 78 | for fr in frameranges: 79 | (start, end) = fr 80 | ic = IConvert("layer_%s_%s" % (start, end), start=start, end=end, 81 | image_input=args.i, image_output=args.o, **options) 82 | j.addLayer(ic) 83 | 84 | runner.setJob(j) 85 | runner.setArg("frame_range", '1-1') 86 | runner.launch() 87 | 88 | 89 | def parseArgs(self, args): 90 | """ Given args, parse them and return as key, value dictionary """ 91 | # too late in the night to think of a cleverer way to do this 92 | result = dict() 93 | if args.threads: 94 | result['threads'] = args.threads 95 | if args.d: 96 | result['d'] = args.d 97 | if args.g: 98 | result['g'] = args.g 99 | 100 | if args.tile: 101 | tiles = args.tile.split(':') 102 | result['tile'] = tiles 103 | if args.scanline: 104 | result['scanline'] = True 105 | if args.compression: 106 | result['compression'] = args.compression 107 | if args.quality: 108 | result['quality'] = args.quality 109 | if args.no_copy_image: 110 | result['no_copy_image']=True 111 | if args.adjust_time: 112 | result['adjust_time'] = True 113 | if args.caption: 114 | result['caption'] = args.caption 115 | if args.keyword: 116 | result['keyword'] = args.keyword 117 | if args.clear_keywords: 118 | result['clear_keywords'] = args.clear_keywords 119 | if args.attrib: 120 | result['attrib'] = args.attrib.split(":") 121 | if args.orientation: 122 | result['orientation'] = args.orientation 123 | if args.rotcw: 124 | result['rotcw'] = True 125 | if args.rotccw: 126 | result['rotccw'] = True 127 | if args.rot180: 128 | result['rot180'] = True 129 | if args.inplace: 130 | result['inplace'] = True 131 | if args.sRGB: 132 | result['sRGB'] = True 133 | if args.separate: 134 | result['separate'] = True 135 | if args.contig: 136 | result['contig'] = True 137 | if args.no_clobber: 138 | result['no_clobber'] = True 139 | 140 | return result 141 | 142 | 143 | if __name__ == "__main__": 144 | app = QConvert() 145 | app.go() 146 | -------------------------------------------------------------------------------- /bin/taskrun: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import traceback 4 | import fileseq 5 | import blueprint 6 | 7 | from blueprint.exception import CommandException 8 | 9 | class Taskrun(blueprint.Application): 10 | 11 | def __init__(self): 12 | blueprint.Application.__init__(self, 13 | "Taskrun - Blueprint task execution tool") 14 | group = self._argparser.add_argument_group("Taskrun Options") 15 | 16 | group.add_argument("script", metavar="SCRIPT", help="Path to blueprint script.") 17 | group.add_argument("-frame", metavar="FRAME", help="The frame number to execute.", default="1001") 18 | 19 | group.add_argument("-layer", metavar="LAYER", 20 | help="Layer to execute task from.") 21 | 22 | group.add_argument("-task", metavar="TASK", 23 | help="Task to execute."); 24 | 25 | def handleArgs(self, args): 26 | job = blueprint.loadScript(args.script) 27 | 28 | try: 29 | if args.layer: 30 | layer = job.getLayer(args.layer) 31 | layer.execute(int(args.frame)) 32 | elif args.task: 33 | task = job.getLayer(args.task) 34 | task.execute() 35 | else: 36 | raise Exception("Unsupported operation, must specify -layer or -task") 37 | 38 | except CommandException, e: 39 | print >> sys.stderr, "%s\n" % e 40 | sys.exit(e.exitStatus) 41 | 42 | except Exception, e: 43 | print >> sys.stderr, "Error running task %s\n" % e 44 | traceback.print_exc(file=sys.stderr) 45 | sys.exit(1) 46 | 47 | 48 | if __name__ == "__main__": 49 | taskrun = Taskrun() 50 | taskrun.go() 51 | -------------------------------------------------------------------------------- /etc/blueprint.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "bp": { 3 | 4 | "scripts_dir": "/usr/local/etc/blueprint", 5 | 6 | "job_name_template": "test-scene.shot-%(USER)s_%(JOB_NAME)s", 7 | 8 | "archive_dir": "%(HOME)s/blueprint/archive/%(JOB_NAME)s", 9 | 10 | "log_dir": "%(HOME)s/blueprint/logs/%(JOB_NAME)s", 11 | 12 | "output_dir": "%(HOME)s/blueprint/rnd", 13 | 14 | "backend": "plowrun", 15 | 16 | "project": "test" 17 | }, 18 | 19 | "env": { 20 | "interpolate": ["USER", "HOME"] 21 | }, 22 | 23 | "system": { 24 | "mkdir": ["/bin/mkdir", "-p"] 25 | }, 26 | 27 | "plugins": [ 28 | { 29 | "name": "plow_outputs", 30 | "module": "blueprint.plugins.plow_outputs", 31 | "enabled": false 32 | }, 33 | { 34 | "name": "test_plugin", 35 | "module": "blueprint.plugins.test_plugin", 36 | "enabled": true 37 | } 38 | ], 39 | 40 | "modules": { 41 | 42 | "maya": { 43 | "padding": 4, 44 | "fnc": "name.#.ext", 45 | "format": "png", 46 | "renderer": "sw", 47 | "camera": "persp" 48 | } 49 | } 50 | } 51 | 52 | 53 | -------------------------------------------------------------------------------- /etc/env_wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Wraps the execution of plow tasks on the render farm. 4 | # 5 | source ${BLUEPRINT_SCRIPTS_PATH}/prerun.sh 6 | 7 | # Set a temporary directory for the process. 8 | export PLOW_TASK_TEMPDIR="${TMPDIR}/${PLOW_JOB_NAME}/${PLOW_TASK_NAME}" 9 | export TMPDIR="$PLOW_TASK_TMPDIR" 10 | 11 | # Make sure it exists 12 | mkdir -p $PLOW_TASK_TEMPDIR 13 | 14 | eval $@ 15 | ret_code=$? 16 | 17 | # Clean up the temp directory 18 | rm -f ${PLOW_TASK_TEMPDIR}/* 19 | 20 | source ${BLUEPRINT_SCRIPTS_PATH}/postrun.sh 21 | 22 | if [ $ret_code != 0 ]; then 23 | exit $ret_code 24 | fi 25 | -------------------------------------------------------------------------------- /etc/postrun.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # The postrun.sh script is for executing your own post task commands. 4 | echo "Executing postrun.sh" -------------------------------------------------------------------------------- /etc/prerun.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # The prerun.sh script is for executing your own pre task commands, for example 4 | # sourcing youe environemt scripts. 5 | echo "Executing prerun.sh" -------------------------------------------------------------------------------- /lib/blueprint/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from app import PluginManager, Application, BlueprintRunner, loadScript 3 | from layer import Layer, Task, PostTask, TaskContainer, TaskIterator, SetupTask, DependType, Depend 4 | from job import Job 5 | 6 | PluginManager.loadAllPlugins() 7 | 8 | -------------------------------------------------------------------------------- /lib/blueprint/app.py: -------------------------------------------------------------------------------- 1 | 2 | import logging 3 | logger = logging.getLogger(__name__) 4 | logging.basicConfig(level=logging.WARN) 5 | 6 | import os 7 | import argparse 8 | import yaml 9 | import pprint 10 | 11 | import conf 12 | 13 | from exception import BlueprintException 14 | 15 | 16 | 17 | __all__ = [ 18 | "Application", 19 | "PluginManager", 20 | "BlueprintRunner", 21 | "loadBackendPlugin", 22 | "loadScript"] 23 | 24 | 25 | class EventManager(object): 26 | pass 27 | 28 | class PluginManager(object): 29 | """ 30 | The PluginManager is used to maintain a list of active plugins. 31 | """ 32 | loaded = [] 33 | 34 | @classmethod 35 | def loadPlugin(cls, module): 36 | if module in cls.loaded: 37 | return 38 | try: 39 | plugin = __import__(module, globals(), locals(), [module]) 40 | except ImportError, e: 41 | logger.warn("Failed to load plugin: %s, %s", module, e) 42 | return 43 | 44 | try: 45 | plugin.init() 46 | except AttributeError, e: 47 | pass 48 | 49 | logger.debug("Initialized plugin %s", plugin) 50 | cls.loaded.append(plugin) 51 | 52 | @classmethod 53 | def runPreLaunch(cls, spec, job): 54 | logger.debug("Running pre-launch plugins on spec: %s" % spec) 55 | for plugin in cls.loaded: 56 | if getattr(plugin, "preLaunch", False): 57 | plugin.preLaunch(spec, job) 58 | 59 | @classmethod 60 | def runAfterInit(cls, layer): 61 | logger.debug("Running after init plugins on %s" % layer) 62 | for plugin in cls.loaded: 63 | if getattr(plugin, "afterInit", False): 64 | plugin.afterInit(layer) 65 | 66 | @classmethod 67 | def runLayerSetup(cls, layer): 68 | logger.debug("Running setup plugins on %s" % layer) 69 | for plugin in cls.loaded: 70 | if getattr(plugin, "layerSetup", False): 71 | plugin.layerSetup(layer) 72 | 73 | @classmethod 74 | def runJobSetup(cls, job): 75 | logger.debug("Running setup plugins on %s" % job) 76 | for plugin in cls.loaded: 77 | if getattr(plugin, "jobSetup", False): 78 | plugin.jobSetup(job) 79 | 80 | @classmethod 81 | def runBeforeExecute(cls, layer): 82 | logger.debug("Running before execute plugins on %s" % layer) 83 | for plugin in cls.loaded: 84 | if getattr(plugin, "beforeExecute", False): 85 | plugin.beforeExecute(layer) 86 | 87 | @classmethod 88 | def runAfterExecute(cls, layer): 89 | logger.debug("Running after execute plugins on %s" % layer) 90 | for plugin in cls.loaded: 91 | if getattr(plugin, "afterExecute", False): 92 | plugin.afterExecute(layer) 93 | 94 | @classmethod 95 | def getActivePlugins(cls): 96 | result = [] 97 | for plugin in conf.get("plugins"): 98 | if not plugin["enabled"]: 99 | continue 100 | result.append(plugin["module"]) 101 | return result 102 | 103 | @classmethod 104 | def getLoadedPlugins(cls): 105 | return cls.loaded 106 | 107 | @classmethod 108 | def loadAllPlugins(cls): 109 | logger.debug("Loading all plugins") 110 | for plugin in cls.getActivePlugins(): 111 | cls.loadPlugin(plugin) 112 | 113 | 114 | class Application(object): 115 | def __init__(self, descr): 116 | 117 | self._runner = BlueprintRunner() 118 | 119 | self._argparser = argparse.ArgumentParser(description=descr) 120 | group = self._argparser.add_argument_group("Logging Options") 121 | group.add_argument("-verbose", action="store_true", 122 | help="Turn on verbose logging.") 123 | group.add_argument("-debug", action="store_true", 124 | help="Turn on debug logging.") 125 | group.add_argument("-host", metavar="HOSTNAME", 126 | help="Specify the server to communicate with, if any.") 127 | group.add_argument("-backend", metavar="BACKEND", 128 | help="Specify the queuing backend plugin.") 129 | 130 | def handleArgs(self, args): 131 | pass 132 | 133 | def go(self): 134 | args = self._argparser.parse_args() 135 | 136 | # Handle the common arguments. 137 | rootLogger = logging.getLogger() 138 | if args.verbose: 139 | rootLogger.setLevel(logging.INFO) 140 | if args.debug: 141 | rootLogger.setLevel(logging.DEBUG) 142 | 143 | if args.host: 144 | self._runner.setArg("host", args.host) 145 | if args.backend: 146 | self._runner.setArg("backend", args.backend) 147 | 148 | # Handle arguments added by specific application. 149 | self.handleArgs(args) 150 | 151 | 152 | # A simple object to use as a BlueprintRunner.getArg() default 153 | # value rather than None, which could be a valid value. 154 | LoadDefault = object() 155 | 156 | class BlueprintRunner(object): 157 | 158 | def __init__(self, job=None, **kwargs): 159 | self.__args = {} 160 | self.__defaults = { 161 | "host": None, 162 | "pause": False, 163 | "backend": conf.get("bp.backend", default="plow"), 164 | "name": "", 165 | "pretend": False, 166 | "script": None, 167 | "range": None, 168 | "env": { } 169 | } 170 | self.__args.update(kwargs) 171 | self.__job = job 172 | 173 | def setArg(self, key, value): 174 | self.__args[key] = value 175 | 176 | def getArg(self, key, default=LoadDefault): 177 | try: 178 | return self.__args[key] 179 | except KeyError: 180 | if default is LoadDefault: 181 | return self.__defaults.get(key) 182 | return default 183 | 184 | def launch(self): 185 | 186 | logger.debug("Blueprint runner args:") 187 | logger.debug(self.__args) 188 | 189 | backend_module = self.getArg("backend") 190 | if not backend_module: 191 | raise BlueprintException("No backend module is set, see defaults.backend setting in config.") 192 | 193 | logging.debug("Launching job with backend: %s" % backend_module) 194 | backend = loadBackendPlugin(backend_module) 195 | 196 | self.setup() 197 | spec = backend.serialize(self) 198 | PluginManager.runPreLaunch(spec, self.__job) 199 | 200 | if self.getArg("pretend"): 201 | pprint.pprint(spec) 202 | else: 203 | return backend.launch(self, spec) 204 | 205 | def setup(self): 206 | job = self.getJob() 207 | job.setup() 208 | 209 | def getJob(self): 210 | if not self.__job: 211 | 212 | if not self.getArg("script"): 213 | raise BlueprintException("A blueprint runner must be provided with a job or script object to run.") 214 | 215 | self.__job = loadScript(self.getArg("script")) 216 | 217 | if self.getArg("range"): 218 | self.__job.setFrameRange(self.getArg("range")) 219 | 220 | if self.getArg("name", None): 221 | self.__job.setName(self.getArg("name")) 222 | 223 | return self.__job 224 | 225 | 226 | def loadBackendPlugin(name): 227 | logger.debug("loading queue backend: %s" % name) 228 | return __import__("blueprint.backend.%s" % name, globals(), locals(), [name]) 229 | 230 | 231 | def loadScript(path): 232 | from blueprint.job import Job 233 | 234 | basename = os.path.basename(path) 235 | if basename == "blueprint.yaml": 236 | Job.Current = yaml.load(file(path, 'r')) 237 | # Yamlized jobs have no session but they 238 | # have a path that points to one so we have 239 | # to bring back the archive. 240 | if Job.Current.getPath(): 241 | Job.Current.loadArchive() 242 | else: 243 | Job.Current = Job(os.path.splitext(basename)[0]) 244 | execfile(path, {}) 245 | 246 | return Job.Current 247 | 248 | 249 | 250 | 251 | 252 | 253 | -------------------------------------------------------------------------------- /lib/blueprint/archive.py: -------------------------------------------------------------------------------- 1 | """The archive provides a mechanism for storing job runtime data.""" 2 | 3 | import os 4 | import logging 5 | import yaml 6 | import shutil 7 | 8 | import blueprint.conf as conf 9 | from blueprint.exception import ArchiveException 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | class Archive(object): 14 | """ 15 | The archive provides a mechanism for storing job/layer runtime data. 16 | It can be used for passing arbitrary metadata between tasks as well 17 | as files. 18 | """ 19 | def __init__(self, job): 20 | self.__job = job 21 | self.__path = os.path.join( 22 | conf.get("bp.archive_dir", JOB_NAME=job.getName()), 23 | job.getId()) 24 | self.__create() 25 | 26 | def __create(self): 27 | logger.debug("Using archive path: %s" % self.__path) 28 | if os.path.exists(self.__path): 29 | return 30 | os.makedirs(self.__path, 0777) 31 | os.mkdir(os.path.join(self.__path, "layers"), 0777) 32 | 33 | def putData(self, name, data, layer=None): 34 | """Put data into the job archive.""" 35 | path = os.path.join(self.getPath(layer), name) 36 | logger.debug("Witing out data %s to path %s" % (name, path)) 37 | fp = open(path, "w") 38 | try: 39 | fp.write(yaml.dump(data)) 40 | finally: 41 | fp.close() 42 | 43 | def getData(self, name, layer=None, default=None): 44 | """ 45 | Get data from the archive or throw an ArchiveException 46 | if data by the given name does not exist. 47 | """ 48 | try: 49 | path = os.path.join(self.getPath(layer), name) 50 | logger.debug("Reading in data %s from path %s" % (name, path)) 51 | stream = open(path, "r") 52 | return yaml.load(stream) 53 | except Exception, e: 54 | if isinstance(default, Exception): 55 | raise default 56 | else: 57 | return default 58 | 59 | def putFile(self, name, path): 60 | """ Copy a file into the job archive.""" 61 | if name == "layers": 62 | raise ArchiveException("layers is a reserved archive name.") 63 | dst = os.path.join(self.getPath(), name) 64 | shutil.copyfile(path, dst) 65 | return dst 66 | 67 | def getFile(self, name): 68 | """ 69 | Return the name of a file in the job archive. 70 | """ 71 | return os.path.join(self.getPath(), name) 72 | 73 | def getPath(self, layer=None): 74 | if layer: 75 | try: 76 | layer_name = layer.getName() 77 | except: 78 | layer_name = str(layer) 79 | path = os.path.join(self.__path, "layers", layer_name) 80 | else: 81 | path = self.__path 82 | 83 | if not os.path.exists(path): 84 | try: 85 | os.mkdir(path, 0777) 86 | except OSError, e: 87 | raise ArchiveException("Failed to make archive dir: " + e) 88 | 89 | return path 90 | 91 | def __str__(self): 92 | return self.__path 93 | 94 | -------------------------------------------------------------------------------- /lib/blueprint/backend/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chadmv/blueprint/0b26e08b64c7dd399889ae4ff60b2c4ae4495d55/lib/blueprint/backend/__init__.py -------------------------------------------------------------------------------- /lib/blueprint/backend/plowrun.py: -------------------------------------------------------------------------------- 1 | import os 2 | import yaml 3 | import getpass 4 | import pprint 5 | import logging 6 | 7 | import plow.client as plow 8 | import blueprint 9 | import blueprint.conf as conf 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | def launch(runner, spec): 14 | """ 15 | Launch the given job spec to plow. 16 | """ 17 | job = spec.launch() 18 | runner.getJob().putData("jobid", job.id) 19 | return job 20 | 21 | def createLayerSpec(layer): 22 | lspec = plow.LayerSpec() 23 | lspec.name = layer.getName() 24 | lspec.tags = layer.getArg("tags", ["unassigned"]) 25 | lspec.isPost = layer.getArg("post", False) 26 | 27 | if layer.getArg("maxRetries"): 28 | lspec.maxRetries = layer.getArg("maxRetries") 29 | if layer.getArg("service"): 30 | lspec.service = layer.getArg("service") 31 | if layer.getArg("memory"): 32 | lspec.minRam = layer.getArg("memory") 33 | return lspec 34 | 35 | def serialize(runner): 36 | """ 37 | Convert the job from the internal blueprint stucture to a plow JobSpec. 38 | """ 39 | job = runner.getJob() 40 | base_name = runner.getArg("name", job.getName()) 41 | job_name = job.getName() 42 | log_dir = job.getLogDir() 43 | 44 | spec = plow.JobSpec() 45 | spec.project = os.environ.get("PLOW_PROJECT", 46 | conf.get("bp.project")) 47 | spec.username = getpass.getuser() 48 | spec.uid = os.getuid() 49 | spec.paused = runner.getArg("pause") 50 | spec.name = job_name 51 | spec.logPath = log_dir 52 | spec.layers = [] 53 | spec.env = { 54 | "BLUEPRINT_SCRIPTS_PATH": conf.get("bp.scripts_dir"), 55 | "BLUEPRINT_ARCHIVE": job.getPath() 56 | } 57 | spec.env.update(runner.getArg("env")) 58 | 59 | for layer in job.getLayers(): 60 | 61 | if isinstance(layer, blueprint.Task): 62 | # These are added via their task containers 63 | continue 64 | 65 | elif isinstance(layer, blueprint.TaskContainer): 66 | 67 | task_cnt_spec = createLayerSpec(layer) 68 | task_cnt_spec.command = [ 69 | conf.get("bp.scripts_dir") + "/env_wrapper.sh", 70 | "taskrun", 71 | "-debug", 72 | "-task", 73 | "%{TASK}", 74 | os.path.join(job.getPath(), "blueprint.yaml") 75 | ] 76 | task_cnt_spec.tasks = [] 77 | spec.layers.append(task_cnt_spec) 78 | 79 | for task in layer.getTasks(): 80 | task_spec = plow.TaskSpec() 81 | task_spec.name = task.getName() 82 | task_spec.depends=[] 83 | task_spec.depends+=setupTaskDepends(job, task) 84 | task_spec.depends+=setupTaskDepends(job, layer) 85 | task_cnt_spec.tasks.append(task_spec) 86 | else: 87 | lspec = createLayerSpec(layer) 88 | lspec.depends = setupLayerDepends(job, layer) 89 | lspec.range = layer.getFrameRange() 90 | lspec.chunk = layer.getChunk() 91 | lspec.command = [ 92 | conf.get("bp.scripts_dir") + "/env_wrapper.sh", 93 | "taskrun", 94 | "-debug", 95 | "-layer", 96 | layer.getName(), 97 | os.path.join(job.getPath(), "blueprint.yaml"), 98 | "-frame", 99 | "%{FRAME}" 100 | ] 101 | spec.layers.append(lspec) 102 | 103 | logger.debug(str(spec)) 104 | return spec 105 | 106 | def setupLayerDepends(job, layer): 107 | result = [] 108 | for depend in layer.getDepends(): 109 | dspec = plow.DependSpec() 110 | 111 | depend_on = job.getLayer(str(depend.dependOn)) 112 | if isinstance(depend_on, (blueprint.Task,)): 113 | # Layer on Task 114 | dspec.type = plow.DependType.LAYER_ON_TASK 115 | dspec.dependentLayer = str(depend.dependent) 116 | dspec.dependOnTask = str(depend_on) 117 | else: 118 | # Layer on Layer + Task by Task 119 | if depend.type == blueprint.DependType.All: 120 | dspec.type = plow.DependType.LAYER_ON_LAYER 121 | elif depend.type == blueprint.DependType.ByTask: 122 | dspec.type = plow.DependType.TASK_BY_TASK 123 | else: 124 | raise Exception("Invalid layer depend type: %s" % depend.type) 125 | dspec.dependentLayer = str(depend.dependent) 126 | dspec.dependOnLayer = str(depend.dependOn) 127 | 128 | result.append(dspec) 129 | return result 130 | 131 | 132 | def setupTaskDepends(job, task): 133 | """ 134 | Setup task dependencies. Tasks can depend on other 135 | tasks or layers. 136 | """ 137 | result = [] 138 | for depend in task.getDepends(): 139 | dspec = plow.DependSpec() 140 | depend_on = job.getLayer(str(depend.dependOn)) 141 | if isinstance(depend_on, (blueprint.Task,)): 142 | # Task on Task 143 | dspec.type = plow.DependType.TASK_ON_TASK 144 | dspec.dependentTask = str(task) 145 | dspec.dependOnTask = str(depend_on) 146 | else: 147 | # Task on Layer 148 | dspec.type = plow.DependType.TASK_ON_LAYER 149 | dspec.dependentTask = str(task) 150 | dspec.dependOnLayer = str(depend.dependOn) 151 | result.append(dspec) 152 | return result 153 | -------------------------------------------------------------------------------- /lib/blueprint/backend/test.py: -------------------------------------------------------------------------------- 1 | """A simple test backend that can be used to manually execute tasks.""" 2 | 3 | def serialize(runner): 4 | pass 5 | 6 | def launch(runner, spec): 7 | pass 8 | -------------------------------------------------------------------------------- /lib/blueprint/conf.py: -------------------------------------------------------------------------------- 1 | """ 2 | Blueprint configuration module. 3 | 4 | Provides a get() function to access the blueprint configuration with a 5 | dot notation style. 6 | 7 | Example: 8 | 9 | maya_renderer = conf.get("modules.maya.renderer") 10 | 11 | """ 12 | import os 13 | import json 14 | import logging 15 | 16 | import blueprint.exception 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | class _Default(object): 21 | def __init__(self): 22 | pass 23 | 24 | class _Raise(object): 25 | def __init__(self): 26 | pass 27 | 28 | __Config = { } 29 | 30 | __EnvMap = { } 31 | 32 | Default = _Default() 33 | 34 | Raise = _Raise() 35 | 36 | def __init(config, envmap): 37 | """ 38 | Initialize the plow configuration. 39 | """ 40 | search_path = [] 41 | override = os.environ.get("BLUEPRINT_CFG") 42 | if override: 43 | search_path.append(override) 44 | else: 45 | search_path.append(os.path.expanduser("~/.blueprint/blueprint.cfg")) 46 | search_path.append("%s/etc/blueprint/blueprint.cfg" % os.environ.get("BLUEPRINT_ROOT", "/usr/local")) 47 | 48 | try: 49 | for path in search_path: 50 | logger.debug("Checking %s for a blueprint configuration." % path) 51 | if os.path.isfile(path): 52 | data = open(path).read() 53 | config.update(json.loads(data)) 54 | envmap.update(dict(((k, os.environ.get(k, "")) 55 | for k in config["env"]["interpolate"]))) 56 | return 57 | except Exception, e: 58 | raise blueprint.exception.BlueprintException( 59 | "Failed to parse plow configuration: %s" % e) 60 | 61 | raise blueprint.exception.BlueprintException( 62 | "Unable to find plow configuration at: %s" % search_path) 63 | 64 | def interp(value, **kwargs): 65 | kwargs.update(__EnvMap) 66 | return value % kwargs 67 | 68 | def get(key, default=Default, **kwargs): 69 | """ 70 | Return the specification configuration value. If the 71 | value is undefined, return the default. If the default is 72 | conf.Raise, then a BlueprintException is thrown. 73 | 74 | If an environment variable named with key.upper().replace(".","_") 75 | exists, that value is returned instead. 76 | 77 | For example, a key of "bp.project" would 78 | translate to the BLUEPRINT_BP_PROJECT environment variable. 79 | """ 80 | env_value = os.environ.get("BLUEPRINT_" + key.upper().replace(".", "_")) 81 | if env_value: 82 | return env_value 83 | 84 | elements = key.split(".") 85 | part = __Config 86 | try: 87 | for e in elements: 88 | part = part[e] 89 | except KeyError: 90 | if default == Raise: 91 | raise blueprint.exception.BlueprintException( 92 | "Invalid configuration option: %s" % key) 93 | return default 94 | 95 | if isinstance(part, basestring): 96 | part = interp(part, **kwargs) 97 | return part 98 | 99 | # Initialize the configuration. 100 | __init(__Config, __EnvMap) 101 | -------------------------------------------------------------------------------- /lib/blueprint/exception.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class BlueprintException(Exception): 4 | pass 5 | 6 | class LayerException(BlueprintException): 7 | pass 8 | 9 | class ArchiveException(BlueprintException): 10 | pass 11 | 12 | class CommandException(BlueprintException): 13 | 14 | def __init__(self, message, exitStatus=1, exitSignal=0): 15 | super(CommandException, self).__init__(message) 16 | self.exitStatus = exitStatus 17 | self.exitSignal = exitSignal -------------------------------------------------------------------------------- /lib/blueprint/io.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import subprocess 3 | import fileseq 4 | import os 5 | 6 | import blueprint.conf as conf 7 | from blueprint.exception import CommandException 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | def getOutputSeq(scene_file, format): 12 | """ 13 | A utility function for converting a scene file name into an output 14 | sequence name. 15 | """ 16 | basename = os.path.splitext(os.path.basename(scene_file))[0] 17 | return os.path.join( 18 | conf.get("bp.output_dir"), 19 | basename, 20 | "%s.#.%s" % (basename, format)) 21 | 22 | def system(cmd, frames=None): 23 | """ 24 | Utility method for shelling out. Takes a shell 25 | command in the form of an array. 26 | """ 27 | cmd = map(str, cmd) 28 | cmdStr = " ".join(cmd) 29 | logger.info("About to run: %s", cmdStr) 30 | p = subprocess.Popen(cmd, shell=False) 31 | ret = p.wait() 32 | 33 | if ret != 0: 34 | raise CommandException( 35 | 'Command exited with a status of %d: "%s"' % (ret, cmdStr), 36 | exitStatus=ret 37 | ) 38 | 39 | def mkdir(path, check=True): 40 | """ 41 | Make the given directory. 42 | """ 43 | if check: 44 | if os.path.exists(path): 45 | return True 46 | 47 | command = conf.get("system.mkdir") 48 | command.append(path) 49 | system(path) 50 | 51 | class FileIO(object): 52 | def __init__(self, path, attrs=None): 53 | self.path = path 54 | self.attrs = attrs or dict() 55 | 56 | @property 57 | def dirname(self): 58 | return os.path.dirname(self.path) 59 | 60 | @property 61 | def basename(self): 62 | return os.path.splitext(os.path.basename(self.path))[0] 63 | 64 | @property 65 | def ext(self): 66 | return os.path.splitext(os.path.basename(self.path))[1] 67 | 68 | def __str__(self): 69 | return "" % (self.path, self.attrs) 70 | 71 | def __repr__(self): 72 | return "" % (self.path, self.attrs) 73 | -------------------------------------------------------------------------------- /lib/blueprint/job.py: -------------------------------------------------------------------------------- 1 | """ 2 | The job class. 3 | """ 4 | import uuid 5 | import os 6 | import logging 7 | 8 | import fileseq 9 | 10 | import blueprint.conf as conf 11 | from blueprint import Layer, Task, TaskContainer 12 | from blueprint.archive import Archive 13 | from blueprint.exception import LayerException 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | class Job(object): 19 | 20 | Current = None 21 | 22 | def __init__(self, name, frange="1001-1001"): 23 | self.setName(name) 24 | self.__id = str(uuid.uuid1()) 25 | self.__layers = [ [], {} ] 26 | self.__archive = None 27 | self.__path = None 28 | self.__range = frange 29 | 30 | def getPath(self): 31 | return self.__path 32 | 33 | def getName(self): 34 | return self.__name 35 | 36 | def setName(self, name): 37 | self.__name = conf.get("bp.job_name_template", JOB_NAME=name) 38 | 39 | def getId(self): 40 | return self.__id 41 | 42 | def getLayer(self, name): 43 | try: 44 | return self.__layers[1][name] 45 | except KeyError: 46 | raise LayerException("Layer %s does not exist." % name) 47 | 48 | def addLayer(self, layer): 49 | 50 | if layer in self.__layers[0]: 51 | logger.debug("The layer/task %s is already in the job." % layer.getName()) 52 | return 53 | 54 | if self.__layers[1].has_key(layer.getName()): 55 | raise LayerException("Invalid layer/task name: %s , duplicate name." % layer) 56 | 57 | if isinstance(layer, Task): 58 | task = layer 59 | task_layer_name = task.getArg("layer", "default") 60 | if task.getArg("post"): 61 | task_layer_name = "%s_post" % task_layer_name 62 | task_layer = self.__layers[1].get(task_layer_name) 63 | if not task_layer: 64 | task_layer = TaskContainer(task_layer_name) 65 | task_layer.setArg("post", task.getArg("post")) 66 | self.addLayer(task_layer) 67 | task_layer.addTask(task) 68 | 69 | self.__layers[0].append(layer) 70 | self.__layers[1][layer.getName()] = layer 71 | layer.setJob(self) 72 | 73 | def getLayers(self): 74 | return self.__layers[0] 75 | 76 | def loadArchive(self): 77 | self.__archive = Archive(self) 78 | 79 | def getArchive(self): 80 | return self.__archive 81 | 82 | def getLogDir(self): 83 | return os.path.join( 84 | conf.get("bp.log_dir", JOB_NAME=self.__name), 85 | self.__id) 86 | 87 | def putData(self, key, value): 88 | return self.__archive.putData(key, value) 89 | 90 | def getData(self, key): 91 | return self.__archive.getData(key) 92 | 93 | def setup(self): 94 | 95 | from blueprint import PluginManager 96 | 97 | self.__archive = Archive(self) 98 | self.__path = self.__archive.getPath() 99 | 100 | for layer in self.__layers[0]: 101 | layer.setup() 102 | 103 | PluginManager.runJobSetup(self) 104 | 105 | archive = self.__archive 106 | try: 107 | self.__archive = None 108 | archive.putData("blueprint.yaml", self) 109 | finally: 110 | self.__archive = archive 111 | 112 | def getFrameRange(self): 113 | """Return the string frame range for this job.""" 114 | return self.__range 115 | 116 | def getFrameSet(self): 117 | """Return a FrameSet for the job's frame range.""" 118 | return fileseq.FrameSet(self.__range) 119 | 120 | def setFrameRange(self, frange): 121 | """Set the job's frame range.""" 122 | self.__range = frange 123 | 124 | def launch(self, **kwargs): 125 | from app import BlueprintRunner 126 | runner = BlueprintRunner(self, **kwargs) 127 | return runner.launch() 128 | -------------------------------------------------------------------------------- /lib/blueprint/layer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | import tempfile 4 | import fileseq 5 | 6 | from collections import namedtuple 7 | 8 | import conf 9 | from io import FileIO, system 10 | from app import PluginManager 11 | from exception import LayerException 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | Depend = namedtuple("Depend", ["dependent", "dependOn", "type", "args"]) 16 | 17 | class DependType(object): 18 | All = "DependAll" 19 | ByTask = "ByTask" 20 | 21 | class LayerAspect(type): 22 | 23 | def __call__(cls, *args, **kwargs): 24 | """ 25 | Intercepts the creation of layer objects and assigns 26 | them to the current job, of there is one. The current job 27 | is set when a job is loaded via script. 28 | """ 29 | from job import Job 30 | layer = super(LayerAspect, cls).__call__(*args, **kwargs) 31 | 32 | if Job.Current: 33 | Job.Current.addLayer(layer) 34 | 35 | layer.afterInit() 36 | return layer 37 | 38 | 39 | class Layer(object): 40 | """ 41 | A base class which implements the core functionality. 42 | """ 43 | __metaclass__ = LayerAspect 44 | 45 | def __init__(self, name, **args): 46 | self.__name = name 47 | self.__args = args 48 | 49 | self.__job = None 50 | self.__req_args = [] 51 | self.__depends = [] 52 | self.__setups = [] 53 | self.__outputs = {} 54 | self.__inputs = {} 55 | 56 | self.__handleDependArg() 57 | self.__loadDefaultArgs() 58 | 59 | def getName(self): 60 | return self.__name 61 | 62 | def setArg(self, name, value): 63 | self.__args[name] = value 64 | 65 | def getArg(self, name, default=None): 66 | return self.__args.get(name, default) 67 | 68 | def isArgSet(self, name): 69 | return self.__args.has_key(name) 70 | 71 | def requireArg(self, name, types=None): 72 | self.__req_args.append((name, types)) 73 | 74 | def dependOn(self, other, args=None): 75 | self.__depends.append(Depend(self, other, DependType.ByTask, args)) 76 | 77 | def dependAll(self, other, args=None): 78 | self.__depends.append(Depend(self, other, DependType.All, args)) 79 | 80 | def getDepends(self): 81 | return list(self.__depends) 82 | 83 | def getJob(self): 84 | return self.__job 85 | 86 | def putData(self, name, data): 87 | if not self.__job: 88 | raise LayerException("The layer %s requires a job to be set setup() is run." % self.__name) 89 | if not self.__job.getArchive(): 90 | raise LayerException("A job archive must exist before dynamic data can be added.") 91 | self.__job.getArchive().putData(name, data, self) 92 | 93 | def getData(self, name, default=None): 94 | return self.__job.getArchive().getData(name, self, default) 95 | 96 | def setJob(self, job): 97 | self.__job = job 98 | 99 | def getOutput(self, name): 100 | return self.__outputs[name] 101 | 102 | def getInput(self, name): 103 | return self.__inputs[name] 104 | 105 | def getOutputs(self): 106 | return self.__outputs.values() 107 | 108 | def getInputs(self): 109 | return self.__inputs.values() 110 | 111 | def addInput(self, name, path, attrs=None): 112 | self.__inputs[name] = FileIO(path, attrs) 113 | 114 | def addOutput(self, name, path, attrs=None): 115 | self.__outputs[name] = FileIO(path, attrs) 116 | 117 | def getSetupTasks(self): 118 | return list(self.__setups) 119 | 120 | def addSetupTask(self, task): 121 | self.__setups.append(task) 122 | 123 | def system(self, cmd): 124 | system(cmd) 125 | 126 | def afterInit(self): 127 | self._afterInit() 128 | PluginManager.runAfterInit(self) 129 | 130 | def setup(self): 131 | # Run the subclass setup 132 | self._setup() 133 | 134 | # Run the plugins 135 | PluginManager.runLayerSetup(self) 136 | 137 | # Flush any inputs/outputs to disk. 138 | self.flushIO() 139 | 140 | def beforeExecute(self): 141 | self.loadIO() 142 | self._beforeExecute() 143 | PluginManager.runBeforeExecute(self) 144 | 145 | def afterExecute(self): 146 | self._afterExecute() 147 | PluginManager.runAfterExecute(self) 148 | 149 | def getTempDir(self): 150 | return tempfile.gettempdir() 151 | 152 | def getDir(self): 153 | return self.__job.getArchive().getPath(self.getName()) 154 | 155 | def system(self, cmd): 156 | system(cmd) 157 | 158 | def loadIO(self): 159 | if self.isSetup(): 160 | logger.info("Loading IO: %s" % self) 161 | self.__inputs = self.getData("inputs", dict()) 162 | self.__outputs = self.getData("outputs", dict()) 163 | 164 | def flushIO(self): 165 | if self.isSetup(): 166 | logger.info("Flushing IO: %s" % self) 167 | self.putData("inputs", self.__inputs) 168 | self.putData("outputs", self.__outputs) 169 | 170 | def _afterInit(self): 171 | """ 172 | _afterInit is called once all the layer args 173 | have their final values. 174 | """ 175 | pass 176 | 177 | def _setup(self): 178 | pass 179 | 180 | def _beforeExecute(self): 181 | pass 182 | 183 | def _afterExecute(self): 184 | pass 185 | 186 | def __loadDefaultArgs(self): 187 | """ 188 | Load in default args for this module from the configuration file 189 | and populate the __defaults. 190 | """ 191 | mod = self.__class__.__name__.lower() 192 | logger.debug("Loading default args for module: %s" % mod) 193 | 194 | default_args = conf.get("modules.%s" % mod, None) 195 | if not default_args: 196 | return 197 | for k, v in default_args.iteritems(): 198 | logger.debug("Setting default %s arg: %s=%s" % (mod, k, v)) 199 | self.setArg(k, v) 200 | 201 | def __handleDependArg(self): 202 | """ 203 | Handles the dependOn kwarg passed in via the constructor. 204 | 205 | foo = Layer("foo", dependOn=["bar", "bing:all"]) 206 | Layer("zing", dependOn=[(foo, Layer.DependAll)]) 207 | """ 208 | if not self.isArgSet("depend"): 209 | return 210 | 211 | depends = self.getArg("depend") 212 | if not isinstance(depends, (list,tuple)): 213 | depends = [depends] 214 | 215 | for dep in depends: 216 | if isinstance(dep, (tuple, list)): 217 | self.dependOn(LayerDepend(self, dep[0], dep[1])) 218 | else: 219 | onLayer = str(dep) 220 | if onLayer.endswith(":all"): 221 | self.dependAll(onLayer.split(":")[0]) 222 | else: 223 | self.dependOn(onLayer) 224 | 225 | def isSetup(self): 226 | if not self.__job: 227 | return False 228 | if not self.__job.getArchive(): 229 | return False 230 | return True 231 | 232 | def __str__(self): 233 | return self.getName() 234 | 235 | class Task(Layer): 236 | """ 237 | Tasks are indiviudal processes with no-frame range. Tasks must be parented 238 | to a layer. 239 | """ 240 | def __init__(self, name, **args): 241 | Layer.__init__(self, name, **args) 242 | self.setArg("layer", "default") 243 | 244 | def _execute(self): 245 | pass 246 | 247 | def execute(self): 248 | self.beforeExecute() 249 | try: 250 | self._execute() 251 | finally: 252 | self.afterExecute() 253 | 254 | class TaskIterator(Layer): 255 | """ 256 | A layer for iterating a command over a range of frame range of tasks. 257 | """ 258 | def __init__(self, name, **args): 259 | Layer.__init__(self, name, **args) 260 | self.__range = args.get("range") 261 | self.__chunk = args.get("chunk", 1) 262 | 263 | def execute(self, frame): 264 | self.beforeExecute() 265 | frameset = self.getLocalFrameSet(frame) 266 | try: 267 | self._execute(frameset) 268 | finally: 269 | self.afterExecute() 270 | 271 | def _execute(self, frameset): 272 | pass 273 | 274 | def getFrameRange(self): 275 | if self.__range: 276 | return self.__range 277 | else: 278 | return self.getJob().getFrameRange() 279 | 280 | def getFrameSet(self): 281 | return fileseq.FrameSet(self.getFrameRange()) 282 | 283 | def setFrameRange(self, frange): 284 | self.__range = frange 285 | 286 | def getLocalFrameSet(self, frame): 287 | """ 288 | Return the local frameset when running in execute mode. 289 | """ 290 | frameset = None 291 | 292 | if self.getChunk() <= 0: 293 | frameset = self.getFrameSet() 294 | elif self.getChunk() >1: 295 | result = [] 296 | 297 | full_range = self.getFrameSet() 298 | end = len(full_range) - 1 299 | 300 | idx = full_range.index(frame) 301 | print "idx :%d" % idx 302 | for i in range(idx, idx+self.getChunk()): 303 | if i > end: 304 | break 305 | result.append(full_range[i]) 306 | frameset = fileseq.FrameSet(",".join(map(str, result))) 307 | else: 308 | frameset = fileseq.FrameSet(str(frame)) 309 | 310 | if frameset is None: 311 | raise LayerException("Unable to determine local frameset.") 312 | 313 | return frameset 314 | 315 | def getChunk(self): 316 | return self.__chunk 317 | 318 | def setChunk(self, size): 319 | self.__chunk = size 320 | 321 | class TaskContainer(Layer): 322 | """ 323 | A container to hold arbitrary tasks. 324 | """ 325 | def __init__(self, name, **args): 326 | Layer.__init__(self, name, **args) 327 | self.__tasks = [] 328 | 329 | def addTask(self, task): 330 | self.__tasks.append(task) 331 | 332 | def getTasks(self): 333 | return self.__tasks 334 | 335 | class SetupTask(Task): 336 | """ 337 | A helper class for setting up task to run before a task iterator or other task. 338 | """ 339 | def __init__(self, layer, **args): 340 | Task.__init__(self, "%s_setup" % layer.getName(), **args) 341 | self.__parent = layer 342 | self.__parent.getJob().addLayer(self) 343 | layer.dependOn(self, DependType.All) 344 | self.setArg("layer", "setup_tasks") 345 | self.setArg("service", "setup_task") 346 | 347 | def getParentLayer(self): 348 | return self.__parent 349 | 350 | def afterExecute(self): 351 | super(SetupTask, self).afterExecute() 352 | self.__parent.flushIO() 353 | 354 | class PostTask(Task): 355 | """ 356 | A helper class for setting up task to run before a task iterator or other task. 357 | """ 358 | def __init__(self, name, **kwargs): 359 | Task.__init__(self, name, **kwargs) 360 | self.setArg("post", True) 361 | 362 | 363 | -------------------------------------------------------------------------------- /lib/blueprint/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chadmv/blueprint/0b26e08b64c7dd399889ae4ff60b2c4ae4495d55/lib/blueprint/modules/__init__.py -------------------------------------------------------------------------------- /lib/blueprint/modules/blender.py: -------------------------------------------------------------------------------- 1 | """Blender integration module""" 2 | import os 3 | import subprocess 4 | import json 5 | import logging 6 | 7 | from blueprint.layer import TaskIterator, SetupTask 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | class BlenderSetup(SetupTask): 12 | """ 13 | The BlenderSetup task will introspect the .blend file and 14 | try to detect and register outputs with blueprint. 15 | """ 16 | def __init__(self, layer, **kwargs): 17 | SetupTask.__init__(self, layer, **kwargs) 18 | 19 | def _execute(self): 20 | 21 | layer = self.getLayer() 22 | 23 | cmd = ["blender"] 24 | cmd.append("-b") 25 | cmd.append(layer.getInput("scene_file").path) 26 | cmd.append("--python") 27 | cmd.append(os.path.join(os.path.dirname(__file__), 28 | "setup", "blender_setup.py")) 29 | 30 | output_path = "%s/blender_outputs_%d.json" % (self.getTempDir(), os.getpid()) 31 | os.environ["PLOW_BLENDER_SETUP_PATH"] = output_path 32 | self.system(cmd) 33 | 34 | logger.info("Loading blender ouputs from: %s" % output_path) 35 | outputs = json.load(open(output_path, "r")) 36 | for output in outputs: 37 | layer.addOutput(output["pass"], output["path"], output) 38 | 39 | class Blender(TaskIterator): 40 | """ 41 | The Blender module renders frames from a blender scene. 42 | """ 43 | def __init__(self, name, **kwargs): 44 | TaskIterator.__init__(self, name, **kwargs) 45 | self.requireArg("scene_file", (str,)) 46 | 47 | self.addInput("scene_file", 48 | os.path.abspath(self.getArg("scene_file"))) 49 | 50 | def _setup(self): 51 | self.addSetupTask(BlenderSetup(self)) 52 | 53 | def _execute(self, frames): 54 | 55 | cmd = ["blender"] 56 | cmd.append("-b") 57 | cmd.append(self.getInput("scene_file").path) 58 | cmd.append("-noaudio") 59 | cmd.append("-noglsl") 60 | cmd.append("-nojoystick") 61 | cmd.extend(("-t", os.environ.get("PLOW_THREADS", "1"))) 62 | 63 | for f in frames: 64 | cmd.extend(("-f", str(f))) 65 | 66 | self.system(cmd) 67 | -------------------------------------------------------------------------------- /lib/blueprint/modules/iconvert.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ OpenImage iconvert module """ 4 | import os 5 | 6 | from blueprint.layer import Layer 7 | from blueprint.layer import Task 8 | 9 | class IConvert(Layer): 10 | """ Layer to batch iconvert 11 | """ 12 | def __init__(self, name, **kwargs): 13 | super(IConvert, self).__init__(name, **kwargs) 14 | self.requireArg("image_input", (str,)) 15 | self.requireArg("image_output", (str,)) 16 | self.requireArg("start", (int,)) 17 | self.requireArg("end", (int,)) 18 | 19 | self.addInput("image_input", self.getArg("image_input")) 20 | self.addInput("image_output", self.getArg("image_output")) 21 | 22 | 23 | def _execute(self, frames): 24 | """ Execute frames """ 25 | 26 | start = self.getArg("start") 27 | end = self.getArg("end") 28 | i_input = str(self.getInput("image_input").path) 29 | i_output = str(self.getInput("image_output").path) 30 | convert_cmd = self.build_command() 31 | 32 | # for every start/end 33 | for f in xrange(start, end+1): 34 | cmd = list(convert_cmd) 35 | cmd.append(i_input % f) 36 | cmd.append(i_output % f) 37 | 38 | print cmd 39 | self.system(cmd) 40 | 41 | return 42 | 43 | def build_command(self): 44 | """ Build iconvert command options """ 45 | cmd = ["iconvert"] 46 | 47 | supported_options = ["v", "threads", "d", "g", "tile", "scanline", "compression", 48 | "quality", "no_copy_image", "adjust_time", "caption", 49 | "keyword", "clear_keywords", "attrib", "orientation", 50 | "rotcw", "rotccw", "rot180", "inplace", "sRGB", "separate", 51 | "contig", "no_clobber"] 52 | 53 | options_with_value = ["threads", "d", "g", "tile", "compression", 54 | "quality", "caption", "keyword", "attrib", 55 | "orientation"] 56 | 57 | for opt in supported_options: 58 | if self.isArgSet(opt): 59 | if len(opt) == 1: 60 | iopt = "-%s" % opt 61 | else: 62 | iopt = "--%s" % opt 63 | 64 | cmd.append(iopt) 65 | if opt in options_with_value: 66 | value = self.getArg(opt) 67 | if isinstance(value, tuple): 68 | cmd.append(value[0]) 69 | cmd.append(value[1]) 70 | else: 71 | cmd.append(value) 72 | 73 | return cmd 74 | 75 | 76 | 77 | if __name__ == "__main__": 78 | pass 79 | 80 | -------------------------------------------------------------------------------- /lib/blueprint/modules/maya.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import logging 4 | 5 | import blueprint.io as io 6 | from blueprint.layer import SetupTask, TaskIterator 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | class MayaSetup(SetupTask): 11 | def __init__(self, parent, **kwargs): 12 | SetupTask.__init__(self, parent, **kwargs) 13 | 14 | def _execute(self, *args): 15 | pass 16 | 17 | class Maya(TaskIterator): 18 | 19 | def __init__(self, name, **kwargs): 20 | TaskIterator.__init__(self, name, **kwargs) 21 | self.requireArg("scene", (str,)) 22 | self.setArg("service", "maya") 23 | 24 | def _execute(self, frames): 25 | 26 | output_seq = io.getOutputSeq( 27 | self.getArg("scene"), self.getArg("format")) 28 | 29 | cmd = [ 30 | "Render", 31 | "-r", self.getArg("renderer"), 32 | "-s", str(frames[0]), 33 | "-e", str(frames[-1]), 34 | "-b", self.getArg("chunk", "1"), 35 | "-of", self.getArg("format", "png"), 36 | "-rd", os.path.dirname(output_seq), 37 | "-fnc", self.getArg("fnc"), 38 | "-pad", self.getArg("padding"), 39 | "-cam", self.getArg("camera"), 40 | self.getArg("scene") 41 | ] 42 | self.system(cmd) 43 | 44 | 45 | -------------------------------------------------------------------------------- /lib/blueprint/modules/nuke.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import logging 4 | import tempfile 5 | import json 6 | 7 | import blueprint.io as io 8 | from blueprint.layer import SetupTask, TaskIterator 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | """ 13 | This is the base of a dynamically generated script that is used 14 | to walk a nuke scene, gather outputs, and register then with 15 | blueprint. 16 | """ 17 | NUKE_SETUP_SCRIPT = """ 18 | 19 | import json 20 | nuke.scriptReadFile('%(nuke_script_path)s') 21 | 22 | only_nodes = frozenset(%(node_list)s) 23 | outputs = [] 24 | for write_node in nuke.allNodes('Write'): 25 | # Skip over disabled nodes 26 | if write_node.knob('disable').value(): 27 | continue 28 | if not only_nodes or write_node.name() in only_nodes: 29 | outputs.append([write_node['file'].getText(), { 30 | 'node': write_node.name(), 31 | 'colorspace': write_node['colorspace'].value(), 32 | 'file_type': write_node['file_type'].value(), 33 | 'datatype': write_node['datatype'].value(), 34 | 'compression:': write_node['compression'].value() 35 | }]) 36 | 37 | json.dump(outputs, open('%(tmp_data_path)s', 'w')) 38 | """ 39 | 40 | class NukeSetup(SetupTask): 41 | def __init__(self, parent, **kwargs): 42 | SetupTask.__init__(self, parent, **kwargs) 43 | 44 | def _execute(self, *args): 45 | 46 | tmp_dir = tempfile.gettempdir() 47 | 48 | opts = { 49 | "nuke_script_path": self.getParentLayer().getArg("script"), 50 | "tmp_data_path": "%s/nuke_setup_data.json" % tmp_dir, 51 | "node_list": str(self.getArg("nodes", list())) 52 | } 53 | 54 | # Apply the opts to the setup script code and write 55 | # the script out to the task's temp area. 56 | setup_script_code = NUKE_SETUP_SCRIPT % opts 57 | setup_script_path = "%s/nuke_setup_script.py" % tmp_dir 58 | 59 | # Execute nuke on the script we just wrote out. 60 | open(setup_script_path, "w").write(setup_script_code) 61 | cmd = ["nuke"] 62 | cmd.extend(("-t", setup_script_path)) 63 | self.system(cmd) 64 | 65 | # Load in the file nuke wrote out and register the outputs 66 | # with blueprint. 67 | outputs = json.load(open(opts["tmp_data_path"], "r")) 68 | for path, attrs in outputs: 69 | self.getParentLayer().addOutput(attrs["node"], path, attrs) 70 | 71 | class Nuke(TaskIterator): 72 | 73 | def __init__(self, name, **kwargs): 74 | TaskIterator.__init__(self, name, **kwargs) 75 | self.requireArg("script", (str,)) 76 | self.setArg("service", "nuke") 77 | 78 | def _setup(self): 79 | NukeSetup(self) 80 | 81 | def _execute(self, frames): 82 | 83 | cmd = [ 84 | "nuke", 85 | "-V", 86 | "-x", 87 | "-c", self.getArg("c", "8G"), 88 | "-m", os.getenv("PLOW_THREADS"), 89 | "-f", 90 | "-F", "%s-%s" %(frames[0], frames[-1]), 91 | ] 92 | if self.getArg("nodes"): 93 | cmd.extend(("-X", ",".join(self.getArg("nodes")))) 94 | 95 | cmd.append(self.getArg("script")) 96 | self.system(cmd) 97 | 98 | -------------------------------------------------------------------------------- /lib/blueprint/modules/setup/blender_setup.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import json 4 | import bpy 5 | 6 | # If nodes are not enabled...we can enable them and use this to figure out outputs. 7 | # bpy.context.scene.use_nodes = True 8 | #[('Composite', bpy.data...nodes["Composite"]), ('Render Layers', bpy.data...nodes["Render Layers"])] 9 | 10 | def getOutputFileNodes(): 11 | result = [] 12 | for ntype, item in bpy.context.scene.node_tree.nodes.items(): 13 | if item.type == "OUTPUT_FILE": 14 | result.append(item) 15 | return result 16 | 17 | def setup(): 18 | result = [] 19 | 20 | for node in getOutputFileNodes(): 21 | for output in node.file_slots.items(): 22 | result.append({ 23 | "path": os.path.join(node.base_path, output[1].path), 24 | "node": node.name, 25 | "pass": "%s_%s" % (node.name, output[1].path.split(".")[0]), 26 | "res_x": bpy.context.scene.render.resolution_x, 27 | "res_y": bpy.context.scene.render.resolution_y, 28 | "file_format": node.file_slots.items()[0][1].format.file_format, 29 | "color_depth": node.file_slots.items()[0][1].format.color_depth 30 | }) 31 | 32 | path = os.environ["PLOW_BLENDER_SETUP_PATH"] 33 | json.dump(result, open(path, "w")) 34 | 35 | setup(); -------------------------------------------------------------------------------- /lib/blueprint/modules/shell.py: -------------------------------------------------------------------------------- 1 | 2 | import logging 3 | 4 | from blueprint.layer import TaskIterator, Task 5 | from blueprint.io import system 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | class Shell(TaskIterator): 10 | 11 | def __init__(self, name, **args): 12 | """ 13 | Executes a shell command over a given frame range. 14 | """ 15 | TaskIterator.__init__(self, name, **args) 16 | self.requireArg("cmd", (list, tuple)) 17 | self.setArg("service", "shell") 18 | 19 | def _execute(self, frames): 20 | for frame in frames: 21 | system(self.getArg("cmd")) 22 | 23 | 24 | class ShellCommand(Task): 25 | 26 | def __init__(self, name, **args): 27 | """ 28 | Executes a command. 29 | """ 30 | Task.__init__(self, name, **args) 31 | self.requireArg("cmd", (list, tuple)) 32 | self.setArg("service", "shell") 33 | 34 | def _execute(self): 35 | system(self.getArg("cmd")) 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /lib/blueprint/plugins/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chadmv/blueprint/0b26e08b64c7dd399889ae4ff60b2c4ae4495d55/lib/blueprint/plugins/__init__.py -------------------------------------------------------------------------------- /lib/blueprint/plugins/plow_outputs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Plow outputs plugin. 3 | 4 | Registers outputs with the plow server. 5 | """ 6 | import os 7 | import logging 8 | logger = logging.getLogger(__name__) 9 | 10 | import blueprint 11 | 12 | def afterExecute(layer): 13 | 14 | # When a setup task is complete outputs should 15 | # be registerd with its parent layer. 16 | if not isinstance(layer, (blueprint.SetupTask,)): 17 | return 18 | 19 | import plow 20 | 21 | parent = layer.getParentLayer() 22 | logger.info("Registering %d outputs" % len(parent.getOutputs())) 23 | for output in parent.getOutputs(): 24 | logger.info("Registering output with plow: %s" % output.path) 25 | 26 | layer = plow.client.get_layer_by_id(os.environ.get("PLOW_LAYER_ID")) 27 | layer.add_output(output.path, output.attrs) 28 | 29 | -------------------------------------------------------------------------------- /lib/blueprint/plugins/test_plugin.py: -------------------------------------------------------------------------------- 1 | 2 | import logging 3 | 4 | logger = logging.getLogger(__name__) 5 | 6 | class Init: 7 | Layer = [] 8 | Loaded = False 9 | AfterInit = False 10 | LayerSetup = False 11 | JobSetup = False 12 | BeforeExecute = False 13 | AfterExecute = False 14 | PreLaunch = False 15 | 16 | def initLayer(layer): 17 | logger.info("initializing %s plugin on layer %s" % (__name__, layer)) 18 | Init.Layer.append(layer) 19 | 20 | def init(): 21 | logger.info("init up %s plugin." % __name__) 22 | Init.Loaded = True 23 | 24 | def afterInit(layer): 25 | logger.info("afterInit %s plugin." % __name__) 26 | Init.AfterInit = True 27 | 28 | def layerSetup(layer): 29 | logger.info("layer setup %s plugin." % __name__) 30 | Init.LayerSetup = True 31 | 32 | def jobSetup(layer): 33 | logger.info("job setup %s plugin." % __name__) 34 | Init.JobSetup = True 35 | 36 | def beforeExecute(layer): 37 | logger.info("beforeExecute %s plugin." % __name__) 38 | Init.BeforeExecute = True 39 | 40 | def afterExecute(layer): 41 | logger.info("afterExecute %s plugin." % __name__) 42 | Init.AfterExecute = True 43 | 44 | def preLaunch(spec, job): 45 | logger.info("PreLaunch %s plugin." % __name__) 46 | Init.PreLaunch = True 47 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import sys 5 | import glob 6 | import shutil 7 | import logging 8 | from setuptools import setup, find_packages, Command 9 | from setuptools.command.install import install as _install 10 | from distutils import log 11 | 12 | 13 | ETC_INSTALL_DIR = "/usr/local/etc/blueprint" 14 | 15 | 16 | class InstallCommand(_install): 17 | 18 | def run(self): 19 | _install.run(self) 20 | 21 | self.announce("Running post-install", log.INFO) 22 | 23 | src_paths = set() 24 | exist_paths = set() 25 | src = "etc" 26 | dst = ETC_INSTALL_DIR 27 | 28 | if not os.path.exists(dst): 29 | os.makedirs(dst) 30 | 31 | for name in os.listdir(src): 32 | src_path = os.path.join(src, name) 33 | dst_path = os.path.join(dst, name) 34 | 35 | if not os.path.exists(dst_path): 36 | self.copy_file(src_path, dst_path) 37 | else: 38 | self.announce("{0} already exists".format(dst_path), log.INFO) 39 | 40 | # 41 | # Setup 42 | # 43 | setup(name='plow-blueprint', 44 | version='0.1.3', 45 | 46 | package_dir = {'': 'lib', 'tests':'tests'}, 47 | packages=find_packages('lib') + find_packages(), 48 | zip_safe=False, 49 | 50 | scripts=glob.glob("bin/*"), 51 | 52 | cmdclass={"install": InstallCommand}, 53 | 54 | test_suite="tests.test_all", 55 | 56 | author='Matt Chambers', 57 | author_email='yougotrooted@gmail.com', 58 | url='https://github.com/sqlboy/blueprint', 59 | description='A Python library for distributing computing using the Plow Render Farm', 60 | ) 61 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chadmv/blueprint/0b26e08b64c7dd399889ae4ff60b2c4ae4495d55/tests/__init__.py -------------------------------------------------------------------------------- /tests/common.py: -------------------------------------------------------------------------------- 1 | import blueprint 2 | 3 | class TestLayer(blueprint.TaskIterator): 4 | def __init__(self, name, **kwargs): 5 | blueprint.TaskIterator.__init__(self, name, **kwargs) 6 | self.afterInitSet = False 7 | self.setupSet = False 8 | self.beforeExecuteSet = False 9 | self.executeSet = False 10 | self.afterExecuteSet = False 11 | 12 | def _afterInit(self): 13 | self.afterInitSet = True 14 | 15 | def _setup(self): 16 | self.setupSet = True 17 | 18 | def _beforeExecute(self): 19 | self.beforeExecuteSet = True 20 | 21 | def _execute(self, frameset): 22 | self.executeSet = True 23 | 24 | def _afterExecute(self): 25 | self.afterExecuteSet = True 26 | 27 | class TestTask(blueprint.Task): 28 | def __init__(self, name, **kwargs): 29 | blueprint.Task.__init__(self, name, **kwargs) 30 | self.afterInitSet = False 31 | self.setupSet = False 32 | self.beforeExecuteSet = False 33 | self.executeSet = False 34 | self.afterExecuteSet = False 35 | 36 | def _afterInit(self): 37 | self.afterInitSet = True 38 | 39 | def _setup(self): 40 | self.setupSet = True 41 | 42 | def _beforeExecute(self): 43 | self.beforeExecuteSet = True 44 | 45 | def _execute(self): 46 | self.executeSet = True 47 | 48 | def _afterExecute(self): 49 | self.afterExecuteSet = True -------------------------------------------------------------------------------- /tests/scenes/sphere_v1.mb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chadmv/blueprint/0b26e08b64c7dd399889ae4ff60b2c4ae4495d55/tests/scenes/sphere_v1.mb -------------------------------------------------------------------------------- /tests/scripts/maya.bps: -------------------------------------------------------------------------------- 1 | 2 | from blueprint.modules.maya import Maya 3 | 4 | Maya("sphere", scene="/Users/matt/Documents/maya/projects/default/scenes/sphere_v1.mb") 5 | 6 | -------------------------------------------------------------------------------- /tests/scripts/sleep.bps: -------------------------------------------------------------------------------- 1 | from blueprint.modules.shell import Shell 2 | 3 | Shell("foo1", cmd=["sleep", "0.1"]) 4 | Shell("foo2", cmd=["sleep", "0.1"]) 5 | Shell("foo3", cmd=["sleep", "0.1"]) 6 | -------------------------------------------------------------------------------- /tests/setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | 4 | logging.basicConfig(level=logging.INFO) 5 | 6 | os.environ["BLUEPRINT_CFG"] = os.path.dirname(__file__) + "/../etc/blueprint.cfg" 7 | 8 | def getTestScene(name): 9 | return os.path.dirname(__file__) + "/scenes/" + name 10 | -------------------------------------------------------------------------------- /tests/test_all.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import unittest 4 | 5 | import os 6 | import sys 7 | 8 | import logging 9 | logging.basicConfig(level=logging.INFO) 10 | 11 | import setup 12 | 13 | TEST_DIR = os.path.abspath(os.path.dirname(__file__)) 14 | SRC_DIR = os.path.join(TEST_DIR, "../lib") 15 | 16 | sys.path.append(SRC_DIR) 17 | sys.path.append(TEST_DIR) 18 | 19 | os.chdir(TEST_DIR) 20 | 21 | 22 | TESTS = [ 23 | 'test_app', 24 | 'test_layer', 25 | 'test_taskrun', 26 | # 'test_modules', # this depends on blender. can't include it 27 | ] 28 | 29 | 30 | def additional_tests(): 31 | suite = unittest.TestSuite() 32 | suite.addTest(unittest.TestLoader().loadTestsFromNames(TESTS)) 33 | return suite 34 | 35 | 36 | if __name__ == "__main__": 37 | suite = additional_tests() 38 | unittest.TextTestRunner(verbosity=2).run(suite) 39 | 40 | -------------------------------------------------------------------------------- /tests/test_app.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import json 4 | import unittest 5 | import logging 6 | 7 | import blueprint 8 | 9 | from common import TestLayer, TestTask 10 | 11 | class PluginManagerTests(unittest.TestCase): 12 | 13 | def testInit(self): 14 | """Test that the test plugin is being initialized.""" 15 | self.assertTrue(blueprint.plugins.test_plugin.Init.Loaded) 16 | 17 | def testAfterInit(self): 18 | """Test that after_init is being run by the plugin manager.""" 19 | l = TestLayer("test2") 20 | self.assertTrue(blueprint.plugins.test_plugin.Init.AfterInit) 21 | 22 | def testSetup(self): 23 | """Test that setup is being run by the plugin manager.""" 24 | j = blueprint.Job("foo") 25 | l = TestLayer("test2") 26 | j.addLayer(l) 27 | j.setup() 28 | self.assertTrue(blueprint.plugins.test_plugin.Init.LayerSetup) 29 | self.assertTrue(blueprint.plugins.test_plugin.Init.JobSetup) 30 | 31 | def testBeforeExecute(self): 32 | """Test that before execute is being run by the plugin manager.""" 33 | l = TestLayer("test2") 34 | l.beforeExecute() 35 | self.assertTrue(blueprint.plugins.test_plugin.Init.BeforeExecute) 36 | 37 | def testAfterExecute(self): 38 | """Test that after execute is being run by the plugin manager.""" 39 | l = TestLayer("test2") 40 | l.afterExecute() 41 | self.assertTrue(blueprint.plugins.test_plugin.Init.AfterExecute) 42 | 43 | if __name__ == "__main__": 44 | suite = unittest.TestSuite() 45 | for t in (PluginManagerTests,): 46 | suite.addTest(unittest.TestLoader().loadTestsFromTestCase(t)) 47 | unittest.TextTestRunner(verbosity=2).run(suite) 48 | 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /tests/test_archive.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import unittest 3 | import os 4 | 5 | import setup 6 | import blueprint 7 | 8 | class ArchiveTests(unittest.TestCase): 9 | 10 | def setUp(self): 11 | self.job = blueprint.Job("test") 12 | self.layer = blueprint.Layer("test") 13 | self.job.setup() 14 | 15 | def testArchivePath(self): 16 | p = self.job.getArchive() 17 | self.assertTrue(self.job.getName() in p.getPath()) 18 | self.assertTrue(os.path.isdir(p.getPath())) 19 | 20 | def testPutGetData(self): 21 | p = self.job.getArchive() 22 | p.putData("foo_str", "a string") 23 | p.putData("foo_list", ["a", "list"]) 24 | p.putData("foo_dict", {"a": "dict"}) 25 | 26 | self.assertEquals("a string", p.getData("foo_str")) 27 | self.assertEquals(["a", "list"], p.getData("foo_list")) 28 | self.assertEquals({"a": "dict"}, p.getData("foo_dict")) 29 | 30 | def testPutGetFile(self): 31 | p = self.job.getArchive() 32 | path = p.putFile("sleep", os.path.dirname(__file__) + "/scripts/sleep.bps") 33 | self.assertEquals(path, p.getFile("sleep")) 34 | self.assertTrue(path.startswith(p.getPath())) 35 | 36 | if __name__ == "__main__": 37 | suite = unittest.TestLoader().loadTestsFromTestCase(ArchiveTests) 38 | unittest.TextTestRunner(verbosity=2).run(suite) 39 | -------------------------------------------------------------------------------- /tests/test_conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import unittest 3 | import os 4 | 5 | import setup 6 | import blueprint 7 | 8 | class ConfTests(unittest.TestCase): 9 | 10 | def testGetModule(self): 11 | """ 12 | Test getting a module configuration. 13 | """ 14 | cfg = blueprint.conf.get("modules.maya") 15 | self.assertEquals(4, cfg["padding"]) 16 | self.assertEquals("name.#.ext", cfg["fnc"]) 17 | self.assertEquals("sw", cfg["renderer"]) 18 | self.assertEquals("png", cfg["format"]) 19 | 20 | def testGetDefaults(self): 21 | """ 22 | Test getting the blueprint defaults 23 | """ 24 | cfg = blueprint.conf.get("bp") 25 | self.assertEquals("test-scene.shot-%(USER)s_%(JOB_NAME)s", 26 | cfg["job_name_template"]) 27 | 28 | def testAutoInterpolate(self): 29 | """ 30 | Test automatic interpolation. 31 | """ 32 | home = os.environ.get("HOME", "") 33 | value = blueprint.conf.get("bp.output_dir") 34 | self.assertEquals("%s/blueprint/rnd" % home, value) 35 | 36 | def testPassInterpolate(self): 37 | """ 38 | Test passing values to interpolate 39 | """ 40 | user = os.environ.get("USER", "") 41 | value = blueprint.conf.get("bp.job_name_template", JOB_NAME="comp") 42 | self.assertEquals("test-scene.shot-%s_%s" % (user, "comp"), value) 43 | 44 | def testInterpolateFunction(self): 45 | """ 46 | Manual test of the interpolation function. 47 | """ 48 | s = "%(a)s + %(b)s" 49 | self.assertEquals("1 + 2", blueprint.conf.interp(s, a="1", b="2")) 50 | 51 | def testGetDefaultValue(self): 52 | """ 53 | Test fetching an invalid key. 54 | """ 55 | self.assertEquals(blueprint.conf.Default, blueprint.conf.get("nothing")) 56 | 57 | def testGetRaiseException(self): 58 | """ 59 | Test throwing an exception. 60 | """ 61 | self.assertRaises(blueprint.exception.BlueprintException, 62 | blueprint.conf.get, "nothing", default=blueprint.conf.Raise) 63 | 64 | def testEnvOverride(self): 65 | """ 66 | Test the environment override feature. 67 | """ 68 | os.environ["BLUEPRINT_TEST_TEST_TEST"] = "a" 69 | self.assertEquals("a", blueprint.conf.get("test.test.test")) 70 | 71 | if __name__ == "__main__": 72 | suite = unittest.TestLoader().loadTestsFromTestCase(ConfTests) 73 | unittest.TextTestRunner(verbosity=2).run(suite) -------------------------------------------------------------------------------- /tests/test_job.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import unittest 3 | import os 4 | 5 | import setup 6 | import blueprint 7 | 8 | class JobTests(unittest.TestCase): 9 | 10 | def setUp(self): 11 | self.job = blueprint.Job("test") 12 | self.layer = blueprint.Layer("test") 13 | self.job.addLayer(self.layer) 14 | 15 | def testSimpleGetters(self): 16 | self.job.getName() 17 | self.job.getId() 18 | self.job.getLogDir() 19 | 20 | def testGetLayer(self): 21 | l = self.job.getLayer("test") 22 | self.assertEquals(self.layer, l) 23 | 24 | def testGetLayers(self): 25 | layers = self.job.getLayers() 26 | self.assertTrue(self.layer in self.job.getLayers()) 27 | 28 | def testAddLayer(self): 29 | l = blueprint.Layer("add_twice") 30 | self.job.addLayer(l) 31 | # The second time should be ignored 32 | self.job.addLayer(l) 33 | 34 | def testGetSetFrameRange(self): 35 | self.assertEquals("1001-1001", self.job.getFrameRange()) 36 | self.assertEquals(1001, self.job.getFrameSet()[0]) 37 | self.assertEquals(1001, self.job.getFrameSet()[-1]) 38 | self.job.setFrameRange("1-10") 39 | self.assertEquals("1-10", self.job.getFrameRange()) 40 | self.assertEquals(1, self.job.getFrameSet()[0]) 41 | self.assertEquals(10, self.job.getFrameSet()[-1]) 42 | 43 | if __name__ == "__main__": 44 | suite = unittest.TestLoader().loadTestsFromTestCase(JobTests) 45 | unittest.TextTestRunner(verbosity=2).run(suite) 46 | -------------------------------------------------------------------------------- /tests/test_layer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import json 4 | import unittest 5 | import logging 6 | 7 | import setup 8 | import blueprint 9 | 10 | from common import TestLayer, TestTask 11 | 12 | class LayerTests(unittest.TestCase): 13 | 14 | def setUp(self): 15 | self.job = blueprint.Job("test") 16 | self.layer = blueprint.TaskIterator("test") 17 | 18 | def testCreateAndGet(self): 19 | """Create a new layer and add it to a job in non-script mode.""" 20 | self.job.addLayer(self.layer) 21 | self.assertEquals(self.layer, self.job.getLayer("test")) 22 | self.assertEquals("test", self.layer.getName()) 23 | 24 | def testGetSetArgs(self): 25 | """Test Get/Set args""" 26 | value = (1,2,3) 27 | self.layer.setArg("foo", value) 28 | self.assertEquals(value, self.layer.getArg("foo")) 29 | self.assertTrue(self.layer.isArgSet("foo")) 30 | self.assertFalse(self.layer.isArgSet("bar")) 31 | 32 | def testAddDepend(self): 33 | """Testing adding a dependency.""" 34 | self.assertEquals(0, len(self.layer.getDepends())) 35 | other = blueprint.Layer("test2") 36 | self.layer.dependOn(other, blueprint.DependType.All) 37 | self.assertEquals(1, len(self.layer.getDepends())) 38 | 39 | def testAddDependByTaskWithConstructor(self): 40 | """Test setup depend by task with constructor.""" 41 | l1 = TestLayer("testLayerA") 42 | l2 = TestLayer("testLayerB", depend="testLayerA") 43 | self.assertEquals(blueprint.DependType.ByTask, l2.getDepends()[0].type) 44 | 45 | def testAddDependAllWithConstructor(self): 46 | """Test setup depend:all with constructor""" 47 | l1 = TestLayer("testLayerA") 48 | l2 = TestLayer("testLayerB", depend="testLayerA:all") 49 | self.assertEquals(blueprint.DependType.All, l2.getDepends()[0].type) 50 | 51 | def testAddOutput(self): 52 | """Test adding an output.""" 53 | self.assertEquals(0, len(self.layer.getOutputs())) 54 | self.layer.addOutput("comp", "/foo/bar.#.dpx") 55 | self.layer.getOutput("comp") 56 | self.assertEquals(1, len(self.layer.getOutputs())) 57 | 58 | def testAddInput(self): 59 | """Test adding an input.""" 60 | self.assertEquals(0, len(self.layer.getInputs())) 61 | self.layer.addInput("scene", "/foo/bar.blender") 62 | self.layer.getInput("scene") 63 | self.assertEquals(1, len(self.layer.getInputs())) 64 | 65 | def testAfterInit(self): 66 | """Test that after_init is being run by the metaclass.""" 67 | l = TestLayer("test2") 68 | self.assertTrue(l.afterInitSet) 69 | 70 | def testSetup(self): 71 | """Test that _setup is being called.""" 72 | # To call setup you must have a job 73 | l = TestLayer("test2") 74 | self.job.addLayer(l) 75 | self.assertFalse(l.setupSet) 76 | self.job.setup() 77 | self.assertTrue(l.setupSet) 78 | 79 | def testExecute(self): 80 | """Test that _execute is being called.""" 81 | l = TestLayer("test2") 82 | self.assertFalse(l.executeSet) 83 | l.execute(1) 84 | self.assertTrue(l.executeSet) 85 | 86 | def testBeforeExecute(self): 87 | """Test that _beforeExecute is being called.""" 88 | l = TestLayer("test2") 89 | self.assertFalse(l.beforeExecuteSet) 90 | l.beforeExecute() 91 | self.assertTrue(l.beforeExecuteSet) 92 | 93 | def testAfterExecute(self): 94 | """Test that _afterExecute is being called.""" 95 | l = TestLayer("test2") 96 | self.assertFalse(l.afterExecuteSet) 97 | l.afterExecute() 98 | self.assertTrue(l.afterExecuteSet) 99 | 100 | def testSetFrameRange(self): 101 | """Test setting the frame range attr.""" 102 | self.job.addLayer(self.layer) 103 | self.assertEquals("1001-1001", self.layer.getFrameRange()) 104 | self.assertEquals(1001, self.layer.getFrameSet()[0]) 105 | self.assertEquals(1001, self.layer.getFrameSet()[-1]) 106 | self.layer.setFrameRange("1-10") 107 | self.assertEquals("1-10", self.layer.getFrameRange()) 108 | self.assertEquals(1, self.layer.getFrameSet()[0]) 109 | self.assertEquals(10, self.layer.getFrameSet()[-1]) 110 | 111 | def testGetLocalFrameSetChunked(self): 112 | """Test getting the local frameset.""" 113 | l = TestLayer("test2", chunk=10, range="1-20") 114 | frameset = l.getLocalFrameSet(1) 115 | self.assertEquals("1-10", str(frameset.normalize())) 116 | frameset = l.getLocalFrameSet(11) 117 | self.assertEquals("11-20", str(frameset.normalize())) 118 | 119 | def testGetLocalFrameSetSingle(self): 120 | """Test getting the local frameset.""" 121 | l = TestLayer("test2") 122 | frameset = l.getLocalFrameSet(1) 123 | print frameset 124 | 125 | class TaskTests(unittest.TestCase): 126 | 127 | def testCreateTask(self): 128 | t = TestTask("test") 129 | 130 | if __name__ == "__main__": 131 | suite = unittest.TestSuite() 132 | for t in (LayerTests, TaskTests): 133 | suite.addTest(unittest.TestLoader().loadTestsFromTestCase(t)) 134 | unittest.TextTestRunner(verbosity=2).run(suite) 135 | 136 | 137 | 138 | -------------------------------------------------------------------------------- /tests/test_module_blender.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import unittest 4 | import logging 5 | 6 | import fileseq 7 | import blueprint.modules.blender as blender 8 | 9 | logging.basicConfig(level=logging.DEBUG) 10 | 11 | class BlenderModuleTests(unittest.TestCase): 12 | 13 | scene_file = os.path.join(os.path.dirname(__file__), "data/test.blend") 14 | 15 | def testInitialize(self): 16 | """Initialize a Blender instance.""" 17 | b = blender.Blender("comp", scene_file=self.scene_file) 18 | b._setup() 19 | 20 | self.assertEquals(1, len(b.getSetupTasks())) 21 | self.assertEquals(1, len(b.getDepends())) 22 | 23 | def testExecuteSetupTask(self): 24 | """Execute the setup task.""" 25 | b = blender.Blender("comp", scene_file=self.scene_file) 26 | b._setup() 27 | 28 | # Grab the setup task and execute it. 29 | b.getSetupTasks()[0]._execute() 30 | # Now the outputs should be available. 31 | outputs = b.getOutputs() 32 | # Assert we have two outputs. 33 | self.assertEquals(4, len(outputs)) 34 | 35 | def testExecuteRenderTask(self): 36 | b = blender.Blender("comp", scene_file=self.scene_file) 37 | b._setup() 38 | b.getSetupTasks()[0]._execute() 39 | b._execute(fileseq.FrameSet("1-2")) 40 | 41 | if __name__ == "__main__": 42 | suite = unittest.TestSuite() 43 | for t in (BlenderModuleTests,): 44 | suite.addTest(unittest.TestLoader().loadTestsFromTestCase(t)) 45 | unittest.TextTestRunner(verbosity=2).run(suite) -------------------------------------------------------------------------------- /tests/test_module_maya.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import unittest 3 | import os 4 | 5 | import setup 6 | import blueprint 7 | import fileseq 8 | 9 | from blueprint.modules.maya import Maya 10 | 11 | class MayaModuleTests(unittest.TestCase): 12 | 13 | def setUp(self): 14 | self.job = blueprint.Job("maya_test") 15 | self.layer = Maya("foo", scene=setup.getTestScene("sphere_v1.mb")) 16 | self.job.addLayer(self.layer) 17 | 18 | def testSetup(self): 19 | self.job.setup() 20 | self.job.getLayer("foo").execute(fileseq.FrameSet("1-1")) 21 | 22 | 23 | if __name__ == "__main__": 24 | suite = unittest.TestLoader().loadTestsFromTestCase(MayaModuleTests) 25 | unittest.TextTestRunner(verbosity=2).run(suite) 26 | -------------------------------------------------------------------------------- /tests/test_taskrun.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import blueprint 4 | 5 | from common import TestLayer, TestTask 6 | 7 | class TaskrunTests(unittest.TestCase): 8 | 9 | def setUp(self): 10 | self.job = blueprint.Job("test") 11 | 12 | def testRunTask(self): 13 | task = TestTask("test") 14 | self.job.addLayer(task) 15 | self.job.setup() 16 | task.execute() 17 | 18 | if __name__ == "__main__": 19 | suite = unittest.TestSuite() 20 | for t in (TaskrunTests,): 21 | suite.addTest(unittest.TestLoader().loadTestsFromTestCase(t)) 22 | unittest.TextTestRunner(verbosity=2).run(suite) 23 | 24 | 25 | --------------------------------------------------------------------------------