├── .gitignore ├── .travis.yml ├── MANIFEST.in ├── README.md ├── archive └── test │ ├── creat │ ├── a.c │ ├── b.c │ └── build.py │ ├── intercept_command_line │ └── build.py │ ├── lstat │ ├── a.c │ ├── b.c │ └── build.py │ ├── md5hasher │ ├── test.py │ ├── testdir │ │ └── testfile │ ├── testdirlink │ ├── testfile │ ├── testlink │ └── testlink_nofile │ ├── mkdir │ ├── build.py │ └── existingdir │ │ └── existingfile │ ├── rename │ ├── build.py │ └── originalfile │ └── symlink │ ├── build.py │ ├── testdir │ └── testfile │ └── testfile ├── benchmark.py ├── fabricate.py ├── setup.cfg ├── setup.py ├── test-requirement.txt └── test ├── conftest.py ├── test_create.py ├── test_fabricate.py ├── test_mkdir.py ├── test_rename.py └── test_symlink.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.orig 2 | *.pyc 3 | 4 | # virtual env 5 | venv/ 6 | 7 | # testing 8 | build_dir/ 9 | .cache/ 10 | 11 | # Distribution / packaging 12 | .Python 13 | env/ 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | language: python 3 | python: 4 | - "2.6" 5 | - "2.7" 6 | - "3.3" 7 | - "3.4" 8 | - "3.5" 9 | - "3.6" 10 | # command to install dependencies 11 | before_install: 12 | - sudo apt-get update -qq 13 | - sudo apt-get install -y strace 14 | install: "pip install -r test-requirement.txt" 15 | # command to run tests 16 | script: pytest -v 17 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.md *.py 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fabricate # 2 | 3 | [![Build 4 | Status](https://travis-ci.org/brushtechnology/fabricate.svg?branch=master)](https://travis-ci.org/brushtechnology/fabricate) 5 | 6 | Note: We are looking for a new project maintainer, as things have stagnated somewhat recently. Please [see here for details](https://github.com/brushtechnology/fabricate/issues/101). 7 | 8 | **fabricate** is a build tool that finds dependencies automatically for any language. It's small and just works. No hidden stuff behind your back. It was inspired by Bill McCloskey's make replacement, memoize, but fabricate works on [Windows](https://github.com/brushtechnology/fabricate/wiki/HowItWorks#windows-issues) as well as Linux. 9 | 10 | [Get fabricate.py now](https://raw.githubusercontent.com/brushtechnology/fabricate/master/fabricate.py), learn [how it works](https://github.com/brushtechnology/fabricate/wiki/HowItWorks), see how to get [in-Python help](https://github.com/brushtechnology/fabricate/wiki/Help), or discuss it on the [mailing list](http://groups.google.com/group/fabricate-users). 11 | 12 | ## Features ## 13 | 14 | * Never have to list dependencies. 15 | * Never have to specify cleanup rules. 16 | * The tool is a single Python file. 17 | * It uses MD5 (not timestamps) to check inputs and outputs. 18 | * You can learn it all in about 10 minutes. 19 | * You can still read your build scripts 3 months later. 20 | * Now supports [parallel building](https://github.com/brushtechnology/fabricate/wiki/ParallelBuilding) 21 | 22 | ## Show me an example! ## 23 | 24 | ```python 25 | from fabricate import * 26 | 27 | sources = ['program', 'util'] 28 | 29 | def build(): 30 | compile() 31 | link() 32 | 33 | def compile(): 34 | for source in sources: 35 | run('gcc', '-c', source+'.c') 36 | 37 | def link(): 38 | objects = [s+'.o' for s in sources] 39 | run('gcc', '-o', 'program', objects) 40 | 41 | def clean(): 42 | autoclean() 43 | 44 | main() 45 | ``` 46 | 47 | This isn't the simplest build script you can make with fabricate (see [other examples](https://github.com/brushtechnology/fabricate/wiki/Examples)), but it's surprisingly close to some of the more complex scripts we use in real life. Things to note: 48 | 49 | * It's an **ordinary Python file.** Use the clarity and power of Python. 50 | * **No implicit stuff** like CCFLAGS. 51 | * **Explicit is better:** you tell fabricate what commands to run, and it runs them -- but only if their inputs or outputs have changed. 52 | * Where you'd use targets in make, you just **use Python functions** -- `build()` is the default. 53 | * You can **easily "autoclean"** any build outputs -- fabricate finds build outputs automatically, just like it finds dependencies. 54 | 55 | ## Using fabricate options ## 56 | 57 | The best way to get started is to take one of the examples linked above and modify it to suit your project. But you're bound to want to use some of the options built into fabricate. To get a list of these: 58 | ``` 59 | from fabricate import * 60 | 61 | help(main) 62 | help(Builder) 63 | ``` 64 | 65 | ## Using fabricate as a script, a la memoize ## 66 | 67 | You can also use fabricate.py as a script and enter commands directly on the command line (see [command line options](https://github.com/brushtechnology/fabricate/wiki/CommandLineOptions)). In the following, each `gcc` command will only be run if its dependencies have changed: 68 | 69 | ``` 70 | fabricate.py gcc -c program.c 71 | fabricate.py gcc -c util.c 72 | fabricate.py gcc -o program program.o util.o 73 | ``` 74 | 75 | ## Why not use make? ## 76 | 77 | For a start, fabricate won't say "`*** missing separator`" if you use spaces instead of tabs. And you'll never need to enter dependencies manually, like this: 78 | 79 | ``` 80 | files.o : files.c defs.h buffer.h command.h 81 | cc -c files.c 82 | ``` 83 | 84 | Instead, you just tell fabricate to `run('cc', 'file.c')` and it'll figure out what that command's inputs and outputs are. Next time you build, the command will only get run if its inputs have changed, or if its outputs have been modified or aren't there. 85 | 86 | And you can use Python's readable string functions instead of producing write-only make rules, like this one from the make docs: 87 | 88 | ``` 89 | %.d : %.c 90 | @set -e; rm -f $@; $(CC) -M $(CPPFLAGS) $< > $@.$$$$; \ 91 | sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' < $@.$$$$ > $@; rm -f $@.$$$$ 92 | ``` 93 | 94 | ## What about SCons? ## 95 | 96 | SCons tempted us at first too. It's Python ... isn't it? But just before it sucks you in, you realise it's actually [quite hard](http://stackoverflow.com/questions/1074062/) to do simple things explicitly. 97 | 98 | Python says that _explicit is better than implicit_ for a reason, and with fabricate, we've made it so you tell it what you want. It won't do things behind your back based on the [83 different tools](http://www.scons.org/doc/HTML/scons-user/a9626.html) it may or may not know about. 99 | 100 | ## Credits ## 101 | 102 | fabricate is inspired by [Bill McCloskey's memoize](http://www.eecs.berkeley.edu/~billm/memoize.html), but fabricate works under Windows as well by using file access times instead of strace if strace is not available on your file system. Read more about [how fabricate works.](https://github.com/brushtechnology/fabricate/wiki/HowItWorks) 103 | 104 | fabricate was originally developed by [Ben Hoyt](https://github.com/benhoyt) at [Brush Technology](http://brush.co.nz/) for in-house use, and we then released into the wild. It now has a small but dedicated user base and is actively being maintained and improved by Simon Alford, with help from other fabricate users. 105 | 106 | ## License ## 107 | 108 | Like memoize, fabricate is released under a [New BSD license](https://github.com/brushtechnology/fabricate/wiki/License). fabricate is 109 | Copyright (c) 2009 Brush Technology. 110 | -------------------------------------------------------------------------------- /archive/test/creat/a.c: -------------------------------------------------------------------------------- 1 | int a() { 2 | return 1; 3 | } 4 | 5 | -------------------------------------------------------------------------------- /archive/test/creat/b.c: -------------------------------------------------------------------------------- 1 | int b { 2 | return 2; 3 | } 4 | -------------------------------------------------------------------------------- /archive/test/creat/build.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | 5 | sys.path.append('../../') 6 | 7 | from fabricate import * 8 | 9 | def build(): 10 | run('tar', 'czvf', 'foo.tar.gz', 'a.c', 'b.c') 11 | 12 | def clean(): 13 | autoclean() 14 | 15 | main() 16 | -------------------------------------------------------------------------------- /archive/test/intercept_command_line/build.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | if __name__ == '__main__': 4 | import sys 5 | sys.path.append('../../') 6 | import fabricate 7 | 8 | default='myfab' 9 | 10 | def myfab(): 11 | fabricate.run('touch', 'testfile') 12 | 13 | def clean(): 14 | fabricate.autoclean() 15 | 16 | if len(sys.argv) > 1: 17 | default=sys.argv[1] 18 | 19 | fabricate.main(command_line=[default]) 20 | -------------------------------------------------------------------------------- /archive/test/lstat/a.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brushtechnology/fabricate/24a3acfc86129fda56c48a6ec1594e2cd5cd8e0c/archive/test/lstat/a.c -------------------------------------------------------------------------------- /archive/test/lstat/b.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brushtechnology/fabricate/24a3acfc86129fda56c48a6ec1594e2cd5cd8e0c/archive/test/lstat/b.c -------------------------------------------------------------------------------- /archive/test/lstat/build.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | 5 | sys.path.append('../../') 6 | 7 | from fabricate import * 8 | 9 | def build(): 10 | run('tar', 'czvf', 'foo.tar.gz', 'a.c', 'b.c') 11 | 12 | def clean(): 13 | autoclean() 14 | 15 | main() 16 | -------------------------------------------------------------------------------- /archive/test/md5hasher/test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | 4 | import sys 5 | import os 6 | 7 | sys.path.append('../../') 8 | 9 | from fabricate import * 10 | 11 | # Run the md5hasher tests 12 | if __name__ == '__main__': 13 | 14 | print(md5_hasher('nofile')) 15 | print(md5_hasher('testfile')) 16 | print(md5_hasher('testdir')) 17 | print(md5_hasher('testlink')) 18 | print(md5_hasher('testdirlink')) 19 | print(md5_hasher('testlink_nofile')) 20 | 21 | sys.exit(0) 22 | -------------------------------------------------------------------------------- /archive/test/md5hasher/testdir/testfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brushtechnology/fabricate/24a3acfc86129fda56c48a6ec1594e2cd5cd8e0c/archive/test/md5hasher/testdir/testfile -------------------------------------------------------------------------------- /archive/test/md5hasher/testdirlink: -------------------------------------------------------------------------------- 1 | testdir/ -------------------------------------------------------------------------------- /archive/test/md5hasher/testfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brushtechnology/fabricate/24a3acfc86129fda56c48a6ec1594e2cd5cd8e0c/archive/test/md5hasher/testfile -------------------------------------------------------------------------------- /archive/test/md5hasher/testlink: -------------------------------------------------------------------------------- 1 | testfile -------------------------------------------------------------------------------- /archive/test/md5hasher/testlink_nofile: -------------------------------------------------------------------------------- 1 | nofile -------------------------------------------------------------------------------- /archive/test/mkdir/build.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | 5 | sys.path.append('../../') 6 | 7 | from multiprocessing import freeze_support 8 | from fabricate import * 9 | 10 | def build(): 11 | # Make lots of directories to check ordered delete 12 | run('mkdir', 'testdir', group='testdir') 13 | run('mkdir', 'testdir/a', group='a', after='testdir') 14 | run('mkdir', 'testdir/b', group='b', after='testdir') 15 | run('mkdir', 'testdir/c', group='c', after='testdir') 16 | run('mkdir', 'testdir/c/f', group='f', after='c') 17 | run('mkdir', 'testdir/c/e', group='e', after='c') 18 | run('mkdir', 'testdir/c/d', group='d', after='c') 19 | 20 | # put some files in them to ensure content deleted before dir 21 | run('touch', 'testdir/f1', after='testdir') 22 | run('touch', 'testdir/f2', after='testdir') 23 | run('touch', 'testdir/b/f1', after='b') 24 | run('touch', 'testdir/b/f2', after='b') 25 | run('touch', 'testdir/c/d/f1', after='d') 26 | run('touch', 'testdir/c/d/f2', after='d') 27 | 28 | # make a dir that alreay exists 29 | run('mkdir', '-p', 'testdir/c/d', after='d') 30 | 31 | # make a dir that already partialy exists 32 | run('mkdir', '-p', 'testdir/c/g', after='c') 33 | 34 | # make a dir that already partialy exists but should not be deleted 35 | run('mkdir', '-p', 'existingdir/a') 36 | 37 | 38 | def clean(): 39 | autoclean() 40 | 41 | if __name__ == "__main__": 42 | freeze_support() 43 | main(parallel_ok=True) 44 | -------------------------------------------------------------------------------- /archive/test/mkdir/existingdir/existingfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brushtechnology/fabricate/24a3acfc86129fda56c48a6ec1594e2cd5cd8e0c/archive/test/mkdir/existingdir/existingfile -------------------------------------------------------------------------------- /archive/test/rename/build.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | 5 | sys.path.append('../../') 6 | 7 | from fabricate import * 8 | 9 | def build(): 10 | run('mv', 'originalfile', 'testfile') 11 | 12 | def clean(): 13 | autoclean() 14 | # remake the original file 15 | shell('touch', 'originalfile') 16 | 17 | main() 18 | -------------------------------------------------------------------------------- /archive/test/rename/originalfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brushtechnology/fabricate/24a3acfc86129fda56c48a6ec1594e2cd5cd8e0c/archive/test/rename/originalfile -------------------------------------------------------------------------------- /archive/test/symlink/build.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | 5 | sys.path.append('../../') 6 | 7 | from fabricate import * 8 | 9 | def build(): 10 | run('ln', '-s', 'testfile', 'testlink') 11 | run('ln', '-s', 'testdir', 'testlink_dir') 12 | run('ln', '-s', 'nofile', 'testlink_nofile') 13 | 14 | def clean(): 15 | autoclean() 16 | 17 | main() 18 | -------------------------------------------------------------------------------- /archive/test/symlink/testdir/testfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brushtechnology/fabricate/24a3acfc86129fda56c48a6ec1594e2cd5cd8e0c/archive/test/symlink/testdir/testfile -------------------------------------------------------------------------------- /archive/test/symlink/testfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brushtechnology/fabricate/24a3acfc86129fda56c48a6ec1594e2cd5cd8e0c/archive/test/symlink/testfile -------------------------------------------------------------------------------- /benchmark.py: -------------------------------------------------------------------------------- 1 | """Simple benchmark for fabricate.""" 2 | from __future__ import print_function 3 | 4 | import os 5 | import shutil 6 | import sys 7 | import time 8 | 9 | import fabricate 10 | 11 | COMPILER = None 12 | BUILD_DIR = 'benchproject' 13 | NUM_SOURCE_FILES = 100 14 | NUM_SOURCE_LINES = 1000 15 | 16 | NUM_HEADER_FILES = 10 17 | NUM_HEADER_LINES = 10000 18 | 19 | if sys.platform == 'win32': 20 | # time.clock() is much more accurate under Windows 21 | get_time = time.clock 22 | else: 23 | get_time = time.time 24 | 25 | def delete_deps(): 26 | if os.path.exists(os.path.join(BUILD_DIR, '.deps')): 27 | os.remove(os.path.join(BUILD_DIR, '.deps')) 28 | 29 | def generate(): 30 | if not os.path.exists(BUILD_DIR): 31 | os.mkdir(BUILD_DIR) 32 | delete_deps() 33 | 34 | for source_index in range(NUM_SOURCE_FILES): 35 | lines = [] 36 | 37 | for header_index in range(NUM_HEADER_FILES): 38 | lines.append('#include "header%d.h"' % header_index) 39 | 40 | if source_index == 0: 41 | lines.append('#include ') 42 | lines.append('int main(void) {') 43 | for source_index2 in range(NUM_SOURCE_FILES): 44 | for line_index in range(NUM_SOURCE_LINES): 45 | lines.append(' printf("%%d ", func%d_%d(42, 24));' % 46 | (source_index2, line_index)) 47 | lines.append('}') 48 | 49 | for line_index in range(NUM_SOURCE_LINES): 50 | lines.append('int func%d_%d(int x, int y) { return x * %d + y * %d; }' % 51 | (source_index, line_index, source_index, line_index)) 52 | 53 | filename = os.path.join(BUILD_DIR, 'source%d.c' % source_index) 54 | f = open(filename, 'w') 55 | f.write('\n'.join(lines)) 56 | f.close() 57 | 58 | for header_index in range(NUM_HEADER_FILES): 59 | lines = [] 60 | lines.append('#ifndef HEADER%d_H' % header_index) 61 | lines.append('#define HEADER%d_H' % header_index) 62 | 63 | if header_index == 0: 64 | for source_index2 in range(NUM_SOURCE_FILES): 65 | for line_index in range(NUM_SOURCE_LINES): 66 | lines.append('int func%d_%d(int x, int y);' % 67 | (source_index2, line_index)) 68 | 69 | for line_index in range(NUM_HEADER_LINES): 70 | lines.append('typedef int type_%d_%d;' % (header_index, line_index)) 71 | 72 | lines.append('#endif') 73 | 74 | filename = os.path.join(BUILD_DIR, 'header%d.h' % header_index) 75 | f = open(filename, 'w') 76 | f.write('\n'.join(lines)) 77 | f.close() 78 | 79 | def benchmark(runner, jobs): 80 | if runner == 'always_runner': 81 | delete_deps() 82 | 83 | para = (', parallel_ok=True, jobs=%d' % jobs) if jobs > 1 else '' 84 | build_file = r""" 85 | from fabricate import * 86 | 87 | sources = [ 88 | %s 89 | ] 90 | 91 | def build(): 92 | compile() 93 | after() 94 | link() 95 | 96 | def compile(): 97 | for source in sources: 98 | run(%s, '-c', source + '.c') 99 | 100 | def link(): 101 | objects = [s + '.o' for s in sources] 102 | run(%s, '-o', 'benchmark', objects) 103 | 104 | def clean(): 105 | autoclean() 106 | 107 | main(runner='%s'%s) 108 | """ % (',\n '.join("'source%d'" % i for i in range(NUM_SOURCE_FILES)), 109 | repr(COMPILER), 110 | repr(COMPILER), 111 | runner, 112 | para) 113 | 114 | filename = os.path.join(BUILD_DIR, 'build.py') 115 | f = open(filename, 'w') 116 | f.write(build_file) 117 | f.close() 118 | 119 | time0 = get_time() 120 | filename = os.path.join(BUILD_DIR, 'build.py') 121 | fabricate.shell('python', filename, '-q') 122 | elapsed_time = get_time() - time0 123 | return elapsed_time 124 | 125 | def benchmake(jobs): 126 | makefile = """ 127 | OBJECTS = \\ 128 | \t%s 129 | 130 | benchmark: $(OBJECTS) 131 | \t"%s" -o benchmark $(OBJECTS) 132 | 133 | %%.o: %%.c 134 | \t"%s" -c $< -o $@ 135 | 136 | %%.c: \\ 137 | \t%s 138 | """ % (' \\\n\t'.join('source%d.o' % i for i in range(NUM_SOURCE_FILES)), 139 | COMPILER, 140 | COMPILER, 141 | ' \\\n\t'.join('header%d.h' % i for i in range(NUM_HEADER_FILES))) 142 | 143 | filename = os.path.join(BUILD_DIR, 'Makefile') 144 | f = open(filename, 'w') 145 | f.write(makefile) 146 | f.close() 147 | 148 | time0 = get_time() 149 | filename = os.path.join(BUILD_DIR, 'build.py') 150 | job_arg = '-j%d' % jobs 151 | fabricate.shell('make', job_arg, '-s', '-C', BUILD_DIR) 152 | elapsed_time = get_time() - time0 153 | return elapsed_time 154 | 155 | def clean(): 156 | if os.path.exists(BUILD_DIR): 157 | shutil.rmtree(BUILD_DIR) 158 | 159 | def usage(): 160 | print('Usage: benchmark.py compiler generate|benchmark [runner=smart_runner [jobs=1]]|benchmake [jobs=1]|clean') 161 | sys.exit(1) 162 | 163 | if __name__ == '__main__': 164 | if len(sys.argv) < 3: 165 | usage() 166 | orig_cwd = os.getcwd() 167 | os.chdir(os.path.dirname(__file__)) 168 | jobs = 1 169 | try: 170 | COMPILER = sys.argv[1] 171 | if sys.argv[2] == 'generate': 172 | generate() 173 | elif sys.argv[2] == 'benchmark': 174 | if len(sys.argv) > 3: 175 | runner = sys.argv[3] 176 | else: 177 | runner = 'smart_runner' 178 | if len(sys.argv) > 4: 179 | jobs = int(sys.argv[4]) 180 | print(benchmark(runner, jobs)) 181 | elif sys.argv[2] == 'benchmake': 182 | if len(sys.argv) > 3: 183 | jobs = int(sys.argv[3]) 184 | print(benchmake(jobs)) 185 | elif sys.argv[2] == 'clean': 186 | clean() 187 | else: 188 | usage() 189 | finally: 190 | os.chdir(orig_cwd) 191 | -------------------------------------------------------------------------------- /fabricate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Build tool that finds dependencies automatically for any language. 4 | 5 | fabricate is a build tool that finds dependencies automatically for any 6 | language. It's small and just works. No hidden stuff behind your back. It was 7 | inspired by Bill McCloskey's make replacement, memoize, but fabricate works on 8 | Windows as well as Linux. 9 | 10 | Read more about how to use it and how it works on the project page: 11 | https://github.com/brushtechnology/fabricate/ 12 | 13 | Like memoize, fabricate is released under a "New BSD license". fabricate is 14 | copyright (c) 2009 Brush Technology. Full text of the license is here: 15 | https://github.com/brushtechnology/fabricate/wiki/License 16 | 17 | To get help on fabricate functions: 18 | from fabricate import * 19 | help(function) 20 | 21 | """ 22 | 23 | from __future__ import with_statement, print_function, unicode_literals 24 | 25 | # fabricate version number 26 | __version__ = '1.29.3' 27 | 28 | # if version of .deps file has changed, we know to not use it 29 | deps_version = 2 30 | 31 | import atexit 32 | import optparse 33 | import os 34 | import platform 35 | import re 36 | import shlex 37 | import stat 38 | import subprocess 39 | import sys 40 | import tempfile 41 | import time 42 | import threading # NB uses old camelCase names for backward compatibility 43 | import traceback 44 | # multiprocessing module only exists on Python >= 2.6 45 | try: 46 | import multiprocessing 47 | except ImportError: 48 | class MultiprocessingModule(object): 49 | def __getattr__(self, name): 50 | raise NotImplementedError("multiprocessing module not available, can't do parallel builds") 51 | multiprocessing = MultiprocessingModule() 52 | 53 | # compatibility 54 | PY3 = sys.version_info[0] >= 3 55 | if PY3: 56 | string_types = str 57 | threading_condition = threading.Condition 58 | else: 59 | string_types = basestring 60 | try: 61 | threading_condition = threading._Condition 62 | except (ImportError, AttributeError): 63 | threading_condition = threading.Condition 64 | 65 | # so you can do "from fabricate import *" to simplify your build script 66 | __all__ = ['setup', 'run', 'autoclean', 'main', 'shell', 'fabricate_version', 67 | 'memoize', 'outofdate', 'parse_options', 'after', 68 | 'ExecutionError', 'md5_hasher', 'mtime_hasher', 69 | 'Runner', 'AtimesRunner', 'StraceRunner', 'AlwaysRunner', 70 | 'SmartRunner', 'Builder'] 71 | 72 | import textwrap 73 | 74 | __doc__ += "Exported functions are:\n" + ' ' + '\n '.join(textwrap.wrap(', '.join(__all__), 80)) 75 | 76 | 77 | 78 | FAT_atime_resolution = 24*60*60 # resolution on FAT filesystems (seconds) 79 | FAT_mtime_resolution = 2 80 | 81 | # NTFS resolution is < 1 ms 82 | # We assume this is considerably more than time to run a new process 83 | 84 | NTFS_atime_resolution = 0.0002048 # resolution on NTFS filesystems (seconds) 85 | NTFS_mtime_resolution = 0.0002048 # is actually 0.1us but python's can be 86 | # as low as 204.8us due to poor 87 | # float precision when storing numbers 88 | # as big as NTFS file times can be 89 | # (float has 52-bit precision and NTFS 90 | # FILETIME has 63-bit precision, so 91 | # we've lost 11 bits = 2048) 92 | 93 | # So we can use md5func in old and new versions of Python without warnings 94 | try: 95 | import hashlib 96 | md5func = hashlib.md5 97 | except ImportError: 98 | import md5 99 | md5func = md5.new 100 | 101 | # Use json, or pickle on older Python versions if simplejson not installed 102 | try: 103 | import json 104 | except ImportError: 105 | try: 106 | import simplejson as json 107 | except ImportError: 108 | import cPickle 109 | # needed to ignore the indent= argument for pickle's dump() 110 | class PickleJson: 111 | def load(self, f): 112 | return cPickle.load(f) 113 | def dump(self, obj, f, indent=None, sort_keys=None): 114 | return cPickle.dump(obj, f) 115 | json = PickleJson() 116 | 117 | def printerr(message): 118 | """ Print given message to stderr with a line feed. """ 119 | print(message, file=sys.stderr) 120 | 121 | class PathError(Exception): 122 | pass 123 | 124 | class ExecutionError(Exception): 125 | """ Raised by shell() and run() if command returns non-zero exit code. """ 126 | pass 127 | 128 | def args_to_list(args): 129 | """ Return a flat list of the given arguments for shell(). """ 130 | arglist = [] 131 | for arg in args: 132 | if arg is None: 133 | continue 134 | if isinstance(arg, (list, tuple)): 135 | arglist.extend(args_to_list(arg)) 136 | else: 137 | if not isinstance(arg, string_types): 138 | arg = str(arg) 139 | arglist.append(arg) 140 | return arglist 141 | 142 | def shell(*args, **kwargs): 143 | r""" Run a command: program name is given in first arg and command line 144 | arguments in the rest of the args. Iterables (lists and tuples) in args 145 | are recursively converted to separate arguments, non-string types are 146 | converted with str(arg), and None is ignored. For example: 147 | 148 | >>> def tail(input, n=3, flags=None): 149 | >>> args = ['-n', n] 150 | >>> return shell('tail', args, flags, input=input) 151 | >>> tail('a\nb\nc\nd\ne\n') 152 | 'c\nd\ne\n' 153 | >>> tail('a\nb\nc\nd\ne\n', 2, ['-v']) 154 | '==> standard input <==\nd\ne\n' 155 | 156 | Keyword arguments kwargs are interpreted as follows: 157 | 158 | "input" is a string to pass standard input into the process (or the 159 | default of None to use parent's stdin, eg: the keyboard) 160 | "silent" is True (default) to return process's standard output as a 161 | string, or False to print it as it comes out 162 | "shell" set to True will run the command via the shell (/bin/sh or 163 | COMSPEC) instead of running the command directly (the default) 164 | "ignore_status" set to True means ignore command status code -- i.e., 165 | don't raise an ExecutionError on nonzero status code 166 | Any other kwargs are passed directly to subprocess.Popen 167 | Raises ExecutionError(message, output, status) if the command returns 168 | a non-zero status code. """ 169 | try: 170 | return _shell(args, **kwargs) 171 | finally: 172 | sys.stderr.flush() 173 | sys.stdout.flush() 174 | 175 | def _shell(args, input=None, silent=True, shell=False, ignore_status=False, **kwargs): 176 | if input: 177 | stdin = subprocess.PIPE 178 | else: 179 | stdin = None 180 | if silent: 181 | stdout = subprocess.PIPE 182 | else: 183 | stdout = None 184 | arglist = args_to_list(args) 185 | if not arglist: 186 | raise TypeError('shell() takes at least 1 argument (0 given)') 187 | if shell: 188 | # handle subprocess.Popen quirk where subsequent args are passed 189 | # to bash instead of to our command 190 | command = subprocess.list2cmdline(arglist) 191 | else: 192 | command = arglist 193 | try: 194 | proc = subprocess.Popen(command, stdin=stdin, stdout=stdout, 195 | stderr=subprocess.STDOUT, shell=shell, **kwargs) 196 | except OSError as e: 197 | # Work around the problem that Windows Popen doesn't say what file it couldn't find 198 | if platform.system() == 'Windows' and e.errno == 2 and e.filename is None: 199 | e.filename = arglist[0] 200 | raise e 201 | output, stderr = proc.communicate(input) 202 | status = proc.wait() 203 | if status and not ignore_status: 204 | raise ExecutionError('%r exited with status %d' 205 | % (os.path.basename(arglist[0]), status), 206 | output, status) 207 | if silent: 208 | return output 209 | 210 | def md5_hasher(filename): 211 | """ Return MD5 hash of given filename if it is a regular file or 212 | a symlink with a hashable target, or the MD5 hash of the 213 | target_filename if it is a symlink without a hashable target, 214 | or the MD5 hash of the filename if it is a directory, or None 215 | if file doesn't exist. 216 | 217 | Note: Pyhton versions before 3.2 do not support os.readlink on 218 | Windows so symlinks without a hashable target fall back to 219 | a hash of the filename if the symlink target is a directory, 220 | or None if the symlink is broken""" 221 | if not isinstance(filename, bytes): 222 | filename = filename.encode('utf-8') 223 | try: 224 | f = open(filename, 'rb') 225 | try: 226 | return md5func(f.read()).hexdigest() 227 | finally: 228 | f.close() 229 | except IOError: 230 | if hasattr(os, 'readlink') and os.path.islink(filename): 231 | return md5func(os.readlink(filename)).hexdigest() 232 | elif os.path.isdir(filename): 233 | return md5func(filename).hexdigest() 234 | return None 235 | 236 | def mtime_hasher(filename): 237 | """ Return modification time of file, or None if file doesn't exist. """ 238 | try: 239 | st = os.stat(filename) 240 | return repr(st.st_mtime) 241 | except (IOError, OSError): 242 | return None 243 | 244 | class RunnerUnsupportedException(Exception): 245 | """ Exception raise by Runner constructor if it is not supported 246 | on the current platform.""" 247 | pass 248 | 249 | class Runner(object): 250 | def __call__(self, *args, **kwargs): 251 | """ Run command and return (dependencies, outputs), where 252 | dependencies is a list of the filenames of files that the 253 | command depended on, and output is a list of the filenames 254 | of files that the command modified. The input is passed 255 | to shell()""" 256 | raise NotImplementedError("Runner subclass called but subclass didn't define __call__") 257 | 258 | def actual_runner(self): 259 | """ Return the actual runner object (overriden in SmartRunner). """ 260 | return self 261 | 262 | def ignore(self, name): 263 | return self._builder.ignore.search(name) 264 | 265 | class AtimesRunner(Runner): 266 | def __init__(self, builder): 267 | self._builder = builder 268 | self.atimes = AtimesRunner.has_atimes(self._builder.dirs) 269 | if self.atimes == 0: 270 | raise RunnerUnsupportedException( 271 | 'atimes are not supported on this platform') 272 | 273 | @staticmethod 274 | def access_file(filename): 275 | """ Access (read a byte from) file to try to update its access time. """ 276 | f = open(filename) 277 | f.read(1) 278 | f.close() 279 | 280 | @staticmethod 281 | def file_has_atimes(filename): 282 | return AtimesRunner.fs_item_has_atimes(filename, AtimesRunner.access_file) 283 | 284 | @staticmethod 285 | def access_dir(dir): 286 | os.walk(dir) 287 | 288 | @staticmethod 289 | def dir_has_atimes(dir): 290 | return AtimesRunner.fs_item_has_atimes(dir, AtimesRunner.access_dir) 291 | 292 | @staticmethod 293 | def fs_item_has_atimes(filename, changer): 294 | """ Return whether the given filesystem supports access time updates for 295 | this file. Return: 296 | - 0 if no a/mtimes not updated 297 | - 1 if the atime resolution is at least one day and 298 | the mtime resolution at least 2 seconds (as on FAT filesystems) 299 | - 2 if the atime and mtime resolutions are both < ms 300 | (NTFS filesystem has 100 ns resolution). """ 301 | 302 | def access_file(filename): 303 | """ Access (read a byte from) file to try to update its access time. """ 304 | f = open(filename) 305 | f.read(1) 306 | f.close() 307 | 308 | initial = os.stat(filename) 309 | os.utime(filename, ( 310 | initial.st_atime-FAT_atime_resolution, 311 | initial.st_mtime-FAT_mtime_resolution)) 312 | 313 | adjusted = os.stat(filename) 314 | changer(filename) 315 | after = os.stat(filename) 316 | 317 | # Check that a/mtimes actually moved back by at least resolution and 318 | # updated by a file access. 319 | # add NTFS_atime_resolution to account for float resolution factors 320 | # Comment on resolution/2 in atimes_runner() 321 | if initial.st_atime-adjusted.st_atime > FAT_atime_resolution+NTFS_atime_resolution or \ 322 | initial.st_mtime-adjusted.st_mtime > FAT_mtime_resolution+NTFS_atime_resolution or \ 323 | initial.st_atime==adjusted.st_atime or \ 324 | initial.st_mtime==adjusted.st_mtime or \ 325 | not after.st_atime-FAT_atime_resolution/2 > adjusted.st_atime: 326 | return 0 327 | 328 | os.utime(filename, ( 329 | initial.st_atime-NTFS_atime_resolution, 330 | initial.st_mtime-NTFS_mtime_resolution)) 331 | adjusted = os.stat(filename) 332 | 333 | # Check that a/mtimes actually moved back by at least resolution 334 | # Note: != comparison here fails due to float rounding error 335 | # double NTFS_atime_resolution to account for float resolution factors 336 | if initial.st_atime-adjusted.st_atime > NTFS_atime_resolution*2 or \ 337 | initial.st_mtime-adjusted.st_mtime > NTFS_mtime_resolution*2 or \ 338 | initial.st_atime==adjusted.st_atime or \ 339 | initial.st_mtime==adjusted.st_mtime: 340 | return 1 341 | 342 | return 2 343 | 344 | @staticmethod 345 | def exists(path): 346 | if not os.path.exists(path): 347 | # Note: in linux, error may not occur: strace runner doesn't check 348 | raise PathError("build dirs specified a non-existant path '%s'" % path) 349 | 350 | @staticmethod 351 | def has_atimes(paths): 352 | """ Return whether a file created in each path supports atimes and mtimes. 353 | Return value is the same as used by file_has_atimes 354 | Note: for speed, this only tests files created at the top directory 355 | of each path. A safe assumption in most build environments. 356 | In the unusual case that any sub-directories are mounted 357 | on alternate file systems that don't support atimes, the build may 358 | fail to identify a dependency """ 359 | 360 | atimes = 2 # start by assuming we have best atimes 361 | for path in paths: 362 | AtimesRunner.exists(path) 363 | handle, filename = tempfile.mkstemp(dir=path) 364 | try: 365 | try: 366 | f = os.fdopen(handle, 'wb') 367 | except: 368 | os.close(handle) 369 | raise 370 | try: 371 | f.write(b'x') # need a byte in the file for access test 372 | finally: 373 | f.close() 374 | atimes = min(atimes, AtimesRunner.file_has_atimes(filename)) 375 | finally: 376 | os.remove(filename) 377 | dir = tempfile.mkdtemp(dir=path) 378 | try: 379 | atimes = min(atimes, AtimesRunner.dir_has_atimes(dir)) 380 | finally: 381 | os.rmdir(dir) 382 | return atimes 383 | 384 | def _file_times(self, path, depth): 385 | """ Helper function for file_times(). 386 | Return a dict of file times, recursing directories that don't 387 | start with self._builder.ignoreprefix """ 388 | 389 | AtimesRunner.exists(path) 390 | names = os.listdir(path) 391 | times = {} 392 | ignoreprefix = self._builder.ignoreprefix 393 | for name in names: 394 | if ignoreprefix and name.startswith(ignoreprefix): 395 | continue 396 | if path == '.': 397 | fullname = name 398 | else: 399 | fullname = os.path.join(path, name) 400 | st = os.stat(fullname) 401 | if stat.S_ISDIR(st.st_mode): 402 | if depth > 1: 403 | times.update(self._file_times(fullname, depth-1)) 404 | elif stat.S_ISREG(st.st_mode): 405 | times[fullname] = st.st_atime, st.st_mtime 406 | return times 407 | 408 | def file_times(self): 409 | """ Return a dict of "filepath: (atime, mtime)" entries for each file 410 | in self._builder.dirs. "filepath" is the absolute path, "atime" is 411 | the access time, "mtime" the modification time. 412 | Recurse directories that don't start with 413 | self._builder.ignoreprefix and have depth less than 414 | self._builder.dirdepth. """ 415 | 416 | times = {} 417 | for path in self._builder.dirs: 418 | times.update(self._file_times(path, self._builder.dirdepth)) 419 | return times 420 | 421 | def _utime(self, filename, atime, mtime): 422 | """ Call os.utime but ignore permission errors """ 423 | try: 424 | os.utime(filename, (atime, mtime)) 425 | except OSError as e: 426 | # ignore permission errors -- we can't build with files 427 | # that we can't access anyway 428 | if e.errno != 1: 429 | raise 430 | 431 | def _age_atimes(self, filetimes): 432 | """ Age files' atimes and mtimes to be at least FAT_xx_resolution old. 433 | Only adjust if the given filetimes dict says it isn't that old, 434 | and return a new dict of filetimes with the ages adjusted. """ 435 | adjusted = {} 436 | now = time.time() 437 | for filename, entry in filetimes.items(): 438 | if now-entry[0] < FAT_atime_resolution or now-entry[1] < FAT_mtime_resolution: 439 | entry = entry[0] - FAT_atime_resolution, entry[1] - FAT_mtime_resolution 440 | self._utime(filename, entry[0], entry[1]) 441 | adjusted[filename] = entry 442 | return adjusted 443 | 444 | def __call__(self, *args, **kwargs): 445 | """ Run command and return its dependencies and outputs, using before 446 | and after access times to determine dependencies. """ 447 | 448 | # For Python pre-2.5, ensure os.stat() returns float atimes 449 | old_stat_float = os.stat_float_times() 450 | os.stat_float_times(True) 451 | 452 | originals = self.file_times() 453 | if self.atimes == 2: 454 | befores = originals 455 | atime_resolution = 0 456 | mtime_resolution = 0 457 | else: 458 | befores = self._age_atimes(originals) 459 | atime_resolution = FAT_atime_resolution 460 | mtime_resolution = FAT_mtime_resolution 461 | shell_keywords = dict(silent=False) 462 | shell_keywords.update(kwargs) 463 | shell(*args, **shell_keywords) 464 | afters = self.file_times() 465 | deps = [] 466 | outputs = [] 467 | for name in afters: 468 | if name in befores: 469 | # if file exists before+after && mtime changed, add to outputs 470 | # Note: Can't just check that atimes > than we think they were 471 | # before because os might have rounded them to a later 472 | # date than what we think we set them to in befores. 473 | # So we make sure they're > by at least 1/2 the 474 | # resolution. This will work for anything with a 475 | # resolution better than FAT. 476 | if afters[name][1]-mtime_resolution/2 > befores[name][1]: 477 | if not self.ignore(name): 478 | outputs.append(name) 479 | elif afters[name][0]-atime_resolution/2 > befores[name][0]: 480 | # otherwise add to deps if atime changed 481 | if not self.ignore(name): 482 | deps.append(name) 483 | else: 484 | # file created (in afters but not befores), add as output 485 | if not self.ignore(name): 486 | outputs.append(name) 487 | 488 | if self.atimes < 2: 489 | # Restore atimes of files we didn't access: not for any functional 490 | # reason -- it's just to preserve the access time for the user's info 491 | for name in deps: 492 | originals.pop(name) 493 | for name in originals: 494 | original = originals[name] 495 | if original != afters.get(name, None): 496 | self._utime(name, original[0], original[1]) 497 | 498 | os.stat_float_times(old_stat_float) # restore stat_float_times value 499 | return deps, outputs 500 | 501 | class StraceProcess(object): 502 | def __init__(self, cwd='.', delayed=False): 503 | self.cwd = cwd 504 | self.deps = set() 505 | self.outputs = set() 506 | self.delayed = delayed 507 | self.delayed_lines = [] 508 | 509 | def add_dep(self, dep): 510 | self.deps.add(dep) 511 | 512 | def add_output(self, output): 513 | self.outputs.add(output) 514 | 515 | def add_delayed_line(self, line): 516 | self.delayed_lines.append(line) 517 | 518 | def __str__(self): 519 | return '' % \ 520 | (self.cwd, self.deps, self.outputs) 521 | 522 | def _call_strace(self, *args, **kwargs): 523 | """ Top level function call for Strace that can be run in parallel """ 524 | return self(*args, **kwargs) 525 | 526 | class StraceRunner(Runner): 527 | keep_temps = False 528 | 529 | def __init__(self, builder, build_dir=None): 530 | self.strace_system_calls = StraceRunner.get_strace_system_calls() 531 | if self.strace_system_calls is None: 532 | raise RunnerUnsupportedException('strace is not available') 533 | self._builder = builder 534 | self.temp_count = 0 535 | self.build_dir = os.path.abspath(build_dir or os.getcwd()) 536 | 537 | @staticmethod 538 | def get_strace_system_calls(): 539 | """ Return None if this system doesn't have strace, otherwise 540 | return a comma seperated list of system calls supported by strace. """ 541 | if platform.system() == 'Windows': 542 | # even if windows has strace, it's probably a dodgy cygwin one 543 | return None 544 | possible_system_calls = ['open','openat','stat', 'stat64', 'lstat', 'lstat64', 545 | 'execve','exit_group','chdir','mkdir','rename','clone','vfork', 546 | 'fork','symlink','creat'] 547 | valid_system_calls = [] 548 | try: 549 | # check strace is installed and if it supports each type of call 550 | for system_call in possible_system_calls: 551 | proc = subprocess.Popen(['strace', '-e', 'trace=' + system_call], stderr=subprocess.PIPE) 552 | stdout, stderr = proc.communicate() 553 | proc.wait() 554 | if b'invalid system call' not in stderr: 555 | valid_system_calls.append(system_call) 556 | except OSError: 557 | return None 558 | return ','.join(valid_system_calls) 559 | 560 | # Regular expressions for parsing of strace log 561 | _open_re = re.compile(r'(?P\d+)\s+open\("(?P[^"]*)", (?P[^,)]*)') 562 | _openat_re = re.compile(r'(?P\d+)\s+openat\([^,]*, "(?P[^"]*)", (?P[^,)]*)') 563 | _stat_re = re.compile(r'(?P\d+)\s+l?stat(?:64)?\("(?P[^"]*)", .*') # stat,lstat,stat64,lstat64 564 | _execve_re = re.compile(r'(?P\d+)\s+execve\("(?P[^"]*)", .*') 565 | _creat_re = re.compile(r'(?P\d+)\s+creat\("(?P[^"]*)", .*') 566 | _mkdir_re = re.compile(r'(?P\d+)\s+mkdir\("(?P[^"]*)", .*\)\s*=\s(?P-?[0-9]*).*') 567 | _rename_re = re.compile(r'(?P\d+)\s+rename\("[^"]*", "(?P[^"]*)"\)') 568 | _symlink_re = re.compile(r'(?P\d+)\s+symlink\("[^"]*", "(?P[^"]*)"\)') 569 | _kill_re = re.compile(r'(?P\d+)\s+killed by.*') 570 | _chdir_re = re.compile(r'(?P\d+)\s+chdir\("(?P[^"]*)"\)') 571 | _exit_group_re = re.compile(r'(?P\d+)\s+exit_group\((?P.*)\).*') 572 | _clone_re = re.compile(r'(?P\d+)\s+(clone|fork|vfork)\(.*\)\s*=\s*(?P\d*)') 573 | 574 | # Regular expressions for detecting interrupted lines in strace log 575 | # 3618 clone( 576 | # 3618 <... clone resumed> child_stack=0, flags=CLONE, child_tidptr=0x7f83deffa780) = 3622 577 | _unfinished_start_re = re.compile(r'(?P\d+)(?P.*)$') 578 | _unfinished_end_re = re.compile(r'(?P\d+)\s+\<\.\.\..*\>(?P.*)') 579 | 580 | def _do_strace(self, args, kwargs, outfile, outname): 581 | """ Run strace on given command args/kwargs, sending output to file. 582 | Return (status code, list of dependencies, list of outputs). """ 583 | shell_keywords = dict(silent=False) 584 | shell_keywords.update(kwargs) 585 | try: 586 | shell('strace', '-fo', outname, '-e', 587 | 'trace=' + self.strace_system_calls, 588 | args, **shell_keywords) 589 | except ExecutionError as e: 590 | # if strace failed to run, re-throw the exception 591 | # we can tell this happend if the file is empty 592 | outfile.seek(0, os.SEEK_END) 593 | if outfile.tell() == 0: 594 | raise e 595 | else: 596 | # reset the file postion for reading 597 | outfile.seek(0) 598 | 599 | self.status = 0 600 | processes = {} # dictionary of processes (key = pid) 601 | unfinished = {} # list of interrupted entries in strace log 602 | for line in outfile: 603 | self._match_line(line, processes, unfinished) 604 | 605 | # collect outputs and dependencies from all processes 606 | deps = set() 607 | outputs = set() 608 | for pid, process in processes.items(): 609 | deps = deps.union(process.deps) 610 | outputs = outputs.union(process.outputs) 611 | 612 | return self.status, list(deps), list(outputs) 613 | 614 | def _match_line(self, line, processes, unfinished): 615 | # look for split lines 616 | unfinished_start_match = self._unfinished_start_re.match(line) 617 | unfinished_end_match = self._unfinished_end_re.match(line) 618 | if unfinished_start_match: 619 | pid = unfinished_start_match.group('pid') 620 | body = unfinished_start_match.group('body') 621 | unfinished[pid] = pid + ' ' + body 622 | return 623 | elif unfinished_end_match: 624 | pid = unfinished_end_match.group('pid') 625 | body = unfinished_end_match.group('body') 626 | if pid not in unfinished: 627 | # Looks like we need to hande an strace bug here 628 | # I think it is safe to ignore as I have only seen futex calls which strace should not output 629 | printerr('fabricate: Warning: resume without unfinished in strace output (strace bug?), \'%s\'' % line.strip()) 630 | return 631 | line = unfinished[pid] + body 632 | del unfinished[pid] 633 | 634 | is_output = False 635 | open_match = self._open_re.match(line) 636 | openat_match = self._openat_re.match(line) 637 | stat_match = self._stat_re.match(line) 638 | execve_match = self._execve_re.match(line) 639 | creat_match = self._creat_re.match(line) 640 | mkdir_match = self._mkdir_re.match(line) 641 | symlink_match = self._symlink_re.match(line) 642 | rename_match = self._rename_re.match(line) 643 | clone_match = self._clone_re.match(line) 644 | 645 | kill_match = self._kill_re.match(line) 646 | if kill_match: 647 | return None, None, None 648 | 649 | match = None 650 | if execve_match: 651 | pid = execve_match.group('pid') 652 | match = execve_match # Executables can be dependencies 653 | if pid not in processes and len(processes) == 0: 654 | # This is the first process so create dict entry 655 | processes[pid] = StraceProcess() 656 | elif clone_match: 657 | pid = clone_match.group('pid') 658 | pid_clone = clone_match.group('pid_clone') 659 | if pid not in processes: 660 | # Simple case where there are no delayed lines 661 | processes[pid] = StraceProcess(processes[pid_clone].cwd) 662 | else: 663 | # Some line processing was delayed due to an interupted clone_match 664 | processes[pid].cwd = processes[pid_clone].cwd # Set the correct cwd 665 | processes[pid].delayed = False # Set that matching is no longer delayed 666 | for delayed_line in processes[pid].delayed_lines: 667 | # Process all the delayed lines 668 | self._match_line(delayed_line, processes, unfinished) 669 | processes[pid].delayed_lines = [] # Clear the lines 670 | elif open_match: 671 | match = open_match 672 | mode = match.group('mode') 673 | if 'O_WRONLY' in mode or 'O_RDWR' in mode: 674 | # it's an output file if opened for writing 675 | is_output = True 676 | elif openat_match: 677 | match = openat_match 678 | mode = match.group('mode') 679 | if 'O_WRONLY' in mode or 'O_RDWR' in mode: 680 | # it's an output file if opened for writing 681 | is_output = True 682 | elif stat_match: 683 | match = stat_match 684 | elif creat_match: 685 | match = creat_match 686 | # a created file is an output file 687 | is_output = True 688 | elif mkdir_match: 689 | match = mkdir_match 690 | if match.group('result') == '0': 691 | # a created directory is an output file 692 | is_output = True 693 | elif symlink_match: 694 | match = symlink_match 695 | # the created symlink is an output file 696 | is_output = True 697 | elif rename_match: 698 | match = rename_match 699 | # the destination of a rename is an output file 700 | is_output = True 701 | 702 | if match: 703 | name = match.group('name') 704 | pid = match.group('pid') 705 | if not self._matching_is_delayed(processes, pid, line): 706 | cwd = processes[pid].cwd 707 | if cwd != '.': 708 | name = os.path.join(cwd, name) 709 | 710 | # normalise path name to ensure files are only listed once 711 | name = os.path.normpath(name) 712 | 713 | # if it's an absolute path name under the build directory, 714 | # make it relative to build_dir before saving to .deps file 715 | if os.path.isabs(name) and name.startswith(self.build_dir): 716 | name = name[len(self.build_dir):] 717 | name = name.lstrip(os.path.sep) 718 | 719 | if (self._builder._is_relevant(name) 720 | and not self.ignore(name) 721 | and os.path.lexists(name)): 722 | if is_output: 723 | processes[pid].add_output(name) 724 | else: 725 | processes[pid].add_dep(name) 726 | 727 | match = self._chdir_re.match(line) 728 | if match: 729 | pid = match.group('pid') 730 | if not self._matching_is_delayed(processes, pid, line): 731 | processes[pid].cwd = os.path.join(processes[pid].cwd, match.group('cwd')) 732 | 733 | match = self._exit_group_re.match(line) 734 | if match: 735 | self.status = int(match.group('status')) 736 | 737 | def _matching_is_delayed(self, processes, pid, line): 738 | # Check if matching is delayed and cache a delayed line 739 | if pid not in processes: 740 | processes[pid] = StraceProcess(delayed=True) 741 | 742 | process = processes[pid] 743 | if process.delayed: 744 | process.add_delayed_line(line) 745 | return True 746 | else: 747 | return False 748 | 749 | def __call__(self, *args, **kwargs): 750 | """ Run command and return its dependencies and outputs, using strace 751 | to determine dependencies (by looking at what files are opened or 752 | modified). """ 753 | ignore_status = kwargs.pop('ignore_status', False) 754 | if self.keep_temps: 755 | outname = 'strace%03d.txt' % self.temp_count 756 | self.temp_count += 1 757 | handle = os.open(outname, os.O_CREAT) 758 | else: 759 | handle, outname = tempfile.mkstemp() 760 | 761 | try: 762 | try: 763 | outfile = os.fdopen(handle, 'r') 764 | except: 765 | os.close(handle) 766 | raise 767 | try: 768 | status, deps, outputs = self._do_strace(args, kwargs, outfile, outname) 769 | if status is None: 770 | raise ExecutionError( 771 | '%r was killed unexpectedly' % args[0], '', -1) 772 | finally: 773 | outfile.close() 774 | finally: 775 | if not self.keep_temps: 776 | os.remove(outname) 777 | 778 | if status and not ignore_status: 779 | raise ExecutionError('%r exited with status %d' 780 | % (os.path.basename(args[0]), status), 781 | '', status) 782 | return list(deps), list(outputs) 783 | 784 | class AlwaysRunner(Runner): 785 | def __init__(self, builder): 786 | pass 787 | 788 | def __call__(self, *args, **kwargs): 789 | """ Runner that always runs given command, used as a backup in case 790 | a system doesn't have strace or atimes. """ 791 | shell_keywords = dict(silent=False) 792 | shell_keywords.update(kwargs) 793 | shell(*args, **shell_keywords) 794 | return None, None 795 | 796 | class SmartRunner(Runner): 797 | """ Smart command runner that uses StraceRunner if it can, 798 | otherwise AtimesRunner if available, otherwise AlwaysRunner. """ 799 | def __init__(self, builder): 800 | self._builder = builder 801 | try: 802 | self._runner = StraceRunner(self._builder) 803 | except RunnerUnsupportedException: 804 | try: 805 | self._runner = AtimesRunner(self._builder) 806 | except RunnerUnsupportedException: 807 | self._runner = AlwaysRunner(self._builder) 808 | 809 | def actual_runner(self): 810 | return self._runner 811 | 812 | def __call__(self, *args, **kwargs): 813 | return self._runner(*args, **kwargs) 814 | 815 | class _running(object): 816 | """ Represents a task put on the parallel pool 817 | and its results when complete """ 818 | def __init__(self, async_result, command): 819 | """ "async_result" is the AsyncResult object returned from pool.apply_async 820 | "command" is the command that was run """ 821 | self.async_result = async_result 822 | self.command = command 823 | self.results = None 824 | 825 | class _after(object): 826 | """ Represents something waiting on completion of some previous commands """ 827 | def __init__(self, afters, do): 828 | """ "afters" is a group id or a iterable of group ids to wait on 829 | "do" is either a tuple representing a command (group, command, 830 | arglist, kwargs) or a threading.Condition to be released """ 831 | self.afters = afters 832 | self.do = do 833 | self.done = False 834 | 835 | class _Groups(object): 836 | """ Thread safe mapping object whose values are lists of _running 837 | or _after objects and a count of how many have *not* completed """ 838 | class value(object): 839 | """ the value type in the map """ 840 | def __init__(self, val=None): 841 | self.count = 0 # count of items not yet completed. 842 | # This also includes count_in_false number 843 | self.count_in_false = 0 # count of commands which is assigned 844 | # to False group, but will be moved 845 | # to this group. 846 | self.items = [] # items in this group 847 | if val is not None: 848 | self.items.append(val) 849 | self.ok = True # True if no error from any command in group so far 850 | 851 | def __init__(self): 852 | self.groups = {False: self.value()} 853 | self.lock = threading.Lock() 854 | 855 | def item_list(self, id): 856 | """ Return copy of the value list """ 857 | with self.lock: 858 | return self.groups[id].items[:] 859 | 860 | def remove(self, id): 861 | """ Remove the group """ 862 | with self.lock: 863 | del self.groups[id] 864 | 865 | def remove_item(self, id, val): 866 | with self.lock: 867 | self.groups[id].items.remove(val) 868 | 869 | def add(self, id, val): 870 | with self.lock: 871 | if id in self.groups: 872 | self.groups[id].items.append(val) 873 | else: 874 | self.groups[id] = self.value(val) 875 | self.groups[id].count += 1 876 | 877 | def ensure(self, id): 878 | """if id does not exit, create it without any value""" 879 | with self.lock: 880 | if not id in self.groups: 881 | self.groups[id] = self.value() 882 | 883 | def get_count(self, id): 884 | with self.lock: 885 | if id not in self.groups: 886 | return 0 887 | return self.groups[id].count 888 | 889 | def dec_count(self, id): 890 | with self.lock: 891 | c = self.groups[id].count - 1 892 | if c < 0: 893 | raise ValueError 894 | self.groups[id].count = c 895 | return c 896 | 897 | def get_ok(self, id): 898 | with self.lock: 899 | return self.groups[id].ok 900 | 901 | def set_ok(self, id, to): 902 | with self.lock: 903 | self.groups[id].ok = to 904 | 905 | def ids(self): 906 | with self.lock: 907 | return self.groups.keys() 908 | 909 | # modification to reserve blocked commands to corresponding groups 910 | def inc_count_for_blocked(self, id): 911 | with self.lock: 912 | if not id in self.groups: 913 | self.groups[id] = self.value() 914 | self.groups[id].count += 1 915 | self.groups[id].count_in_false += 1 916 | 917 | def add_for_blocked(self, id, val): 918 | # modification of add(), in order to move command from False group 919 | # to actual group 920 | with self.lock: 921 | # id must be registered before 922 | self.groups[id].items.append(val) 923 | # count does not change (already considered 924 | # in inc_count_for_blocked), but decrease count_in_false. 925 | c = self.groups[id].count_in_false - 1 926 | if c < 0: 927 | raise ValueError 928 | self.groups[id].count_in_false = c 929 | 930 | 931 | # pool of processes to run parallel jobs, must not be part of any object that 932 | # is pickled for transfer to these processes, ie it must be global 933 | _pool = None 934 | # object holding results, must also be global 935 | _groups = _Groups() 936 | # results collecting thread 937 | _results = None 938 | _stop_results = threading.Event() 939 | 940 | class _todo(object): 941 | """ holds the parameters for commands waiting on others """ 942 | def __init__(self, group, command, arglist, kwargs): 943 | self.group = group # which group it should run as 944 | self.command = command # string command 945 | self.arglist = arglist # command arguments 946 | self.kwargs = kwargs # keywork args for the runner 947 | 948 | def _results_handler( builder, delay=0.01): 949 | """ Body of thread that stores results in .deps and handles 'after' 950 | conditions 951 | "builder" the builder used """ 952 | try: 953 | while not _stop_results.isSet(): 954 | # go through the lists and check any results available 955 | for id in _groups.ids(): 956 | if id is False: continue # key of False is _afters not _runnings 957 | for r in _groups.item_list(id): 958 | if r.results is None and r.async_result.ready(): 959 | try: 960 | d, o = r.async_result.get() 961 | except ExecutionError as e: 962 | r.results = e 963 | _groups.set_ok(id, False) 964 | message, data, status = e 965 | printerr("fabricate: " + message) 966 | else: 967 | builder.done(r.command, d, o) # save deps 968 | r.results = (r.command, d, o) 969 | _groups.dec_count(id) 970 | # check if can now schedule things waiting on the after queue 971 | for a in _groups.item_list(False): 972 | still_to_do = sum(_groups.get_count(g) for g in a.afters) 973 | no_error = all(_groups.get_ok(g) for g in a.afters) 974 | if False in a.afters: 975 | still_to_do -= 1 # don't count yourself of course 976 | if still_to_do == 0: 977 | if isinstance(a.do, _todo): 978 | if no_error: 979 | async_result = _pool.apply_async(_call_strace, a.do.arglist, 980 | a.do.kwargs) 981 | _groups.add_for_blocked(a.do.group, _running(async_result, a.do.command)) 982 | else: 983 | # Mark the command as not done due to errors 984 | r = _running(None, a.do.command) 985 | _groups.add_for_blocked(a.do.group, r) 986 | r.results = False 987 | _groups.set_ok(a.do.group, False) 988 | _groups.dec_count(a.do.group) 989 | elif isinstance(a.do, threading_condition): 990 | # is this only for threading_condition in after()? 991 | a.do.acquire() 992 | # only mark as done if there is no error 993 | a.done = no_error 994 | a.do.notify() 995 | a.do.release() 996 | # else: #are there other cases? 997 | _groups.remove_item(False, a) 998 | _groups.dec_count(False) 999 | 1000 | _stop_results.wait(delay) 1001 | except Exception: 1002 | etype, eval, etb = sys.exc_info() 1003 | printerr("Error: exception " + repr(etype) + " at line " + str(etb.tb_lineno)) 1004 | traceback.print_tb(etb) 1005 | finally: 1006 | if not _stop_results.isSet(): 1007 | # oh dear, I am about to die for unexplained reasons, stop the whole 1008 | # app otherwise the main thread hangs waiting on non-existant me, 1009 | # Note: sys.exit() only kills me 1010 | printerr("Error: unexpected results handler exit") 1011 | os._exit(1) 1012 | 1013 | class Builder(object): 1014 | """ The Builder. 1015 | 1016 | You may supply a "runner" class to change the way commands are run 1017 | or dependencies are determined. For an example, see: 1018 | https://github.com/brushtechnology/fabricate/wiki/HowtoMakeYourOwnRunner 1019 | 1020 | A "runner" must be a subclass of Runner and must have a __call__() 1021 | function that takes a command as a list of args and returns a tuple of 1022 | (deps, outputs), where deps is a list of rel-path'd dependency files 1023 | and outputs is a list of rel-path'd output files. The default runner 1024 | is SmartRunner, which automatically picks one of StraceRunner, 1025 | AtimesRunner, or AlwaysRunner depending on your system. 1026 | A "runner" class may have an __init__() function that takes the 1027 | builder as a parameter. 1028 | """ 1029 | 1030 | def __init__(self, runner=None, dirs=None, dirdepth=100, ignoreprefix='.', 1031 | ignore=None, hasher=md5_hasher, depsname='.deps', 1032 | quiet=False, debug=False, inputs_only=False, parallel_ok=False): 1033 | """ Initialise a Builder with the given options. 1034 | 1035 | "runner" specifies how programs should be run. It is either a 1036 | callable compatible with the Runner class, or a string selecting 1037 | one of the standard runners ("atimes_runner", "strace_runner", 1038 | "always_runner", or "smart_runner"). 1039 | "dirs" is a list of paths to look for dependencies (or outputs) in 1040 | if using the strace or atimes runners. 1041 | "dirdepth" is the depth to recurse into the paths in "dirs" (default 1042 | essentially means infinitely). Set to 1 to just look at the 1043 | immediate paths in "dirs" and not recurse at all. This can be 1044 | useful to speed up the AtimesRunner if you're building in a large 1045 | tree and you don't care about all of the subdirectories. 1046 | "ignoreprefix" prevents recursion into directories that start with 1047 | prefix. It defaults to '.' to ignore svn directories. 1048 | Change it to '_svn' if you use _svn hidden directories. 1049 | "ignore" is a regular expression. Any dependency that contains a 1050 | regex match is ignored and not put into the dependency list. 1051 | Note that the regex may be VERBOSE (spaces are ignored and # line 1052 | comments allowed -- use \ prefix to insert these characters) 1053 | "hasher" is a function which returns a string which changes when 1054 | the contents of its filename argument changes, or None on error. 1055 | Default is md5_hasher, but can also be mtime_hasher. 1056 | "depsname" is the name of the JSON dependency file to load/save. 1057 | "quiet" set to True tells the builder to not display the commands being 1058 | executed (or other non-error output). 1059 | "debug" set to True makes the builder print debug output, such as why 1060 | particular commands are being executed 1061 | "inputs_only" set to True makes builder only re-build if input hashes 1062 | have changed (ignores output hashes); use with tools that touch 1063 | files that shouldn't cause a rebuild; e.g. g++ collect phase 1064 | "parallel_ok" set to True to indicate script is safe for parallel running 1065 | """ 1066 | if dirs is None: 1067 | dirs = ['.'] 1068 | self.dirs = dirs 1069 | self.dirdepth = dirdepth 1070 | self.ignoreprefix = ignoreprefix 1071 | if ignore is None: 1072 | ignore = r'$x^' # something that can't match 1073 | self.ignore = re.compile(ignore, re.VERBOSE) 1074 | self.depsname = depsname 1075 | self.hasher = hasher 1076 | self.quiet = quiet 1077 | self.debug = debug 1078 | self.inputs_only = inputs_only 1079 | self.checking = False 1080 | self.hash_cache = {} 1081 | 1082 | # instantiate runner after the above have been set in case it needs them 1083 | if runner is not None: 1084 | self.set_runner(runner) 1085 | elif hasattr(self, 'runner'): 1086 | # For backwards compatibility, if a derived class has 1087 | # defined a "runner" method then use it: 1088 | pass 1089 | else: 1090 | self.runner = SmartRunner(self) 1091 | 1092 | is_strace = isinstance(self.runner.actual_runner(), StraceRunner) 1093 | self.parallel_ok = parallel_ok and is_strace and _pool is not None 1094 | if self.parallel_ok: 1095 | global _results 1096 | _results = threading.Thread(target=_results_handler, 1097 | args=[self]) 1098 | _results.setDaemon(True) 1099 | _results.start() 1100 | atexit.register(self._join_results_handler) 1101 | StraceRunner.keep_temps = False # unsafe for parallel execution 1102 | 1103 | def echo(self, message): 1104 | """ Print message, but only if builder is not in quiet mode. """ 1105 | if not self.quiet: 1106 | print(message) 1107 | 1108 | def echo_command(self, command, echo=None): 1109 | """ Show a command being executed. Also passed run's "echo" arg 1110 | so you can override what's displayed. 1111 | """ 1112 | if echo is not None: 1113 | command = str(echo) 1114 | self.echo(command) 1115 | 1116 | def echo_delete(self, filename, error=None): 1117 | """ Show a file being deleted. For subclassing Builder and overriding 1118 | this function, the exception is passed in if an OSError occurs 1119 | while deleting a file. """ 1120 | if error is None: 1121 | self.echo('deleting %s' % filename) 1122 | else: 1123 | self.echo_debug('error deleting %s: %s' % (filename, error.strerror)) 1124 | 1125 | def echo_debug(self, message): 1126 | """ Print message, but only if builder is in debug mode. """ 1127 | if self.debug: 1128 | print('DEBUG: ' + message) 1129 | 1130 | def _run(self, *args, **kwargs): 1131 | after = kwargs.pop('after', None) 1132 | group = kwargs.pop('group', True) 1133 | echo = kwargs.pop('echo', None) 1134 | arglist = args_to_list(args) 1135 | if not arglist: 1136 | raise TypeError('run() takes at least 1 argument (0 given)') 1137 | # we want a command line string for the .deps file key and for display 1138 | command = subprocess.list2cmdline(arglist) 1139 | if not self.cmdline_outofdate(command): 1140 | if self.parallel_ok: 1141 | _groups.ensure(group) 1142 | return command, None, None 1143 | 1144 | # if just checking up-to-date-ness, set flag and do nothing more 1145 | self.outofdate_flag = True 1146 | if self.checking: 1147 | if self.parallel_ok: 1148 | _groups.ensure(group) 1149 | return command, None, None 1150 | 1151 | # use runner to run command and collect dependencies 1152 | self.echo_command(command, echo=echo) 1153 | if self.parallel_ok: 1154 | arglist.insert(0, self.runner) 1155 | if after is not None: 1156 | if not isinstance(after, (list, tuple)): 1157 | after = [after] 1158 | # This command is registered to False group firstly, 1159 | # but the actual group of this command should 1160 | # count this blocked command as well as usual commands 1161 | _groups.inc_count_for_blocked(group) 1162 | _groups.add(False, 1163 | _after(after, _todo(group, command, arglist, 1164 | kwargs))) 1165 | else: 1166 | async_result = _pool.apply_async(_call_strace, arglist, kwargs) 1167 | _groups.add(group, _running(async_result, command)) 1168 | return None 1169 | else: 1170 | deps, outputs = self.runner(*arglist, **kwargs) 1171 | return self.done(command, deps, outputs) 1172 | 1173 | def run(self, *args, **kwargs): 1174 | """ Run command given in args with kwargs per shell(), but only if its 1175 | dependencies or outputs have changed or don't exist. Return tuple 1176 | of (command_line, deps_list, outputs_list) so caller or subclass 1177 | can use them. 1178 | 1179 | Parallel operation keyword args "after" specifies a group or 1180 | iterable of groups to wait for after they finish, "group" specifies 1181 | the group to add this command to. 1182 | 1183 | Optional "echo" keyword arg is passed to echo_command() so you can 1184 | override its output if you want. 1185 | """ 1186 | try: 1187 | return self._run(*args, **kwargs) 1188 | finally: 1189 | sys.stderr.flush() 1190 | sys.stdout.flush() 1191 | 1192 | def done(self, command, deps, outputs): 1193 | """ Store the results in the .deps file when they are available """ 1194 | if deps is not None or outputs is not None: 1195 | deps_dict = {} 1196 | 1197 | # hash the dependency inputs and outputs 1198 | for dep in deps: 1199 | if dep in self.hash_cache: 1200 | # already hashed so don't repeat hashing work 1201 | hashed = self.hash_cache[dep] 1202 | else: 1203 | hashed = self.hasher(dep) 1204 | if hashed is not None: 1205 | deps_dict[dep] = "input-" + hashed 1206 | # store hash in hash cache as it may be a new file 1207 | self.hash_cache[dep] = hashed 1208 | 1209 | for output in outputs: 1210 | hashed = self.hasher(output) 1211 | if hashed is not None: 1212 | deps_dict[output] = "output-" + hashed 1213 | # update hash cache as this file should already be in 1214 | # there but has probably changed 1215 | self.hash_cache[output] = hashed 1216 | 1217 | self.deps[command] = deps_dict 1218 | 1219 | return command, deps, outputs 1220 | 1221 | def memoize(self, command, **kwargs): 1222 | """ Run the given command, but only if its dependencies have changed -- 1223 | like run(), but returns the status code instead of raising an 1224 | exception on error. If "command" is a string (as per memoize.py) 1225 | it's split into args using shlex.split() in a POSIX/bash style, 1226 | otherwise it's a list of args as per run(). 1227 | 1228 | This function is for compatiblity with memoize.py and is 1229 | deprecated. Use run() instead. """ 1230 | if isinstance(command, string_types): 1231 | args = shlex.split(command) 1232 | else: 1233 | args = args_to_list(command) 1234 | try: 1235 | self.run(args, **kwargs) 1236 | return 0 1237 | except ExecutionError as exc: 1238 | message, data, status = exc 1239 | return status 1240 | 1241 | def outofdate(self, func): 1242 | """ Return True if given build function is out of date. """ 1243 | self.checking = True 1244 | self.outofdate_flag = False 1245 | func() 1246 | self.checking = False 1247 | return self.outofdate_flag 1248 | 1249 | def cmdline_outofdate(self, command): 1250 | """ Return True if given command line is out of date. """ 1251 | if command in self.deps: 1252 | # command has been run before, see if deps have changed 1253 | for dep, oldhash in self.deps[command].items(): 1254 | assert oldhash.startswith('input-') or \ 1255 | oldhash.startswith('output-'), \ 1256 | "%s file corrupt, do a clean!" % self.depsname 1257 | io_type, oldhash = oldhash.split('-', 1) 1258 | 1259 | # make sure this dependency or output hasn't changed 1260 | if dep in self.hash_cache: 1261 | # already hashed so don't repeat hashing work 1262 | newhash = self.hash_cache[dep] 1263 | else: 1264 | # not in hash_cache so make sure this dependency or 1265 | # output hasn't changed 1266 | newhash = self.hasher(dep) 1267 | if newhash is not None: 1268 | # Add newhash to the hash cache 1269 | self.hash_cache[dep] = newhash 1270 | 1271 | if newhash is None: 1272 | self.echo_debug("rebuilding %r, %s %s doesn't exist" % 1273 | (command, io_type, dep)) 1274 | break 1275 | if newhash != oldhash and (not self.inputs_only or io_type == 'input'): 1276 | self.echo_debug("rebuilding %r, hash for %s %s (%s) != old hash (%s)" % 1277 | (command, io_type, dep, newhash, oldhash)) 1278 | break 1279 | else: 1280 | # all dependencies are unchanged 1281 | return False 1282 | else: 1283 | self.echo_debug('rebuilding %r, no dependency data' % command) 1284 | # command has never been run, or one of the dependencies didn't 1285 | # exist or had changed 1286 | return True 1287 | 1288 | def autoclean(self): 1289 | """ Automatically delete all outputs of this build as well as the .deps 1290 | file. """ 1291 | # first build a list of all the outputs from the .deps file 1292 | outputs = [] 1293 | dirs = [] 1294 | for command, deps in self.deps.items(): 1295 | outputs.extend(dep for dep, hashed in deps.items() 1296 | if hashed.startswith('output-')) 1297 | outputs.append(self.depsname) 1298 | self._deps = None 1299 | for output in outputs: 1300 | try: 1301 | os.remove(output) 1302 | except OSError as e: 1303 | if os.path.isdir(output): 1304 | # cache directories to be removed once all other outputs 1305 | # have been removed, as they may be content of the dir 1306 | dirs.append(output) 1307 | else: 1308 | self.echo_delete(output, e) 1309 | else: 1310 | self.echo_delete(output) 1311 | # delete the directories in reverse sort order 1312 | # this ensures that parents are removed after children 1313 | for dir in sorted(dirs, reverse=True): 1314 | try: 1315 | os.rmdir(dir) 1316 | except OSError as e: 1317 | self.echo_delete(dir, e) 1318 | else: 1319 | self.echo_delete(dir) 1320 | 1321 | 1322 | @property 1323 | def deps(self): 1324 | """ Lazy load .deps file so that instantiating a Builder is "safe". """ 1325 | if not hasattr(self, '_deps') or self._deps is None: 1326 | self.read_deps() 1327 | atexit.register(self.write_deps, depsname=os.path.abspath(self.depsname)) 1328 | return self._deps 1329 | 1330 | def read_deps(self): 1331 | """ Read dependency JSON file into deps object. """ 1332 | try: 1333 | f = open(self.depsname) 1334 | try: 1335 | self._deps = json.load(f) 1336 | # make sure the version is correct 1337 | if self._deps.get('.deps_version', 0) != deps_version: 1338 | printerr('Bad %s dependency file version! Rebuilding.' 1339 | % self.depsname) 1340 | self._deps = {} 1341 | self._deps.pop('.deps_version', None) 1342 | finally: 1343 | f.close() 1344 | except IOError: 1345 | self._deps = {} 1346 | 1347 | def write_deps(self, depsname=None): 1348 | """ Write out deps object into JSON dependency file. """ 1349 | if self._deps is None: 1350 | return # we've cleaned so nothing to save 1351 | self.deps['.deps_version'] = deps_version 1352 | if depsname is None: 1353 | depsname = self.depsname 1354 | f = open(depsname, 'w') 1355 | try: 1356 | json.dump(self.deps, f, indent=4, sort_keys=True) 1357 | finally: 1358 | f.close() 1359 | self._deps.pop('.deps_version', None) 1360 | 1361 | _runner_map = { 1362 | 'atimes_runner' : AtimesRunner, 1363 | 'strace_runner' : StraceRunner, 1364 | 'always_runner' : AlwaysRunner, 1365 | 'smart_runner' : SmartRunner, 1366 | } 1367 | 1368 | def set_runner(self, runner): 1369 | """Set the runner for this builder. "runner" is either a Runner 1370 | subclass (e.g. SmartRunner), or a string selecting one of the 1371 | standard runners ("atimes_runner", "strace_runner", 1372 | "always_runner", or "smart_runner").""" 1373 | try: 1374 | self.runner = self._runner_map[runner](self) 1375 | except KeyError: 1376 | if isinstance(runner, string_types): 1377 | # For backwards compatibility, allow runner to be the 1378 | # name of a method in a derived class: 1379 | self.runner = getattr(self, runner) 1380 | else: 1381 | # pass builder to runner class to get a runner instance 1382 | self.runner = runner(self) 1383 | 1384 | def _is_relevant(self, fullname): 1385 | """ Return True if file is in the dependency search directories. """ 1386 | 1387 | # need to abspath to compare rel paths with abs 1388 | fullname = os.path.abspath(fullname) 1389 | for path in self.dirs: 1390 | path = os.path.abspath(path) 1391 | if fullname.startswith(path): 1392 | rest = fullname[len(path):] 1393 | # files in dirs starting with ignoreprefix are not relevant 1394 | if os.sep+self.ignoreprefix in os.sep+os.path.dirname(rest): 1395 | continue 1396 | # files deeper than dirdepth are not relevant 1397 | if rest.count(os.sep) > self.dirdepth: 1398 | continue 1399 | return True 1400 | return False 1401 | 1402 | def _join_results_handler(self): 1403 | """Stops then joins the results handler thread""" 1404 | _stop_results.set() 1405 | _results.join() 1406 | 1407 | # default Builder instance, used by helper run() and main() helper functions 1408 | default_builder = None 1409 | default_command = 'build' 1410 | 1411 | # save the setup arguments for use by main() 1412 | _setup_builder = None 1413 | _setup_default = None 1414 | _setup_kwargs = {} 1415 | 1416 | def setup(builder=None, default=None, **kwargs): 1417 | """ NOTE: setup functionality is now in main(), setup() is kept for 1418 | backward compatibility and should not be used in new scripts. 1419 | 1420 | Setup the default Builder (or an instance of given builder if "builder" 1421 | is not None) with the same keyword arguments as for Builder(). 1422 | "default" is the name of the default function to run when the build 1423 | script is run with no command line arguments. """ 1424 | global _setup_builder, _setup_default, _setup_kwargs 1425 | _setup_builder = builder 1426 | _setup_default = default 1427 | _setup_kwargs = kwargs 1428 | setup.__doc__ += '\n\n' + Builder.__init__.__doc__ 1429 | 1430 | def _set_default_builder(): 1431 | """ Set default builder to Builder() instance if it's not yet set. """ 1432 | global default_builder 1433 | if default_builder is None: 1434 | default_builder = Builder() 1435 | 1436 | def run(*args, **kwargs): 1437 | """ Run the given command, but only if its dependencies have changed. Uses 1438 | the default Builder. Return value as per Builder.run(). If there is 1439 | only one positional argument which is an iterable treat each element 1440 | as a command, returns a list of returns from Builder.run(). 1441 | """ 1442 | _set_default_builder() 1443 | if len(args) == 1 and isinstance(args[0], (list, tuple)): 1444 | return [default_builder.run(*a, **kwargs) for a in args[0]] 1445 | return default_builder.run(*args, **kwargs) 1446 | 1447 | def after(*args): 1448 | """ wait until after the specified command groups complete and return 1449 | results, or None if not parallel """ 1450 | _set_default_builder() 1451 | if getattr(default_builder, 'parallel_ok', False): 1452 | if len(args) == 0: 1453 | args = _groups.ids() # wait on all 1454 | cond = threading.Condition() 1455 | cond.acquire() 1456 | a = _after(args, cond) 1457 | _groups.add(False, a) 1458 | cond.wait() 1459 | if not a.done: 1460 | sys.exit(1) 1461 | results = [] 1462 | ids = _groups.ids() 1463 | for a in args: 1464 | if a in ids and a is not False: 1465 | r = [] 1466 | for i in _groups.item_list(a): 1467 | r.append(i.results) 1468 | results.append((a,r)) 1469 | return results 1470 | else: 1471 | return None 1472 | 1473 | def autoclean(): 1474 | """ Automatically delete all outputs of the default build. """ 1475 | _set_default_builder() 1476 | default_builder.autoclean() 1477 | 1478 | def memoize(command, **kwargs): 1479 | _set_default_builder() 1480 | return default_builder.memoize(command, **kwargs) 1481 | 1482 | memoize.__doc__ = Builder.memoize.__doc__ 1483 | 1484 | def outofdate(command): 1485 | """ Return True if given command is out of date and needs to be run. """ 1486 | _set_default_builder() 1487 | return default_builder.outofdate(command) 1488 | 1489 | # save options for use by main() if parse_options called earlier by user script 1490 | _parsed_options = None 1491 | 1492 | # default usage message 1493 | _usage = '[options] build script functions to run' 1494 | 1495 | def parse_options(usage=_usage, extra_options=None, command_line=None): 1496 | """ Parse command line options and return (parser, options, args). """ 1497 | parser = optparse.OptionParser(usage='Usage: %prog '+usage, 1498 | version='%prog '+__version__) 1499 | parser.disable_interspersed_args() 1500 | parser.add_option('-t', '--time', action='store_true', 1501 | help='use file modification times instead of MD5 sums') 1502 | parser.add_option('-d', '--dir', action='append', 1503 | help='add DIR to list of relevant directories') 1504 | parser.add_option('-c', '--clean', action='store_true', 1505 | help='autoclean build outputs before running') 1506 | parser.add_option('-q', '--quiet', action='store_true', 1507 | help="don't echo commands, only print errors") 1508 | parser.add_option('-D', '--debug', action='store_true', 1509 | help="show debug info (why commands are rebuilt)") 1510 | parser.add_option('-k', '--keep', action='store_true', 1511 | help='keep temporary strace output files') 1512 | parser.add_option('-j', '--jobs', type='int', 1513 | help='maximum number of parallel jobs') 1514 | if extra_options: 1515 | # add any user-specified options passed in via main() 1516 | for option in extra_options: 1517 | parser.add_option(option) 1518 | if command_line is not None: 1519 | options, args = parser.parse_args(command_line) 1520 | else: 1521 | options, args = parser.parse_args() 1522 | global _parsed_options 1523 | _parsed_options = (parser, options, args) 1524 | return _parsed_options 1525 | 1526 | def fabricate_version(min=None, max=None): 1527 | """ If min is given, assert that the running fabricate is at least that 1528 | version or exit with an error message. If max is given, assert that 1529 | the running fabricate is at most that version. Return the current 1530 | fabricate version string. This function was introduced in v1.14; 1531 | for prior versions, the version string is available only as module 1532 | local string fabricate.__version__ """ 1533 | 1534 | if min is not None and float(__version__) < min: 1535 | sys.stderr.write(("fabricate is version %s. This build script " 1536 | "requires at least version %.2f") % (__version__, min)) 1537 | sys.exit() 1538 | if max is not None and float(__version__) > max: 1539 | sys.stderr.write(("fabricate is version %s. This build script " 1540 | "requires at most version %.2f") % (__version__, max)) 1541 | sys.exit() 1542 | return __version__ 1543 | 1544 | def main(globals_dict=None, build_dir=None, extra_options=None, builder=None, 1545 | default=None, jobs=1, command_line=None, **kwargs): 1546 | """ Run the default function or the function(s) named in the command line 1547 | arguments. Call this at the end of your build script. If one of the 1548 | functions returns nonzero, main will exit with the last nonzero return 1549 | value as its status code. 1550 | 1551 | "builder" is the class of builder to create, default (None) is the 1552 | normal builder 1553 | "command_line" is an optional list of command line arguments that can 1554 | be used to prevent the default parsing of sys.argv. Used to intercept 1555 | and modify the command line passed to the build script. 1556 | "default" is the default user script function to call, None = 'build' 1557 | "extra_options" is an optional list of options created with 1558 | optparse.make_option(). The pseudo-global variable main.options 1559 | is set to the parsed options list. 1560 | "kwargs" is any other keyword arguments to pass to the builder """ 1561 | global default_builder, default_command, _pool 1562 | 1563 | kwargs.update(_setup_kwargs) 1564 | if _parsed_options is not None and command_line is None: 1565 | parser, options, actions = _parsed_options 1566 | else: 1567 | parser, options, actions = parse_options(extra_options=extra_options, command_line=command_line) 1568 | kwargs['quiet'] = options.quiet 1569 | kwargs['debug'] = options.debug 1570 | if options.time: 1571 | kwargs['hasher'] = mtime_hasher 1572 | if options.dir: 1573 | kwargs['dirs'] = options.dir 1574 | if options.keep: 1575 | StraceRunner.keep_temps = options.keep 1576 | main.options = options 1577 | if options.jobs is not None: 1578 | jobs = options.jobs 1579 | if default is not None: 1580 | default_command = default 1581 | if default_command is None: 1582 | default_command = _setup_default 1583 | if not actions: 1584 | actions = [default_command] 1585 | 1586 | original_path = os.getcwd() 1587 | if None in [globals_dict, build_dir]: 1588 | try: 1589 | frame = sys._getframe(1) 1590 | except: 1591 | printerr("Your Python version doesn't support sys._getframe(1),") 1592 | printerr("call main(globals(), build_dir) explicitly") 1593 | sys.exit(1) 1594 | if globals_dict is None: 1595 | globals_dict = frame.f_globals 1596 | if build_dir is None: 1597 | build_file = frame.f_globals.get('__file__', None) 1598 | if build_file: 1599 | build_dir = os.path.dirname(build_file) 1600 | if build_dir: 1601 | if not options.quiet and os.path.abspath(build_dir) != original_path: 1602 | print("Entering directory '%s'" % build_dir) 1603 | os.chdir(build_dir) 1604 | if _pool is None and jobs > 1: 1605 | _pool = multiprocessing.Pool(jobs) 1606 | 1607 | use_builder = Builder 1608 | if _setup_builder is not None: 1609 | use_builder = _setup_builder 1610 | if builder is not None: 1611 | use_builder = builder 1612 | default_builder = use_builder(**kwargs) 1613 | 1614 | if options.clean: 1615 | default_builder.autoclean() 1616 | 1617 | status = 0 1618 | try: 1619 | for action in actions: 1620 | if '(' not in action: 1621 | action = action.strip() + '()' 1622 | name = action.split('(')[0].split('.')[0] 1623 | if name in globals_dict: 1624 | this_status = eval(action, globals_dict) 1625 | if this_status: 1626 | status = int(this_status) 1627 | else: 1628 | printerr('%r command not defined!' % action) 1629 | sys.exit(1) 1630 | after() # wait till the build commands are finished 1631 | except ExecutionError as exc: 1632 | message, data, status = exc.args 1633 | printerr('fabricate: ' + message) 1634 | finally: 1635 | _stop_results.set() # stop the results gatherer so I don't hang 1636 | if not options.quiet and os.path.abspath(build_dir) != original_path: 1637 | print("Leaving directory '%s' back to '%s'" % (build_dir, original_path)) 1638 | os.chdir(original_path) 1639 | sys.exit(status) 1640 | 1641 | def cli(): 1642 | # if called as a script, emulate memoize.py -- run() command line 1643 | parser, options, args = parse_options('[options] command line to run') 1644 | status = 0 1645 | if args: 1646 | status = memoize(args) 1647 | elif not options.clean: 1648 | parser.print_help() 1649 | status = 1 1650 | # autoclean may have been used 1651 | sys.exit(status) 1652 | 1653 | if __name__ == '__main__': 1654 | cli() 1655 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [tool:pytest] 2 | addopts = --ignore venv --ignore "*.egg-info" 3 | 4 | [pytest-watch] 5 | ignore = ./venv ./*.egg-info ./dist ./tmp ./build ./htmlcov /coverage_results 6 | 7 | [bdist_wheel] 8 | universal=1 9 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | from fabricate import __version__ 4 | 5 | # see https://github.com/pypa/pypi-legacy/issues/148 6 | try: 7 | import pypandoc 8 | long_description = pypandoc.convert('README.md', 'rst') 9 | except ImportError: 10 | long_description = open('README.md').read() 11 | 12 | setup( 13 | name='fabricate', 14 | version=__version__, 15 | description='The better build tool. Finds dependencies automatically for any language.', 16 | long_description=long_description, 17 | license='New BSD License', 18 | maintainer='Chris Coetzee', 19 | maintainer_email='chriscz93@gmail.com', 20 | url='https://github.com/brushtechnology/fabricate/', 21 | py_modules=['fabricate'], 22 | 23 | entry_points={ 24 | 'console_scripts': [ 25 | 'fabricate = fabricate:cli', 26 | ], 27 | }, 28 | 29 | extras_require=dict( 30 | build=['twine', 'wheel', 'setuptools-git', 'pypandoc'], 31 | ), 32 | 33 | keywords='fabricate make python build', 34 | classifiers=[ 35 | 'License :: OSI Approved :: BSD License', 36 | 'Programming Language :: Python', 37 | 'Topic :: Software Development :: Build Tools', 38 | 'Programming Language :: Python :: 2.6', 39 | 'Programming Language :: Python :: 2.7', 40 | 'Programming Language :: Python :: 3.3', 41 | 'Programming Language :: Python :: 3.4', 42 | 'Programming Language :: Python :: 3.5', 43 | 'Programming Language :: Python :: 3.6', 44 | ], 45 | # https://pypi.python.org/pypi?%3Aaction=list_classifiers 46 | platforms=[ 47 | 'Operating System :: Microsoft :: Windows', 48 | 'Operating System :: MacOS', 49 | 'Operating System :: POSIX :: Linux' 50 | ] 51 | ) 52 | 53 | 54 | -------------------------------------------------------------------------------- /test-requirement.txt: -------------------------------------------------------------------------------- 1 | mock==2.0.0 2 | plumbum==1.6.3 3 | pytest==4.3.0 4 | pytest-mock==1.5.0 5 | -------------------------------------------------------------------------------- /test/conftest.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pytest 3 | import os 4 | import shutil 5 | import atexit 6 | import json 7 | import inspect 8 | import tempfile 9 | 10 | sys.path.append('.') 11 | 12 | from fabricate import * 13 | import fabricate 14 | 15 | from _pytest.monkeypatch import MonkeyPatch 16 | 17 | 18 | __all__ = [ 'runner_list', 'assert_same_json', 'assert_json_equality', 'FabricateBuild'] 19 | 20 | possible_runner_list = [StraceRunner, AtimesRunner] 21 | runner_list = [] 22 | temp_dir = tempfile.mkdtemp() 23 | for r in possible_runner_list: 24 | try: 25 | if r(fabricate.Builder(dirs=[temp_dir])): 26 | runner_list.append(r) 27 | except fabricate.RunnerUnsupportedException: 28 | runner_list.append(pytest.param(r, marks=pytest.mark.skip)) 29 | 30 | try: 31 | string_types = (basestring,) 32 | except NameError: 33 | string_types = (str,) 34 | 35 | @pytest.fixture(autouse=True) 36 | def mock_env(request, mocker): 37 | mocker.patch('sys.exit' 38 | ) # prevent sys.exit from existing so as to do other tests 39 | 40 | @pytest.fixture 41 | def cleandir(): 42 | """ Should the build directory be cleaned at the end of each test """ 43 | return True 44 | 45 | 46 | @pytest.fixture 47 | def builddir(request, cleandir): 48 | bdir = os.path.join("build_dir", "%s-%s" % (request.module.__name__, 49 | request.function.__name__)) 50 | try: 51 | shutil.rmtree(bdir) 52 | except OSError: 53 | pass 54 | os.makedirs(bdir) 55 | 56 | def fin(): 57 | if cleandir: 58 | shutil.rmtree(bdir, ignore_errors=True) 59 | 60 | request.addfinalizer(fin) 61 | return bdir 62 | 63 | 64 | def assert_same_json(depfile, depref): 65 | """ Are the json in '.deps' `depfile` and the dict in `depref` equivalent 66 | modulo the md5sum values """ 67 | assert_json_equality(depfile, depref, structural_only=True) 68 | 69 | def assert_json_equality(depfile, depref, structural_only=False): 70 | """ Are the json in '.deps' `depfile` and the dict in `depref` equivalent """ 71 | def _replace_md5(d): 72 | for k in d: 73 | if isinstance(d[k], dict): 74 | _replace_md5(d[k]) 75 | else: 76 | if isinstance(d[k], string_types): 77 | if d[k].startswith("input-"): 78 | d[k] = d[k][:6] 79 | elif d[k].startswith("output-"): 80 | d[k] = d[k][:7] 81 | with open(depfile, 'r') as depfd: 82 | out = json.load(depfd) 83 | if structural_only: 84 | _replace_md5(out) 85 | _replace_md5(depref) 86 | assert out == depref 87 | 88 | 89 | class FabricateBuild(object): 90 | """ 91 | Simple wrapper class for builds during testing 92 | """ 93 | EXCLUDED_NAMES = set(['to_dict', 'main', 'EXCLUDED_NAMES', '_main_kwargs']) 94 | 95 | def __init__(self, **kwargs): 96 | """ 97 | Any kwargs passed will be passed to fabricate.main when a call to .main is made 98 | """ 99 | self._main_kwargs = kwargs 100 | 101 | def build(self): 102 | pass 103 | 104 | def clean(self): 105 | pass 106 | 107 | def main(self, *args, **kwargs): 108 | """execute the fabricate.main function with default 109 | kwargs as given to __init__ and 110 | globals_dict=self.to_dict() 111 | """ 112 | 113 | kwargs['globals_dict'] = kwargs.pop('globals_dict', self.to_dict()) 114 | 115 | for name in self._main_kwargs: 116 | kwargs[name] = kwargs.pop(name, self._main_kwargs[name]) 117 | 118 | # --- intercept any exit atexit functions 119 | exithandlers = [] 120 | def atexit_register(func, *targs, **kargs): 121 | exithandlers.append((func, targs, kargs)) 122 | return func 123 | 124 | def run_exitfuncs(): 125 | exc_info = None 126 | while exithandlers: 127 | func, targs, kargs = exithandlers.pop() 128 | try: 129 | func(*targs, **kargs) 130 | except SystemExit: 131 | exc_info = sys.exc_info() 132 | except: 133 | import traceback 134 | print >> sys.stderr, "Error in mock_env.run_exitfuncs:" 135 | traceback.print_exc() 136 | exc_info = sys.exc_info() 137 | 138 | if exc_info is not None: 139 | raise (exc_info[0], exc_info[1], exc_info[2]) 140 | 141 | monkeypatch = MonkeyPatch() 142 | monkeypatch.setattr(atexit, 'register', atexit_register) 143 | 144 | try: 145 | fabricate.main(*args, **kwargs) 146 | run_exitfuncs() 147 | finally: 148 | monkeypatch.undo() 149 | 150 | 151 | def to_dict(self): 152 | dct = {} 153 | 154 | # filter out special names 155 | for name in dir(self): 156 | if name.startswith('__'): 157 | continue 158 | 159 | if name in self.EXCLUDED_NAMES: 160 | continue 161 | 162 | dct[name] = getattr(self, name) 163 | 164 | return dct 165 | -------------------------------------------------------------------------------- /test/test_create.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pytest 3 | import plumbum.cmd as sh 4 | from plumbum import local 5 | import os 6 | 7 | from fabricate import * 8 | from conftest import * 9 | 10 | class BuildFile(FabricateBuild): 11 | 12 | def build(self): 13 | run('tar', 'czvf', 'foo.tar.gz', 'a.c', 'b.c') 14 | 15 | def clean(self): 16 | autoclean() 17 | 18 | 19 | @pytest.mark.parametrize("runner", runner_list) 20 | def test_create(builddir, runner): 21 | # prepare needed files 22 | with local.cwd(builddir): 23 | (sh.echo["a.c"] > "a.c")() 24 | (sh.echo["b.c"] > "b.c")() 25 | 26 | builder = BuildFile(build_dir=builddir, runner=runner) 27 | builder.main(command_line=['-c', '-D', 'build']) 28 | 29 | expected_json = { 30 | 'tar czvf foo.tar.gz a.c b.c': { 31 | 'b.c': 'input-', 32 | 'foo.tar.gz': 'output-', 33 | 'a.c': 'input-' 34 | }, 35 | '.deps_version': 2 36 | } 37 | 38 | # assertions 39 | with local.cwd(builddir): 40 | assert_same_json('.deps', expected_json) 41 | assert os.path.isfile('foo.tar.gz') 42 | assert sh.tar("tf", 'foo.tar.gz') == "a.c\nb.c\n" 43 | print(sh.ls("-al")) 44 | assert '"a.c": "input-' in sh.cat(".deps") 45 | sys.exit.assert_called_once_with(0) 46 | 47 | # Modify a.c to force rebuilding 48 | (sh.echo["newline"] > "a.c")() 49 | 50 | builder.main(command_line=['-D', 'build']) 51 | 52 | with local.cwd(builddir): 53 | sh.tar("df", "foo.tar.gz") # ensure tar diff return no difference 54 | -------------------------------------------------------------------------------- /test/test_fabricate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import sys 5 | import pytest 6 | import plumbum.cmd as sh 7 | from plumbum import local 8 | import os 9 | from copy import copy 10 | 11 | 12 | from fabricate import * 13 | from fabricate import md5func 14 | from conftest import * 15 | 16 | EMPTY_FILE_MD5 = 'd41d8cd98f00b204e9800998ecf8427e' 17 | 18 | def test_md5_hasher(builddir): 19 | with local.cwd(builddir): 20 | sh.touch('testfile') 21 | sh.mkdir('testdir') 22 | sh.touch('testdir/testfile') 23 | sh.ln('-s', 'testdir', 'testdirlink') 24 | sh.ln('-s', 'testfile', 'testlink') 25 | sh.ln('-s', 'nofile', 'testlink_nofile') 26 | assert md5_hasher('nofile') == None 27 | assert md5_hasher('testfile') == EMPTY_FILE_MD5 28 | assert md5_hasher('testdir') == md5func('testdir'.encode('utf-8')).hexdigest() 29 | assert md5_hasher('testlink') == EMPTY_FILE_MD5 30 | assert md5_hasher('testdirlink') == md5func('testdir'.encode('utf-8')).hexdigest() 31 | assert md5_hasher('testlink_nofile') == md5func('nofile'.encode('utf-8')).hexdigest() 32 | 33 | 34 | -------------------------------------------------------------------------------- /test/test_mkdir.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pytest 3 | import plumbum.cmd as sh 4 | from plumbum import local 5 | import os 6 | 7 | from fabricate import * 8 | from conftest import * 9 | 10 | class BuildFile(FabricateBuild): 11 | def build(self): 12 | # Make lots of directories to check ordered delete 13 | run('mkdir', 'testdir', group='testdir') 14 | run('mkdir', 'testdir/a', group='a', after='testdir') 15 | run('mkdir', 'testdir/b', group='b', after='testdir') 16 | run('mkdir', 'testdir/c', group='c', after='testdir') 17 | run('mkdir', 'testdir/c/f', group='f', after='c') 18 | run('mkdir', 'testdir/c/e', group='e', after='c') 19 | run('mkdir', 'testdir/c/d', group='d', after='c') 20 | 21 | # put some files in them to ensure content deleted before dir 22 | run('touch', 'testdir/f1', after='testdir') 23 | run('touch', 'testdir/f2', after='testdir') 24 | run('touch', 'testdir/b/f1', after='b') 25 | run('touch', 'testdir/b/f2', after='b') 26 | run('touch', 'testdir/c/d/f1', after='d') 27 | run('touch', 'testdir/c/d/f2', after='d') 28 | 29 | # make a dir that alreay exists 30 | run('mkdir', '-p', 'testdir/c/d', after='d') 31 | 32 | # make a dir that already partialy exists 33 | run('mkdir', '-p', 'testdir/c/g', after='c') 34 | 35 | # make a dir that already partialy exists but should not be deleted 36 | run('mkdir', '-p', 'existingdir/a') 37 | 38 | def clean(self): 39 | autoclean() 40 | 41 | 42 | @pytest.mark.parametrize("runner", runner_list) 43 | def test_mkdir(builddir, runner): 44 | # prepare needed files 45 | with local.cwd(builddir): 46 | sh.mkdir('existingdir') 47 | sh.touch('existingdir/existingfile') 48 | 49 | builder = BuildFile(build_dir=builddir, runner=runner) 50 | builder.main(command_line=['-D', 'build']) #, parallel_ok=True) 51 | 52 | expected_json = { 53 | ".deps_version": 2, 54 | "mkdir -p existingdir/a": { 55 | "existingdir": "input-ae394c47b4ccf49007dc9ec847f657b9", 56 | "existingdir/a": "output-16873f5a4ba5199a8b51f812d159e37e" 57 | }, 58 | "mkdir -p testdir/c/d": { 59 | "testdir": "input-3ca0a3620b59afb57cf5fd77cee6432c", 60 | "testdir/c": "input-54a9057bcd619534a49f669dd5ed3078", 61 | "testdir/c/d": "input-fdb1b8414eeab993acc5623371c43a71" 62 | }, 63 | "mkdir -p testdir/c/g": { 64 | "testdir": "input-3ca0a3620b59afb57cf5fd77cee6432c", 65 | "testdir/c": "input-54a9057bcd619534a49f669dd5ed3078", 66 | "testdir/c/g": "output-c512be1476c9253326e479827c491f7f" 67 | }, 68 | "mkdir testdir": { 69 | "testdir": "output-3ca0a3620b59afb57cf5fd77cee6432c" 70 | }, 71 | "mkdir testdir/a": { 72 | "testdir/a": "output-832651e32363cb4b115b074240cd08b5" 73 | }, 74 | "mkdir testdir/b": { 75 | "testdir/b": "output-0432d5c3dc41495725df46eeeedb1386" 76 | }, 77 | "mkdir testdir/c": { 78 | "testdir/c": "output-54a9057bcd619534a49f669dd5ed3078" 79 | }, 80 | "mkdir testdir/c/d": { 81 | "testdir/c/d": "output-fdb1b8414eeab993acc5623371c43a71" 82 | }, 83 | "mkdir testdir/c/e": { 84 | "testdir/c/e": "output-eadea986453292aaa62ccde2312c3413" 85 | }, 86 | "mkdir testdir/c/f": { 87 | "testdir/c/f": "output-5d7c7f98e6d795bbb252f6866c8d7850" 88 | }, 89 | "touch testdir/b/f1": { 90 | "testdir/b/f1": "output-d41d8cd98f00b204e9800998ecf8427e" 91 | }, 92 | "touch testdir/b/f2": { 93 | "testdir/b/f2": "output-d41d8cd98f00b204e9800998ecf8427e" 94 | }, 95 | "touch testdir/c/d/f1": { 96 | "testdir/c/d/f1": "output-d41d8cd98f00b204e9800998ecf8427e" 97 | }, 98 | "touch testdir/c/d/f2": { 99 | "testdir/c/d/f2": "output-d41d8cd98f00b204e9800998ecf8427e" 100 | }, 101 | "touch testdir/f1": { 102 | "testdir/f1": "output-d41d8cd98f00b204e9800998ecf8427e" 103 | }, 104 | "touch testdir/f2": { 105 | "testdir/f2": "output-d41d8cd98f00b204e9800998ecf8427e" 106 | } 107 | } 108 | 109 | # assertions 110 | with local.cwd(builddir): 111 | assert_json_equality('.deps', expected_json) 112 | assert os.path.isdir('testdir/c/g') 113 | assert os.path.isfile('testdir/c/d/f2') 114 | assert os.path.isdir('existingdir/a') 115 | sys.exit.assert_called_once_with(0) 116 | 117 | builder.main(command_line=['-D', 'clean']) 118 | #parallel_ok=True, 119 | #jobs=4, 120 | 121 | with local.cwd(builddir): 122 | assert not os.path.isdir('testdir') 123 | assert os.path.isdir('existingdir') 124 | assert os.path.isfile('existingdir/existingfile') 125 | assert not os.path.isdir('existingdir/a') 126 | 127 | 128 | -------------------------------------------------------------------------------- /test/test_rename.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pytest 3 | import plumbum.cmd as sh 4 | from plumbum import local 5 | import os 6 | 7 | from fabricate import * 8 | from conftest import * 9 | 10 | 11 | class BuildFile(FabricateBuild): 12 | def build(self): 13 | run('mv', 'originalfile', 'testfile') 14 | 15 | def clean(self): 16 | autoclean() 17 | # remake the original file 18 | shell('touch', 'originalfile') 19 | 20 | 21 | # Builder.done compute the hash after the file has been removed ! 22 | # Thus dependency is lost, and the mv command is not applied when originalfile 23 | # is changed 24 | @pytest.mark.xfail 25 | @pytest.mark.parametrize("runner", runner_list) 26 | def test_rename(builddir, runner): 27 | # prepare needed files 28 | with local.cwd(builddir): 29 | sh.touch('originalfile') 30 | 31 | builder = BuildFile(build_dir=builddir, runner=runner) 32 | 33 | ###### First build ########## 34 | builder.main(command_line=['-D', 'build']) 35 | 36 | expected_json = { 37 | ".deps_version": 2, 38 | "mv originalfile testfile": { 39 | "originalfile": "input-d41d8cd98f00b204e9800998ecf8427e", 40 | "testfile": "output-d41d8cd98f00b204e9800998ecf8427e" 41 | } 42 | } 43 | 44 | # assertions 45 | with local.cwd(builddir): 46 | assert_json_equality('.deps', expected_json) 47 | assert os.path.isfile('testfile') 48 | sys.exit.assert_called_once_with(0) 49 | 50 | # update original file to check the rebuild 51 | (sh.echo["newline"] > "originalfile")() 52 | 53 | ###### Second build ########## 54 | builder.main(command_line=['-D', 'build']) 55 | 56 | expected_json = { 57 | ".deps_version": 2, 58 | "mv originalfile testfile": { 59 | "originalfile": "input-321060ae067e2a25091be3372719e053", 60 | "testfile": "output-321060ae067e2a25091be3372719e053" 61 | } 62 | } 63 | 64 | with local.cwd(builddir): 65 | assert_json_equality('.deps', expected_json) 66 | assert "newline" in sh.cat('testfile') 67 | 68 | ###### Cleaning ########## 69 | builder.main(command_line=['-D', 'clean']) 70 | 71 | with local.cwd(builddir): 72 | assert not os.isfile('testfile') 73 | assert os.isfile('originalfile') 74 | -------------------------------------------------------------------------------- /test/test_symlink.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pytest 3 | import plumbum.cmd as sh 4 | from plumbum import local 5 | import os 6 | 7 | from fabricate import * 8 | from conftest import * 9 | 10 | class BuildFile(FabricateBuild): 11 | def build(self): 12 | run('ln', '-s', 'testfile', 'testlink') 13 | run('ln', '-s', 'testdir', 'testlink_dir') 14 | run('ln', '-s', 'nofile', 'testlink_nofile') 15 | 16 | def clean(self): 17 | autoclean() 18 | 19 | @pytest.mark.parametrize("runner", runner_list) 20 | def test_symlink(builddir, runner): 21 | builder = BuildFile(build_dir=builddir, runner=runner) 22 | 23 | with local.cwd(builddir): 24 | sh.touch('testfile') 25 | sh.mkdir('testdir') 26 | 27 | ###### First build ########## 28 | builder.main(command_line=['-D', 'build']) 29 | 30 | expected_json = { 31 | ".deps_version": 2, 32 | "ln -s nofile testlink_nofile": { 33 | "testlink_nofile": "output-" 34 | }, 35 | "ln -s testdir testlink_dir": { 36 | "testlink_dir": "output-" 37 | }, 38 | "ln -s testfile testlink": { 39 | "testlink": "output-" 40 | } 41 | } 42 | 43 | # assertions 44 | with local.cwd(builddir): 45 | assert_same_json('.deps', expected_json) 46 | assert os.path.islink('testlink') 47 | assert os.path.realpath('testlink').endswith('/testfile') 48 | assert os.path.islink('testlink_dir') 49 | assert os.path.realpath('testlink_dir').endswith('/testdir') 50 | assert os.path.islink('testlink_nofile') 51 | assert not os.path.isfile('testlink_nofile') 52 | sys.exit.assert_called_once_with(0) 53 | 54 | ###### Cleaning ########## 55 | builder.main(command_line=['-D', 'clean']) 56 | 57 | with local.cwd(builddir): 58 | assert not os.path.isfile('.deps') 59 | assert not os.path.islink('testlink') 60 | assert not os.path.islink('testlink_dir') 61 | assert not os.path.islink('testlink_nofile') 62 | 63 | --------------------------------------------------------------------------------