├── MANIFEST.in ├── .gitignore ├── tests ├── VersionInfo.rc ├── nondep │ ├── README.txt │ ├── hello.c │ └── __mbuild ├── circular-dep │ ├── m.h │ ├── main.c │ ├── b.h │ ├── c.h │ ├── z.h │ ├── a.h │ ├── find.py │ └── mfile.py ├── hello.c ├── spew ├── negtime.py ├── delay.c ├── foo.c ├── die.py ├── timed3.py ├── die2.py ├── foo.py ├── stdin.py ├── 1.py ├── sleep.py ├── 3.py ├── find.py ├── a.py ├── c.py ├── 2.py ├── b.py ├── timed4.py ├── use-icl-win.py └── nodag.py ├── README.md ├── Security.md ├── do-dist ├── templates └── find_mbuild.py ├── setup.py ├── mbuild ├── header_tag.py ├── osenv.py ├── arar.py ├── plan.py ├── scanner.py ├── dfs.py ├── __init__.py ├── base.py ├── doxygen.py ├── build_env.py ├── work_queue.py └── util.py ├── conanfile.py ├── CONTRIBUTING.md └── LICENSE /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include m*.py 2 | include __init__.py 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *~ 3 | *# 4 | .#* 5 | build/ 6 | *.egg-info/ 7 | -------------------------------------------------------------------------------- /tests/VersionInfo.rc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intelxed/mbuild/HEAD/tests/VersionInfo.rc -------------------------------------------------------------------------------- /tests/nondep/README.txt: -------------------------------------------------------------------------------- 1 | this is not a dependence driven build. 2 | it is for testing various msvs environments. 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | MBUILD USER INFO 2 | ----------------- 3 | See: 4 | 5 | https://intelxed.github.io/ 6 | 7 | and 8 | 9 | https://github.com/intelxed/mbuild 10 | 11 | Documentation generation manual: 12 | http://epydoc.sourceforge.net/epydoc.html 13 | 14 | To generate documentation from a check'ed out source tree, see "build-doc". 15 | -------------------------------------------------------------------------------- /Security.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation. 3 | 4 | ## Reporting a Vulnerability 5 | Please report any security vulnerabilities in this project [utilizing the guidelines here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html). -------------------------------------------------------------------------------- /tests/circular-dep/m.h: -------------------------------------------------------------------------------- 1 | /* BEGIN_LEGAL 2 | 3 | Copyright (c) 2016 Intel Corporation 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | END_LEGAL */ 18 | #if !defined(MMM_1) 19 | # define MMM_1 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /tests/circular-dep/main.c: -------------------------------------------------------------------------------- 1 | /* BEGIN_LEGAL 2 | 3 | Copyright (c) 2016 Intel Corporation 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | END_LEGAL */ 18 | #include "z.h" 19 | 20 | int main() { 21 | return 0; 22 | } 23 | -------------------------------------------------------------------------------- /tests/circular-dep/b.h: -------------------------------------------------------------------------------- 1 | /* BEGIN_LEGAL 2 | 3 | Copyright (c) 2016 Intel Corporation 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | END_LEGAL */ 18 | #if !defined(BBB_2) 19 | # define BBB_2 20 | 21 | # include "c.h" 22 | #endif 23 | -------------------------------------------------------------------------------- /tests/circular-dep/c.h: -------------------------------------------------------------------------------- 1 | /* BEGIN_LEGAL 2 | 3 | Copyright (c) 2016 Intel Corporation 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | END_LEGAL */ 18 | #if !defined(CCC_1) 19 | # define CCC_1 20 | 21 | # include "a.h" 22 | #endif 23 | -------------------------------------------------------------------------------- /tests/circular-dep/z.h: -------------------------------------------------------------------------------- 1 | /* BEGIN_LEGAL 2 | 3 | Copyright (c) 2016 Intel Corporation 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | END_LEGAL */ 18 | 19 | 20 | #if !defined(ZZZ_1) 21 | # define ZZZ_1 22 | 23 | # include "a.h" 24 | #endif 25 | -------------------------------------------------------------------------------- /tests/circular-dep/a.h: -------------------------------------------------------------------------------- 1 | /* BEGIN_LEGAL 2 | 3 | Copyright (c) 2016 Intel Corporation 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | END_LEGAL */ 18 | 19 | 20 | #if !defined(AAA_1) 21 | # define AAA_1 22 | 23 | # include "b.h" 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /tests/hello.c: -------------------------------------------------------------------------------- 1 | /*BEGIN_LEGAL 2 | 3 | Copyright (c) 2016 Intel Corporation 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | END_LEGAL */ 18 | #include 19 | int main() { 20 | printf("Hello world\n"); 21 | 22 | return 0; 23 | } 24 | -------------------------------------------------------------------------------- /tests/nondep/hello.c: -------------------------------------------------------------------------------- 1 | /* BEGIN_LEGAL 2 | 3 | Copyright (c) 2016 Intel Corporation 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | END_LEGAL */ 18 | #include 19 | int main() { 20 | printf("hello world\n"); 21 | return 0; 22 | } 23 | -------------------------------------------------------------------------------- /tests/spew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #BEGIN_LEGAL 3 | # 4 | #Copyright (c) 2017 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | #END_LEGAL 19 | from __future__ import print_function 20 | for i in range(0,1000*1000*100): 21 | print(i) 22 | -------------------------------------------------------------------------------- /tests/negtime.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #BEGIN_LEGAL 3 | # 4 | #Copyright (c) 2017 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | #END_LEGAL 19 | from __future__ import print_function 20 | import find 21 | import mbuild 22 | print (mbuild.get_elapsed_time(10,0)) 23 | -------------------------------------------------------------------------------- /tests/delay.c: -------------------------------------------------------------------------------- 1 | /*BEGIN_LEGAL 2 | 3 | Copyright (c) 2016 Intel Corporation 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | END_LEGAL */ 18 | int main(int argc, char** argv) { 19 | unsigned int i,j,n; 20 | n = atoi(argv[1]); 21 | for(i=0;i 53 | 54 | Use your real name (sorry, no pseudonyms or anonymous contributions.) 55 | 56 | If you set your `user.name` and `user.email` git configs, you can sign your 57 | commit automatically with `git commit -s`. 58 | 59 | -------------------------------------------------------------------------------- /tests/nondep/__mbuild: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- python -*- 3 | #BEGIN_LEGAL 4 | # 5 | #Copyright (c) 2016 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | #END_LEGAL 20 | import sys 21 | import os 22 | import glob 23 | 24 | def find_dir(d): 25 | idir = os.getcwd() 26 | last_idir = '' 27 | while idir != last_idir: 28 | mfile = os.path.join(idir,d) 29 | if os.path.exists(mfile): 30 | return mfile 31 | last_idir = idir 32 | idir = os.path.dirname(idir) 33 | print "Could not find %s file, looking upwards"% (mfile) 34 | sys.exit(1) 35 | 36 | #sys.path = [find_dir('mbuild')] + sys.path 37 | sys.path.append('../..') 38 | import mbuild 39 | 40 | 41 | def build(env, phase='BUILD',terminate_on_errors=False): 42 | """Build everything in the work queue""" 43 | okay = env.work_queue.build(die_on_errors=False) 44 | if not okay: 45 | if terminate_on_errors: 46 | mbuild.die("[%s] failed." % phase) 47 | else: 48 | mbuild.msgb("[%s] failed." % phase) 49 | else: 50 | mbuild.msgb(phase, "succeeded") 51 | 52 | def compile_and_link(env,fn): 53 | env['file']=fn 54 | (base,ext) = os.path.splitext(fn) 55 | if ext in ['.cpp','.C']: 56 | env['x_compiler'] = "%(CXX)s" 57 | env['tflags'] = "%(CXXFLAGS)s" 58 | else: 59 | env['x_compiler'] = "%(CC)s" 60 | env['tflags'] = "%(CCFLAGS)s" 61 | env['exe'] = "%s.%s.%s.exe" % (base, env['build_os'],env['host_cpu']) 62 | cmd = "%(x_compiler)s %(tflags)s %(file)s %(EXEOUT)s%(exe)s" 63 | cmd = env.expand_string(cmd) 64 | env.work_queue.add(mbuild.command_t(cmd)) 65 | 66 | 67 | def work(): 68 | env = mbuild.env_t() 69 | env.parser.add_option("--build", 70 | dest="build", action="store_true", default=False, 71 | help="Build tests") 72 | 73 | env.parse_args() 74 | env.work_queue = mbuild.work_queue_t(env['jobs']) 75 | 76 | if env['build']: 77 | for s in glob.glob('*.c'): 78 | compile_and_link(env,s) 79 | build(env) 80 | 81 | if __name__ == "__main__": 82 | work() 83 | 84 | 85 | -------------------------------------------------------------------------------- /mbuild/arar.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- python -*- 3 | # Repackage a bunch of static libs as one big static library. 4 | #BEGIN_LEGAL 5 | # 6 | #Copyright (c) 2024 Intel Corporation 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | #END_LEGAL 21 | from __future__ import print_function 22 | import os 23 | import sys 24 | import shutil 25 | import re 26 | 27 | from .base import * 28 | from .util import * 29 | 30 | class arar_error(Exception): 31 | def __init__(self, value): 32 | self.value = value 33 | def _str__(self): 34 | return repr(self.value) 35 | 36 | def repack(files, ar='ar', target='liball.a', verbose=False): 37 | """For linux only. Repackage the list of files using ar as the 38 | archiver program. The input files list can contain .a or .o 39 | files. The output library name is supplied by the target keyword 40 | argument. This will raise an exception arar_error in the event of 41 | a problem, setting the exception value field with an explanation.""" 42 | import glob 43 | pid= os.getpid() 44 | #error=os.system(ar + " --version") 45 | tdir = 'tmp.arar.%d' % (pid) 46 | if os.path.exists(tdir): 47 | raise arar_error('Conflict with existing temporary directory: %s' % \ 48 | (tdir)) 49 | os.mkdir(tdir) 50 | # work in a temporary subdirectory 51 | os.chdir(tdir) 52 | doto = [] 53 | for arg in files: 54 | if re.search(r'[.]o$', arg): 55 | if arg[0] == '/': 56 | doto.append(arg) 57 | else: 58 | doto.append(os.path.join('..',arg)) 59 | continue 60 | if arg[0] == '/': 61 | cmd = "%s x %s" % (ar,arg) 62 | else: 63 | cmd = "%s x ../%s" % (ar,arg) 64 | if verbose: 65 | print(u"EXTRACTING %s" % (cmd)) 66 | error= os.system(cmd) 67 | if error: 68 | raise arar_error('Extract failed for command %s' % (cmd)) 69 | files = glob.glob('*.o') + doto 70 | local_target = os.path.basename(target) 71 | cmd = "%s rcv %s %s" % (ar, local_target, " ".join(files)) 72 | if verbose: 73 | print(u"RECOMBINING %s" % (cmd)) 74 | error=os.system(cmd) 75 | if error: 76 | raise arar_error('Recombine failed') 77 | 78 | os.chdir('..') 79 | os.rename(os.path.join(tdir,local_target), target) 80 | if verbose: 81 | print(u"CREATED %s" % (target)) 82 | shutil.rmtree(tdir) 83 | 84 | 85 | -------------------------------------------------------------------------------- /mbuild/plan.py: -------------------------------------------------------------------------------- 1 | # -*- python -*- 2 | #BEGIN_LEGAL 3 | # 4 | #Copyright (c) 2016 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | #END_LEGAL 19 | 20 | """Intermediate data structure produced by builders and sent to the 21 | dependence directed acyclic graph (DAG) that sequences execution. 22 | 23 | Users who create their own builders to call python functions should emit 24 | an plan_t object and add it to the DAG. 25 | """ 26 | 27 | class plan_t(object): 28 | """ 29 | An object that the builders create and is passed to the DAG L{dag_t} to 30 | order the tasks. This is used exclusively to create 31 | L{command_t}'s. 32 | """ 33 | def __init__(self, command, args=None, env=None, input=None, output=None, name=None): 34 | """ 35 | Create an input record for the L{dag_t} describing a 36 | command. The command can be a string to execute or a python 37 | function or a list of strings and python functions. The python 38 | function will be passed two arguments: args and env. args is 39 | typically a list, but could be anything. 40 | 41 | The input and output lists of files are used by the L{dag_t} to 42 | order this command relative to other commands. 43 | 44 | When the command is a python function, the python function is 45 | called with two arguments: args and an env of type 46 | L{env_t}. The args can be anything but are typically the 47 | inputs to the python function and any information required to 48 | generate the corresponding outputs. The python functions return 49 | a 2-typle (retcode, stdout). 50 | 51 | The input list: When the command is a python function, the 52 | plan_t's input list contains at least the input files names 53 | passed via args variable. The input list can be a superset 54 | containing more stuff that might trigger the command 55 | execution. 56 | 57 | If the command does not produce a specific output, you can 58 | specify a dummy file name to allow sequencing relative to 59 | other commands. 60 | 61 | @type command: string or python function or a list 62 | @param command: string or python function. 63 | 64 | @type args: list 65 | @param args: (optional) arguments to the command if it is a python function 66 | 67 | @type env: L{env_t} 68 | @param env: (optional) an environment to pass to the python function 69 | 70 | @type input: list 71 | @param input: (optional) files upon which this command depends. 72 | 73 | @type output: list 74 | @param output: (optional) files which depend on this command. 75 | 76 | @type name: string 77 | @param name: (optional) short name to be used to identify the work/task 78 | """ 79 | self.command = command 80 | self.args = args 81 | self.env = env 82 | self.input = input 83 | self.output = output 84 | self.name = name 85 | 86 | def __str__(self): 87 | s = [] 88 | if self.name: 89 | s.append('NAME: ' + str(self.name)) 90 | s.append('CMD: ' + str(self.command)) 91 | s.append('INPUT: ' + str(self.input)) 92 | s.append('OUTPUT: ' + str(self.output)) 93 | return " ".join(s) 94 | -------------------------------------------------------------------------------- /mbuild/scanner.py: -------------------------------------------------------------------------------- 1 | # -*- python -*- 2 | #BEGIN_LEGAL 3 | # 4 | #Copyright (c) 2022 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | #END_LEGAL 19 | 20 | """Function for header include scanning""" 21 | from __future__ import print_function 22 | import re 23 | import os 24 | import sys 25 | from . import base 26 | from . import util 27 | 28 | 29 | class mbuild_header_record_t: 30 | """Stores information about headers that we find""" 31 | def __init__(self, fn, found=True): 32 | self.file_name = fn 33 | self.system = False 34 | self.found = found 35 | def __str__(self): 36 | s = '' 37 | s = self.file_name 38 | if self.system: 39 | s += ' system' 40 | if not self.found: 41 | s += ' not-found' 42 | return s 43 | 44 | def mbuild_compute_path(hname, search_path): 45 | """Return the full path of the header hname, if found and None 46 | otherwise. Search the path in order and see if we find the file""" 47 | for p in search_path: 48 | tname = util.join(p,hname) 49 | tname = os.path.realpath(tname) 50 | #base.msgb("TESTING", tname) 51 | if os.path.exists(tname): 52 | return tname 53 | return None 54 | 55 | # FIXME: ignoring system headers for now. 56 | mbuild_include_pattern = re.compile(r'^[ \t]*#[ \t]*include[ \t]+"(?P[^"]+)"') 57 | mbuild_nasm_include_pattern = re.compile(r'^[ \t]*%include[ \t]+"(?P[^"]+)"') 58 | 59 | is_py2 = sys.version[0] == '2' 60 | def _open_errors(fn): 61 | if is_py2: 62 | return open(fn, 'r') 63 | else: 64 | return open(fn, 'r', errors='ignore') 65 | 66 | def mbuild_scan(fn, search_path): 67 | """Given a file name fn, and a list of search paths, scan for 68 | headers in fn and return a list of mbuild_header_record_t's. The 69 | header records indicate if the file is a system include based on 70 | <> symbols or if the file was missing. If the file cannot be 71 | found, we assume it is in the assumed_directory.""" 72 | global mbuild_include_pattern 73 | global mbuild_nasm_include_pattern 74 | 75 | all_names = [] 76 | 77 | if not os.path.exists(fn): 78 | return all_names 79 | 80 | source_path = os.path.dirname(fn) 81 | if source_path == '': 82 | source_path = '.' 83 | aug_search_path = [source_path] + search_path 84 | 85 | with _open_errors(fn) as f: 86 | for line in f: 87 | hgroup = mbuild_include_pattern.match(line) 88 | if not hgroup: 89 | hgroup = mbuild_nasm_include_pattern.match(line) 90 | if hgroup: 91 | hname = hgroup.group('hdr') 92 | full_name = mbuild_compute_path(hname, aug_search_path) 93 | if full_name: 94 | if full_name == fn: 95 | # self loop. compilation will fail unless C-preprocessor has 96 | # guards against self-include. We'll assume that and ignore 97 | # this file. 98 | base.msgb("IGNORING CYCLIC SELF-INCLUDE", fn) 99 | continue 100 | hr = mbuild_header_record_t(full_name) 101 | else: 102 | hr = mbuild_header_record_t(hname, found=False) 103 | all_names.append(hr) 104 | return all_names 105 | 106 | 107 | 108 | def _test_scan(): 109 | paths = ["/home/mjcharne/proj/learn/" ] 110 | all_headers = mbuild_scan("/home/mjcharne/proj/learn/foo.cpp", paths) 111 | for hr in all_headers: 112 | print (hr) 113 | 114 | if __name__ == '__main__': 115 | _test_scan() 116 | 117 | -------------------------------------------------------------------------------- /mbuild/dfs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # FILE: dfs.py 3 | #BEGIN_LEGAL 4 | # 5 | #Copyright (c) 2022 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | #END_LEGAL 20 | 21 | """This file provides a node_t type and a dfs() routine that prints out 22 | cycles found in a graph represented as a list of node_t objects. 23 | """ 24 | from __future__ import print_function 25 | _dfs_verbose = False 26 | 27 | class node_t(object): 28 | def __init__(self,name='no-name-for-node'): 29 | self.name = name 30 | self.afters = [] 31 | self.befores = [] 32 | self.zero() 33 | 34 | # The colors are: 35 | # 0 = white (unvisited), 36 | # 1=grey (discovered, visiting), 37 | # 2=black (finalized) 38 | self.color = 0 39 | 40 | self.discover = 0 41 | self.finalize = 0 42 | self.predecessor = None 43 | def zero(self): 44 | self.color = 0 45 | def add_successor(self, s): 46 | self.afters.append(s) 47 | s.befores.append(self) 48 | def add_ancestor(self, s): 49 | self.befores.append(s) 50 | s.afters.append(self) 51 | def __str__(self): 52 | s = [] 53 | s.append("TARGET: %s\n\t" % self.name) 54 | s.append("discovered %d finalized %d\n\t" % (self.discover, self.finalize)) 55 | s.extend(["\t\n{}".format(x.name) for x in self.afters]) 56 | return ''.join(s) 57 | 58 | 59 | 60 | def _print_cycle(last_visit, grey_loop_closer): 61 | pad = '' 62 | p = last_visit 63 | while 1: 64 | print (pad, p.name) 65 | if p == grey_loop_closer: 66 | break 67 | p = p.predecessor 68 | pad += ' ' 69 | 70 | def _visit(n): 71 | global _dfs_time 72 | n.color = 1 73 | n.discover = _dfs_time 74 | if _dfs_verbose: 75 | print ("visiting %s" % str(n)) 76 | _dfs_time += 1 77 | retval = False 78 | for a in n.afters: 79 | if a.color == 0: 80 | a.predecessor = n 81 | retval |= _visit(a) 82 | elif a.color == 1: 83 | # a back-edge 84 | print ("cycle") 85 | _print_cycle(n,a) 86 | retval = True 87 | n.color = 2 88 | n.finalize = _dfs_time 89 | _dfs_time += 1 90 | return retval 91 | 92 | def dfs(nodes): 93 | """Depth first search a list of node_t objects. Print out cycles. 94 | @rtype: bool 95 | @return: True if cycles were detected. 96 | """ 97 | global _dfs_time 98 | _dfs_time = 0 99 | for t in nodes: 100 | t.zero() 101 | cycle = False 102 | for n in nodes: 103 | if n.color == 0: 104 | cycle |= _visit(n) 105 | return cycle 106 | 107 | ####################################################### 108 | 109 | # stuff for a strongly connected components algorithm -- currently 110 | # unused. 111 | 112 | def _node_cmp(aa,bb): 113 | return aa.finalize.__cmp__(bb.finalize) 114 | 115 | def _visit_transpose(n): 116 | global _dfs_time 117 | n.color = 1 118 | if _dfs_verbose: 119 | print ("visiting %s" % str(n)) 120 | for a in n.befores: 121 | if a.color == 0: 122 | _visit_transpose(a) 123 | n.color = 2 124 | 125 | 126 | def dfs_transpose(nodes): 127 | global _dfs_time 128 | _dfs_time = 0 129 | for t in nodes: 130 | t.zero() 131 | nodes.sort(cmp=_node_cmp) 132 | for n in nodes: 133 | if n.color == 0: 134 | _visit_transpose(n) 135 | if _dfs_verbose: 136 | print ("====") 137 | 138 | #################################################### 139 | 140 | def _test_dfs(): 141 | node1 = node_t('1') 142 | node2 = node_t('2') 143 | node3 = node_t('3') 144 | node4 = node_t('4') 145 | node1.add_successor(node2) 146 | node1.add_successor(node3) 147 | node3.add_successor(node4) 148 | node4.add_successor(node1) 149 | 150 | nodes = [ node1, node2, node3, node4 ] 151 | cycle = dfs(nodes) 152 | if cycle: 153 | print ("CYCLE DETECTED") 154 | #print ("VISIT TRANSPOSE") 155 | #dfs_transpose(nodes) 156 | 157 | # print ("NODES\n", "\n".join(map(str,nodes))) 158 | 159 | if __name__ == '__main__': 160 | _test_dfs() 161 | -------------------------------------------------------------------------------- /mbuild/__init__.py: -------------------------------------------------------------------------------- 1 | #BEGIN_LEGAL 2 | # 3 | #Copyright (c) 2024 Intel Corporation 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | #END_LEGAL 18 | # __init__.py 19 | """This is mbuild: a simple portable dependence-based build-system 20 | written in python. 21 | 22 | mbuild is a python-based build system very similar to scons with some 23 | philosophical features of make. mbuild exposes the scan and build phases 24 | allowing them to be repeated as necessary. Multiple DAGs can be 25 | built, one during each scan phase. 26 | 27 | Conceptually there are 3 major components to mbuild: 28 | - The environment L{env_t} 29 | - The directed acyclic graph L{dag_t} 30 | - The work queue L{work_queue_t} 31 | 32 | Using the environment L{env_t} you customize your build configuration 33 | and construct names for your source files, object files, executables, 34 | etc. The environment contains builder methods that create L{plan_t} 35 | objects. There are builders for C, C++, static and dynamic libraries, 36 | assembly files and linking programs. The environment and builders 37 | support string substitution. 38 | 39 | The L{plan_t} objects are passed to the L{dag_t} which stores the 40 | dependences that order execution. The L{plan_t} objects describe work 41 | that needs to be done. Plans typically contain a command line strings 42 | (with all substitutions done), but can also be python functions that 43 | will be executed during the build. 44 | 45 | Using the L{plan_t} objects, the L{dag_t} creates L{command_t} 46 | objects that are passed to the L{work_queue_t} to ultimately build the 47 | target or targets. 48 | 49 | Your build file can have multiple environments, DAGS and work queues. 50 | 51 | 52 | Using the environment dictionary 53 | ================================ 54 | 55 | You can bind or augmenting environment variables from the command 56 | line. For example, one can say C{build_cpu=ia32} on an x86-64 system 57 | to change the default compilation behavior. Similarly, one can say 58 | C{CXXFLAGS+=-g} to add the C{-g} flag to the existing C{CXXFLAGS} 59 | variable. 60 | 61 | Dynamic substitution is also used. Patterns of the form %(I{string})s 62 | will substitute I{string} dynamically before it is used. The 63 | expansion can happen directly from the environment and is 64 | recursive. The expansion can also use dictionaries that are variables 65 | in the environment. A dictionary in the environment is really a tuple 66 | of the key-variable and the dictionary itself. 67 | 68 | For example:: 69 | 70 | env['opt_flag'] = ( 'opt', {'noopt':'', 71 | '0':'%(OPTOPT)s0', 72 | '1':'%(OPTOPT)s1', 73 | '2':'%(OPTOPT)s2', 74 | '3':'%(OPTOPT)s3', 75 | '4':'%(OPTOPT)s4'} ) 76 | 77 | env['OPTOPT'] = ( 'compiler', { 'gnu':'-O', 78 | 'ms':'/O'}) 79 | 80 | 81 | env['CXXFLAGS'] += ' %(opt_flag)s' 82 | 83 | The C{OPTOPT} variable depends on C{env['compiler']}. 84 | If C{env['compiler']='gnu'} then C{env['OPTOPT']} expands to C{-O}. 85 | If C{env['compiler']='ms'} then C{env['OPTOPT']} expands to C{/O}. 86 | 87 | If the C{opt} variable is set "C{opt=3}" on the command line, or equivalently 88 | if C{env['opt']='3'} is 89 | set in the script, 90 | then if the C{env['compiler']='gnu'} in the environment at the time of expansion, 91 | then the flag in the 92 | C{CXXFLAGS} will be C{-O3}. If C{env['compiler']='ms'} at the time of expansion, 93 | then the optimiation 94 | flag would be C{/O3}. If C{opt=noopt} (on the command line) then there will be no 95 | optimization flag in the C{CXXFLAGS}. 96 | 97 | 98 | Introspection 99 | ============= 100 | 101 | The L{command_t} that are executed during the build have their output 102 | (stdout/stderr) stored in the L{dag_t}. After a build it is possible 103 | to collect the commands using the L{dag_t.results} function and analyze the 104 | output. This is very handy for test and validation suites. 105 | """ 106 | 107 | from .base import * 108 | from .dag import * 109 | from .work_queue import * 110 | from .env import * 111 | from .util import * 112 | from .plan import * 113 | from .arar import * 114 | from .doxygen import doxygen_run, doxygen_args, doxygen_env 115 | 116 | __all__ = [ 'base', 117 | 'dag', 118 | 'work_queue', 119 | 'env', 120 | 'util', 121 | 'plan', 122 | 'msvs', 123 | 'arar', 124 | 'doxygen', 125 | 'dfs'] 126 | 127 | 128 | import time 129 | def mbuild_exit(): 130 | """mbuild's exit function""" 131 | 132 | import atexit 133 | atexit.register(mbuild_exit) 134 | -------------------------------------------------------------------------------- /mbuild/base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- python -*- 3 | #BEGIN_LEGAL 4 | # 5 | #Copyright (c) 2024 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | #END_LEGAL 20 | """Base functionality: messages, verbosity, python version checking""" 21 | 22 | import os 23 | import sys 24 | import traceback 25 | import locale 26 | 27 | _MBUILD_ENCODING = locale.getpreferredencoding() 28 | def unicode_encoding() -> str: 29 | return _MBUILD_ENCODING 30 | 31 | PY3 = sys.version_info > (3,) 32 | def is_python3(): 33 | return PY3 34 | 35 | _mbuild_verbose_level = 1 36 | def verbose(level=0) -> bool: 37 | """Return True if the configured message level supplied is >= the 38 | level arguement 39 | @param level: int 40 | @param level: the verbosity level at which this function should return True 41 | 42 | @rtype: bool 43 | @return: True iff the level argument is >= current verbosity level 44 | """ 45 | global _mbuild_verbose_level 46 | if _mbuild_verbose_level >= level: 47 | return True 48 | return False 49 | def set_verbosity(v): 50 | """Set the global verbosity level. 0=quiet, 99=very very noisy""" 51 | global _mbuild_verbose_level 52 | _mbuild_verbose_level = v 53 | 54 | def get_verbosity(): 55 | """Return the global verbosity level. 0=quiet, 99=very very noisy""" 56 | global _mbuild_verbose_level 57 | return _mbuild_verbose_level 58 | 59 | def bracket(s,m='') -> str: 60 | """add a bracket around s and append m. 61 | @rtype: string 62 | @return: a bracketed string s and a suffixed message m 63 | """ 64 | return u'[{}] {}'.format(s,m) 65 | 66 | def error_msg(s,t): 67 | """Emit '[s] t' to stderr with a newline""" 68 | print(bracket(s,t)) 69 | 70 | def msg(s, pad=''): 71 | """Emit s to stdout with a newline""" 72 | print(pad, end='') 73 | print(s) 74 | 75 | def msgn(s, pad=''): 76 | """Emit s to stdout without a newline""" 77 | print(pad, end='') 78 | print(s, end='') 79 | 80 | def msgb(s,t='',pad=''): 81 | """a bracketed string s sent to stdout, followed by a string t""" 82 | msg(bracket(s,t), pad=pad) 83 | 84 | def vmsg(v,s,pad=''): 85 | """If verbosity v is sufficient, emit s to stdout with a newline""" 86 | # someone could pass unicode as pad... 87 | if verbose(v): 88 | msg(s,pad=pad) 89 | 90 | def vmsgb(v,s,t='',pad=''): 91 | """If verbosity v is sufficient, emit a bracketed string s sent to 92 | stdout, followed by a string t""" 93 | vmsg(v,bracket(s,t),pad=pad) 94 | 95 | def cond_die(v, cmd, msg): 96 | """Conditionally die, if v is not zero. Print the msg and the cmd. 97 | @type v: int 98 | @param v: we die if v is not 0 99 | 100 | @type cmd: string 101 | @param cmd: a command to print 102 | 103 | @type msg: string 104 | @param msg: a message to print before the command 105 | """ 106 | if v != 0: 107 | s = msg + "\n [CMD] " + cmd 108 | die(s) 109 | 110 | def die(m,s=''): 111 | """Emit an error message m (and optionally s) and exit with a return 112 | value 1""" 113 | msgb("MBUILD ERROR", "%s %s\n\n" % (m,s) ) 114 | etype, value, tb = sys.exc_info() 115 | if tb is None: 116 | stack = traceback.extract_stack()[:-1] 117 | traceback.print_list(stack, file=sys.stdout) 118 | else: 119 | traceback.print_exception(etype, value, tb, file=sys.stdout) 120 | sys.exit(1) 121 | 122 | def warn(m): 123 | """Emit an warning message""" 124 | msgb("MBUILD WARNING", m) 125 | 126 | def get_python_version(): 127 | """Return the python version as an integer 128 | @rtype: int 129 | @return: major * 100000 + minor + 1000 + fixlevel 130 | """ 131 | tuple = sys.version_info 132 | major = int(tuple[0]) 133 | minor = int(tuple[1]) 134 | fix = int(tuple[2]) 135 | vnum = major *100000 + minor * 1000 + fix 136 | return vnum 137 | 138 | def get_python_version_tuple(): 139 | """Return the python version as a tuple (major,minor,fixlevel) 140 | @rtype: tuple 141 | @return: (major,minor,fixlevel) 142 | """ 143 | 144 | tuple = sys.version_info 145 | major = int(tuple[0]) 146 | minor = int(tuple[1]) 147 | fix = int(tuple[2]) 148 | return (major,minor,fix) 149 | 150 | def check_python_version(maj,minor,fix=0): 151 | """Return true if the current python version at least the one 152 | specified by the arguments. 153 | @rtype: bool 154 | @return: True/False 155 | """ 156 | t = get_python_version_tuple() 157 | if t[0] > maj: 158 | return True 159 | if t[0] == maj and t[1] > minor: 160 | return True 161 | if t[0] == maj and t[1] == minor and t[2] >= fix: 162 | return True 163 | return False 164 | 165 | 166 | 167 | try: 168 | if check_python_version(3,9) == False: 169 | die("MBUILD error: Need Python version 3.9 or later.") 170 | except: 171 | die("MBUILD error: Need Python version 3.9 or later.") 172 | 173 | import platform 174 | _on_mac = False 175 | _on_native_windows = False 176 | _on_windows = False # cygwin or native windows 177 | _on_cygwin = False 178 | _on_linux = False 179 | _on_freebsd = False 180 | _on_netbsd = False 181 | _operating_system_name = platform.system() 182 | if _operating_system_name.find('CYGWIN') != -1: 183 | _on_cygwin = True 184 | _on_windows = True 185 | elif _operating_system_name == 'Microsoft' or _operating_system_name == 'Windows': 186 | _on_native_windows = True 187 | _on_windows = True 188 | elif _operating_system_name == 'Linux': 189 | _on_linux = True 190 | elif _operating_system_name == 'FreeBSD': 191 | _on_freebsd = True 192 | elif _operating_system_name == 'NetBSD': 193 | _on_netbsd = True 194 | elif _operating_system_name == 'Darwin': 195 | _on_mac = True 196 | else: 197 | die("Could not detect operating system type: " + _operating_system_name) 198 | 199 | def on_native_windows(): 200 | """ 201 | @rtype: bool 202 | @return: True iff on native windows win32/win64 203 | """ 204 | global _on_native_windows 205 | return _on_native_windows 206 | 207 | def on_windows(): 208 | """ 209 | @rtype: bool 210 | @return: True iff on windows cygwin/win32/win64 211 | """ 212 | global _on_windows 213 | return _on_windows 214 | 215 | def on_mac(): 216 | """ 217 | @rtype: bool 218 | @return: True iff on mac 219 | """ 220 | global _on_mac 221 | return _on_mac 222 | 223 | ###### 224 | 225 | 226 | def bytes2unicode(bs): 227 | """Convert a bytes object to unicode""" 228 | return bs.decode(unicode_encoding()) 229 | 230 | def ensure_string(x): 231 | # strings in python2 turn up as bytes 232 | # strings in python3 show up as strings and are unicode 233 | if isinstance(x,bytes): 234 | return bytes2unicode(x) 235 | if isinstance(x,list): 236 | o = [] 237 | for y in x: 238 | if isinstance(y,bytes): 239 | o.append( bytes2unicode(y) ) 240 | else: 241 | o.append( y ) 242 | return o 243 | return x 244 | 245 | def uappend(lst,s): 246 | """Make sure s is unicode before adding it to the list lst""" 247 | lst.append(ensure_string(s)) 248 | 249 | def is_stringish(x): 250 | if isinstance(x,bytes) or isinstance(x,str): 251 | return True 252 | return False 253 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | -------------------------------------------------------------------------------- /mbuild/doxygen.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- python -*- 3 | #BEGIN_LEGAL 4 | # 5 | #Copyright (c) 2021 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | #END_LEGAL 20 | 21 | 22 | ############################################################################ 23 | ## START OF IMPORTS SETUP 24 | ############################################################################ 25 | 26 | import sys 27 | import os 28 | import re 29 | import copy 30 | import glob 31 | import types 32 | 33 | try: 34 | from . import base 35 | from . import dag 36 | from . import util 37 | from . import plan 38 | except: 39 | s = "\nXED ERROR: mfile.py could not find mbuild." + \ 40 | " Should be a sibling of the xed2 directory.\n\n" 41 | sys.stderr.write(s) 42 | sys.exit(1) 43 | 44 | 45 | ########################################################################### 46 | ## DOXYGEN SUPPORT 47 | ########################################################################### 48 | 49 | def _doxygen_version_okay(s, want_major, want_minor, want_fix): 50 | values = s.split('.') 51 | 52 | maj =int(values[0]) 53 | minor = int(values[1]) 54 | fix = 0 55 | if len(values) > 2: 56 | # remove everything after the dash for things like: 'Doxygen 57 | # 1.5.1-p1' 58 | values[2] = re.sub(r'-.*$','',values[2]) 59 | try: 60 | fix = int(values[2]) 61 | except ValueError as v: 62 | pass 63 | if (maj > 1) or \ 64 | (maj == want_major and minor > want_minor) or \ 65 | (maj == want_major and minor == want_minor and fix >= want_fix): 66 | return True 67 | return False 68 | 69 | def _find_doxygen(env): 70 | """Find the right version of doxygen. Return a tuple of the 71 | command name and a boolean indicating whether or not the version 72 | checked out.""" 73 | 74 | if env['doxygen_cmd'] == '': 75 | doxygen_cmd_intel = "/usr/intel/bin/doxygen" 76 | doxygen_cmd_cygwin = "C:/cygwin/bin/doxygen" 77 | doxygen_cmd_mac = \ 78 | "/Applications/Doxygen.app/Contents/Resources/doxygen" 79 | doxygen_cmd = "doxygen" 80 | 81 | if env['build_os'] == 'win': 82 | if os.path.exists(doxygen_cmd_cygwin): 83 | doxygen_cmd = doxygen_cmd_cygwin 84 | else: 85 | base.msgb('DOXYGEN',"Could not find cygwin's doxygen," + 86 | "trying doxygen from PATH") 87 | elif env['build_os'] == 'lin': 88 | if base.verbose(2): 89 | base.msgb("CHECKING FOR", doxygen_cmd_intel) 90 | if os.path.exists(doxygen_cmd_intel): 91 | doxygen_cmd = doxygen_cmd_intel 92 | elif env['build_os'] == 'mac': 93 | if base.verbose(2): 94 | base.msgb("CHECKING FOR", doxygen_cmd_mac) 95 | if os.path.exists(doxygen_cmd_mac): 96 | doxygen_cmd = doxygen_cmd_mac 97 | else: 98 | doxygen_cmd = env['doxygen_cmd'] 99 | 100 | doxygen_cmd = env.escape_string(doxygen_cmd) 101 | doxygen_okay = False 102 | if base.verbose(2): 103 | base.msgb('Checking doxygen version','...') 104 | if base.check_python_version(2,4): 105 | try: 106 | (retval, output, error_output) = \ 107 | util.run_command(doxygen_cmd + " --version") 108 | if retval==0: 109 | if len(output) > 0: 110 | first_line = output[0].strip() 111 | if base.verbose(2): 112 | base.msgb("Doxygen version", first_line) 113 | doxygen_okay = _doxygen_version_okay(first_line, 1,4,6) 114 | else: 115 | for o in output: 116 | base.msgb("Doxygen-version-check STDOUT", o) 117 | if error_output: 118 | for line in error_output: 119 | base.msgb("STDERR ",line.rstrip()) 120 | except: 121 | base.die("Doxygen required by the command line options " + 122 | "but no doxygen found") 123 | 124 | return (doxygen_cmd, doxygen_okay) 125 | 126 | 127 | def _replace_match(istring, mtch, newstring, group_name): 128 | """This is a lame way of avoiding regular expression backslashing 129 | issues""" 130 | x1= mtch.start(group_name) 131 | x2= mtch.end(group_name) 132 | ostring = istring[0:x1] + newstring + istring[x2:] 133 | return ostring 134 | 135 | 136 | def _customize_doxygen_file(env, subs): 137 | 138 | """Change the $(*) strings to the proper value in the config file. 139 | Returns True on success""" 140 | 141 | # doxygen wants quotes around paths with spaces 142 | for k,s in iter(subs.items()): 143 | if re.search(' ',s): 144 | if not re.search('^".*"$',s): 145 | base.die("Doxygen requires quotes around strings with spaces: [%s]->[%s]" % 146 | ( k,s)) 147 | return False 148 | 149 | # input and output files 150 | try: 151 | lines = open(env['doxygen_config']).readlines() 152 | except: 153 | base.msgb("Could not open input file: " + env['doxygen_config']) 154 | return False 155 | 156 | env['doxygen_config_customized'] = \ 157 | env.build_dir_join(os.path.basename(env['doxygen_config']) + '.customized') 158 | try: 159 | ofile = open(env['doxygen_config_customized'],'w') 160 | except: 161 | base.msgb("Could not open output file: " + env['doxygen_config_customized']) 162 | return False 163 | 164 | # compile the patterns 165 | rsubs = {} 166 | for k,v in iter(subs.items()): 167 | rsubs[k]=re.compile(r'(?P[$][(]' + k + '[)])') 168 | 169 | olines = [] 170 | for line in lines: 171 | oline = line 172 | for k,p in iter(rsubs.items()): 173 | #print ('searching for', k, 'to replace it with', subs[k]) 174 | m = p.search(oline) 175 | while m: 176 | #print ('replacing', k, 'with', subs[k]) 177 | oline = _replace_match(oline, m, subs[k], 'tag') 178 | m = p.search(oline) 179 | olines.append(oline) 180 | 181 | 182 | try: 183 | for line in olines: 184 | ofile.write(line) 185 | except: 186 | ofile.close() 187 | base.msgb("Could not write output file: " + env['doxygen_config_customized']) 188 | return False 189 | 190 | ofile.close() 191 | return True 192 | 193 | def _build_doxygen_main(args, env): 194 | """Customize the doxygen input file. Run the doxygen command, copy 195 | in any images, and put the output in the right place.""" 196 | 197 | if isinstance(args, list): 198 | if len(args) < 2: 199 | base.die("Need subs dictionary and dummy file arg for the doxygen command " + 200 | "to indicate its processing") 201 | else: 202 | base.die("Need a list for _build_doxygen_main with the subs " + 203 | "dictionary and the dummy file name") 204 | 205 | (subs,dummy_file) = args 206 | 207 | (doxygen_cmd, doxygen_okay) = _find_doxygen(env) 208 | if not doxygen_okay: 209 | msg = 'No good doxygen available on this system; ' + \ 210 | 'Your command line arguments\n\trequire it to be present. ' + \ 211 | 'Consider dropping the "doc" and "doc-build" options\n\t or ' + \ 212 | 'specify a path to doxygen with the --doxygen knob.\n\n\n' 213 | return (1, [msg]) # failure 214 | else: 215 | env['DOXYGEN'] = doxygen_cmd 216 | 217 | try: 218 | okay = _customize_doxygen_file(env, subs) 219 | except: 220 | base.die("CUSTOMIZE DOXYGEN INPUT FILE FAILED") 221 | if not okay: 222 | return (1, ['Doxygen customization failed']) 223 | 224 | cmd = env['DOXYGEN'] + ' ' + \ 225 | env.escape_string(env['doxygen_config_customized']) 226 | if base.verbose(2): 227 | base.msgb("RUN DOXYGEN", cmd) 228 | (retval, output, error_output) = util.run_command(cmd) 229 | 230 | for line in output: 231 | base.msgb("DOX",line.rstrip()) 232 | if error_output: 233 | for line in error_output: 234 | base.msgb("DOX-ERROR",line.rstrip()) 235 | if retval != 0: 236 | base.msgb("DOXYGEN FAILED") 237 | base.die("Doxygen run failed. Retval=", str(retval)) 238 | util.touch(dummy_file) 239 | base.msgb("DOXYGEN","succeeded") 240 | return (0, []) # success 241 | 242 | 243 | ########################################################################### 244 | # Doxygen build 245 | ########################################################################### 246 | def _empty_dir(d): 247 | """return True if the directory d does not exist or if it contains no 248 | files/subdirectories.""" 249 | if not os.path.exists(d): 250 | return True 251 | for (root, subdirs, subfiles) in os.walk(d): 252 | if len(subfiles) or len(subdirs): 253 | return False 254 | return True 255 | 256 | def _make_doxygen_reference_manual(env, doxygen_inputs, subs, work_queue, 257 | hash_file_name='dox'): 258 | """Install the doxygen reference manual the doyxgen_output_dir 259 | directory. doxygen_inputs is a list of files """ 260 | 261 | dox_dag = dag.dag_t(hash_file_name,env=env) 262 | 263 | # so that the scanner can find them 264 | dirs = {} 265 | for f in doxygen_inputs: 266 | dirs[os.path.dirname(f)]=True 267 | for d in dirs.keys(): 268 | env.add_include_dir(d) 269 | 270 | # make sure the config and top file are in the inptus list 271 | doxygen_inputs.append(env['doxygen_config']) 272 | doxygen_inputs.append(env['doxygen_top_src']) 273 | 274 | dummy = env.build_dir_join('dummy-doxygen-' + hash_file_name) 275 | 276 | # Run it via the builder to make it dependence driven 277 | run_always = False 278 | if _empty_dir(env['doxygen_install']): 279 | run_always = True 280 | 281 | if run_always: 282 | _build_doxygen_main([subs,dummy], env) 283 | else: 284 | c1 = plan.plan_t(command=_build_doxygen_main, 285 | args= [subs,dummy], 286 | env= env, 287 | input= doxygen_inputs, 288 | output= dummy) 289 | dox1 = dox_dag.add(env,c1) 290 | 291 | okay = work_queue.build(dag=dox_dag) 292 | phase = "DOXYGEN" 293 | if not okay: 294 | base.die("[%s] failed. dying..." % phase) 295 | if base.verbose(2): 296 | base.msgb(phase, "build succeeded") 297 | 298 | 299 | ############################################################ 300 | 301 | def doxygen_env(env): 302 | """Add the doxygen variables to the environment""" 303 | doxygen_defaults = dict( doxygen_config='', 304 | doxygen_top_src='', 305 | doxygen_install='', 306 | doxygen_cmd='' ) 307 | env.update_dict(doxygen_defaults) 308 | 309 | def doxygen_args(env): 310 | """Add the knobs to the command line knobs parser""" 311 | 312 | env.parser.add_option("--doxygen-install", 313 | dest="doxygen_install", 314 | action="store", 315 | default='', 316 | help="Doxygen installation directory") 317 | 318 | env.parser.add_option("--doxygen-config", 319 | dest="doxygen_config", 320 | action="store", 321 | default='', 322 | help="Doxygen config file") 323 | 324 | env.parser.add_option("--doxygen-top-src", 325 | dest="doxygen_top_src", 326 | action="store", 327 | default='', 328 | help="Doxygen top source file") 329 | 330 | env.parser.add_option("--doxygen-cmd", 331 | dest="doxygen_cmd", 332 | action="store", 333 | default='', 334 | help="Doxygen command name") 335 | 336 | 337 | def doxygen_run(env, inputs, subs, work_queue, hash_file_name='dox'): 338 | """Run doxygen assuming certain values are in the environment env. 339 | 340 | @type env: env_t 341 | @param env: the environment 342 | 343 | @type inputs: list 344 | @param inputs: list of input files to scan for dependences 345 | 346 | @type subs: dictionary 347 | @param subs: replacements in the config file 348 | 349 | @type work_queue: work_queue_t 350 | @param work_queue: a work queue for the build 351 | 352 | @type hash_file_name: string 353 | @param hash_file_name: used for the dummy file and mbuild hash suffix 354 | """ 355 | _make_doxygen_reference_manual(env, inputs, subs, work_queue, hash_file_name) 356 | 357 | 358 | 359 | 360 | 361 | -------------------------------------------------------------------------------- /mbuild/build_env.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- python -*- 3 | #BEGIN_LEGAL 4 | # 5 | #Copyright (c) 2025 Intel Corporation 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | #END_LEGAL 20 | 21 | """Setup functions for the ms/gnu compiler environment""" 22 | 23 | import os 24 | import sys 25 | import platform 26 | from .base import * 27 | from .util import * 28 | from .env import * 29 | from . import msvs 30 | 31 | 32 | def set_compiler_env_common(env): 33 | """Set up some common stuff that depends heavily on the compiler setting""" 34 | 35 | # This whole section was really an experiment in how dynamically I 36 | # could do substitutions. 37 | 38 | env['debug_flag'] = ( 'debug', { True: '%(DEBUGFLAG)s', 39 | False:''}) 40 | env['debug_flag_link'] = ( 'debug', { True: '%(DEBUGFLAG_LINK)s', 41 | False:''}) 42 | 43 | win_shared_compile_dict = ('compiler', 44 | {'ms': ('mbuild_mscrt', {False: '', True: ('debug', {True: '/MDd', False: '/MD'})}), 45 | 'icl': ('mbuild_mscrt', {False: '', True: ('debug', {True: '/MDd', False: '/MD'})}), 46 | 'otherwise': '', 47 | }) 48 | 49 | shared_compile_dict = ( 'host_os', { 'android': '-fPIC', 50 | 'lin': '-fPIC', 51 | 'win': win_shared_compile_dict, 52 | 'bsd': '-fPIC', 53 | 'otherwise': '', 54 | }) 55 | 56 | win_static_compile_dict = ( 'compiler', 57 | {'ms': ('mbuild_mscrt', {False: '', True: ('debug', {True: '/MTd', False: '/MT'})}), 58 | 'icl': ('mbuild_mscrt', {False: '', True: ('debug', {True: '/MTd', False: '/MT'})}), 59 | 'otherwise': '', 60 | }) 61 | 62 | static_compile_dict = ( 'host_os', { 'android': '', 63 | 'lin': '', 64 | 'win': win_static_compile_dict, 65 | 'bsd': '', 66 | 'otherwise': '', 67 | }) 68 | 69 | env['shared_compile_flag'] = ( 'shared', { True: shared_compile_dict, 70 | False: static_compile_dict 71 | }) 72 | 73 | shared_link_dict = ('compiler', { 'ms':'/dll', 74 | 'icl':'/dll', 75 | 'icc':'-shared', 76 | 'icx':'-shared', 77 | 'gnu':'-shared',}) 78 | 79 | env['shared_link'] = ( 'shared', { True: shared_link_dict, 80 | False:''}) 81 | 82 | env['OPTOPT'] = ( 'compiler', { 'gnu':'-O', 83 | 'clang':'-O', 84 | 'iclang':'-O', 85 | 'icc':'-O', 86 | 'icl':'/O', 87 | 'icx':'-O', 88 | 'ms':'/O'}) 89 | 90 | env['nologo'] = ( 'compiler', { 'gnu':'', 91 | 'clang':'', 92 | 'iclang':'', 93 | 'icc':'', 94 | 'icl':'/nologo', 95 | 'icx':'', 96 | 'ms':'/nologo'}) 97 | flags = '' 98 | flags += ' %(debug_flag)s' 99 | flags += ' %(nologo)s' 100 | flags += ' %(opt_flag)s' 101 | flags += ' %(shared_compile_flag)s' 102 | env['CCFLAGS'] = flags 103 | env['CXXFLAGS'] = flags 104 | env['LINKFLAGS'] += ' %(debug_flag_link)s' 105 | 106 | def add_gnu_arch_flags(d): 107 | """Accept a dictionary, return a string""" 108 | if d['compiler'] in ['gnu','clang'] and d['gcc_version'] != '2.96': # FIXME: iclang? 109 | if d['host_cpu'] == 'x86-64': 110 | return '-m64' 111 | elif d['host_cpu'] == 'ia32': 112 | return '-m32' 113 | return '' 114 | 115 | 116 | def set_env_gnu(env): 117 | """Example of setting up the GNU GCC environment for compilation""" 118 | set_compiler_env_common(env) 119 | 120 | env['opt_flag'] = ( 'opt', {'noopt':'', 121 | 's':'%(OPTOPT)ss', 122 | '0':'%(OPTOPT)s0', 123 | '1':'%(OPTOPT)s1', 124 | '2':'%(OPTOPT)s2', 125 | '3':'%(OPTOPT)s3', 126 | '4':'%(OPTOPT)s4'} ) 127 | 128 | # lazy toolchain and other env var (f) expansion 129 | mktool = lambda f: "%(toolchain)s%(" + f + ")s" 130 | 131 | if env['CXX_COMPILER'] == '': 132 | env['CXX_COMPILER'] = ( 'compiler', { 'gnu':'g++', 133 | 'icc':'icpc', 134 | 'icx':'icpx', 135 | 'iclang':'icl++', 136 | 'clang':'clang++'}) 137 | if env['CC_COMPILER'] == '': 138 | env['CC_COMPILER'] = ( 'compiler', { 'gnu':'gcc', 139 | 'icc':'icc', 140 | 'icx':'icx', 141 | 'iclang':'icl', 142 | 'clang':'clang' }) 143 | if env['ASSEMBLER'] == '': 144 | env['ASSEMBLER'] = ( 'compiler', { 'gnu':'gcc', 145 | 'icc':'icc', 146 | 'icx':'icx', 147 | 'iclang':'icl', 148 | 'clang':'yasm' }) 149 | 150 | if env['LINKER'] == '': 151 | env['LINKER'] = '%(CXX_COMPILER)s' # FIXME C++ or C? 152 | if env['ARCHIVER'] == '': 153 | env['ARCHIVER'] = ( 'compiler', { 'gnu': 'ar', # or GAR?? 154 | 'icc' : 'xiar', 155 | 'icx' : 'xiar', 156 | 'iclang' : 'xiar', 157 | 'clang':'llvm-ar' }) 158 | if env['RANLIB_CMD'] == '': 159 | env['RANLIB_CMD'] = 'ranlib' 160 | 161 | if env['CC'] == '': 162 | env['CC'] = mktool('CC_COMPILER') 163 | if env['CXX'] == '': 164 | env['CXX'] = mktool('CXX_COMPILER') 165 | if env['AS'] == '': 166 | env['AS'] = mktool('ASSEMBLER') 167 | if env['LINK'] == '': 168 | env['LINK'] = mktool('LINKER') 169 | if env['AR'] == '': 170 | env['AR'] = mktool('ARCHIVER') 171 | if env['RANLIB'] == '': 172 | env['RANLIB'] = mktool('RANLIB_CMD') 173 | 174 | # if using gcc to compile include -c. If using gas, omit the -c 175 | env['ASFLAGS'] = ' -c' 176 | 177 | env['ARFLAGS'] = "rcv" 178 | env['STATIC'] = ( 'static', { True : "-static", 179 | False : "" } ) 180 | env['LINKFLAGS'] += " %(STATIC)s" 181 | 182 | env['GNU64'] = add_gnu_arch_flags # dynamically called function during variable expansion! 183 | s = ' %(GNU64)s' 184 | env['CCFLAGS'] += s 185 | env['CXXFLAGS'] += s 186 | env['LINKFLAGS'] += s 187 | # if using gcc to compile use -m64, otherwise if gas is used, omit the -m64. 188 | env['ASFLAGS'] += s 189 | 190 | env['DEBUGFLAG'] = '-g' 191 | env['DEBUGFLAG_LINK'] = '-g' 192 | env['COPT'] = '-c' 193 | env['DOPT'] = '-D' 194 | env['ASDOPT'] = '-D' 195 | env['IOPT'] = '-I' 196 | env['ISYSOPT'] = '-isystem ' # trailing space required 197 | env['LOPT'] = '-L' 198 | 199 | env['COUT'] = '-o ' 200 | env['ASMOUT'] = '-o ' 201 | env['LIBOUT'] = ' ' # nothing when using gar/ar 202 | env['LINKOUT'] = '-o ' 203 | env['EXEOUT'] = '-o ' 204 | if env.on_mac() or env.on_windows(): 205 | env['DLLOPT'] = '-shared' # '-dynamiclib' 206 | else: 207 | env['DLLOPT'] = '-shared -Wl,-soname,%(SOLIBNAME)s' 208 | 209 | env['OBJEXT'] = '.o' 210 | if env.on_windows(): 211 | env['EXEEXT'] = '.exe' 212 | env['DLLEXT'] = '.dll' 213 | env['LIBEXT'] = '.lib' 214 | env['PDBEXT'] = '.pdb' 215 | elif env.on_mac(): 216 | env['EXEEXT'] = '' 217 | env['DLLEXT'] = '.dylib' 218 | env['LIBEXT'] = '.a' 219 | env['PDBEXT'] = '' 220 | else: 221 | env['EXEEXT'] = '' 222 | env['DLLEXT'] = '.so' 223 | env['LIBEXT'] = '.a' 224 | env['PDBEXT'] = '' 225 | 226 | def find_ms_toolchain(env): 227 | if env['msvs_version']: 228 | env['setup_msvc']=True 229 | 230 | if env['vc_dir'] == '' and not env['setup_msvc']: 231 | if 'MSVCDir' in os.environ: 232 | vs_dir = os.environ['MSVCDir'] 233 | if os.path.exists(vs_dir): 234 | env['vc_dir'] = vs_dir 235 | elif 'VCINSTALLDIR' in os.environ: 236 | vc_dir = os.environ['VCINSTALLDIR'] 237 | if os.path.exists(vc_dir): 238 | env['vc_dir'] = vc_dir 239 | msvs7 = os.path.join(env['vc_dir'],"Vc7") 240 | if os.path.exists(msvs7): 241 | env['vc_dir'] = msvs7 242 | elif 'VSINSTALLDIR' in os.environ: 243 | vs_dir = os.environ['VSINSTALLDIR'] 244 | if os.path.exists(vs_dir): 245 | env['vc_dir'] = os.path.join(vs_dir, 'VC') 246 | elif 'MSVCDIR' in os.environ: 247 | vs_dir = os.environ['MSVCDIR'] 248 | if os.path.exists(vs_dir): 249 | env['vc_dir'] = vs_dir 250 | 251 | # Before DEV15, the VCINSTALLDIR was sufficient to find the 252 | # compiler. But with DEV15, they locate the compiler more deeply 253 | # in to the file system and we need more information including the 254 | # build number. The DEV15 installation sets the env var 255 | # VCToolsInstallDir with that information. The headers and 256 | # libraries change location too so relying on VCINTALLDIR is 257 | # insufficient. So if people run with (1) mbuild's setup of DEV15 258 | # or (2) the MSVS command prompt, they should be fine. But 259 | # anything else is probably questionable. 260 | 261 | incoming_setup = True # presume system setup by user 262 | if env['vc_dir'] == '' or env['setup_msvc']: 263 | incoming_setup = False 264 | env['vc_dir'] = msvs.set_msvs_env(env) 265 | 266 | # toolchain is the bin directory of the compiler with a trailing slash 267 | if env['toolchain'] == '': 268 | if incoming_setup: 269 | # relying on user-setup env (say MSVS cmd.exe or vcvars-equiv bat file) 270 | if os.environ['VisualStudioVersion'] in ['15.0','16.0','17.0']: 271 | env['msvs_version'] = str(int(float(os.environ['VisualStudioVersion']))) 272 | msvs.set_msvc_compilers(env, os.environ['VCToolsInstallDir']) 273 | if env['compiler']=='ms': 274 | env['toolchain'] = msvs.pick_compiler(env) 275 | 276 | 277 | 278 | def _check_set_rc(env, sdk): 279 | def _path_check_rc_cmd(env): 280 | if os.path.exists(env.expand('%(RC_CMD)s')): 281 | return True 282 | return False 283 | 284 | if env['host_cpu'] == 'x86-64': 285 | env['RC_CMD'] = os.path.join(sdk,'x64','rc.exe') 286 | else: 287 | env['RC_CMD'] = os.path.join(sdk,'x86','rc.exe') 288 | if not _path_check_rc_cmd(env): 289 | env['RC_CMD'] = os.path.join(sdk,'rc.exe') 290 | return _path_check_rc_cmd(env) 291 | 292 | 293 | def _find_rc_cmd(env): 294 | r""" 295 | Finding the rc executable is a bit of a nightmare. 296 | 297 | In MSVS2005(VC8): 298 | C:/Program Files (x86)/Microsoft Visual Studio 8/VC 299 | bin/rc.exe 300 | or 301 | PlatformSDK/Bin/win64/AMD64/rc.exe 302 | which is $VCINSTALLDIR/bin or 303 | $VCINSTALLDIR/PlatformSDK/bin/win64/AMD64 304 | We do not bother attempting to find that version of rc. 305 | Put it on your path or set env['RC_CMD'] if you need it. 306 | 307 | In MSVS2008(VC9), MSVS2010 (VC10) and MSVS2012 (VC11): 308 | have rc.exe in the SDK directory, though the location varies 309 | a little for the 32b version. 310 | 311 | With winsdk10 (used by MSVS2017/DEV15), rc.exe moved around from 312 | version to version of the sdk. In the early versions of the SDK, 313 | the rc.exe is located in: 314 | 315 | C:\Program Files (x86)\Windows Kits\10\bin\{x86,x64} 316 | 317 | However, in later versions (starting with 10.0.16299.0), they 318 | placed the rc.exe in the numbered subdirectory: 319 | 320 | C:\Program Files (x86)\Windows Kits\10\bin\10.0.16299.0\{x86,x64} 321 | """ 322 | sdks = [] # list of directories to search 323 | 324 | def _add_bin(s): 325 | return os.path.join(s,'bin') 326 | 327 | if 'rc_winkit_number' in env: # set up by msvs.py for dev14, dev15 328 | p = "{}/bin/{}".format( env['rc_winkit'], 329 | env['rc_winkit_number']) 330 | sdks.append(p) 331 | 332 | if 'rc_winkit' in env: # set up by msvs.py for dev14, dev15 333 | sdks.append(_add_bin(env['rc_winkit'])) 334 | pass 335 | 336 | if 'WindowsSdkDir' in env: 337 | sdks.append( _add_bin(env['WindowsSdkDir'])) 338 | elif 'WindowsSdkDir' in os.environ: 339 | sdks.append( _add_bin(os.environ['WindowsSdkDir'])) 340 | 341 | for k in sdks: 342 | if _check_set_rc(env,k): 343 | # found a good one...work with that. 344 | return 345 | 346 | if env['host_cpu'] == 'x86-64': 347 | warn("Could not find 64b RC command in SDK directory; assuming on PATH") 348 | else: 349 | warn("Could not find 32b RC command in SDK directory; assuming on PATH") 350 | # hope the user puts the location of RC on their PATH 351 | env['RC_CMD'] = 'rc' 352 | 353 | 354 | 355 | def set_env_ms(env): 356 | """Example of setting up the MSVS environment for compilation""" 357 | set_compiler_env_common(env) 358 | 359 | # FIXME: allow combinations of options 360 | env['opt_flag'] = ( 'opt', {'noopt':'', 361 | '0':'%(OPTOPT)sd', 362 | '1':'%(OPTOPT)s1', 363 | '2':'%(OPTOPT)s2', 364 | '3':'%(OPTOPT)s2', # map O3 and O4 to O2 365 | '4':'%(OPTOPT)s2', # map O3 and O4 to O2 366 | 'b':'%(OPTOPT)sb', 367 | 'i':'%(OPTOPT)si', 368 | 's':'%(OPTOPT)ss', 369 | 'x':'%(OPTOPT)sx', 370 | 'd':'%(OPTOPT)sd', 371 | 'g':'%(OPTOPT)sg'} ) 372 | 373 | env['ASFLAGS'] = '/c /nologo ' 374 | if env['compiler'] != 'icx': 375 | env['LINKFLAGS'] += ' /nologo' 376 | env['ARFLAGS'] = '/nologo' 377 | 378 | env['link_prefix'] = ('use_compiler_to_link', { True:'/link', 379 | False:'' }) 380 | if env['host_cpu'] == 'ia32': 381 | env['LINKFLAGS'] += ' %(link_prefix)s ' 382 | if env['compiler'] != 'icx': 383 | env['LINKFLAGS'] += ' /MACHINE:X86 ' 384 | env['ARFLAGS'] += ' /MACHINE:X86' 385 | elif env['host_cpu'] == 'x86-64': 386 | env['LINKFLAGS'] += ' %(link_prefix)s ' 387 | if env['compiler'] != 'icx': 388 | env['LINKFLAGS'] += ' /MACHINE:X64 ' 389 | env['ARFLAGS'] += ' /MACHINE:X64' 390 | 391 | env['favor'] = ( 'compiler', { 'ms' : ' /favor:EM64T', 392 | 'otherwise' : '' }) 393 | env['CXXFLAGS'] += ' %(favor)s' 394 | env['CCFLAGS'] += ' %(favor)s' 395 | 396 | elif env['host_cpu'] == 'ipf': 397 | env['LINKFLAGS'] += ' %(link_prefix)s /MACHINE:IA64' 398 | env['ARFLAGS'] += ' /MACHINE:IA64' 399 | 400 | env['COPT'] = '/c' 401 | env['DOPT'] = '/D' 402 | env['ASDOPT'] = '/D' 403 | 404 | # I use '-I' instead of '/I' because it simplifies use of YASM 405 | # which requires -I for includes. 406 | env['IOPT'] = '-I' # -I or /I works with MSVS8. 407 | env['ISYSOPT'] = '-I' # MSVS has not -isystem so we use -I 408 | env['LOPT'] = '%(link_prefix)s /LIBPATH:' 409 | 410 | 411 | # Some options differ when using the compiler to link programs. 412 | # Note: /Zi has parallel-build synchronization bugs 413 | env['DEBUGFLAG'] = '/Z7' 414 | env['DEBUGFLAG_LINK'] = ('use_compiler_to_link', { True:'/Z7', # of /Zi 415 | False:'/debug'}) 416 | if env['compiler'] != 'icx': 417 | env['COUT'] = '/Fo' 418 | else: 419 | env['COUT'] = '-o ' 420 | env['ASMOUT'] = '/Fo' 421 | env['LIBOUT'] = '/out:' 422 | env['EXEOUT'] = '/Fe' 423 | if env['compiler'] != 'icx': 424 | env['LINKOUT'] = ('use_compiler_to_link',{ True:'/Fo', 425 | False:'/OUT:'}) 426 | else: 427 | env['LINKOUT'] = '-o ' 428 | env['DLLOPT'] = '/dll' 429 | env['OBJEXT'] = '.obj' 430 | env['LIBEXT'] = '.lib' 431 | env['DLLEXT'] = '.dll' 432 | env['EXEEXT'] = '.exe' 433 | env['PDBEXT'] = '.pdb' 434 | env['PDBEXT'] = '.pdb' 435 | env['RCEXT'] = '.rc' 436 | env['RESEXT'] = '.res' 437 | 438 | find_ms_toolchain(env) 439 | 440 | if env['ASSEMBLER'] == '': 441 | if env['host_cpu'] == 'ia32': 442 | env['ASSEMBLER'] = 'ml.exe' 443 | elif env['host_cpu'] == 'x86-64': 444 | env['ASSEMBLER'] = 'ml64.exe' 445 | 446 | if env['CXX_COMPILER'] == '': 447 | env['CXX_COMPILER'] = ( 'compiler', { 'ms':'cl.exe', 448 | 'icl':'icl.exe', 449 | 'icx':'icpx.exe'}) 450 | if env['CC_COMPILER'] == '': 451 | env['CC_COMPILER'] = ( 'compiler', { 'ms':'cl.exe', 452 | 'icl':'icl.exe', 453 | 'icx':'icx.exe'}) 454 | if env['LINKER'] == '': 455 | env['LINKER'] = ( 'compiler', { 'ms': 'link.exe', 456 | 'icl' :'xilink.exe', 457 | 'icx':'icpx.exe'}) 458 | 459 | # old versions of RC do not accept the /nologo switch 460 | env['rcnologo'] = ( 'msvs_version', { 'otherwise':' /nologo', 461 | '6':'', 462 | '7':'', 463 | '8':'', 464 | '9':'' }) 465 | env['RCFLAGS'] = " %(rcnologo)s" 466 | 467 | 468 | if env['RC_CMD'] == '': 469 | _find_rc_cmd(env) 470 | 471 | if env['RC'] == '': 472 | env['RC'] = quote('%(RC_CMD)s') 473 | 474 | if env['ARCHIVER'] == '': 475 | env['ARCHIVER'] =( 'compiler', { 'ms': 'lib.exe', 476 | 'icl' : 'xilib.exe'}) 477 | # lazy toolchain and other env var (f) expansion 478 | mktool = lambda f: "%(toolchain)s%(" + f + ")s" 479 | 480 | if env['CXX'] == '': 481 | env['CXX'] = quote(mktool('CXX_COMPILER')) 482 | if env['CC'] == '': 483 | env['CC'] = quote(mktool('CC_COMPILER')) 484 | if env['AS'] == '': 485 | env['AS'] = quote(mktool('ASSEMBLER')) 486 | if env['LINK'] == '': 487 | env['LINK'] = quote(mktool('LINKER')) 488 | if env['AR'] == '': 489 | env['AR'] = quote(mktool('ARCHIVER')) 490 | 491 | 492 | 493 | def yasm_support(env): 494 | """Initialize the YASM support based on the env's host_os and host_cpu""" 495 | # FIXME: android??? 496 | yasm_formats={} 497 | yasm_formats['win'] = { 'ia32': 'win32', 'x86-64': 'win64'} 498 | yasm_formats['lin'] = { 'ia32': 'elf32', 'x86-64': 'elf64'} 499 | yasm_formats['bsd'] = { 'ia32': 'elf32', 'x86-64': 'elf64'} 500 | yasm_formats['mac'] = { 'ia32': 'macho32', 'x86-64': 'macho64'} 501 | env['ASDOPT']='-D' 502 | try: 503 | env['ASFLAGS'] = ' -f' + yasm_formats[env['host_os']][env['host_cpu']] 504 | env['ASMOUT'] = '-o ' 505 | env['AS'] = 'yasm' 506 | except: 507 | die("YASM does not know what format to use for build O/S: %s and target CPU: %s" % 508 | (env['host_os'], env['host_cpu'])) 509 | 510 | 511 | 512 | def set_env_clang(env): 513 | set_env_gnu(env) 514 | 515 | 516 | def set_env_icc(env): 517 | """Example of setting up the Intel ICC environment for compilation""" 518 | set_env_gnu(env) 519 | 520 | def set_env_icx(env): 521 | """Example of setting up the Intel ICX environment for compilation""" 522 | if env.on_windows(): 523 | set_env_ms(env) 524 | else: 525 | set_env_gnu(env) 526 | 527 | def set_env_iclang(env): 528 | """Example of setting up the Intel iclang (aka mac icl) environment for compilation""" 529 | set_env_gnu(env) 530 | 531 | def set_env_icl(env): 532 | """Example of setting up the Intel ICL (windows) environment for compilation""" 533 | set_env_ms(env) 534 | -------------------------------------------------------------------------------- /mbuild/work_queue.py: -------------------------------------------------------------------------------- 1 | # -*- python -*- 2 | #BEGIN_LEGAL 3 | # 4 | #Copyright (c) 2024 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | #END_LEGAL 19 | 20 | """Command objects and parallel work queue""" 21 | from __future__ import print_function 22 | import os 23 | import sys 24 | import types 25 | is_py2 = sys.version[0] == '2' 26 | if is_py2: 27 | import Queue as queue 28 | else: 29 | import queue as queue 30 | from threading import Thread 31 | from collections import deque 32 | 33 | from .base import * 34 | from .util import * 35 | from .dag import * 36 | 37 | 38 | ############################################################################ 39 | class dir_cmd_t(object): 40 | """For holding a directory and a command. When you call 41 | execute(), it changes to the directory an executes the command""" 42 | 43 | def __init__(self, dir, command, output_file=None): 44 | self.dir= dir 45 | self.command= command 46 | self.output_file = output_file 47 | def __str__(self): 48 | return "DIR: %s\nCOMMAND: %s" % (self.dir, self.command) 49 | 50 | def execute(self,args=None, env=None): 51 | """Change to the specified directory and execute the command, 52 | unbufferred""" 53 | orig = os.getcwd() 54 | try: 55 | msgb("CHDIR TO", self.dir) 56 | os.chdir(self.dir) 57 | except: 58 | return (-1, ["no such dir: " + self.dir]) 59 | msgb("EXECUTING", self.command) 60 | if self.output_file: 61 | (retcode, out, err) = \ 62 | run_command_output_file(self.command, self.output_file) 63 | msgb("WROTE", self.output_file) 64 | else: 65 | (retcode, out, err) = run_command_unbufferred(self.command) 66 | os.chdir(orig) 67 | if not err: 68 | err = [] 69 | if not out: 70 | out = [] 71 | if err: 72 | return (retcode, out+err) 73 | else: 74 | return (retcode, out) 75 | 76 | class command_t(object): 77 | """The primary data structure used to track jobs in this script. It 78 | is created when you add L{plan_t} objects to the DAG 79 | L{dag_t}.""" 80 | 81 | _ids = 0 82 | 83 | def __init__(self, 84 | command=None, 85 | args=None, 86 | xenv=None, 87 | unbufferred=False, 88 | output_file_name=None, 89 | shell_executable=None, 90 | directory=None, 91 | name=None, 92 | show_output=True, 93 | osenv=None, 94 | seconds=0, 95 | input_file_name=None): 96 | """ 97 | This is the unit of work for the L{work_queue_t}. These are 98 | typically created by the L{dag_t} but they can also be created 99 | by hand and added to the L{work_queue_t} to execute arbitrary 100 | commands. 101 | 102 | @type command: string or python function, or a list of both 103 | @param command: command line string to execute or a python function 104 | 105 | @type args: anything 106 | @param args: (optional) typically a list of arguments for the python function. 107 | 108 | @type xenv: L{env_t} 109 | @param xenv: (optional) environment for used by the python 110 | function. Passed as the second argument to the python function. 111 | 112 | @type osenv: dictionary 113 | @param osenv: (optional) the environment that will be set in the new subprocess. 114 | 115 | @type unbufferred: L{bool} 116 | @param unbufferred: (optional) true if the output should be unbufferred. 117 | 118 | @type output_file_name: string 119 | @param output_file_name: (optional) file name for stderr/stdout 120 | 121 | @type show_output: L{bool} 122 | @param show_output: (optional) show output, default True 123 | 124 | @type input_file_name: string 125 | @param input_file_name: (optional) file name for stdin 126 | 127 | """ 128 | self.id = command_t._ids 129 | command_t._ids += 1 130 | # store the command as a list 131 | if isinstance(command,list): 132 | self.command = command 133 | else: 134 | self.command = [ command ] 135 | self.name = name 136 | self.shell_executable = shell_executable 137 | self.args = args 138 | self.xenv = xenv 139 | self.osenv = osenv 140 | self.exit_status = 0 141 | self.output = [] 142 | self.stderr = [] 143 | self.unbufferred = unbufferred 144 | self.input_file_name = input_file_name 145 | self.output_file_name = output_file_name 146 | self.start_time = 0 147 | self.end_time = 0 148 | self.directory = directory 149 | self.show_output = show_output 150 | self.input_file_name = input_file_name 151 | 152 | # Has this command be submitted to the work queue? 153 | self.submitted = False 154 | 155 | # executed is set to True when this command tries to execute. 156 | self.executed = False 157 | 158 | # all prerequisite commands are ready 159 | self.ready = False 160 | 161 | # completed is set to True when this command exits successfully. 162 | self.completed = False 163 | 164 | # things that depend on this command completing sucessfully 165 | self.after_me = [] 166 | 167 | # things that must complete before this command can run 168 | self.before_me = [] 169 | 170 | # from the file DAG. A list of inputs upon which this command depends 171 | self.inputs = [] 172 | # from the file DAG. A list of things generated by this command 173 | self.targets = [] 174 | 175 | # used for special signals to the worker threads to tell them to 176 | # shut down. 177 | self.terminator = False 178 | self.timeout = seconds 179 | 180 | def failed(self): 181 | """ 182 | Return the exit status. 183 | @rtype: bool 184 | @return: True if the command failed (exit status != 0) 185 | """ 186 | if self.exit_status != 0: 187 | return True 188 | return False 189 | 190 | def _complete(self): 191 | self.completed = True 192 | 193 | def _ready(self): 194 | """Return true if all things that must execute before this node 195 | have completed and false otherwise. Updates self.ready.""" 196 | if self.ready: 197 | return True 198 | 199 | for n in self.before_me: 200 | if not n.completed: 201 | return False 202 | 203 | self.ready=True 204 | return True 205 | 206 | def is_python_command(self, i=0): 207 | """Return true if the command list element is a python function 208 | @rtype: bool 209 | """ 210 | if isinstance(self.command[i],types.FunctionType): 211 | return True 212 | return False 213 | 214 | def is_dir_cmd(self, i=0): 215 | """Return true if the command list element is a python dir_cmd_t object 216 | @rtype: bool 217 | """ 218 | if isinstance(self.command[i],dir_cmd_t): 219 | return True 220 | return False 221 | 222 | def has_python_subcommand(self): 223 | """Return true if the command list has a python function 224 | @rtype: bool 225 | """ 226 | for c in self.command: 227 | if isinstance(c,types.FunctionType): 228 | return True 229 | return False 230 | 231 | def is_command_line(self, i=0): 232 | """Return true if the command list element is normal string command 233 | line. 234 | @rtype: bool 235 | """ 236 | if not isinstance(self.command[i],types.FunctionType) and \ 237 | not isinstance(self.command[i],dir_cmd_t): 238 | return True 239 | return False 240 | 241 | def dagkey(self): 242 | s = [] 243 | for i in self.command: 244 | if not isinstance(i,types.FunctionType): 245 | s.append(i) 246 | t = "MBUILD_COMMAND_KEY " + (" - ".join(s)) 247 | return t 248 | 249 | def hash(self): 250 | s = [] 251 | for i in self.command: 252 | if not isinstance(i,types.FunctionType): 253 | s.append(i) 254 | t = " - ".join(s) 255 | h = hash_string(t.encode(unicode_encoding())) 256 | return h 257 | 258 | def add_before_me(self,n): 259 | """Make the current command execute after command n 260 | @type n: L{command_t} 261 | @param n: another (earlier) command 262 | """ 263 | if isinstance(n,list): 264 | for x in n: 265 | self.before_me.append(x) 266 | x.after_me.append(self) 267 | else: 268 | self.before_me.append(n) 269 | n.after_me.append(self) 270 | 271 | def add_after_me(self,n): 272 | """Make the current command execute before command n. 273 | @type n: L{command_t} 274 | @param n: another (later) command 275 | """ 276 | if isinstance(n, list): 277 | for x in n: 278 | self.after_me.append(x) 279 | x.before_me.append(self) 280 | else: 281 | self.after_me.append(n) 282 | n.before_me.append(self) 283 | 284 | def _check_afters(self): 285 | """Return a list of after nodes that are as-yet not submitted 286 | but now ready""" 287 | ready = [] 288 | for x in self.after_me: 289 | if not x.submitted and x._ready(): 290 | ready.append(x) 291 | return ready 292 | 293 | def elapsed_time(self): 294 | """Return the elapsed time as an number of seconds""" 295 | if self.end_time == None: 296 | self.end_time = get_time() 297 | return self.end_time - self.start_time 298 | 299 | def elapsed(self): 300 | """Return the elapsed time. 301 | @rtype: string 302 | @returns: the elapsed wall clock time of execution. 303 | """ 304 | if self.end_time == None: 305 | self.end_time = get_time() 306 | elapsed = get_elapsed_time(self.start_time, self.end_time) 307 | return elapsed 308 | 309 | def dump_cmd(self): 310 | return self._pretty_cmd_str() 311 | 312 | def stderr_exists(self): 313 | if self.stderr and len(self.stderr) > 0: 314 | if len(self.stderr) == 1 and len(self.stderr[0]) == 0: 315 | return False 316 | return True 317 | return False 318 | 319 | def stdout_exists(self): 320 | if self.output and len(self.output) > 0: 321 | if len(self.output) == 1 and len(self.output[0]) == 0: 322 | return False 323 | return True 324 | return False 325 | 326 | def _pretty_cmd_str(self): 327 | s = [] 328 | for cmd in self.command: 329 | if isinstance(cmd,types.FunctionType): 330 | s.append("PYTHON FN: " + cmd.__name__) 331 | elif is_stringish(cmd): 332 | s.append(cmd) 333 | else: 334 | s.append(str(cmd)) 335 | return " ;;;; ".join(s) 336 | 337 | 338 | def dump(self, tab_output=False, show_output=True): 339 | s = [] 340 | nl = '\n' 341 | if verbose(2): 342 | pass 343 | elif self.failed(): 344 | pass 345 | elif self.targets: 346 | s.append(bracket('TARGET ', " ".join(self.targets))) 347 | s.append(nl) 348 | if self.name: 349 | s.append(bracket('NAME ', self.name)) 350 | s.append(nl) 351 | if self.command: 352 | s.append(bracket('COMMAND ', self._pretty_cmd_str())) 353 | s.append(nl) 354 | else: 355 | s.append( bracket('COMMAND ', 'none') ) 356 | s.append(nl) 357 | if self.args: 358 | args_string = str(self.args) 359 | print_limit = 400 360 | if len(args_string) > print_limit: 361 | args_string = args_string[:print_limit] 362 | s.append(bracket('ARGS ', args_string)) 363 | s.append(nl) 364 | if self.xenv: 365 | s.append(bracket('ENV ', 'some env')) 366 | s.append(nl) 367 | #if self.submitted: 368 | # s.append(bracket('START_TIME ', self.start_time)) 369 | # s.append(nl) 370 | if self.input_file_name: 371 | s.append(bracket('INPUT_FILE ', self.input_file_name)) 372 | s.append(nl) 373 | 374 | if self.completed or self.failed(): 375 | if self.exit_status != 0: 376 | s.append(bracket('EXIT_STATUS ', str(self.exit_status))) 377 | s.append(nl) 378 | if self.elapsed_time() > 1: 379 | s.append(bracket('ELAPSED_TIME', self.elapsed())) 380 | s.append(nl) 381 | if self.input_file_name: 382 | s.append(bracket('INPUT FILE', self.input_file_name)) 383 | s.append(nl) 384 | if self.output_file_name: 385 | s.append(bracket('OUTPUT FILE', self.output_file_name)) 386 | s.append(nl) 387 | 388 | # stdout and stderr frequently have unicode 389 | s = ensure_string(s) 390 | if self.unbufferred == False and self.output_file_name==None: 391 | if show_output and self.show_output and self.stdout_exists(): 392 | uappend(s,bracket('OUTPUT')) 393 | uappend(s,nl) 394 | for line in self.output: 395 | if tab_output: 396 | uappend(s,'\t') 397 | uappend(s,line) 398 | if show_output and self.show_output and self.stderr_exists(): 399 | uappend(s,bracket('STDERR')) 400 | uappend(s,nl) 401 | 402 | for line in self.stderr: 403 | if tab_output: 404 | uappend(s,'\t') 405 | uappend(s,line) 406 | return u"".join(s) 407 | 408 | def __str__(self): 409 | return self.dump() 410 | 411 | def _extend_output(self, lines): 412 | if lines: 413 | util_add_to_list(self.output,ensure_string(lines)) 414 | 415 | def _extend_stderr(self, lines): 416 | if lines: 417 | util_add_to_list(self.stderr,ensure_string(lines)) 418 | 419 | def _extend_output_stderr(self, output, stderr): 420 | self._extend_output(output) 421 | self._extend_stderr(stderr) 422 | 423 | 424 | def execute(self): 425 | """Execute the command whether it be a python function or a 426 | command string. This is executed by worker threads but is made 427 | available here for potential debugging. Record execution exit/return 428 | status and output. 429 | 430 | Sets the exit_status, output and stderr error fields of the 431 | command object. 432 | """ 433 | self.executed = True 434 | self.start_time = get_time() 435 | self.output = [] 436 | self.stderr = [] 437 | for cmd in self.command: 438 | try: 439 | if isinstance(cmd, dir_cmd_t): 440 | # execute dir_cmd_t objects 441 | (self.exit_status, output) = cmd.execute( self.args, self.xenv ) 442 | self._extend_output(output) 443 | 444 | elif isinstance(cmd,types.FunctionType): 445 | # execute python functions 446 | (self.exit_status, output) = cmd( self.args, self.xenv ) 447 | self._extend_output(output) 448 | 449 | elif is_stringish(cmd): 450 | # execute command strings 451 | if self.output_file_name: 452 | (self.exit_status, output, stderr) = \ 453 | run_command_output_file(cmd, 454 | self.output_file_name, 455 | shell_executable=self.shell_executable, 456 | directory=self.directory, 457 | osenv=self.osenv, 458 | input_file_name=self.input_file_name) 459 | self._extend_output_stderr(output,stderr) 460 | 461 | elif self.unbufferred: 462 | (self.exit_status, output, stderr) = \ 463 | run_command_unbufferred(cmd, 464 | shell_executable= 465 | self.shell_executable, 466 | directory = self.directory, 467 | osenv = self.osenv, 468 | input_file_name=self.input_file_name) 469 | self._extend_output_stderr(output, stderr) 470 | else: 471 | # execute timed_cmd_t objects 472 | (self.exit_status, output, stderr) = \ 473 | run_command_timed(cmd, 474 | shell_executable=self.shell_executable, 475 | directory = self.directory, 476 | osenv = self.osenv, 477 | seconds=self.timeout, 478 | input_file_name = self.input_file_name) 479 | self._extend_output_stderr(output, stderr) 480 | 481 | else: 482 | self.exit_status = 1 483 | self._extend_output("Unhandled command object: " + self.dump()) 484 | 485 | # stop if something failed 486 | if self.exit_status != 0: 487 | break; 488 | except Exception as e: 489 | self.exit_status = 1 490 | self._extend_stderr(u"Execution error for: %s\n%s" % (str(e), self.dump())) 491 | break 492 | 493 | self.end_time = get_time() 494 | 495 | 496 | 497 | def _worker_one_task(incoming,outgoing): 498 | """A thread. Takes stuff from the incoming queue and puts stuff on 499 | the outgoing queue. calls execute for each command it takes off the 500 | in queue. Return False when we receive a terminator command""" 501 | #msgb("WORKER WAITING") 502 | item = incoming.get() 503 | #msgb("WORKER GOT A TASK") 504 | if item.terminator: 505 | outgoing.put(item) 506 | return False 507 | item.execute() 508 | incoming.task_done() 509 | outgoing.put(item) 510 | return True 511 | 512 | def _worker(incoming,outgoing): 513 | """A thread. Takes stuff from the incoming queue and puts stuff on 514 | the outgoing queue. calls execute for each command it takes off the 515 | in queue. Return when we get a terminator command""" 516 | keep_going = True 517 | while keep_going: 518 | keep_going = _worker_one_task(incoming, outgoing) 519 | 520 | class work_queue_t(object): 521 | """This stores the threads and controls their execution""" 522 | def __init__(self, max_parallelism=4): 523 | """ 524 | @type max_parallelism: int 525 | @param max_parallelism: the number of worker threads to start 526 | """ 527 | max_parallelism = int(max_parallelism) 528 | if max_parallelism <= 0: 529 | die("Bad value for --jobs option: " + str(max_parallelism)) 530 | self.max_parallelism = max_parallelism 531 | self.use_threads = True 532 | self.threads = [] 533 | 534 | # worker threads can add stuff to the new_queue so we 535 | # use an MT-safe queue. 536 | self.new_queue = queue.Queue(0) 537 | self.out_queue = queue.Queue(0) 538 | self.back_queue = queue.Queue(0) 539 | self.pending_commands = deque() 540 | 541 | self.message_delay = 10 542 | self.min_message_delay = 10 543 | self.message_delay_delta = 10 544 | 545 | self.job_num = 0 546 | self.pending = 0 547 | self._clean_slate() 548 | 549 | if self.use_threads: 550 | if len(self.threads) == 0: 551 | self._start_daemons() 552 | 553 | def _empty_queue(self, q): 554 | while not q.empty(): 555 | item = q.get_nowait() 556 | 557 | def _cleanup(self): 558 | """After a failed build we want to clean up our any in-progress state 559 | so we can re-use the work queue object""" 560 | 561 | # the new_queue, job_num and pending get updated by add() before we build. 562 | # so we must clean them up after every build. Also good hygene to clean out 563 | # the task queues that we use to talk to the workers. 564 | self.pending_commands = deque() 565 | self._empty_queue(self.new_queue) 566 | self._empty_queue(self.out_queue) 567 | self._empty_queue(self.back_queue) 568 | self.job_num = 0 569 | self.pending = 0 570 | 571 | def _clean_slate(self): 572 | self.running_commands = [] 573 | self.all_commands = [] 574 | self.running = 0 575 | self.sent = 0 576 | self.finished = 0 577 | self.errors = 0 578 | self.dag = None 579 | 580 | # for message limiting in _status() 581 | self.last_time = 0 582 | self.last_pending = 0 583 | self.last_finished = 0 584 | self.last_running = 0 585 | 586 | self.start_time = get_time() 587 | self.end_time = None 588 | 589 | # we set dying to to True when we are trying to stop because of an error 590 | self.dying = False 591 | 592 | self._empty_queue(self.out_queue) 593 | self._empty_queue(self.back_queue) 594 | 595 | 596 | def clear_commands(self): 597 | """Remove any previously remembered commands""" 598 | self.all_commands = [] 599 | def commands(self): 600 | """Return list of all commands involved in last build""" 601 | return self.all_commands 602 | 603 | def elapsed_time(self): 604 | """Return the elapsed time as an a number""" 605 | if self.end_time == None: 606 | self.end_time = get_time() 607 | return self.end_time - self.start_time 608 | 609 | def elapsed(self): 610 | """Return the elapsed time as a pretty string 611 | @rtype: string 612 | @returns: the elapsed wall clock time of execution. 613 | """ 614 | if self.end_time == None: 615 | self.end_time = get_time() 616 | elapsed = get_elapsed_time(self.start_time, self.end_time) 617 | return elapsed 618 | 619 | def _terminate(self): 620 | """Shut everything down. Kill the worker threads if any were 621 | being used. This is called when the work_queue_t is garbage 622 | collected, but can be called directly.""" 623 | self.dying = True 624 | if self.use_threads: 625 | self._stop_daemons() 626 | self._join_threads() 627 | 628 | def _start_daemons(self): 629 | """Start up a bunch of daemon worker threads to process jobs from 630 | the queue.""" 631 | for i in range(self.max_parallelism): 632 | t = Thread(target=_worker, args=(self.out_queue, self.back_queue)) 633 | t.setDaemon(True) 634 | t.start() 635 | self.threads.append(t) 636 | 637 | def _stop_daemons(self): 638 | """Send terminator objects to all the workers""" 639 | for i in range(self.max_parallelism): 640 | t = command_t() 641 | t.terminator = True 642 | if verbose(4): 643 | msgb("SENT TERMINATOR", str(i)) 644 | self._start_a_job(t) 645 | 646 | def _join_threads(self): 647 | """Use this when not running threads in daemon-mode""" 648 | for t in self.threads: 649 | t.join() 650 | if verbose(4): 651 | msgb("WORKER THREAD TERMINATED") 652 | self.threads = [] 653 | 654 | def _add_one(self,command): 655 | """Add a single command of type L{command_t} to the list 656 | of jobs to run.""" 657 | # FIXME: make this take a string and build a command_t 658 | 659 | if command.completed: 660 | if verbose(5): 661 | msgb("SKIPPING COMPLETED CMD", str(command.command)) 662 | msgb("SKIPPING COMPLETED CMD", str(command.command)) 663 | self.add(command._check_afters()) 664 | return 665 | if command.submitted: 666 | if verbose(5): 667 | msgb("SKIPPING SUBMITTED CMD", str(command.command)) 668 | msgb("SKIPPING SUBMITTED CMD", str(command.command)) 669 | return 670 | command.submitted = True 671 | if verbose(6): 672 | msgb("WQ ADDING", str(command.command)) 673 | self.job_num += 1 674 | self.new_queue.put( command ) 675 | self.pending += 1 676 | 677 | def add_sequential(self,command_strings, unbufferred=False): 678 | """ 679 | Add a list of command strings as sequential tasks to the work queue. 680 | 681 | @type command_strings: list of strings 682 | @param command_strings: command strings to add to the L{work_queue_t} 683 | 684 | @rtype: list of L{command_t} 685 | @return: the commands created 686 | """ 687 | last_cmd = None 688 | cmds = [] 689 | for c in command_strings: 690 | co = command_t(c, unbufferred=unbufferred) 691 | cmds.append(co) 692 | self.add(co) 693 | if last_cmd: 694 | last_cmd.add_after_me(co) 695 | last_cmd = co 696 | return cmds 697 | 698 | def add(self,command): 699 | """Add a command or list of commands of type L{command_t} 700 | to the list of jobs to run. 701 | 702 | @type command: L{command_t} 703 | @param command: the command to run 704 | """ 705 | if verbose(5): 706 | msgb("ADD CMD", str(type(command))) 707 | 708 | if command: 709 | if isinstance(command,list): 710 | for c in command: 711 | if verbose(5): 712 | msgb("ADD CMD", str(type(c))) 713 | self._add_one(c) 714 | else: 715 | self._add_one(command) 716 | 717 | def _done(self): 718 | if self.running > 0: 719 | return False 720 | if not self.dying and self.pending > 0: 721 | return False 722 | return True 723 | 724 | def _status(self): 725 | if self.show_progress or verbose(2): 726 | s = ( '[STATUS] RUNNING: %d PENDING: %d COMPLETED: %d ' + 727 | 'ERRORS: %d ELAPSED: %s %s' ) 728 | s = ( 'R: %d P: %d C: %d E: %d / %s %s' ) 729 | cur_time = get_time() 730 | 731 | changed = False 732 | if (self.running != self.last_running or 733 | self.pending != self.last_pending or 734 | self.finished != self.last_finished): 735 | changed = True 736 | 737 | if (changed or 738 | # have we waited sufficiently long? 739 | cur_time >= self.last_time + self.message_delay): 740 | 741 | # speed back up when anything finishes 742 | if self.finished != self.last_finished: 743 | self.message_delay = self.min_message_delay 744 | elif self.last_time != 0: 745 | # only printing because of timeout delay, so 746 | # we increase the time a little bit. 747 | self.message_delay += self.min_message_delay 748 | 749 | # store the other limiters for next time 750 | self.last_time = cur_time 751 | self.last_pending = self.pending 752 | self.last_finished = self.finished 753 | self.last_running = self.running 754 | 755 | vmsg(1, s % (self.running, 756 | self.pending, 757 | self.finished, 758 | self.errors, 759 | get_elapsed_time(self.start_time, get_time()), 760 | self._command_names())) 761 | 762 | def _start_more_jobs(self): 763 | """If there are jobs to start and we didn't hit our parallelism 764 | limit, start more jobs""" 765 | 766 | # copy from new_queue to pending_commands to avoid data 767 | # race on iterating over pending commands. 768 | started = False 769 | while not self.new_queue.empty(): 770 | self.pending_commands.append( self.new_queue.get() ) 771 | 772 | ready = deque() 773 | for cmd in self.pending_commands: 774 | if cmd._ready(): 775 | ready.append(cmd) 776 | 777 | while self.running < self.max_parallelism and ready: 778 | cmd = ready.popleft() 779 | # FIXME: small concern that this could be slow 780 | self.pending_commands.remove(cmd) 781 | if verbose(3): 782 | msgb("LAUNCHING", cmd.dump_cmd()) 783 | self._start_a_job(cmd) 784 | self.pending -= 1 785 | started = True 786 | return started 787 | 788 | def _start_a_job(self,cmd): 789 | """Private function to kick off a command""" 790 | self.out_queue.put(cmd) 791 | self.running_commands.append(cmd) 792 | if not cmd.terminator: 793 | self.all_commands.append(cmd) 794 | self.sent += 1 795 | self.running += 1 796 | 797 | def _command_names(self): 798 | s = [] 799 | anonymous_jobs = 0 800 | for r in self.running_commands: 801 | if hasattr(r,'name') and r.name: 802 | s.append(r.name) 803 | else: 804 | anonymous_jobs += 1 805 | if s: 806 | if anonymous_jobs: 807 | s.append('%d-anonymous' % (anonymous_jobs)) 808 | return '[' + ' '.join(s) + ']' 809 | else: 810 | return '' 811 | 812 | def _wait_for_jobs(self): 813 | """Return one command object when it finishes, or None on timeout (or 814 | other non-keyboard-interrupt exceptions).""" 815 | if self.running > 0: 816 | try: 817 | cmd = self.back_queue.get(block=True, timeout=self.join_timeout) 818 | self.running -= 1 819 | self.finished += 1 820 | self.running_commands.remove(cmd) 821 | self.back_queue.task_done() 822 | return cmd 823 | except queue.Empty: 824 | return None 825 | except KeyboardInterrupt: 826 | msgb('INTERRUPT') 827 | self._terminate() 828 | self.dying = True 829 | sys.exit(1) 830 | return None # NOT REACHED 831 | except: 832 | return None 833 | return None 834 | 835 | def build(self, 836 | dag=None, 837 | targets=None, 838 | die_on_errors=True, 839 | show_output=True, 840 | error_limit=0, 841 | show_progress=False, 842 | show_errors_only=False, 843 | join_timeout=10.0): 844 | """ 845 | This makes the work queue start building stuff. If no targets 846 | are specified then all the targets are considered and built if 847 | necessary. All commands that get run or generated are stored in 848 | the all_commands attribute. That attribute gets re-initialized 849 | on each call to build. 850 | 851 | @type dag: L{dag_t} 852 | @param dag: the dependence tree object 853 | 854 | @type targets: list 855 | @param targets: specific targets to build 856 | 857 | @type die_on_errors: bool 858 | @param die_on_errors: keep going or die on errors 859 | 860 | @type show_output: bool 861 | @param show_output: show stdout/stderr (or just buffer it in 862 | memory for later processing). Setting this to False is good for 863 | avoiding voluminous screen output. The default is True. 864 | 865 | @type show_progress: bool 866 | @param show_progress: show the running/pending/completed/errors msgs 867 | 868 | @type show_errors_only: bool 869 | @param show_errors_only: normally print the commands as they complete. 870 | If True, only show the commands that fail. 871 | 872 | @type join_timeout: float 873 | @param join_timeout: how long to wait for thread to terminate. default 10s 874 | """ 875 | self._clean_slate() 876 | 877 | self.show_progress = show_progress 878 | self.join_timeout = join_timeout 879 | self.errors = 0 880 | self.show_errors_only = show_errors_only 881 | self.message_delay = self.min_message_delay 882 | self.last_time = 0 883 | self.clear_commands() 884 | self.dag = dag 885 | if self.dag: 886 | for x in self.dag._leaves_with_changes(targets): 887 | self.add(x.creator) 888 | okay = self._build_blind(die_on_errors, show_output, error_limit) 889 | if okay and self.dag: 890 | did_not_build = self.dag.check_for_skipped() 891 | if len(did_not_build) > 0: 892 | # some stuff did not build, force an error status return 893 | msgb("ERROR: DID NOT BUILD SOME STUFF", "\n\t".join(did_not_build)) 894 | if self.dag: 895 | print(self.dag.dump()) 896 | self.end_time = get_time() 897 | self._cleanup() 898 | return False 899 | # normal exit path 900 | self.end_time = get_time() 901 | if self.dag: 902 | self.dag.dag_write_signatures() 903 | self._cleanup() 904 | return okay 905 | 906 | def _build_blind(self, die_on_errors=True, show_output=True, error_limit=0): 907 | """Start running the commands that are pending and kick off 908 | dependent jobs as those complete. If die_on_errors is True, the 909 | default, we stop running new jobs after one job returns a nonzero 910 | status. Returns True if no errors""" 911 | if self.use_threads: 912 | return self._build_blind_threads(die_on_errors, 913 | show_output, 914 | error_limit) 915 | else: 916 | return self._build_blind_no_threads(die_on_errors, 917 | show_output, 918 | error_limit) 919 | 920 | def _build_blind_threads(self, 921 | die_on_errors=True, 922 | show_output=True, 923 | error_limit=0): 924 | """Start running the commands that are pending and kick off 925 | dependent jobs as those complete. If die_on_errors is True, the 926 | default, we stop running new jobs after one job returns a nonzero 927 | status. Returns True if no errors""" 928 | okay = True 929 | started = False 930 | while 1: 931 | c = None 932 | if started: 933 | c = self._wait_for_jobs() 934 | if c: 935 | if verbose(4): 936 | msgb("JOB COMPLETED") 937 | if c.failed(): 938 | self.errors += 1 939 | okay = False 940 | if die_on_errors or (error_limit != 0 and 941 | self.errors > error_limit): 942 | warn("Command execution failed. " + 943 | "Waiting for remaining jobs and exiting.") 944 | self.dying = True 945 | 946 | if not self.dying: 947 | started |= self._start_more_jobs() 948 | self._status() 949 | 950 | if c and not self.dying: 951 | c._complete() 952 | # Command objects can depend on each other 953 | # directly. Enable execution of dependent commands. 954 | if verbose(4): 955 | msgb("ADD CMD-AFTERS") 956 | self.add(c._check_afters()) 957 | # Or we might find new commands from the file DAG. 958 | if self.dag: 959 | for x in self.dag._enable_successors(c): 960 | self.add(x.creator) 961 | if c: 962 | if self.show_errors_only==False or c.failed(): 963 | print(c.dump(show_output=show_output)) 964 | elif c.targets: 965 | for x in c.targets: 966 | vmsg(1, u'\tBUILT: {}'.format(x)) 967 | if self._done(): 968 | break; 969 | return okay 970 | 971 | def _build_blind_no_threads(self, die_on_errors=True, 972 | show_output=True, error_limit=0): 973 | """Start running the commands that are pending and kick off 974 | dependent jobs as those complete. If die_on_errors is True, the 975 | default, we stop running new jobs after one job returns a nonzero 976 | status. Returns True if no errors""" 977 | okay = True 978 | while 1: 979 | started = False 980 | if not self.dying: 981 | started = self._start_more_jobs() 982 | if started: 983 | self._status() 984 | 985 | # EXECUTE THE TASK OURSELVES 986 | if self.running > 0: 987 | _worker_one_task(self.out_queue, self.back_queue) 988 | c = self._wait_for_jobs() 989 | if c: 990 | if verbose(4): 991 | msgb("JOB COMPLETED") 992 | if c.failed(): 993 | okay = False 994 | self.errors += 1 995 | if die_on_errors or (error_limit !=0 and 996 | self.errors > error_limit): 997 | warn("Command execution failed. " + 998 | "Waiting for remaining jobs and exiting.") 999 | self.dying = True 1000 | if not self.dying: 1001 | c._complete() 1002 | # Command objects can depende on each other 1003 | # directly. Enable execution of dependent commands. 1004 | if verbose(4): 1005 | msgb("ADD CMD-AFTERS") 1006 | self.add(c._check_afters()) 1007 | # Or we might find new commands from the file DAG. 1008 | if self.dag: 1009 | for x in self.dag._enable_successors(c): 1010 | self.add(x.creator) 1011 | if self.show_errors_only==False or c.failed(): 1012 | print(c.dump(show_output=show_output)) 1013 | self._status() 1014 | if self._done(): 1015 | break; 1016 | return okay 1017 | 1018 | 1019 | 1020 | -------------------------------------------------------------------------------- /mbuild/util.py: -------------------------------------------------------------------------------- 1 | # -*- python -*- 2 | #BEGIN_LEGAL 3 | # 4 | #Copyright (c) 2024 Intel Corporation 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | #END_LEGAL 19 | 20 | """Basic useful utilities: file copying, removal, permissions, 21 | path-name manipulation, and command execution.""" 22 | 23 | import os 24 | import re 25 | import glob 26 | import io # for io.open 27 | import sys 28 | import shutil 29 | import stat 30 | import types 31 | import time 32 | import subprocess 33 | import tempfile 34 | import shlex 35 | import traceback 36 | try: 37 | import cPickle as apickle 38 | except: 39 | import pickle as apickle 40 | 41 | from .base import * 42 | 43 | 44 | def find_python(env): 45 | """return path to NON cygwin""" 46 | pycmd = sys.executable # use whatever the user invoked us with 47 | if env.on_windows() and env.on_cygwin(): 48 | # avoid cygwin python 49 | if pycmd in ['/usr/bin/python', '/bin/python']: 50 | python_commands = [ 'c:/python27/python.exe', 51 | 'c:/python26/python.exe', 52 | 'c:/python25/python.exe' ] 53 | pycmd = None 54 | for p in python_commands: 55 | if os.path.exists(p): 56 | return p 57 | if not pycmd: 58 | die("Could not find win32 python at these locations: %s" % 59 | "\n\t" + "\n\t".join(python_commands)) 60 | 61 | return pycmd 62 | def copy_file(src,tgt): 63 | """Copy src to tgt.""" 64 | if verbose(2): 65 | msgb("COPY", tgt + " <- " + src) 66 | shutil.copy(src,tgt) 67 | def move_file(src,tgt): 68 | """Move/Rename src to tgt.""" 69 | if verbose(2): 70 | msgb("MOVE", src + " -> " + tgt) 71 | shutil.move(src,tgt) 72 | def symlink(env,src,tgt): 73 | """Make a symlink from src to target. Not available on windows.""" 74 | if env.on_windows(): 75 | die("symlink() not available on windows") 76 | if verbose(2): 77 | msgb("SYMLINK", src + " -> " + tgt) 78 | os.symlink(src,tgt) 79 | 80 | def copy_tree(src,tgt, ignore_patterns=None, symlinks=False): 81 | """Copy the tree at src to tgt. This will first remove tgt if it 82 | already exists.""" 83 | if verbose(2): 84 | msgb("COPYTREE", tgt + " <- " + src) 85 | if not os.path.exists(src): 86 | error_msg("SRC TREE DOES NOT EXIST", src) 87 | raise Exception 88 | if os.path.exists(tgt): 89 | if verbose(2): 90 | msgb("Removing existing target tree", tgt) 91 | shutil.rmtree(tgt, ignore_errors=True) 92 | if verbose(2): 93 | msgb("Copying to tree", tgt) 94 | if ignore_patterns: 95 | sp = shutil.ignore_patterns(ignore_patterns) 96 | else: 97 | sp = None 98 | shutil.copytree(src,tgt,ignore=sp, symlinks=symlinks) 99 | if verbose(2): 100 | msgb("Done copying tree", tgt) 101 | 102 | def cmkdir(path_to_dir, exist_ok = False): 103 | """Make a directory if it does not exist""" 104 | if not os.path.exists(path_to_dir): 105 | if verbose(2): 106 | msgb("MKDIR", path_to_dir) 107 | os.makedirs(path_to_dir,exist_ok=exist_ok) 108 | def list2string(ls): 109 | """Print a list as a string""" 110 | s = " ".join(ls) 111 | return s 112 | 113 | def util_add_to_list(olst, v): 114 | """Add v to olst. v can be a list or a non-list object. If v is a 115 | list, extend olst. If v is not a list, append to olst. """ 116 | if isinstance(v,list): 117 | olst.extend(v) 118 | else: 119 | olst.append(v) 120 | 121 | def remove_file(fn, env=None, quiet=True): 122 | """Remove a file or link if it exists. env parameter is not used.""" 123 | if os.path.exists(fn): 124 | make_writable(fn) 125 | if os.path.exists(fn) or os.path.lexists(fn): 126 | if not quiet: 127 | vmsgb(2, "REMOVING", fn) 128 | os.unlink(fn) 129 | return (0, []) 130 | def remove_tree(dir_name, env=None, dangerous=False): 131 | """Remove a directory if it exists. env parameter is not 132 | used. This will not remove a directory that has a .svn or .git 133 | subdirectory indicating it is a source directory. Warning: It does 134 | not look recursively for .svn/.git subdirectories. 135 | @type dir_name: string 136 | @param dir_name: a directory name 137 | @type env: L{env_t} 138 | @param env: optional. Not currently used. 139 | @type dangerous: bool 140 | @param dangerous: optional. If True,will delete anything including svn trees!! BE CAREFUL! default False. 141 | """ 142 | 143 | def _important_file(dir_name): 144 | for idir in ['.git', '.svn']: 145 | if os.path.exists(os.path.join(dir_name, idir)): 146 | return True 147 | return False 148 | 149 | vmsgb(2, "CHECKING", dir_name) 150 | if os.path.exists(dir_name): 151 | if not dangerous and _important_file(dir_name): 152 | s = 'Did not remove directory {} because of a .svn/.git subdirectory'.format(dir_name) 153 | warn(s) 154 | return (1, [ s ]) 155 | vmsgb(2, "REMOVING", dir_name) 156 | make_writable(dir_name) 157 | shutil.rmtree(dir_name, ignore_errors = True) 158 | return (0, []) 159 | def remove_files(lst, env=None): 160 | """Remove all the files in the list of files, lst. The env 161 | parameter is not used""" 162 | for fn in lst: 163 | remove_file(fn) 164 | return (0, []) 165 | 166 | def remove_files_glob(lst,env=None): 167 | """Remove all files in the list of wild card expressions. The env 168 | parameter is not used""" 169 | for fn_glob in lst: 170 | #msgb("REMOVING", fn_glob) 171 | for file_name in glob(fn_glob): 172 | remove_file(file_name) 173 | return (0, []) 174 | 175 | def remove_files_from_tree(dir, file_patterns): 176 | """Remove files that match the re object compiled pattern provided""" 177 | for (dir, subdirs, subfiles) in os.walk(dir): 178 | for file_name in subfiles: 179 | fn = os.path.join(dir,file_name) 180 | if file_patterns.search(fn): 181 | remove_file(fn) 182 | 183 | 184 | _readable_by_all = stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH 185 | _readable_by_ug = stat.S_IRUSR|stat.S_IRGRP 186 | _executable_by_all = stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH 187 | _executable_by_ug = stat.S_IXUSR|stat.S_IXGRP 188 | _writeable_by_me = stat.S_IWUSR 189 | _rwx_by_me = stat.S_IWUSR| stat.S_IRUSR|stat.S_IXUSR 190 | _writeable_by_ug = stat.S_IWUSR|stat.S_IWGRP 191 | 192 | def make_writable(fn): 193 | """Make the file or directory readable/writable/executable by me""" 194 | global _rwx_by_me 195 | os.chmod(fn, _rwx_by_me) 196 | 197 | def make_executable(fn): 198 | """Make the file or directory readable & executable by user/group, writable by user""" 199 | global _executable_by_ug 200 | global _readable_by_ug 201 | global _writeable_by_me 202 | os.chmod(fn, _readable_by_ug|_writeable_by_me|_executable_by_ug) 203 | 204 | def modify_dir_tree(path, dir_fn=None, file_fn=None): 205 | """Walk the tree rooted at path and apply the function dir_fn to 206 | directories and file_fn to files. This is intended for doing 207 | recursive chmods, etc.""" 208 | if dir_fn: 209 | dir_fn(path) 210 | for (dir, subdirs, subfiles) in os.walk(path): 211 | if dir_fn: 212 | for subdir in subdirs: 213 | dir_fn(os.path.join(dir,subdir)) 214 | if file_fn: 215 | for file_name in subfiles: 216 | file_fn(os.path.join(dir,file_name)) 217 | 218 | 219 | def make_read_only(fn): 220 | """Make the file fn read-only""" 221 | global _readable_by_all 222 | os.chmod(fn, _readable_by_all) 223 | 224 | def make_web_accessible(fn): 225 | """Make the file readable by all and writable by the current owner""" 226 | global _readable_by_all 227 | global _writeable_by_me 228 | if verbose(8): 229 | msgb("make_web_accessible", fn) 230 | os.chmod(fn, _writeable_by_me|_readable_by_all) 231 | def make_web_accessible_dir(dir): 232 | """Make the directory readable and executable by all and writable 233 | by the current owner""" 234 | global _readable_by_all 235 | global _executable_by_all 236 | global _writeable_by_me 237 | if verbose(8): 238 | msgb("make_web_accessible_dir", dir) 239 | os.chmod(dir, _writeable_by_me|_readable_by_all|_executable_by_all) 240 | 241 | def make_documentation_tree_accessible(dir): 242 | """Make the directory teree rooted at dir web-accessible. That is, 243 | the directories are readable and executable by anyone and the 244 | files are readable by anyone.""" 245 | msgb("CHMOD TREE", dir) 246 | modify_dir_tree(dir, make_web_accessible_dir, make_web_accessible) 247 | 248 | 249 | 250 | def prefix_files(dir,input_files): 251 | """Add dir on to the front of the input file or files. Works with 252 | strings or lists of strings. 253 | @type dir: string 254 | @param dir: prefix directory 255 | 256 | @type input_files: string or list of strings 257 | @param input_files: name(s) of files 258 | 259 | @rtype: string or list of strings 260 | @return: input file(s) prefixed with dir sp 261 | """ 262 | if isinstance(input_files,list): 263 | new_files = [join(dir,x) for x in input_files] 264 | return new_files 265 | elif is_stringish(input_files): 266 | new_file = join(dir, input_files) 267 | return new_file 268 | die("Unhandled type in prefix_files: "+ str(type(input_files))) 269 | 270 | def quote(fn): 271 | """Add quotes around the file nameed fn. Return a string""" 272 | return "\"%s\"" % fn 273 | 274 | def qdip(fn): 275 | """Add quotes to a string if there are spaces in the name""" 276 | if re.search(' ',fn): 277 | return '"%s"' % fn 278 | return fn 279 | 280 | 281 | def touch(fn): 282 | """Open a file for append. Write nothing to it""" 283 | vmsgb(1, "TOUCH", fn) 284 | f=open(fn,"a") 285 | f.close() 286 | 287 | ############################################################ 288 | if on_native_windows(): 289 | _mysep = "\\" 290 | else: 291 | _mysep = "/" 292 | 293 | def myjoin( *args ): 294 | """join all the args supplied as arguments using _mysep as the 295 | separator. _mysep is a backslash on native windows and a forward 296 | slash everywhere else. 297 | @type args: strings 298 | @param args: path component strings 299 | 300 | @rtype: string 301 | @return: string with _mysep slashes 302 | """ 303 | s = '' 304 | first = True 305 | for a in args: 306 | if first: 307 | first = False 308 | else: 309 | s = s + _mysep 310 | s = s + a 311 | return s 312 | 313 | def strip_quotes(a): 314 | """Conditionally remove leading/trailing quotes from a string 315 | @type a: string 316 | @param a: a string potentially with quotes 317 | 318 | @rtype: string 319 | @return: same string without the leading and trailing quotes 320 | """ 321 | ln = len(a) 322 | if ln >= 2: 323 | strip_quotes = False 324 | if a[0] == '"' and a[-1] == '"': 325 | strip_quotes=True 326 | elif a[0] == "'" and a[-1] == "'": 327 | strip_quotes=True 328 | if strip_quotes: 329 | b = a[1:ln-1] 330 | return b 331 | return a 332 | 333 | def join( *args ): 334 | """join all the args supplied as arguments using a forward slash as 335 | the separator 336 | 337 | @type args: strings 338 | @param args: path component strings 339 | 340 | @rtype: string 341 | @return: string with forward-slashes 342 | """ 343 | s = '' 344 | first = True 345 | for a in args: 346 | ln = len(s) 347 | if first: 348 | first = False 349 | elif ln == 0 or s[-1] != '/': 350 | # if the last character is not a fwd slash already, add a slash 351 | s = s + '/' 352 | a = strip_quotes(a) 353 | s = s + a 354 | return s 355 | 356 | 357 | def flip_slashes(s): 358 | """convert to backslashes to _mysep slashes. _mysep slashes are 359 | defined to be backslashes on native windows and forward slashes 360 | everywhere else. 361 | @type s: string or list of strings 362 | @param s: path name(s) 363 | 364 | @rtype: string or list of strings 365 | @return: string(s) with _mysep slashes 366 | """ 367 | 368 | if on_native_windows(): 369 | return s 370 | if isinstance(s, list): 371 | return list(map(flip_slashes, s)) 372 | t = re.sub(r'\\',_mysep,s,0) # replace all 373 | return t 374 | 375 | def posix_slashes(s): 376 | """convert to posix slashes. Do not flip slashes immediately before spaces 377 | @type s: string or list of strings 378 | @param s: path name(s) 379 | 380 | @rtype: string or list of strings 381 | @return: string(s) with forward slashes 382 | """ 383 | if isinstance(s,list): 384 | return list(map(posix_slashes, s)) 385 | #t = re.sub(r'\\','/',s,0) # replace all 386 | last = len(s)-1 387 | t=[] 388 | for i,a in enumerate(s): 389 | x=a 390 | if a == '\\': 391 | if i == last: 392 | x = '/' 393 | elif s[i+1] != ' ': 394 | x = '/' 395 | t.append(x) 396 | return ''.join(t) 397 | 398 | def glob(*s): 399 | """If multiple arguments are passed, we run them through mbuild.join() 400 | first. Run the normal glob.glob() on s but make sure all the 401 | slashes are flipped forward afterwards. This is shorthand for 402 | posix_slashes(glob.glob(s)) """ 403 | import glob 404 | if len(s) > 1: 405 | t = join(*s) 406 | else: 407 | t = s[0] 408 | return posix_slashes(glob.glob(t)) 409 | 410 | def cond_add_quotes(s): 411 | """If there are spaces in the input string s, put quotes around the 412 | string and return it... if there are not already quotes in the 413 | string. 414 | 415 | @type s: string 416 | @param s: path name 417 | 418 | @rtype: string 419 | @return: string with quotes, if necessary 420 | """ 421 | if re.search(r'[ ]',s) and not ( re.search(r'["].*["]',s) or 422 | re.search(r"['].*[']",s) ): 423 | return '\"' + s + '\"' 424 | return s 425 | 426 | def escape_string(s): 427 | return cond_add_quotes(s) 428 | 429 | def escape_special_characters(s): 430 | """Add a backslash before characters that have special meanings in 431 | regular expressions. Python does not handle backslashes in regular 432 | expressions or substitution text so they must be escaped before 433 | processing.""" 434 | 435 | special_chars = r'\\' 436 | new_string = '' 437 | for c in s: 438 | if c in special_chars: 439 | new_string += '\\' 440 | new_string += c 441 | return new_string 442 | 443 | ############################################################### 444 | 445 | if check_python_version(2,5): 446 | import hashlib 447 | hasher = hashlib.sha1 448 | else: 449 | import sha 450 | hasher = sha.new 451 | 452 | def hash_list(list_of_strings): 453 | """Compute a sha1 hash of a list of strings and return the hex digest""" 454 | m = hasher() 455 | for l in list_of_strings: 456 | m.update(l.encode(unicode_encoding())) 457 | return m.hexdigest() 458 | 459 | 460 | def hash_file(fn): 461 | if not os.path.exists(fn): 462 | return None 463 | m = hasher() 464 | with open(fn,'rb') as afile: 465 | buf = afile.read() 466 | m.update(buf) 467 | return m.hexdigest() 468 | 469 | 470 | 471 | def write_signatures(fn,d): 472 | """Write a dictionary of d[file]=hash to the specified file""" 473 | # FIXME: binary protocol 2, binary file write DOES NOT WORK ON win32/win64 474 | f = open(fn,"wb") 475 | apickle.dump(d,f) 476 | f.close() 477 | 478 | def read_signatures(fn): 479 | """Return a dictionary of d[file]=hash from the specified file""" 480 | try: 481 | f = open(fn,"rb") 482 | d = apickle.load(f) 483 | f.close() 484 | return d 485 | except: 486 | return None 487 | 488 | 489 | def hash_string(s): 490 | """Compute a sha1 hash of a string and return the hex digest""" 491 | if check_python_version(2,5): 492 | m = hashlib.sha1() 493 | else: 494 | m = sha.new() 495 | m.update(s) 496 | d = m.hexdigest() 497 | return d 498 | 499 | 500 | def hash_files(list_of_files, fn): 501 | """Hash the files in the list of files and write the hashes to fn""" 502 | d = {} 503 | for f in list_of_files: 504 | d[f] = hash_file(f) 505 | write_signatures(fn,d) 506 | 507 | def file_hashes_are_valid(list_of_files, fn): 508 | """Return true iff the old hashes in the file fn are valid for all 509 | of the specified list of files.""" 510 | if not os.path.exists(fn): 511 | return False 512 | d = read_signatures(fn) 513 | if d == None: 514 | return False 515 | for f in list_of_files: 516 | if os.path.exists(f): 517 | nhash = hash_file(f) 518 | else: 519 | return False 520 | if nhash == None: 521 | return False 522 | if f not in d: 523 | return False 524 | elif d[f] != nhash: 525 | return False; 526 | return True 527 | 528 | ############################################################### 529 | # Time functions 530 | def get_time_str(): 531 | """@rtype: string 532 | @returns: current time as string 533 | """ 534 | # include time zone 535 | return time.strftime('%Y-%m-%d %H:%M:%S %Z') 536 | 537 | def get_time(): 538 | """@rtype: float 539 | @returns: current time as float 540 | """ 541 | return time.time() 542 | 543 | def get_elapsed_time(start_time, end_time=None): 544 | """compute the elapsed time in seconds or minutes 545 | @type start_time: float 546 | @param start_time: starting time. 547 | @type end_time: float 548 | @param end_time: ending time. 549 | @rtype: string 550 | """ 551 | if end_time == None: 552 | end_time = get_time() 553 | seconds = end_time - start_time 554 | negative_prefix = '' 555 | if seconds < 0: 556 | negative_prefix = '-' 557 | seconds = -seconds 558 | if seconds < 120: 559 | if int(seconds) == 0: 560 | milli_seconds = seconds * 1000 561 | timestr = "%d" % int(milli_seconds) 562 | suffix = " msecs" 563 | else: 564 | timestr = "%d" % int(seconds) 565 | suffix = " secs" 566 | else: 567 | minutes = int(seconds/60.0) 568 | remainder_seconds = int(seconds - (minutes*60)) 569 | timestr = "%.d:%02d" % (minutes,remainder_seconds) 570 | suffix = " min:sec" 571 | return "".join([negative_prefix, timestr, suffix]) 572 | 573 | def print_elapsed_time(start_time, end_time=None, prefix=None, current=False): 574 | """print the elapsed time in seconds or minutes. 575 | 576 | @type start_time: float 577 | @param start_time: the starting time 578 | @type end_time: float 579 | @param end_time: the ending time (optional) 580 | @type prefix: string 581 | @param prefix: a string to print at the start of the line (optional) 582 | """ 583 | if end_time == None: 584 | end_time = get_time() 585 | ets = "ELAPSED TIME" 586 | if prefix: 587 | s = "%s %s" % (prefix, ets) 588 | else: 589 | s = ets 590 | 591 | t = get_elapsed_time(start_time, end_time) 592 | if current: 593 | t = t + " / NOW: " + get_time_str() 594 | vmsgb(1,s,t) 595 | 596 | 597 | ############################################################### 598 | def _prepare_cmd(cmd): 599 | """Tokenize the cmd string input. Return as list on non-windows 600 | platforms. On windows, it returns the raw command string.""" 601 | 602 | if on_native_windows(): 603 | # the posix=False is required to keep shlex from eating 604 | # backslashed path characters on windows. But 605 | # the nonposix chokes on /Dfoo="xxx yyy" in that it'll 606 | # split '/Dfoo="xxx' and 'yyy"' in to two different args. 607 | # so we cannot use that 608 | #args = shlex.split(cmd,posix=False) 609 | 610 | # using posix mode (default) means that all commands must must 611 | # forward slashes. So that is annoying and we avoid that 612 | #args = shlex.split(cmd) 613 | 614 | # passing the args through works fine. Make sure not to have 615 | # any carriage returns or leading white space in the supplied 616 | # command. 617 | args = cmd 618 | 619 | else: 620 | args = shlex.split(cmd) 621 | return args 622 | 623 | def _cond_open_input_file(directory,input_file_name): 624 | if input_file_name: 625 | if directory and not os.path.isabs(input_file_name): 626 | fn = os.path.join(directory, input_file_name) 627 | else: 628 | fn = input_file_name 629 | input_file_obj = open(fn,"r") 630 | return input_file_obj 631 | return None 632 | 633 | def run_command(cmd, 634 | separate_stderr=False, 635 | shell_executable=None, 636 | directory=None, 637 | osenv=None, 638 | input_file_name=None, 639 | **kwargs): 640 | """ 641 | Run a command string using the subprocess module. 642 | 643 | @type cmd: string 644 | @param cmd: command line to execut with all args. 645 | @type separate_stderr: bool 646 | @param separate_stderr: If True, the return tuple has a list of stderr lines as the 3rd element 647 | @type shell_executable: string 648 | @param shell_executable: the shell executable 649 | @type directory: string 650 | @param directory: a directory to change to before running the command. 651 | @type osenv: dictionary 652 | @param osenv: dict of environment vars to be passed to the new process 653 | @type input_file_name: string 654 | @param input_file_name: file name to read stdin from. Default none 655 | 656 | @rtype: tuple 657 | @return: (return code, list of stdout lines, list of lines of stderr) 658 | """ 659 | use_shell = False 660 | if verbose(99): 661 | msgb("RUN COMMAND", cmd) 662 | msgb("RUN COMMAND repr", repr(cmd)) 663 | stdout = None 664 | stderr = None 665 | cmd_args = _prepare_cmd(cmd) 666 | try: 667 | input_file_obj = _cond_open_input_file(directory, input_file_name) 668 | errout = subprocess.PIPE if separate_stderr else subprocess.STDOUT 669 | sub = subprocess.Popen(cmd_args, 670 | shell=use_shell, 671 | executable=shell_executable, 672 | stdin = input_file_obj, 673 | stdout = subprocess.PIPE, 674 | stderr = errout, 675 | cwd=directory, 676 | env=osenv, 677 | text=True, 678 | **kwargs) 679 | (stdout, stderr ) = sub.communicate() 680 | if separate_stderr and not isinstance(stderr, list): 681 | stderr = stderr.splitlines(True) 682 | if not isinstance(stdout,list): 683 | stdout = stdout.splitlines(True) 684 | return (sub.returncode, stdout, stderr) 685 | except OSError as e: 686 | s= [u"Execution failed for: %s\n" % (cmd) ] 687 | s.append("Result is %s\n" % (str(e))) 688 | # put the error message in stderr if there is a separate 689 | # stderr, otherwise put it in stdout. 690 | if separate_stderr: 691 | if stderr == None: 692 | stderr = [] 693 | elif not isinstance(stderr,list): 694 | stderr = stderr.splitlines(True) 695 | if stdout == None: 696 | stdout = [] 697 | elif not isinstance(stdout,list): 698 | stdout = stdout.splitlines(True) 699 | stderr = ensure_string(stderr) 700 | stdout = ensure_string(stdout) 701 | if separate_stderr: 702 | stderr.extend(s) 703 | else: 704 | stdout.extend(s) 705 | return (1, stdout, stderr) 706 | 707 | 708 | def run_command_unbufferred(cmd, 709 | prefix_line=None, 710 | shell_executable=None, 711 | directory=None, 712 | osenv=None, 713 | input_file_name=None, 714 | **kwargs): 715 | """ 716 | Run a command string using the subprocess module. 717 | 718 | @type cmd: string 719 | @param cmd: command line to execut with all args. 720 | @type prefix_line: string 721 | @param prefix_line: a string to prefix each output line. Default None 722 | @type shell_executable: string 723 | @param shell_executable: NOT USED BY THIS FUNCTION 724 | @type directory: string 725 | @param directory: a directory to change to before running the command. 726 | @type osenv: dictionary 727 | @param osenv: dict of environment vars to be passed to the new process 728 | @type input_file_name: string 729 | @param input_file_name: file name to read stdin from. Default none 730 | 731 | @rtype: tuple 732 | @return: (return code, list of stdout lines, empty list) 733 | 734 | """ 735 | use_shell = False 736 | if verbose(99): 737 | msgb("RUN COMMAND", cmd) 738 | msgb("RUN COMMAND repr", repr(cmd)) 739 | lines: list = [] 740 | cmd_args = _prepare_cmd(cmd) 741 | try: 742 | input_file_obj = _cond_open_input_file(directory, input_file_name) 743 | sub = subprocess.Popen(cmd_args, 744 | shell=use_shell, 745 | executable=shell_executable, 746 | stdin = input_file_obj, 747 | stdout = subprocess.PIPE, 748 | stderr = subprocess.STDOUT, 749 | env=osenv, 750 | cwd=directory, 751 | text=True, # For a string stdout, stderr and stdin 752 | bufsize=1, # line buffered 753 | **kwargs) 754 | while 1: 755 | # FIXME: 2008-12-05 bad for password prompts without newlines. 756 | line: str = sub.stdout.readline() 757 | if line == '': 758 | break 759 | line = line.rstrip() 760 | if prefix_line: 761 | msgn(prefix_line) 762 | msg(line) 763 | lines.append(line + "\n") 764 | sub.wait() 765 | 766 | if input_file_obj: 767 | input_file_obj.close() 768 | return (sub.returncode, lines, []) 769 | except OSError as e: 770 | uappend(lines, u"Execution failed for: %s\n" % (cmd)) 771 | uappend(lines, u"Result is %s\n" % (str(e))) 772 | return (1, lines, []) 773 | 774 | def run_command_output_file(cmd, 775 | output_file_name, 776 | shell_executable=None, 777 | directory=None, 778 | osenv=None, 779 | input_file_name=None, 780 | **kwargs): 781 | """ 782 | Run a command string using the subprocess module. 783 | 784 | @type cmd: string 785 | @param cmd: command line to execut with all args. 786 | @type output_file_name: string 787 | @param output_file_name: output file name 788 | @type shell_executable: string 789 | @param shell_executable: the shell executable 790 | @type directory: string 791 | @param directory: a directory to change to before running the command. 792 | @type osenv: dictionary 793 | @param osenv: dict of environment vars to be passed to the new process 794 | @type input_file_name: string 795 | @param input_file_name: file name to read stdin from. Default none 796 | 797 | @rtype: tuple 798 | @return: (return code, list of stdout lines) 799 | """ 800 | use_shell = False 801 | if verbose(99): 802 | msgb("RUN COMMAND", cmd) 803 | lines: list = [] 804 | cmd_args = _prepare_cmd(cmd) 805 | try: 806 | output = io.open(output_file_name,"wt",encoding=unicode_encoding()) 807 | input_file_obj = _cond_open_input_file(directory, input_file_name) 808 | sub = subprocess.Popen(cmd_args, 809 | shell=use_shell, 810 | executable=shell_executable, 811 | stdin = input_file_obj, 812 | stdout = subprocess.PIPE, 813 | stderr = subprocess.STDOUT, 814 | env=osenv, 815 | cwd=directory, 816 | text=True, # For a string stdout, stderr and stdin 817 | **kwargs) 818 | 819 | (stdout, stderr) = sub.communicate() 820 | if stdout: 821 | lines = stdout.splitlines(True) 822 | for l in lines: 823 | output.write(l) 824 | output.close() 825 | return (sub.returncode, lines, []) 826 | except OSError as e: 827 | uappend(lines,"Execution failed for: %s\n" % (cmd)) 828 | uappend(lines,"Result is %s\n" % (str(e))) 829 | return (1, lines, []) 830 | except: 831 | print("Unxpected error:", sys.exc_info()[0]) 832 | raise 833 | 834 | def run_cmd_io(cmd, fn_i, fn_o,shell_executable=None, directory=None): 835 | """ 836 | Run a command string using the subprocess module. Read standard 837 | input from fn_i and write stdout/stderr to fn_o. 838 | 839 | @type cmd: string 840 | @param cmd: command line to execut with all args. 841 | @type fn_i: string 842 | @param fn_i: input file name 843 | @type fn_o: string 844 | @param fn_o: output file name 845 | @type shell_executable: string 846 | @param shell_executable: the shell executable 847 | @type directory: string 848 | @param directory: a directory to change to before running the command. 849 | 850 | @rtype: integer 851 | @return: return code 852 | """ 853 | use_shell = False 854 | cmd_args = _prepare_cmd(cmd) 855 | try: 856 | fin = io.open(fn_i, 'rt', encoding=unicode_encoding()) 857 | fout = io.open(fn_o, 'wt', encoding=unicode_encoding()) 858 | sub = subprocess.Popen(cmd_args, 859 | shell=use_shell, 860 | executable=shell_executable, 861 | stdin=fin, 862 | stdout=fout, 863 | stderr=subprocess.STDOUT, 864 | text=True, # For a string stdout, stderr and stdin 865 | cwd=directory) 866 | retval = sub.wait() 867 | fin.close() 868 | fout.close() 869 | return retval 870 | except OSError as e: 871 | die(u"Execution failed for cmd %s\nResult is %s\n" % (cmd,str(e))) 872 | 873 | def find_dir(d): 874 | """Look upwards for a particular filesystem directory d as a 875 | subdirectory of one of the ancestors. Return None on failure""" 876 | dir = os.getcwd() 877 | last = '' 878 | while dir != last: 879 | target_dir = os.path.join(dir,d) 880 | if os.path.exists(target_dir): 881 | return target_dir 882 | last = dir 883 | (dir,tail) = os.path.split(dir) 884 | return None 885 | 886 | def peel_dir(s,n): 887 | """Remove n trailing path components from s by calling 888 | os.path.dirname()""" 889 | t = s 890 | for i in range(0,n): 891 | t = os.path.dirname(t) 892 | return t 893 | 894 | def get_gcc_version(gcc): 895 | """Return the compressed version number of gcc""" 896 | cmd = gcc + " -dumpversion" 897 | try: 898 | (retcode, stdout, stderr) = run_command(cmd) 899 | if retcode == 0: 900 | version = stdout[0] 901 | return version.strip() 902 | except: 903 | return 'unknown' 904 | 905 | def get_clang_version(full_path): 906 | cmd = full_path + " -dM -E -x c " 907 | try: 908 | (retcode, stdout, stderr) = run_command(cmd, 909 | input_file_name="/dev/null") 910 | if retcode == 0: 911 | major=minor=patchlevel='x' 912 | for line in stdout: 913 | line = line.strip() 914 | chunks = line.split() 915 | if len(chunks) == 3: 916 | if chunks[1] == '__clang_major__': 917 | major = chunks[2] 918 | elif chunks[1] == '__clang_minor__': 919 | minor = chunks[2] 920 | elif chunks[1] == '__clang_patchlevel__': 921 | patchlevel = chunks[2] 922 | version = "{}.{}.{}".format(major,minor,patchlevel) 923 | return version 924 | except: 925 | pass 926 | # Try the --version knob 927 | try: 928 | (retcode, stdout, stderr) = run_command(f'{full_path} --version') 929 | if retcode == 0: 930 | for line in stdout: 931 | r = re.search(r'version[ ]+(?P(\d+\.)+\d+)', line.lower()) 932 | if r: 933 | return r.group('version') 934 | except: 935 | pass 936 | return 'unknown' 937 | 938 | # unify names for clang/gcc version checkers 939 | def compute_clang_version(full_path): 940 | return get_clang_version(full_path) 941 | 942 | def compute_gcc_version(full_path): 943 | return get_gcc_version(full_path) 944 | 945 | def gcc_version_test(major,minor,rev,gstr): 946 | """Return True if the specified gcc version string (gstr) is at or 947 | after the specified major,minor,revision args""" 948 | 949 | n = gstr.split('.') 950 | if len(n) not in [2,3]: 951 | die("Cannot compute gcc version from input string: [%s]" % (gstr)) 952 | ga = int(n[0]) 953 | gb = int(n[1]) 954 | if len(n) == 2: 955 | gc = 0 956 | else: 957 | gc = int(n[2]) 958 | 959 | if ga > major: 960 | return True 961 | if ga == major and gb > minor: 962 | return True 963 | if ga == major and gb == minor and gc >= rev: 964 | return True 965 | return False 966 | 967 | import threading 968 | # requires Python2.6 or later 969 | class _timed_command_t(threading.Thread): 970 | """ 971 | Internal function to mbuild util.py. Do not call directly. 972 | 973 | Examples of use 974 | env = os.environ 975 | env['FOOBAR'] = 'hi' 976 | # the command a.out prints out the getenv("FOOBAR") value 977 | rc = _timed_command_t(["./a.out", "5"], seconds=4, env=env) 978 | rc.timed_run() 979 | 980 | rc = _timed_command_t(["/bin/sleep", "5"], seconds=4) 981 | rc.timed_run() 982 | """ 983 | 984 | def __init__(self, cmd, 985 | shell_executable=None, 986 | directory=None, 987 | osenv=None, 988 | seconds=0, 989 | input_file_name=None, 990 | **kwargs): 991 | """The kwargs are for the other parameters to Popen""" 992 | threading.Thread.__init__(self) 993 | self.cmd = cmd 994 | self.kwargs = kwargs 995 | self.seconds = seconds 996 | self.timed_out = False 997 | self.sub = None 998 | self.osenv= osenv 999 | self.input_file_name = input_file_name 1000 | self.directory = directory 1001 | self.shell_executable = shell_executable 1002 | self.exception_type = None 1003 | self.exception_object = None 1004 | self.exception_trace = None 1005 | self.exitcode = 0, 1006 | self.output = "", 1007 | self.stderr = "", 1008 | 1009 | def run(self): # executed by calling start() 1010 | cmd = self.cmd 1011 | #run a python command 1012 | if _is_python_cmd(cmd): 1013 | kwargs = self.kwargs 1014 | xenv = kwargs.get('xenv') 1015 | args_lst = kwargs.get('args_lst') 1016 | if args_lst == None: 1017 | args_lst = [] 1018 | if xenv == None: 1019 | (self.exitcode,self.output,self.stderr) = cmd(*args_lst) 1020 | else: 1021 | (self.exitcode,self.output,self.stderr) = cmd(xenv, *args_lst) 1022 | return 1023 | 1024 | #run an executable 1025 | use_shell = False 1026 | cmd_args = _prepare_cmd(cmd) 1027 | input_file_obj = _cond_open_input_file(self.directory, 1028 | self.input_file_name) 1029 | try: 1030 | self.sub = subprocess.Popen(cmd_args, 1031 | shell=use_shell, 1032 | executable=self.shell_executable, 1033 | cwd=self.directory, 1034 | env=self.osenv, 1035 | stdin = input_file_obj, 1036 | text=True, # For a string stdout, stderr and stdin 1037 | **self.kwargs) 1038 | except: 1039 | (self.exception_type, 1040 | self.exception_object, 1041 | self.exception_trace) = sys.exc_info() 1042 | else: 1043 | self.sub.wait() 1044 | 1045 | def timed_run(self): 1046 | """Returns False if the process times out. Also sets 1047 | self.timed_out to True.""" 1048 | 1049 | self.timed_out=False 1050 | self.start() # calls run() 1051 | if self.seconds: 1052 | self.join(self.seconds) 1053 | else: 1054 | self.join() 1055 | 1056 | if self.is_alive(): 1057 | try: 1058 | if self.sub: 1059 | if on_windows(): 1060 | # On Windows terminate() does not always kill 1061 | # the process So we need specific handling for 1062 | # Windows here. 1063 | kill_cmd = "taskkill /F /T /PID %i" % (self.sub.pid) 1064 | cmd_args = _prepare_cmd(kill_cmd) 1065 | subprocess.Popen(cmd_args, shell=True) 1066 | else: 1067 | self.sub.kill() 1068 | except: 1069 | pass 1070 | 1071 | self.join() 1072 | self.timed_out=True 1073 | return False 1074 | return True 1075 | 1076 | 1077 | def _is_python_cmd(cmd): 1078 | return isinstance(cmd,types.FunctionType) 1079 | 1080 | 1081 | def run_command_timed( cmd, 1082 | shell_executable=None, 1083 | directory=None, 1084 | osenv=None, 1085 | seconds=0, 1086 | input_file_name=None, 1087 | **kwargs ): 1088 | """Run a timed command. kwargs are keyword args for subprocess.Popen. 1089 | 1090 | @type cmd: string or python function 1091 | @param cmd: command to run 1092 | 1093 | @type shell_executable: string 1094 | @param shell_executable: the shell executable 1095 | 1096 | @type directory: string 1097 | @param directory: the directory to run the command in 1098 | 1099 | @type osenv: dictionary 1100 | @param osenv: dict of environment vars to be passed to the new process 1101 | 1102 | @type seconds: number 1103 | @param seconds: maximum execution time in seconds 1104 | 1105 | @type input_file_name: string 1106 | @param input_file_name: input filename when redirecting stdin. 1107 | 1108 | @type kwargs: keyword args 1109 | @param kwargs: keyword args for subprocess.Popen 1110 | 1111 | @rtype: tuple 1112 | return: (return code, list of stdout+stderr lines) 1113 | """ 1114 | 1115 | def _get_exit_code(tc): 1116 | exit_code = 399 1117 | if tc.sub: 1118 | # if tc.sub does not have a returncode, then something went 1119 | # very wrong, usually an exception running the subprocess. 1120 | if hasattr(tc.sub, 'returncode'): 1121 | exit_code = tc.sub.returncode 1122 | return exit_code 1123 | 1124 | # we use a temporary file to hold the output because killing the 1125 | # process disrupts the normal output collection mechanism. 1126 | fo = tempfile.SpooledTemporaryFile() # FIXME: PY3 mode='w+'? 1127 | fe = tempfile.SpooledTemporaryFile() # FIXME: PY3 mode='w+'? 1128 | tc = _timed_command_t(cmd, 1129 | shell_executable, 1130 | directory, 1131 | osenv, 1132 | seconds, 1133 | input_file_name, 1134 | stdout=fo, 1135 | stderr=fe, 1136 | **kwargs) 1137 | 1138 | tc.timed_run() 1139 | 1140 | if _is_python_cmd(tc.cmd): 1141 | exit_code = tc.exitcode 1142 | output = tc.output 1143 | stderr = tc.stderr 1144 | else: 1145 | fo.seek(0) 1146 | output = fo.readlines() 1147 | fo.close() 1148 | output = ensure_string(output) 1149 | 1150 | fe.seek(0) 1151 | stderr = fe.readlines() 1152 | fe.close() 1153 | stderr = ensure_string(stderr) 1154 | exit_code = _get_exit_code(tc) 1155 | 1156 | nl = u'\n' 1157 | if tc.timed_out: 1158 | stderr.extend([ nl, 1159 | u'COMMAND TIMEOUT'+nl, 1160 | u'KILLING PROCCESS'+nl]) 1161 | if tc.exception_type: 1162 | stderr.extend([ nl, 1163 | u'COMMAND ENCOUNTERD AN EXCEPTION' + nl]) 1164 | stderr.extend(traceback.format_exception(tc.exception_type, 1165 | tc.exception_object, 1166 | tc.exception_trace)) 1167 | 1168 | return (exit_code, output, stderr) 1169 | 1170 | 1171 | def make_list_of_str(lst): 1172 | return [ str(x) for x in lst] 1173 | --------------------------------------------------------------------------------