├── README.md ├── manage_externals ├── manic │ ├── __init__.py │ ├── global_constants.py │ ├── repository_factory.py │ ├── repository.py │ ├── externals_status.py │ ├── repository_svn.py │ ├── utils.py │ ├── sourcetree.py │ ├── checkout.py │ ├── repository_git.py │ └── externals_description.py ├── .gitignore ├── .dir_locals.el ├── .travis.yml ├── checkout_externals ├── LICENSE.txt └── README.md └── Externals.cfg /README.md: -------------------------------------------------------------------------------- 1 | # UFS Medium-Range Weather App 2 | -------------------------------------------------------------------------------- /manage_externals/manic/__init__.py: -------------------------------------------------------------------------------- 1 | """Public API for the manage_externals library 2 | """ 3 | 4 | from manic import checkout 5 | from manic.utils import printlog 6 | 7 | __all__ = [ 8 | 'checkout', 'printlog', 9 | ] 10 | -------------------------------------------------------------------------------- /manage_externals/.gitignore: -------------------------------------------------------------------------------- 1 | # directories that are checked out by the tool 2 | cime/ 3 | cime_config/ 4 | components/ 5 | 6 | # generated local files 7 | *.log 8 | 9 | # editor files 10 | *~ 11 | *.bak 12 | 13 | # generated python files 14 | *.pyc 15 | -------------------------------------------------------------------------------- /manage_externals/.dir_locals.el: -------------------------------------------------------------------------------- 1 | ; -*- mode: Lisp -*- 2 | 3 | ((python-mode 4 | . ( 5 | ;; fill the paragraph to 80 columns when using M-q 6 | (fill-column . 80) 7 | 8 | ;; Use 4 spaces to indent in Python 9 | (python-indent-offset . 4) 10 | (indent-tabs-mode . nil) 11 | ))) 12 | 13 | -------------------------------------------------------------------------------- /manage_externals/manic/global_constants.py: -------------------------------------------------------------------------------- 1 | """Globals shared across modules 2 | """ 3 | 4 | from __future__ import absolute_import 5 | from __future__ import unicode_literals 6 | from __future__ import print_function 7 | 8 | import pprint 9 | 10 | EMPTY_STR = '' 11 | LOCAL_PATH_INDICATOR = '.' 12 | VERSION_SEPERATOR = '.' 13 | LOG_FILE_NAME = 'manage_externals.log' 14 | PPRINTER = pprint.PrettyPrinter(indent=4) 15 | 16 | VERBOSITY_DEFAULT = 0 17 | VERBOSITY_VERBOSE = 1 18 | VERBOSITY_DUMP = 2 19 | -------------------------------------------------------------------------------- /Externals.cfg: -------------------------------------------------------------------------------- 1 | [externals_description] 2 | schema_version = 1.0.0 3 | 4 | [model] 5 | repo_url = https://github.com/ufs-community/ufs-weather-model/ 6 | hash = 7a4a7f3d 7 | protocol = git 8 | local_path = src/model 9 | required = True 10 | 11 | [cime] 12 | #hash = aa44652 13 | branch = ufs_release 14 | protocol = git 15 | repo_url = https://github.com/jedwards4b/cime.git 16 | local_path = cime 17 | required = True 18 | 19 | # this layer required for CIME to know how to build 20 | # FV3GFS - this should be merged into the fv3atm 21 | # repository so this extra repo is not needed 22 | [fv3gfs_interface] 23 | branch = jpe_cime_bld 24 | protocol = git 25 | repo_url = https://github.com/ESCOMP/fv3gfs_interface.git 26 | local_path = src/model/FV3/cime 27 | required = True 28 | 29 | [nems_interface] 30 | branch = master 31 | protocol = git 32 | repo_url = https://github.com/ESCOMP/NEMS_interface.git 33 | local_path = src/model/NEMS/cime/ 34 | required = True 35 | -------------------------------------------------------------------------------- /manage_externals/.travis.yml: -------------------------------------------------------------------------------- 1 | # NOTE(bja, 2017-11) travis-ci dosen't support python language builds 2 | # on mac os. As a work around, we use built-in python on linux, and 3 | # declare osx a 'generic' language, and create our own python env. 4 | 5 | language: python 6 | os: linux 7 | python: 8 | - "2.7" 9 | - "3.4" 10 | - "3.5" 11 | - "3.6" 12 | matrix: 13 | include: 14 | - os: osx 15 | language: generic 16 | before_install: 17 | # NOTE(bja, 2017-11) update is slow, 2.7.12 installed by default, good enough! 18 | # - brew update 19 | # - brew outdated python2 || brew upgrade python2 20 | - pip install virtualenv 21 | - virtualenv env -p python2 22 | - source env/bin/activate 23 | install: 24 | - pip install -r test/requirements.txt 25 | before_script: 26 | - git --version 27 | script: 28 | - cd test; make test 29 | - cd test; make lint 30 | after_success: 31 | - cd test; make coverage 32 | - cd test; coveralls 33 | -------------------------------------------------------------------------------- /manage_externals/checkout_externals: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Main driver wrapper around the manic/checkout utility. 4 | 5 | Tool to assemble external respositories represented in an externals 6 | description file. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | from __future__ import unicode_literals 11 | from __future__ import print_function 12 | 13 | import sys 14 | import traceback 15 | 16 | import manic 17 | 18 | if sys.hexversion < 0x02070000: 19 | print(70 * '*') 20 | print('ERROR: {0} requires python >= 2.7.x. '.format(sys.argv[0])) 21 | print('It appears that you are running python {0}'.format( 22 | '.'.join(str(x) for x in sys.version_info[0:3]))) 23 | print(70 * '*') 24 | sys.exit(1) 25 | 26 | 27 | if __name__ == '__main__': 28 | ARGS = manic.checkout.commandline_arguments() 29 | try: 30 | RET_STATUS, _ = manic.checkout.main(ARGS) 31 | sys.exit(RET_STATUS) 32 | except Exception as error: # pylint: disable=broad-except 33 | manic.printlog(str(error)) 34 | if ARGS.backtrace: 35 | traceback.print_exc() 36 | sys.exit(1) 37 | -------------------------------------------------------------------------------- /manage_externals/manic/repository_factory.py: -------------------------------------------------------------------------------- 1 | """Factory for creating and initializing the appropriate repository class 2 | """ 3 | 4 | from __future__ import absolute_import 5 | from __future__ import unicode_literals 6 | from __future__ import print_function 7 | 8 | from .repository_git import GitRepository 9 | from .repository_svn import SvnRepository 10 | from .externals_description import ExternalsDescription 11 | from .utils import fatal_error 12 | 13 | 14 | def create_repository(component_name, repo_info, svn_ignore_ancestry=False): 15 | """Determine what type of repository we have, i.e. git or svn, and 16 | create the appropriate object. 17 | 18 | """ 19 | protocol = repo_info[ExternalsDescription.PROTOCOL].lower() 20 | if protocol == 'git': 21 | repo = GitRepository(component_name, repo_info) 22 | elif protocol == 'svn': 23 | repo = SvnRepository(component_name, repo_info, ignore_ancestry=svn_ignore_ancestry) 24 | elif protocol == 'externals_only': 25 | repo = None 26 | else: 27 | msg = 'Unknown repo protocol "{0}"'.format(protocol) 28 | fatal_error(msg) 29 | return repo 30 | -------------------------------------------------------------------------------- /manage_externals/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017-2018, University Corporation for Atmospheric Research (UCAR) 2 | All rights reserved. 3 | 4 | Developed by: 5 | University Corporation for Atmospheric Research - National Center for Atmospheric Research 6 | https://www2.cesm.ucar.edu/working-groups/sewg 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining 9 | a copy of this software and associated documentation files (the "Software"), 10 | to deal with the Software without restriction, including without limitation 11 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 12 | and/or sell copies of the Software, and to permit persons to whom 13 | the Software is furnished to do so, subject to the following conditions: 14 | 15 | - Redistributions of source code must retain the above copyright notice, 16 | this list of conditions and the following disclaimers. 17 | - Redistributions in binary form must reproduce the above copyright notice, 18 | this list of conditions and the following disclaimers in the documentation 19 | and/or other materials provided with the distribution. 20 | - Neither the names of [Name of Development Group, UCAR], 21 | nor the names of its contributors may be used to endorse or promote 22 | products derived from this Software without specific prior written permission. 23 | 24 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 25 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 28 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 | POSSIBILITY OF SUCH DAMAGE. 35 | -------------------------------------------------------------------------------- /manage_externals/manic/repository.py: -------------------------------------------------------------------------------- 1 | """Base class representation of a repository 2 | """ 3 | 4 | from .externals_description import ExternalsDescription 5 | from .utils import fatal_error 6 | from .global_constants import EMPTY_STR 7 | 8 | 9 | class Repository(object): 10 | """ 11 | Class to represent and operate on a repository description. 12 | """ 13 | 14 | def __init__(self, component_name, repo): 15 | """ 16 | Parse repo externals description 17 | """ 18 | self._name = component_name 19 | self._protocol = repo[ExternalsDescription.PROTOCOL] 20 | self._tag = repo[ExternalsDescription.TAG] 21 | self._branch = repo[ExternalsDescription.BRANCH] 22 | self._hash = repo[ExternalsDescription.HASH] 23 | self._url = repo[ExternalsDescription.REPO_URL] 24 | 25 | if self._url is EMPTY_STR: 26 | fatal_error('repo must have a URL') 27 | 28 | if ((self._tag is EMPTY_STR) and (self._branch is EMPTY_STR) and 29 | (self._hash is EMPTY_STR)): 30 | fatal_error('{0} repo must have a branch, tag or hash element') 31 | 32 | ref_count = 0 33 | if self._tag is not EMPTY_STR: 34 | ref_count += 1 35 | if self._branch is not EMPTY_STR: 36 | ref_count += 1 37 | if self._hash is not EMPTY_STR: 38 | ref_count += 1 39 | if ref_count != 1: 40 | fatal_error('repo {0} must have exactly one of ' 41 | 'tag, branch or hash.'.format(self._name)) 42 | 43 | def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument 44 | """ 45 | If the repo destination directory exists, ensure it is correct (from 46 | correct URL, correct branch or tag), and possibly update the source. 47 | If the repo destination directory does not exist, checkout the correce 48 | branch or tag. 49 | NB: is include as an argument for compatibility with 50 | git functionality (repository_git.py) 51 | """ 52 | msg = ('DEV_ERROR: checkout method must be implemented in all ' 53 | 'repository classes! {0}'.format(self.__class__.__name__)) 54 | fatal_error(msg) 55 | 56 | def status(self, stat, repo_dir_path): # pylint: disable=unused-argument 57 | """Report the status of the repo 58 | 59 | """ 60 | msg = ('DEV_ERROR: status method must be implemented in all ' 61 | 'repository classes! {0}'.format(self.__class__.__name__)) 62 | fatal_error(msg) 63 | 64 | def submodules_file(self, repo_path=None): 65 | # pylint: disable=no-self-use,unused-argument 66 | """Stub for use by non-git VC systems""" 67 | return None 68 | 69 | def url(self): 70 | """Public access of repo url. 71 | """ 72 | return self._url 73 | 74 | def tag(self): 75 | """Public access of repo tag 76 | """ 77 | return self._tag 78 | 79 | def branch(self): 80 | """Public access of repo branch. 81 | """ 82 | return self._branch 83 | 84 | def hash(self): 85 | """Public access of repo hash. 86 | """ 87 | return self._hash 88 | 89 | def name(self): 90 | """Public access of repo name. 91 | """ 92 | return self._name 93 | 94 | def protocol(self): 95 | """Public access of repo protocol. 96 | """ 97 | return self._protocol 98 | -------------------------------------------------------------------------------- /manage_externals/manic/externals_status.py: -------------------------------------------------------------------------------- 1 | """ExternalStatus 2 | 3 | Class to store status and state information about repositories and 4 | create a string representation. 5 | 6 | """ 7 | from __future__ import absolute_import 8 | from __future__ import unicode_literals 9 | from __future__ import print_function 10 | 11 | from .global_constants import EMPTY_STR 12 | from .utils import printlog, indent_string 13 | from .global_constants import VERBOSITY_VERBOSE, VERBOSITY_DUMP 14 | 15 | 16 | class ExternalStatus(object): 17 | """Class to represent the status of a given source repository or tree. 18 | 19 | Individual repositories determine their own status in the 20 | Repository objects. This object is just resposible for storing the 21 | information and passing it up to a higher level for reporting or 22 | global decisions. 23 | 24 | There are two states of concern: 25 | 26 | * If the repository is in-sync with the externals description file. 27 | 28 | * If the repostiory working copy is clean and there are no pending 29 | transactions (e.g. add, remove, rename, untracked files). 30 | 31 | """ 32 | DEFAULT = '-' 33 | UNKNOWN = '?' 34 | EMPTY = 'e' 35 | MODEL_MODIFIED = 's' # a.k.a. out-of-sync 36 | DIRTY = 'M' 37 | 38 | STATUS_OK = ' ' 39 | STATUS_ERROR = '!' 40 | 41 | # source types 42 | OPTIONAL = 'o' 43 | STANDALONE = 's' 44 | MANAGED = ' ' 45 | 46 | def __init__(self): 47 | self.sync_state = self.DEFAULT 48 | self.clean_state = self.DEFAULT 49 | self.source_type = self.DEFAULT 50 | self.path = EMPTY_STR 51 | self.current_version = EMPTY_STR 52 | self.expected_version = EMPTY_STR 53 | self.status_output = EMPTY_STR 54 | 55 | def log_status_message(self, verbosity): 56 | """Write status message to the screen and log file 57 | """ 58 | self._default_status_message() 59 | if verbosity >= VERBOSITY_VERBOSE: 60 | self._verbose_status_message() 61 | if verbosity >= VERBOSITY_DUMP: 62 | self._dump_status_message() 63 | 64 | def _default_status_message(self): 65 | """Return the default terse status message string 66 | """ 67 | msg = '{sync}{clean}{src_type} {path}'.format( 68 | sync=self.sync_state, clean=self.clean_state, 69 | src_type=self.source_type, path=self.path) 70 | printlog(msg) 71 | 72 | def _verbose_status_message(self): 73 | """Return the verbose status message string 74 | """ 75 | clean_str = self.DEFAULT 76 | if self.clean_state == self.STATUS_OK: 77 | clean_str = 'clean sandbox' 78 | elif self.clean_state == self.DIRTY: 79 | clean_str = 'modified sandbox' 80 | 81 | sync_str = 'on {0}'.format(self.current_version) 82 | if self.sync_state != self.STATUS_OK: 83 | sync_str = '{current} --> {expected}'.format( 84 | current=self.current_version, expected=self.expected_version) 85 | msg = ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) 86 | printlog(msg) 87 | 88 | def _dump_status_message(self): 89 | """Return the dump status message string 90 | """ 91 | msg = indent_string(self.status_output, 12) 92 | printlog(msg) 93 | 94 | def safe_to_update(self): 95 | """Report if it is safe to update a repository. Safe is defined as: 96 | 97 | * If a repository is empty, it is safe to update. 98 | 99 | * If a repository exists and has a clean working copy state 100 | with no pending transactions. 101 | 102 | """ 103 | safe_to_update = False 104 | repo_exists = self.exists() 105 | if not repo_exists: 106 | safe_to_update = True 107 | else: 108 | # If the repo exists, it must be in ok or modified 109 | # sync_state. Any other sync_state at this point 110 | # represents a logic error that should have been handled 111 | # before now! 112 | sync_safe = ((self.sync_state == ExternalStatus.STATUS_OK) or 113 | (self.sync_state == ExternalStatus.MODEL_MODIFIED)) 114 | if sync_safe: 115 | # The clean_state must be STATUS_OK to update. Otherwise we 116 | # are dirty or there was a missed error previously. 117 | if self.clean_state == ExternalStatus.STATUS_OK: 118 | safe_to_update = True 119 | return safe_to_update 120 | 121 | def exists(self): 122 | """Determine if the repo exists. This is indicated by: 123 | 124 | * sync_state is not EMPTY 125 | 126 | * if the sync_state is empty, then the valid states for 127 | clean_state are default, empty or unknown. Anything else 128 | and there was probably an internal logic error. 129 | 130 | NOTE(bja, 2017-10) For the moment we are considering a 131 | sync_state of default or unknown to require user intervention, 132 | but we may want to relax this convention. This is probably a 133 | result of a network error or internal logic error but more 134 | testing is needed. 135 | 136 | """ 137 | is_empty = (self.sync_state == ExternalStatus.EMPTY) 138 | clean_valid = ((self.clean_state == ExternalStatus.DEFAULT) or 139 | (self.clean_state == ExternalStatus.EMPTY) or 140 | (self.clean_state == ExternalStatus.UNKNOWN)) 141 | 142 | if is_empty and clean_valid: 143 | exists = False 144 | else: 145 | exists = True 146 | return exists 147 | 148 | 149 | def check_safe_to_update_repos(tree_status): 150 | """Check if *ALL* repositories are in a safe state to update. We don't 151 | want to do a partial update of the repositories then die, leaving 152 | the model in an inconsistent state. 153 | 154 | Note: if there is an update to do, the repositories will by 155 | definiation be out of synce with the externals description, so we 156 | can't use that as criteria for updating. 157 | 158 | """ 159 | safe_to_update = True 160 | for comp in tree_status: 161 | stat = tree_status[comp] 162 | safe_to_update &= stat.safe_to_update() 163 | 164 | return safe_to_update 165 | -------------------------------------------------------------------------------- /manage_externals/README.md: -------------------------------------------------------------------------------- 1 | -- AUTOMATICALLY GENERATED FILE. DO NOT EDIT -- 2 | 3 | [![Build Status](https://travis-ci.org/ESMCI/manage_externals.svg?branch=master)](https://travis-ci.org/ESMCI/manage_externals)[![Coverage Status](https://coveralls.io/repos/github/ESMCI/manage_externals/badge.svg?branch=master)](https://coveralls.io/github/ESMCI/manage_externals?branch=master) 4 | ``` 5 | usage: checkout_externals [-h] [-e [EXTERNALS]] [-o] [-S] [-v] [--backtrace] 6 | [-d] [--no-logging] 7 | 8 | checkout_externals manages checking out groups of externals from revision 9 | control based on a externals description file. By default only the 10 | required externals are checkout out. 11 | 12 | Operations performed by manage_externals utilities are explicit and 13 | data driven. checkout_externals will always make the working copy *exactly* 14 | match what is in the externals file when modifying the working copy of 15 | a repository. 16 | 17 | If checkout_externals isn't doing what you expected, double check the contents 18 | of the externals description file. 19 | 20 | Running checkout_externals without the '--status' option will always attempt to 21 | synchronize the working copy to exactly match the externals description. 22 | 23 | optional arguments: 24 | -h, --help show this help message and exit 25 | -e [EXTERNALS], --externals [EXTERNALS] 26 | The externals description filename. Default: 27 | Externals.cfg. 28 | -o, --optional By default only the required externals are checked 29 | out. This flag will also checkout the optional 30 | externals. 31 | -S, --status Output status of the repositories managed by 32 | checkout_externals. By default only summary 33 | information is provided. Use verbose output to see 34 | details. 35 | -v, --verbose Output additional information to the screen and log 36 | file. This flag can be used up to two times, 37 | increasing the verbosity level each time. 38 | --backtrace DEVELOPER: show exception backtraces as extra 39 | debugging output 40 | -d, --debug DEVELOPER: output additional debugging information to 41 | the screen and log file. 42 | --no-logging DEVELOPER: disable logging. 43 | 44 | ``` 45 | NOTE: checkout_externals *MUST* be run from the root of the source tree it 46 | is managing. For example, if you cloned a repository with: 47 | 48 | $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev 49 | 50 | Then the root of the source tree is /path/to/some-project-dev. If you 51 | obtained a sub-project via a checkout of another project: 52 | 53 | $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev 54 | 55 | and you need to checkout the sub-project externals, then the root of the 56 | source tree is /path/to/some-project-dev. Do *NOT* run checkout_externals 57 | from within /path/to/some-project-dev/sub-project 58 | 59 | The root of the source tree will be referred to as `${SRC_ROOT}` below. 60 | 61 | # Supported workflows 62 | 63 | * Checkout all required components from the default externals 64 | description file: 65 | 66 | $ cd ${SRC_ROOT} 67 | $ ./manage_externals/checkout_externals 68 | 69 | * To update all required components to the current values in the 70 | externals description file, re-run checkout_externals: 71 | 72 | $ cd ${SRC_ROOT} 73 | $ ./manage_externals/checkout_externals 74 | 75 | If there are *any* modifications to *any* working copy according 76 | to the git or svn 'status' command, checkout_externals 77 | will not update any external repositories. Modifications 78 | include: modified files, added files, removed files, or missing 79 | files. 80 | 81 | To avoid this safety check, edit the externals description file 82 | and comment out the modified external block. 83 | 84 | * Checkout all required components from a user specified externals 85 | description file: 86 | 87 | $ cd ${SRC_ROOT} 88 | $ ./manage_externals/checkout_externals --excernals my-externals.cfg 89 | 90 | * Status summary of the repositories managed by checkout_externals: 91 | 92 | $ cd ${SRC_ROOT} 93 | $ ./manage_externals/checkout_externals --status 94 | 95 | ./cime 96 | s ./components/cism 97 | ./components/mosart 98 | e-o ./components/rtm 99 | M ./src/fates 100 | e-o ./tools/PTCLM 101 | 102 | where: 103 | * column one indicates the status of the repository in relation 104 | to the externals description file. 105 | * column two indicates whether the working copy has modified files. 106 | * column three shows how the repository is managed, optional or required 107 | 108 | Column one will be one of these values: 109 | * s : out-of-sync : repository is checked out at a different commit 110 | compared with the externals description 111 | * e : empty : directory does not exist - checkout_externals has not been run 112 | * ? : unknown : directory exists but .git or .svn directories are missing 113 | 114 | Column two will be one of these values: 115 | * M : Modified : modified, added, deleted or missing files 116 | * : blank / space : clean 117 | * - : dash : no meaningful state, for empty repositories 118 | 119 | Column three will be one of these values: 120 | * o : optional : optionally repository 121 | * : blank / space : required repository 122 | 123 | * Detailed git or svn status of the repositories managed by checkout_externals: 124 | 125 | $ cd ${SRC_ROOT} 126 | $ ./manage_externals/checkout_externals --status --verbose 127 | 128 | # Externals description file 129 | 130 | The externals description contains a list of the external 131 | repositories that are used and their version control locations. The 132 | file format is the standard ini/cfg configuration file format. Each 133 | external is defined by a section containing the component name in 134 | square brackets: 135 | 136 | * name (string) : component name, e.g. [cime], [cism], etc. 137 | 138 | Each section has the following keyword-value pairs: 139 | 140 | * required (boolean) : whether the component is a required checkout, 141 | 'true' or 'false'. 142 | 143 | * local_path (string) : component path *relative* to where 144 | checkout_externals is called. 145 | 146 | * protoctol (string) : version control protocol that is used to 147 | manage the component. Valid values are 'git', 'svn', 148 | 'externals_only'. 149 | 150 | Switching an external between different protocols is not 151 | supported, e.g. from svn to git. To switch protocols, you need to 152 | manually move the old working copy to a new location. 153 | 154 | Note: 'externals_only' will only process the external's own 155 | external description file without trying to manage a repository 156 | for the component. This is used for retreiving externals for 157 | standalone components like cam and clm. If the source root of the 158 | externals_only component is the same as the main source root, then 159 | the local path must be set to '.', the unix current working 160 | directory, e. g. 'local_path = .' 161 | 162 | * repo_url (string) : URL for the repository location, examples: 163 | * https://svn-ccsm-models.cgd.ucar.edu/glc 164 | * git@github.com:esmci/cime.git 165 | * /path/to/local/repository 166 | * . 167 | 168 | NOTE: To operate on only the local clone and and ignore remote 169 | repositories, set the url to '.' (the unix current path), 170 | i.e. 'repo_url = .' . This can be used to checkout a local branch 171 | instead of the upstream branch. 172 | 173 | If a repo url is determined to be a local path (not a network url) 174 | then user expansion, e.g. ~/, and environment variable expansion, 175 | e.g. $HOME or $REPO_ROOT, will be performed. 176 | 177 | Relative paths are difficult to get correct, especially for mixed 178 | use repos. It is advised that local paths expand to absolute paths. 179 | If relative paths are used, they should be relative to one level 180 | above local_path. If local path is 'src/foo', the the relative url 181 | should be relative to 'src'. 182 | 183 | * tag (string) : tag to checkout 184 | 185 | * hash (string) : the git hash to checkout. Only applies to git 186 | repositories. 187 | 188 | * branch (string) : branch to checkout from the specified 189 | repository. Specifying a branch on a remote repository means that 190 | checkout_externals will checkout the version of the branch in the remote, 191 | not the the version in the local repository (if it exists). 192 | 193 | Note: one and only one of tag, branch hash must be supplied. 194 | 195 | * externals (string) : used to make manage_externals aware of 196 | sub-externals required by an external. This is a relative path to 197 | the external's root directory. For example, the main externals 198 | description has an external checkout out at 'src/useful_library'. 199 | useful_library requires additional externals to be complete. 200 | Those additional externals are managed from the source root by the 201 | externals description file pointed 'useful_library/sub-xternals.cfg', 202 | Then the main 'externals' field in the top level repo should point to 203 | 'sub-externals.cfg'. 204 | 205 | * Lines begining with '#' or ';' are comments and will be ignored. 206 | 207 | # Obtaining this tool, reporting issues, etc. 208 | 209 | The master repository for manage_externals is 210 | https://github.com/ESMCI/manage_externals. Any issues with this tool 211 | should be reported there. 212 | -------------------------------------------------------------------------------- /manage_externals/manic/repository_svn.py: -------------------------------------------------------------------------------- 1 | """Class for interacting with svn repositories 2 | """ 3 | 4 | from __future__ import absolute_import 5 | from __future__ import unicode_literals 6 | from __future__ import print_function 7 | 8 | import os 9 | import re 10 | import xml.etree.ElementTree as ET 11 | 12 | from .global_constants import EMPTY_STR, VERBOSITY_VERBOSE 13 | from .repository import Repository 14 | from .externals_status import ExternalStatus 15 | from .utils import fatal_error, indent_string, printlog 16 | from .utils import execute_subprocess 17 | 18 | 19 | class SvnRepository(Repository): 20 | """ 21 | Class to represent and operate on a repository description. 22 | 23 | For testing purpose, all system calls to svn should: 24 | 25 | * be isolated in separate functions with no application logic 26 | * of the form: 27 | - cmd = ['svn', ...] 28 | - value = execute_subprocess(cmd, output_to_caller={T|F}, 29 | status_to_caller={T|F}) 30 | - return value 31 | * be static methods (not rely on self) 32 | * name as _svn_subcommand_args(user_args) 33 | 34 | This convention allows easy unit testing of the repository logic 35 | by mocking the specific calls to return predefined results. 36 | 37 | """ 38 | RE_URLLINE = re.compile(r'^URL:') 39 | 40 | def __init__(self, component_name, repo, ignore_ancestry=False): 41 | """ 42 | Parse repo (a XML element). 43 | """ 44 | Repository.__init__(self, component_name, repo) 45 | self._ignore_ancestry = ignore_ancestry 46 | if self._branch: 47 | self._url = os.path.join(self._url, self._branch) 48 | elif self._tag: 49 | self._url = os.path.join(self._url, self._tag) 50 | else: 51 | msg = "DEV_ERROR in svn repository. Shouldn't be here!" 52 | fatal_error(msg) 53 | 54 | # ---------------------------------------------------------------- 55 | # 56 | # Public API, defined by Repository 57 | # 58 | # ---------------------------------------------------------------- 59 | def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument 60 | """Checkout or update the working copy 61 | 62 | If the repo destination directory exists, switch the sandbox to 63 | match the externals description. 64 | 65 | If the repo destination directory does not exist, checkout the 66 | correct branch or tag. 67 | NB: is include as an argument for compatibility with 68 | git functionality (repository_git.py) 69 | 70 | """ 71 | repo_dir_path = os.path.join(base_dir_path, repo_dir_name) 72 | if os.path.exists(repo_dir_path): 73 | cwd = os.getcwd() 74 | os.chdir(repo_dir_path) 75 | self._svn_switch(self._url, self._ignore_ancestry, verbosity) 76 | # svn switch can lead to a conflict state, but it gives a 77 | # return code of 0. So now we need to make sure that we're 78 | # in a clean (non-conflict) state. 79 | self._abort_if_dirty(repo_dir_path, 80 | "Expected clean state following switch") 81 | os.chdir(cwd) 82 | else: 83 | self._svn_checkout(self._url, repo_dir_path, verbosity) 84 | 85 | def status(self, stat, repo_dir_path): 86 | """ 87 | Check and report the status of the repository 88 | """ 89 | self._check_sync(stat, repo_dir_path) 90 | if os.path.exists(repo_dir_path): 91 | self._status_summary(stat, repo_dir_path) 92 | 93 | # ---------------------------------------------------------------- 94 | # 95 | # Internal work functions 96 | # 97 | # ---------------------------------------------------------------- 98 | def _check_sync(self, stat, repo_dir_path): 99 | """Check to see if repository directory exists and is at the expected 100 | url. Return: status object 101 | 102 | """ 103 | if not os.path.exists(repo_dir_path): 104 | # NOTE(bja, 2017-10) this state should have been handled by 105 | # the source object and we never get here! 106 | stat.sync_state = ExternalStatus.STATUS_ERROR 107 | else: 108 | svn_output = self._svn_info(repo_dir_path) 109 | if not svn_output: 110 | # directory exists, but info returned nothing. .svn 111 | # directory removed or incomplete checkout? 112 | stat.sync_state = ExternalStatus.UNKNOWN 113 | else: 114 | stat.sync_state, stat.current_version = \ 115 | self._check_url(svn_output, self._url) 116 | stat.expected_version = '/'.join(self._url.split('/')[3:]) 117 | 118 | def _abort_if_dirty(self, repo_dir_path, message): 119 | """Check if the repo is in a dirty state; if so, abort with a 120 | helpful message. 121 | 122 | """ 123 | 124 | stat = ExternalStatus() 125 | self._status_summary(stat, repo_dir_path) 126 | if stat.clean_state != ExternalStatus.STATUS_OK: 127 | status = self._svn_status_verbose(repo_dir_path) 128 | status = indent_string(status, 4) 129 | errmsg = """In directory 130 | {cwd} 131 | 132 | svn status now shows: 133 | {status} 134 | 135 | ERROR: {message} 136 | 137 | One possible cause of this problem is that there may have been untracked 138 | files in your working directory that had the same name as tracked files 139 | in the new revision. 140 | 141 | To recover: Clean up the above directory (resolving conflicts, etc.), 142 | then rerun checkout_externals. 143 | """.format(cwd=repo_dir_path, message=message, status=status) 144 | 145 | fatal_error(errmsg) 146 | 147 | @staticmethod 148 | def _check_url(svn_output, expected_url): 149 | """Determine the svn url from svn info output and return whether it 150 | matches the expected value. 151 | 152 | """ 153 | url = None 154 | for line in svn_output.splitlines(): 155 | if SvnRepository.RE_URLLINE.match(line): 156 | url = line.split(': ')[1].strip() 157 | break 158 | if not url: 159 | status = ExternalStatus.UNKNOWN 160 | elif url == expected_url: 161 | status = ExternalStatus.STATUS_OK 162 | else: 163 | status = ExternalStatus.MODEL_MODIFIED 164 | 165 | if url: 166 | current_version = '/'.join(url.split('/')[3:]) 167 | else: 168 | current_version = EMPTY_STR 169 | 170 | return status, current_version 171 | 172 | def _status_summary(self, stat, repo_dir_path): 173 | """Report whether the svn repository is in-sync with the model 174 | description and whether the sandbox is clean or dirty. 175 | 176 | """ 177 | svn_output = self._svn_status_xml(repo_dir_path) 178 | is_dirty = self.xml_status_is_dirty(svn_output) 179 | if is_dirty: 180 | stat.clean_state = ExternalStatus.DIRTY 181 | else: 182 | stat.clean_state = ExternalStatus.STATUS_OK 183 | 184 | # Now save the verbose status output incase the user wants to 185 | # see it. 186 | stat.status_output = self._svn_status_verbose(repo_dir_path) 187 | 188 | @staticmethod 189 | def xml_status_is_dirty(svn_output): 190 | """Parse svn status xml output and determine if the working copy is 191 | clean or dirty. Dirty is defined as: 192 | 193 | * modified files 194 | * added files 195 | * deleted files 196 | * missing files 197 | 198 | Unversioned files do not affect the clean/dirty status. 199 | 200 | 'external' is also an acceptable state 201 | 202 | """ 203 | # pylint: disable=invalid-name 204 | SVN_EXTERNAL = 'external' 205 | SVN_UNVERSIONED = 'unversioned' 206 | # pylint: enable=invalid-name 207 | 208 | is_dirty = False 209 | try: 210 | xml_status = ET.fromstring(svn_output) 211 | except BaseException: 212 | fatal_error( 213 | "SVN returned invalid XML message {}".format(svn_output)) 214 | xml_target = xml_status.find('./target') 215 | entries = xml_target.findall('./entry') 216 | for entry in entries: 217 | status = entry.find('./wc-status') 218 | item = status.get('item') 219 | if item == SVN_EXTERNAL: 220 | continue 221 | if item == SVN_UNVERSIONED: 222 | continue 223 | else: 224 | is_dirty = True 225 | break 226 | return is_dirty 227 | 228 | # ---------------------------------------------------------------- 229 | # 230 | # system call to svn for information gathering 231 | # 232 | # ---------------------------------------------------------------- 233 | @staticmethod 234 | def _svn_info(repo_dir_path): 235 | """Return results of svn info command 236 | """ 237 | cmd = ['svn', 'info', repo_dir_path] 238 | output = execute_subprocess(cmd, output_to_caller=True) 239 | return output 240 | 241 | @staticmethod 242 | def _svn_status_verbose(repo_dir_path): 243 | """capture the full svn status output 244 | """ 245 | cmd = ['svn', 'status', repo_dir_path] 246 | svn_output = execute_subprocess(cmd, output_to_caller=True) 247 | return svn_output 248 | 249 | @staticmethod 250 | def _svn_status_xml(repo_dir_path): 251 | """ 252 | Get status of the subversion sandbox in repo_dir 253 | """ 254 | cmd = ['svn', 'status', '--xml', repo_dir_path] 255 | svn_output = execute_subprocess(cmd, output_to_caller=True) 256 | return svn_output 257 | 258 | # ---------------------------------------------------------------- 259 | # 260 | # system call to svn for sideffects modifying the working tree 261 | # 262 | # ---------------------------------------------------------------- 263 | @staticmethod 264 | def _svn_checkout(url, repo_dir_path, verbosity): 265 | """ 266 | Checkout a subversion repository (repo_url) to checkout_dir. 267 | """ 268 | cmd = ['svn', 'checkout', '--quiet', url, repo_dir_path] 269 | if verbosity >= VERBOSITY_VERBOSE: 270 | printlog(' {0}'.format(' '.join(cmd))) 271 | execute_subprocess(cmd) 272 | 273 | @staticmethod 274 | def _svn_switch(url, ignore_ancestry, verbosity): 275 | """ 276 | Switch branches for in an svn sandbox 277 | """ 278 | cmd = ['svn', 'switch', '--quiet'] 279 | if ignore_ancestry: 280 | cmd.append('--ignore-ancestry') 281 | cmd.append(url) 282 | if verbosity >= VERBOSITY_VERBOSE: 283 | printlog(' {0}'.format(' '.join(cmd))) 284 | execute_subprocess(cmd) 285 | -------------------------------------------------------------------------------- /manage_externals/manic/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Common public utilities for manic package 4 | 5 | """ 6 | 7 | from __future__ import absolute_import 8 | from __future__ import unicode_literals 9 | from __future__ import print_function 10 | 11 | import logging 12 | import os 13 | import subprocess 14 | import sys 15 | from threading import Timer 16 | 17 | from .global_constants import LOCAL_PATH_INDICATOR 18 | 19 | # --------------------------------------------------------------------- 20 | # 21 | # screen and logging output and functions to massage text for output 22 | # 23 | # --------------------------------------------------------------------- 24 | 25 | 26 | def log_process_output(output): 27 | """Log each line of process output at debug level so it can be 28 | filtered if necessary. By default, output is a single string, and 29 | logging.debug(output) will only put log info heading on the first 30 | line. This makes it hard to filter with grep. 31 | 32 | """ 33 | output = output.split('\n') 34 | for line in output: 35 | logging.debug(line) 36 | 37 | 38 | def printlog(msg, **kwargs): 39 | """Wrapper script around print to ensure that everything printed to 40 | the screen also gets logged. 41 | 42 | """ 43 | logging.info(msg) 44 | if kwargs: 45 | print(msg, **kwargs) 46 | else: 47 | print(msg) 48 | sys.stdout.flush() 49 | 50 | 51 | def last_n_lines(the_string, n_lines, truncation_message=None): 52 | """Returns the last n lines of the given string 53 | 54 | Args: 55 | the_string: str 56 | n_lines: int 57 | truncation_message: str, optional 58 | 59 | Returns a string containing the last n lines of the_string 60 | 61 | If truncation_message is provided, the returned string begins with 62 | the given message if and only if the string is greater than n lines 63 | to begin with. 64 | """ 65 | 66 | lines = the_string.splitlines(True) 67 | if len(lines) <= n_lines: 68 | return_val = the_string 69 | else: 70 | lines_subset = lines[-n_lines:] 71 | str_truncated = ''.join(lines_subset) 72 | if truncation_message: 73 | str_truncated = truncation_message + '\n' + str_truncated 74 | return_val = str_truncated 75 | 76 | return return_val 77 | 78 | 79 | def indent_string(the_string, indent_level): 80 | """Indents the given string by a given number of spaces 81 | 82 | Args: 83 | the_string: str 84 | indent_level: int 85 | 86 | Returns a new string that is the same as the_string, except that 87 | each line is indented by 'indent_level' spaces. 88 | 89 | In python3, this can be done with textwrap.indent. 90 | """ 91 | 92 | lines = the_string.splitlines(True) 93 | padding = ' ' * indent_level 94 | lines_indented = [padding + line for line in lines] 95 | return ''.join(lines_indented) 96 | 97 | # --------------------------------------------------------------------- 98 | # 99 | # error handling 100 | # 101 | # --------------------------------------------------------------------- 102 | 103 | 104 | def fatal_error(message): 105 | """ 106 | Error output function 107 | """ 108 | logging.error(message) 109 | raise RuntimeError("{0}ERROR: {1}".format(os.linesep, message)) 110 | 111 | 112 | # --------------------------------------------------------------------- 113 | # 114 | # Data conversion / manipulation 115 | # 116 | # --------------------------------------------------------------------- 117 | def str_to_bool(bool_str): 118 | """Convert a sting representation of as boolean into a true boolean. 119 | 120 | Conversion should be case insensitive. 121 | """ 122 | value = None 123 | str_lower = bool_str.lower() 124 | if str_lower in ('true', 't'): 125 | value = True 126 | elif str_lower in ('false', 'f'): 127 | value = False 128 | if value is None: 129 | msg = ('ERROR: invalid boolean string value "{0}". ' 130 | 'Must be "true" or "false"'.format(bool_str)) 131 | fatal_error(msg) 132 | return value 133 | 134 | 135 | REMOTE_PREFIXES = ['http://', 'https://', 'ssh://', 'git@'] 136 | 137 | 138 | def is_remote_url(url): 139 | """check if the user provided a local file path instead of a 140 | remote. If so, it must be expanded to an absolute 141 | path. 142 | 143 | """ 144 | remote_url = False 145 | for prefix in REMOTE_PREFIXES: 146 | if url.startswith(prefix): 147 | remote_url = True 148 | return remote_url 149 | 150 | 151 | def split_remote_url(url): 152 | """check if the user provided a local file path or a 153 | remote. If remote, try to strip off protocol info. 154 | 155 | """ 156 | remote_url = is_remote_url(url) 157 | if not remote_url: 158 | return url 159 | 160 | for prefix in REMOTE_PREFIXES: 161 | url = url.replace(prefix, '') 162 | 163 | if '@' in url: 164 | url = url.split('@')[1] 165 | 166 | if ':' in url: 167 | url = url.split(':')[1] 168 | 169 | return url 170 | 171 | 172 | def expand_local_url(url, field): 173 | """check if the user provided a local file path instead of a 174 | remote. If so, it must be expanded to an absolute 175 | path. 176 | 177 | Note: local paths of LOCAL_PATH_INDICATOR have special meaning and 178 | represent local copy only, don't work with the remotes. 179 | 180 | """ 181 | remote_url = is_remote_url(url) 182 | if not remote_url: 183 | if url.strip() == LOCAL_PATH_INDICATOR: 184 | pass 185 | else: 186 | url = os.path.expandvars(url) 187 | url = os.path.expanduser(url) 188 | if not os.path.isabs(url): 189 | msg = ('WARNING: Externals description for "{0}" contains a ' 190 | 'url that is not remote and does not expand to an ' 191 | 'absolute path. Version control operations may ' 192 | 'fail.\n\nurl={1}'.format(field, url)) 193 | printlog(msg) 194 | else: 195 | url = os.path.normpath(url) 196 | return url 197 | 198 | 199 | # --------------------------------------------------------------------- 200 | # 201 | # subprocess 202 | # 203 | # --------------------------------------------------------------------- 204 | 205 | # Give the user a helpful message if we detect that a command seems to 206 | # be hanging. 207 | _HANGING_SEC = 300 208 | 209 | 210 | def _hanging_msg(working_directory, command): 211 | print(""" 212 | 213 | Command '{command}' 214 | from directory {working_directory} 215 | has taken {hanging_sec} seconds. It may be hanging. 216 | 217 | The command will continue to run, but you may want to abort 218 | manage_externals with ^C and investigate. A possible cause of hangs is 219 | when svn or git require authentication to access a private 220 | repository. On some systems, svn and git requests for authentication 221 | information will not be displayed to the user. In this case, the program 222 | will appear to hang. Ensure you can run svn and git manually and access 223 | all repositories without entering your authentication information. 224 | 225 | """.format(command=command, 226 | working_directory=working_directory, 227 | hanging_sec=_HANGING_SEC)) 228 | 229 | 230 | def execute_subprocess(commands, status_to_caller=False, 231 | output_to_caller=False): 232 | """Wrapper around subprocess.check_output to handle common 233 | exceptions. 234 | 235 | check_output runs a command with arguments and waits 236 | for it to complete. 237 | 238 | check_output raises an exception on a nonzero return code. if 239 | status_to_caller is true, execute_subprocess returns the subprocess 240 | return code, otherwise execute_subprocess treats non-zero return 241 | status as an error and raises an exception. 242 | 243 | """ 244 | cwd = os.getcwd() 245 | msg = 'In directory: {0}\nexecute_subprocess running command:'.format(cwd) 246 | logging.info(msg) 247 | commands_str = ' '.join(commands) 248 | logging.info(commands_str) 249 | return_to_caller = status_to_caller or output_to_caller 250 | status = -1 251 | output = '' 252 | hanging_timer = Timer(_HANGING_SEC, _hanging_msg, 253 | kwargs={"working_directory": cwd, 254 | "command": commands_str}) 255 | hanging_timer.start() 256 | try: 257 | output = subprocess.check_output(commands, stderr=subprocess.STDOUT, 258 | universal_newlines=True) 259 | log_process_output(output) 260 | status = 0 261 | except OSError as error: 262 | msg = failed_command_msg( 263 | 'Command execution failed. Does the executable exist?', 264 | commands) 265 | logging.error(error) 266 | fatal_error(msg) 267 | except ValueError as error: 268 | msg = failed_command_msg( 269 | 'DEV_ERROR: Invalid arguments trying to run subprocess', 270 | commands) 271 | logging.error(error) 272 | fatal_error(msg) 273 | except subprocess.CalledProcessError as error: 274 | # Only report the error if we are NOT returning to the 275 | # caller. If we are returning to the caller, then it may be a 276 | # simple status check. If returning, it is the callers 277 | # responsibility determine if an error occurred and handle it 278 | # appropriately. 279 | if not return_to_caller: 280 | msg_context = ('Process did not run successfully; ' 281 | 'returned status {0}'.format(error.returncode)) 282 | msg = failed_command_msg(msg_context, commands, 283 | output=error.output) 284 | logging.error(error) 285 | logging.error(msg) 286 | log_process_output(error.output) 287 | fatal_error(msg) 288 | status = error.returncode 289 | finally: 290 | hanging_timer.cancel() 291 | 292 | if status_to_caller and output_to_caller: 293 | ret_value = (status, output) 294 | elif status_to_caller: 295 | ret_value = status 296 | elif output_to_caller: 297 | ret_value = output 298 | else: 299 | ret_value = None 300 | 301 | return ret_value 302 | 303 | 304 | def failed_command_msg(msg_context, command, output=None): 305 | """Template for consistent error messages from subprocess calls. 306 | 307 | If 'output' is given, it should provide the output from the failed 308 | command 309 | """ 310 | 311 | if output: 312 | output_truncated = last_n_lines(output, 20, 313 | truncation_message='[... Output truncated for brevity ...]') 314 | errmsg = ('Failed with output:\n' + 315 | indent_string(output_truncated, 4) + 316 | '\nERROR: ') 317 | else: 318 | errmsg = '' 319 | 320 | command_str = ' '.join(command) 321 | errmsg += """In directory 322 | {cwd} 323 | {context}: 324 | {command} 325 | """.format(cwd=os.getcwd(), context=msg_context, command=command_str) 326 | 327 | if output: 328 | errmsg += 'See above for output from failed command.\n' 329 | 330 | return errmsg 331 | -------------------------------------------------------------------------------- /manage_externals/manic/sourcetree.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | FIXME(bja, 2017-11) External and SourceTree have a circular dependancy! 4 | """ 5 | 6 | import errno 7 | import logging 8 | import os 9 | 10 | from .externals_description import ExternalsDescription 11 | from .externals_description import read_externals_description_file 12 | from .externals_description import create_externals_description 13 | from .repository_factory import create_repository 14 | from .repository_git import GitRepository 15 | from .externals_status import ExternalStatus 16 | from .utils import fatal_error, printlog 17 | from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR 18 | from .global_constants import VERBOSITY_VERBOSE 19 | 20 | class _External(object): 21 | """ 22 | _External represents an external object inside a SourceTree 23 | """ 24 | 25 | # pylint: disable=R0902 26 | 27 | def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): 28 | """Parse an external description file into a dictionary of externals. 29 | 30 | Input: 31 | 32 | root_dir : string - the root directory path where 33 | 'local_path' is relative to. 34 | 35 | name : string - name of the ext_description object. may or may not 36 | correspond to something in the path. 37 | 38 | ext_description : dict - source ExternalsDescription object 39 | 40 | svn_ignore_ancestry : bool - use --ignore-externals with svn switch 41 | 42 | """ 43 | self._name = name 44 | self._repo = None 45 | self._externals = EMPTY_STR 46 | self._externals_sourcetree = None 47 | self._stat = ExternalStatus() 48 | # Parse the sub-elements 49 | 50 | # _path : local path relative to the containing source tree 51 | self._local_path = ext_description[ExternalsDescription.PATH] 52 | # _repo_dir : full repository directory 53 | repo_dir = os.path.join(root_dir, self._local_path) 54 | self._repo_dir_path = os.path.abspath(repo_dir) 55 | # _base_dir : base directory *containing* the repository 56 | self._base_dir_path = os.path.dirname(self._repo_dir_path) 57 | # repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path 58 | self._repo_dir_name = os.path.basename(self._repo_dir_path) 59 | assert(os.path.join(self._base_dir_path, self._repo_dir_name) 60 | == self._repo_dir_path) 61 | 62 | self._required = ext_description[ExternalsDescription.REQUIRED] 63 | self._externals = ext_description[ExternalsDescription.EXTERNALS] 64 | # Treat a .gitmodules file as a backup externals config 65 | if not self._externals: 66 | if GitRepository.has_submodules(self._repo_dir_path): 67 | self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME 68 | 69 | repo = create_repository( 70 | name, ext_description[ExternalsDescription.REPO], 71 | svn_ignore_ancestry=svn_ignore_ancestry) 72 | if repo: 73 | self._repo = repo 74 | 75 | if self._externals and (self._externals.lower() != 'none'): 76 | self._create_externals_sourcetree() 77 | 78 | def get_name(self): 79 | """ 80 | Return the external object's name 81 | """ 82 | return self._name 83 | 84 | def get_local_path(self): 85 | """ 86 | Return the external object's path 87 | """ 88 | return self._local_path 89 | 90 | def status(self): 91 | """ 92 | If the repo destination directory exists, ensure it is correct (from 93 | correct URL, correct branch or tag), and possibly update the external. 94 | If the repo destination directory does not exist, checkout the correce 95 | branch or tag. 96 | If load_all is True, also load all of the the externals sub-externals. 97 | """ 98 | 99 | self._stat.path = self.get_local_path() 100 | if not self._required: 101 | self._stat.source_type = ExternalStatus.OPTIONAL 102 | elif self._local_path == LOCAL_PATH_INDICATOR: 103 | # LOCAL_PATH_INDICATOR, '.' paths, are standalone 104 | # component directories that are not managed by 105 | # checkout_externals. 106 | self._stat.source_type = ExternalStatus.STANDALONE 107 | else: 108 | # managed by checkout_externals 109 | self._stat.source_type = ExternalStatus.MANAGED 110 | 111 | ext_stats = {} 112 | 113 | if not os.path.exists(self._repo_dir_path): 114 | self._stat.sync_state = ExternalStatus.EMPTY 115 | msg = ('status check: repository directory for "{0}" does not ' 116 | 'exist.'.format(self._name)) 117 | logging.info(msg) 118 | self._stat.current_version = 'not checked out' 119 | # NOTE(bja, 2018-01) directory doesn't exist, so we cannot 120 | # use repo to determine the expected version. We just take 121 | # a best-guess based on the assumption that only tag or 122 | # branch should be set, but not both. 123 | if not self._repo: 124 | self._stat.expected_version = 'unknown' 125 | else: 126 | self._stat.expected_version = self._repo.tag() + self._repo.branch() 127 | else: 128 | if self._repo: 129 | self._repo.status(self._stat, self._repo_dir_path) 130 | 131 | if self._externals and self._externals_sourcetree: 132 | # we expect externals and they exist 133 | cwd = os.getcwd() 134 | # SourceTree expects to be called from the correct 135 | # root directory. 136 | os.chdir(self._repo_dir_path) 137 | ext_stats = self._externals_sourcetree.status(self._local_path) 138 | os.chdir(cwd) 139 | 140 | all_stats = {} 141 | # don't add the root component because we don't manage it 142 | # and can't provide useful info about it. 143 | if self._local_path != LOCAL_PATH_INDICATOR: 144 | # store the stats under tha local_path, not comp name so 145 | # it will be sorted correctly 146 | all_stats[self._stat.path] = self._stat 147 | 148 | if ext_stats: 149 | all_stats.update(ext_stats) 150 | 151 | return all_stats 152 | 153 | def checkout(self, verbosity, load_all): 154 | """ 155 | If the repo destination directory exists, ensure it is correct (from 156 | correct URL, correct branch or tag), and possibly update the external. 157 | If the repo destination directory does not exist, checkout the correct 158 | branch or tag. 159 | If load_all is True, also load all of the the externals sub-externals. 160 | """ 161 | if load_all: 162 | pass 163 | # Make sure we are in correct location 164 | 165 | if not os.path.exists(self._repo_dir_path): 166 | # repository directory doesn't exist. Need to check it 167 | # out, and for that we need the base_dir_path to exist 168 | try: 169 | os.makedirs(self._base_dir_path) 170 | except OSError as error: 171 | if error.errno != errno.EEXIST: 172 | msg = 'Could not create directory "{0}"'.format( 173 | self._base_dir_path) 174 | fatal_error(msg) 175 | 176 | if self._stat.source_type != ExternalStatus.STANDALONE: 177 | if verbosity >= VERBOSITY_VERBOSE: 178 | # NOTE(bja, 2018-01) probably do not want to pass 179 | # verbosity in this case, because if (verbosity == 180 | # VERBOSITY_DUMP), then the previous status output would 181 | # also be dumped, adding noise to the output. 182 | self._stat.log_status_message(VERBOSITY_VERBOSE) 183 | 184 | if self._repo: 185 | if self._stat.sync_state == ExternalStatus.STATUS_OK: 186 | # If we're already in sync, avoid showing verbose output 187 | # from the checkout command, unless the verbosity level 188 | # is 2 or more. 189 | checkout_verbosity = verbosity - 1 190 | else: 191 | checkout_verbosity = verbosity 192 | 193 | self._repo.checkout(self._base_dir_path, self._repo_dir_name, 194 | checkout_verbosity, self.clone_recursive()) 195 | 196 | def checkout_externals(self, verbosity, load_all): 197 | """Checkout the sub-externals for this object 198 | """ 199 | if self.load_externals(): 200 | if self._externals_sourcetree: 201 | # NOTE(bja, 2018-02): the subtree externals objects 202 | # were created during initial status check. Updating 203 | # the external may have changed which sub-externals 204 | # are needed. We need to delete those objects and 205 | # re-read the potentially modified externals 206 | # description file. 207 | self._externals_sourcetree = None 208 | self._create_externals_sourcetree() 209 | self._externals_sourcetree.checkout(verbosity, load_all) 210 | 211 | def load_externals(self): 212 | 'Return True iff an externals file should be loaded' 213 | load_ex = False 214 | if os.path.exists(self._repo_dir_path): 215 | if self._externals: 216 | if self._externals.lower() != 'none': 217 | load_ex = os.path.exists(os.path.join(self._repo_dir_path, 218 | self._externals)) 219 | 220 | return load_ex 221 | 222 | def clone_recursive(self): 223 | 'Return True iff any .gitmodules files should be processed' 224 | # Try recursive unless there is an externals entry 225 | recursive = not self._externals 226 | 227 | return recursive 228 | 229 | def _create_externals_sourcetree(self): 230 | """ 231 | """ 232 | if not os.path.exists(self._repo_dir_path): 233 | # NOTE(bja, 2017-10) repository has not been checked out 234 | # yet, can't process the externals file. Assume we are 235 | # checking status before code is checkoud out and this 236 | # will be handled correctly later. 237 | return 238 | 239 | cwd = os.getcwd() 240 | os.chdir(self._repo_dir_path) 241 | if self._externals.lower() == 'none': 242 | msg = ('Internal: Attempt to create source tree for ' 243 | 'externals = none in {}'.format(self._repo_dir_path)) 244 | fatal_error(msg) 245 | 246 | if not os.path.exists(self._externals): 247 | if GitRepository.has_submodules(): 248 | self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME 249 | 250 | if not os.path.exists(self._externals): 251 | # NOTE(bja, 2017-10) this check is redundent with the one 252 | # in read_externals_description_file! 253 | msg = ('External externals description file "{0}" ' 254 | 'does not exist! In directory: {1}'.format( 255 | self._externals, self._repo_dir_path)) 256 | fatal_error(msg) 257 | 258 | externals_root = self._repo_dir_path 259 | model_data = read_externals_description_file(externals_root, 260 | self._externals) 261 | externals = create_externals_description(model_data, 262 | parent_repo=self._repo) 263 | self._externals_sourcetree = SourceTree(externals_root, externals) 264 | os.chdir(cwd) 265 | 266 | class SourceTree(object): 267 | """ 268 | SourceTree represents a group of managed externals 269 | """ 270 | 271 | def __init__(self, root_dir, model, svn_ignore_ancestry=False): 272 | """ 273 | Build a SourceTree object from a model description 274 | """ 275 | self._root_dir = os.path.abspath(root_dir) 276 | self._all_components = {} 277 | self._required_compnames = [] 278 | for comp in model: 279 | src = _External(self._root_dir, comp, model[comp], svn_ignore_ancestry) 280 | self._all_components[comp] = src 281 | if model[comp][ExternalsDescription.REQUIRED]: 282 | self._required_compnames.append(comp) 283 | 284 | def status(self, relative_path_base=LOCAL_PATH_INDICATOR): 285 | """Report the status components 286 | 287 | FIXME(bja, 2017-10) what do we do about situations where the 288 | user checked out the optional components, but didn't add 289 | optional for running status? What do we do where the user 290 | didn't add optional to the checkout but did add it to the 291 | status. -- For now, we run status on all components, and try 292 | to do the right thing based on the results.... 293 | 294 | """ 295 | load_comps = self._all_components.keys() 296 | 297 | summary = {} 298 | for comp in load_comps: 299 | printlog('{0}, '.format(comp), end='') 300 | stat = self._all_components[comp].status() 301 | for name in stat.keys(): 302 | # check if we need to append the relative_path_base to 303 | # the path so it will be sorted in the correct order. 304 | if not stat[name].path.startswith(relative_path_base): 305 | stat[name].path = os.path.join(relative_path_base, 306 | stat[name].path) 307 | # store under key = updated path, and delete the 308 | # old key. 309 | comp_stat = stat[name] 310 | del stat[name] 311 | stat[comp_stat.path] = comp_stat 312 | summary.update(stat) 313 | 314 | return summary 315 | 316 | def checkout(self, verbosity, load_all, load_comp=None): 317 | """ 318 | Checkout or update indicated components into the the configured 319 | subdirs. 320 | 321 | If load_all is True, recursively checkout all externals. 322 | If load_all is False, load_comp is an optional set of components to load. 323 | If load_all is True and load_comp is None, only load the required externals. 324 | """ 325 | if verbosity >= VERBOSITY_VERBOSE: 326 | printlog('Checking out externals: ') 327 | else: 328 | printlog('Checking out externals: ', end='') 329 | 330 | if load_all: 331 | load_comps = self._all_components.keys() 332 | elif load_comp is not None: 333 | load_comps = [load_comp] 334 | else: 335 | load_comps = self._required_compnames 336 | 337 | # checkout the primary externals 338 | for comp in load_comps: 339 | if verbosity < VERBOSITY_VERBOSE: 340 | printlog('{0}, '.format(comp), end='') 341 | else: 342 | # verbose output handled by the _External object, just 343 | # output a newline 344 | printlog(EMPTY_STR) 345 | self._all_components[comp].checkout(verbosity, load_all) 346 | printlog('') 347 | 348 | # now give each external an opportunitity to checkout it's externals. 349 | for comp in load_comps: 350 | self._all_components[comp].checkout_externals(verbosity, load_all) 351 | -------------------------------------------------------------------------------- /manage_externals/manic/checkout.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Tool to assemble repositories represented in a model-description file. 5 | 6 | If loaded as a module (e.g., in a component's buildcpp), it can be used 7 | to check the validity of existing subdirectories and load missing sources. 8 | """ 9 | from __future__ import absolute_import 10 | from __future__ import unicode_literals 11 | from __future__ import print_function 12 | 13 | import argparse 14 | import logging 15 | import os 16 | import os.path 17 | import sys 18 | 19 | from manic.externals_description import create_externals_description 20 | from manic.externals_description import read_externals_description_file 21 | from manic.externals_status import check_safe_to_update_repos 22 | from manic.sourcetree import SourceTree 23 | from manic.utils import printlog, fatal_error 24 | from manic.global_constants import VERSION_SEPERATOR, LOG_FILE_NAME 25 | 26 | if sys.hexversion < 0x02070000: 27 | print(70 * '*') 28 | print('ERROR: {0} requires python >= 2.7.x. '.format(sys.argv[0])) 29 | print('It appears that you are running python {0}'.format( 30 | VERSION_SEPERATOR.join(str(x) for x in sys.version_info[0:3]))) 31 | print(70 * '*') 32 | sys.exit(1) 33 | 34 | 35 | # --------------------------------------------------------------------- 36 | # 37 | # User input 38 | # 39 | # --------------------------------------------------------------------- 40 | def commandline_arguments(args=None): 41 | """Process the command line arguments 42 | 43 | Params: args - optional args. Should only be used during systems 44 | testing. 45 | 46 | Returns: processed command line arguments 47 | """ 48 | description = ''' 49 | 50 | %(prog)s manages checking out groups of externals from revision 51 | control based on an externals description file. By default only the 52 | required externals are checkout out. 53 | 54 | Running %(prog)s without the '--status' option will always attempt to 55 | synchronize the working copy to exactly match the externals description. 56 | ''' 57 | 58 | epilog = ''' 59 | ``` 60 | NOTE: %(prog)s *MUST* be run from the root of the source tree it 61 | is managing. For example, if you cloned a repository with: 62 | 63 | $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev 64 | 65 | Then the root of the source tree is /path/to/some-project-dev. If you 66 | obtained a sub-project via a checkout of another project: 67 | 68 | $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev 69 | 70 | and you need to checkout the sub-project externals, then the root of the 71 | source tree remains /path/to/some-project-dev. Do *NOT* run %(prog)s 72 | from within /path/to/some-project-dev/sub-project 73 | 74 | The root of the source tree will be referred to as `${SRC_ROOT}` below. 75 | 76 | 77 | # Supported workflows 78 | 79 | * Checkout all required components from the default externals 80 | description file: 81 | 82 | $ cd ${SRC_ROOT} 83 | $ ./manage_externals/%(prog)s 84 | 85 | * To update all required components to the current values in the 86 | externals description file, re-run %(prog)s: 87 | 88 | $ cd ${SRC_ROOT} 89 | $ ./manage_externals/%(prog)s 90 | 91 | If there are *any* modifications to *any* working copy according 92 | to the git or svn 'status' command, %(prog)s 93 | will not update any external repositories. Modifications 94 | include: modified files, added files, removed files, or missing 95 | files. 96 | 97 | To avoid this safety check, edit the externals description file 98 | and comment out the modified external block. 99 | 100 | * Checkout all required components from a user specified externals 101 | description file: 102 | 103 | $ cd ${SRC_ROOT} 104 | $ ./manage_externals/%(prog)s --externals my-externals.cfg 105 | 106 | * Status summary of the repositories managed by %(prog)s: 107 | 108 | $ cd ${SRC_ROOT} 109 | $ ./manage_externals/%(prog)s --status 110 | 111 | ./cime 112 | s ./components/cism 113 | ./components/mosart 114 | e-o ./components/rtm 115 | M ./src/fates 116 | e-o ./tools/PTCLM 117 | 118 | 119 | where: 120 | * column one indicates the status of the repository in relation 121 | to the externals description file. 122 | * column two indicates whether the working copy has modified files. 123 | * column three shows how the repository is managed, optional or required 124 | 125 | Column one will be one of these values: 126 | * s : out-of-sync : repository is checked out at a different commit 127 | compared with the externals description 128 | * e : empty : directory does not exist - %(prog)s has not been run 129 | * ? : unknown : directory exists but .git or .svn directories are missing 130 | 131 | Column two will be one of these values: 132 | * M : Modified : modified, added, deleted or missing files 133 | * : blank / space : clean 134 | * - : dash : no meaningful state, for empty repositories 135 | 136 | Column three will be one of these values: 137 | * o : optional : optionally repository 138 | * : blank / space : required repository 139 | 140 | * Detailed git or svn status of the repositories managed by %(prog)s: 141 | 142 | $ cd ${SRC_ROOT} 143 | $ ./manage_externals/%(prog)s --status --verbose 144 | 145 | # Externals description file 146 | 147 | The externals description contains a list of the external 148 | repositories that are used and their version control locations. The 149 | file format is the standard ini/cfg configuration file format. Each 150 | external is defined by a section containing the component name in 151 | square brackets: 152 | 153 | * name (string) : component name, e.g. [cime], [cism], etc. 154 | 155 | Each section has the following keyword-value pairs: 156 | 157 | * required (boolean) : whether the component is a required checkout, 158 | 'true' or 'false'. 159 | 160 | * local_path (string) : component path *relative* to where 161 | %(prog)s is called. 162 | 163 | * protoctol (string) : version control protocol that is used to 164 | manage the component. Valid values are 'git', 'svn', 165 | 'externals_only'. 166 | 167 | Switching an external between different protocols is not 168 | supported, e.g. from svn to git. To switch protocols, you need to 169 | manually move the old working copy to a new location. 170 | 171 | Note: 'externals_only' will only process the external's own 172 | external description file without trying to manage a repository 173 | for the component. This is used for retrieving externals for 174 | standalone components like cam and ctsm which also serve as 175 | sub-components within a larger project. If the source root of the 176 | externals_only component is the same as the main source root, then 177 | the local path must be set to '.', the unix current working 178 | directory, e. g. 'local_path = .' 179 | 180 | * repo_url (string) : URL for the repository location, examples: 181 | * https://svn-ccsm-models.cgd.ucar.edu/glc 182 | * git@github.com:esmci/cime.git 183 | * /path/to/local/repository 184 | * . 185 | 186 | NOTE: To operate on only the local clone and and ignore remote 187 | repositories, set the url to '.' (the unix current path), 188 | i.e. 'repo_url = .' . This can be used to checkout a local branch 189 | instead of the upstream branch. 190 | 191 | If a repo url is determined to be a local path (not a network url) 192 | then user expansion, e.g. ~/, and environment variable expansion, 193 | e.g. $HOME or $REPO_ROOT, will be performed. 194 | 195 | Relative paths are difficult to get correct, especially for mixed 196 | use repos. It is advised that local paths expand to absolute paths. 197 | If relative paths are used, they should be relative to one level 198 | above local_path. If local path is 'src/foo', the the relative url 199 | should be relative to 'src'. 200 | 201 | * tag (string) : tag to checkout 202 | 203 | * hash (string) : the git hash to checkout. Only applies to git 204 | repositories. 205 | 206 | * branch (string) : branch to checkout from the specified 207 | repository. Specifying a branch on a remote repository means that 208 | %(prog)s will checkout the version of the branch in the remote, 209 | not the the version in the local repository (if it exists). 210 | 211 | Note: one and only one of tag, branch hash must be supplied. 212 | 213 | * externals (string) : used to make manage_externals aware of 214 | sub-externals required by an external. This is a relative path to 215 | the external's root directory. For example, if LIBX is often used 216 | as a sub-external, it might have an externals file (for its 217 | externals) called Externals_LIBX.cfg. To use libx as a standalone 218 | checkout, it would have another file, Externals.cfg with the 219 | following entry: 220 | 221 | [ libx ] 222 | local_path = . 223 | protocol = externals_only 224 | externals = Externals_LIBX.cfg 225 | required = True 226 | 227 | Now, %(prog)s will process Externals.cfg and also process 228 | Externals_LIBX.cfg as if it was a sub-external. 229 | 230 | * Lines beginning with '#' or ';' are comments and will be ignored. 231 | 232 | # Obtaining this tool, reporting issues, etc. 233 | 234 | The master repository for manage_externals is 235 | https://github.com/ESMCI/manage_externals. Any issues with this tool 236 | should be reported there. 237 | 238 | # Troubleshooting 239 | 240 | Operations performed by manage_externals utilities are explicit and 241 | data driven. %(prog)s will always attempt to make the working copy 242 | *exactly* match what is in the externals file when modifying the 243 | working copy of a repository. 244 | 245 | If %(prog)s is not doing what you expected, double check the contents 246 | of the externals description file or examine the output of 247 | ./manage_externals/%(prog)s --status 248 | 249 | ''' 250 | 251 | parser = argparse.ArgumentParser( 252 | description=description, epilog=epilog, 253 | formatter_class=argparse.RawDescriptionHelpFormatter) 254 | 255 | # 256 | # user options 257 | # 258 | parser.add_argument("components", nargs="*", 259 | help="Specific component(s) to checkout. By default, " 260 | "all required externals are checked out.") 261 | 262 | parser.add_argument('-e', '--externals', nargs='?', 263 | default='Externals.cfg', 264 | help='The externals description filename. ' 265 | 'Default: %(default)s.') 266 | 267 | parser.add_argument('-o', '--optional', action='store_true', default=False, 268 | help='By default only the required externals ' 269 | 'are checked out. This flag will also checkout the ' 270 | 'optional externals.') 271 | 272 | parser.add_argument('-S', '--status', action='store_true', default=False, 273 | help='Output the status of the repositories managed by ' 274 | '%(prog)s. By default only summary information ' 275 | 'is provided. Use the verbose option to see details.') 276 | 277 | parser.add_argument('-v', '--verbose', action='count', default=0, 278 | help='Output additional information to ' 279 | 'the screen and log file. This flag can be ' 280 | 'used up to two times, increasing the ' 281 | 'verbosity level each time.') 282 | 283 | parser.add_argument('--svn-ignore-ancestry', action='store_true', default=False, 284 | help='By default, subversion will abort if a component is ' 285 | 'already checked out and there is no common ancestry with ' 286 | 'the new URL. This flag passes the "--ignore-ancestry" flag ' 287 | 'to the svn switch call. (This is not recommended unless ' 288 | 'you are sure about what you are doing.)') 289 | 290 | # 291 | # developer options 292 | # 293 | parser.add_argument('--backtrace', action='store_true', 294 | help='DEVELOPER: show exception backtraces as extra ' 295 | 'debugging output') 296 | 297 | parser.add_argument('-d', '--debug', action='store_true', default=False, 298 | help='DEVELOPER: output additional debugging ' 299 | 'information to the screen and log file.') 300 | 301 | logging_group = parser.add_mutually_exclusive_group() 302 | 303 | logging_group.add_argument('--logging', dest='do_logging', 304 | action='store_true', 305 | help='DEVELOPER: enable logging.') 306 | logging_group.add_argument('--no-logging', dest='do_logging', 307 | action='store_false', default=False, 308 | help='DEVELOPER: disable logging ' 309 | '(this is the default)') 310 | 311 | if args: 312 | options = parser.parse_args(args) 313 | else: 314 | options = parser.parse_args() 315 | return options 316 | 317 | 318 | # --------------------------------------------------------------------- 319 | # 320 | # main 321 | # 322 | # --------------------------------------------------------------------- 323 | def main(args): 324 | """ 325 | Function to call when module is called from the command line. 326 | Parse externals file and load required repositories or all repositories if 327 | the --all option is passed. 328 | 329 | Returns a tuple (overall_status, tree_status). overall_status is 0 330 | on success, non-zero on failure. tree_status gives the full status 331 | *before* executing the checkout command - i.e., the status that it 332 | used to determine if it's safe to proceed with the checkout. 333 | """ 334 | if args.do_logging: 335 | logging.basicConfig(filename=LOG_FILE_NAME, 336 | format='%(levelname)s : %(asctime)s : %(message)s', 337 | datefmt='%Y-%m-%d %H:%M:%S', 338 | level=logging.DEBUG) 339 | 340 | program_name = os.path.basename(sys.argv[0]) 341 | logging.info('Beginning of %s', program_name) 342 | 343 | load_all = False 344 | if args.optional: 345 | load_all = True 346 | 347 | root_dir = os.path.abspath(os.getcwd()) 348 | external_data = read_externals_description_file(root_dir, args.externals) 349 | external = create_externals_description( 350 | external_data, components=args.components) 351 | 352 | for comp in args.components: 353 | if comp not in external.keys(): 354 | fatal_error( 355 | "No component {} found in {}".format( 356 | comp, args.externals)) 357 | 358 | source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry) 359 | printlog('Checking status of externals: ', end='') 360 | tree_status = source_tree.status() 361 | printlog('') 362 | 363 | if args.status: 364 | # user requested status-only 365 | for comp in sorted(tree_status.keys()): 366 | tree_status[comp].log_status_message(args.verbose) 367 | else: 368 | # checkout / update the external repositories. 369 | safe_to_update = check_safe_to_update_repos(tree_status) 370 | if not safe_to_update: 371 | # print status 372 | for comp in sorted(tree_status.keys()): 373 | tree_status[comp].log_status_message(args.verbose) 374 | # exit gracefully 375 | msg = """The external repositories labeled with 'M' above are not in a clean state. 376 | 377 | The following are two options for how to proceed: 378 | 379 | (1) Go into each external that is not in a clean state and issue either 380 | an 'svn status' or a 'git status' command. Either revert or commit 381 | your changes so that all externals are in a clean state. (Note, 382 | though, that it is okay to have untracked files in your working 383 | directory.) Then rerun {program_name}. 384 | 385 | (2) Alternatively, you do not have to rely on {program_name}. Instead, you 386 | can manually update out-of-sync externals (labeled with 's' above) 387 | as described in the configuration file {config_file}. 388 | 389 | 390 | The external repositories labeled with '?' above are not under version 391 | control using the expected protocol. If you are sure you want to switch 392 | protocols, and you don't have any work you need to save from this 393 | directory, then run "rm -rf [directory]" before re-running the 394 | checkout_externals tool. 395 | """.format(program_name=program_name, config_file=args.externals) 396 | 397 | printlog('-' * 70) 398 | printlog(msg) 399 | printlog('-' * 70) 400 | else: 401 | if not args.components: 402 | source_tree.checkout(args.verbose, load_all) 403 | for comp in args.components: 404 | source_tree.checkout(args.verbose, load_all, load_comp=comp) 405 | printlog('') 406 | 407 | logging.info('%s completed without exceptions.', program_name) 408 | # NOTE(bja, 2017-11) tree status is used by the systems tests 409 | return 0, tree_status 410 | -------------------------------------------------------------------------------- /manage_externals/manic/repository_git.py: -------------------------------------------------------------------------------- 1 | """Class for interacting with git repositories 2 | """ 3 | 4 | from __future__ import absolute_import 5 | from __future__ import unicode_literals 6 | from __future__ import print_function 7 | 8 | import copy 9 | import os 10 | 11 | from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR 12 | from .global_constants import VERBOSITY_VERBOSE 13 | from .repository import Repository 14 | from .externals_status import ExternalStatus 15 | from .externals_description import ExternalsDescription, git_submodule_status 16 | from .utils import expand_local_url, split_remote_url, is_remote_url 17 | from .utils import fatal_error, printlog 18 | from .utils import execute_subprocess 19 | 20 | 21 | class GitRepository(Repository): 22 | """Class to represent and operate on a repository description. 23 | 24 | For testing purpose, all system calls to git should: 25 | 26 | * be isolated in separate functions with no application logic 27 | * of the form: 28 | - cmd = ['git', ...] 29 | - value = execute_subprocess(cmd, output_to_caller={T|F}, 30 | status_to_caller={T|F}) 31 | - return value 32 | * be static methods (not rely on self) 33 | * name as _git_subcommand_args(user_args) 34 | 35 | This convention allows easy unit testing of the repository logic 36 | by mocking the specific calls to return predefined results. 37 | 38 | """ 39 | 40 | def __init__(self, component_name, repo): 41 | """ 42 | Parse repo (a XML element). 43 | """ 44 | Repository.__init__(self, component_name, repo) 45 | self._gitmodules = None 46 | self._submods = None 47 | 48 | # ---------------------------------------------------------------- 49 | # 50 | # Public API, defined by Repository 51 | # 52 | # ---------------------------------------------------------------- 53 | def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): 54 | """ 55 | If the repo destination directory exists, ensure it is correct (from 56 | correct URL, correct branch or tag), and possibly update the source. 57 | If the repo destination directory does not exist, checkout the correct 58 | branch or tag. 59 | """ 60 | repo_dir_path = os.path.join(base_dir_path, repo_dir_name) 61 | repo_dir_exists = os.path.exists(repo_dir_path) 62 | if (repo_dir_exists and not os.listdir( 63 | repo_dir_path)) or not repo_dir_exists: 64 | self._clone_repo(base_dir_path, repo_dir_name, verbosity) 65 | self._checkout_ref(repo_dir_path, verbosity, recursive) 66 | gmpath = os.path.join(repo_dir_path, 67 | ExternalsDescription.GIT_SUBMODULES_FILENAME) 68 | if os.path.exists(gmpath): 69 | self._gitmodules = gmpath 70 | self._submods = git_submodule_status(repo_dir_path) 71 | else: 72 | self._gitmodules = None 73 | self._submods = None 74 | 75 | def status(self, stat, repo_dir_path): 76 | """ 77 | If the repo destination directory exists, ensure it is correct (from 78 | correct URL, correct branch or tag), and possibly update the source. 79 | If the repo destination directory does not exist, checkout the correct 80 | branch or tag. 81 | """ 82 | self._check_sync(stat, repo_dir_path) 83 | if os.path.exists(repo_dir_path): 84 | self._status_summary(stat, repo_dir_path) 85 | 86 | def submodules_file(self, repo_path=None): 87 | if repo_path is not None: 88 | gmpath = os.path.join(repo_path, 89 | ExternalsDescription.GIT_SUBMODULES_FILENAME) 90 | if os.path.exists(gmpath): 91 | self._gitmodules = gmpath 92 | self._submods = git_submodule_status(repo_path) 93 | 94 | return self._gitmodules 95 | 96 | # ---------------------------------------------------------------- 97 | # 98 | # Internal work functions 99 | # 100 | # ---------------------------------------------------------------- 101 | def _clone_repo(self, base_dir_path, repo_dir_name, verbosity): 102 | """Prepare to execute the clone by managing directory location 103 | """ 104 | cwd = os.getcwd() 105 | os.chdir(base_dir_path) 106 | self._git_clone(self._url, repo_dir_name, verbosity) 107 | os.chdir(cwd) 108 | 109 | def _current_ref(self): 110 | """Determine the *name* associated with HEAD. 111 | 112 | If we're on a branch, then returns the branch name; otherwise, 113 | if we're on a tag, then returns the tag name; otherwise, returns 114 | the current hash. Returns an empty string if no reference can be 115 | determined (e.g., if we're not actually in a git repository). 116 | """ 117 | ref_found = False 118 | 119 | # If we're on a branch, then use that as the current ref 120 | branch_found, branch_name = self._git_current_branch() 121 | if branch_found: 122 | current_ref = branch_name 123 | ref_found = True 124 | 125 | if not ref_found: 126 | # Otherwise, if we're exactly at a tag, use that as the 127 | # current ref 128 | tag_found, tag_name = self._git_current_tag() 129 | if tag_found: 130 | current_ref = tag_name 131 | ref_found = True 132 | 133 | if not ref_found: 134 | # Otherwise, use current hash as the current ref 135 | hash_found, hash_name = self._git_current_hash() 136 | if hash_found: 137 | current_ref = hash_name 138 | ref_found = True 139 | 140 | if not ref_found: 141 | # If we still can't find a ref, return empty string. This 142 | # can happen if we're not actually in a git repo 143 | current_ref = '' 144 | 145 | return current_ref 146 | 147 | def _check_sync(self, stat, repo_dir_path): 148 | """Determine whether a git repository is in-sync with the model 149 | description. 150 | 151 | Because repos can have multiple remotes, the only criteria is 152 | whether the branch or tag is the same. 153 | 154 | """ 155 | if not os.path.exists(repo_dir_path): 156 | # NOTE(bja, 2017-10) condition should have been determined 157 | # by _Source() object and should never be here! 158 | stat.sync_state = ExternalStatus.STATUS_ERROR 159 | else: 160 | git_dir = os.path.join(repo_dir_path, '.git') 161 | if not os.path.exists(git_dir): 162 | # NOTE(bja, 2017-10) directory exists, but no git repo 163 | # info.... Can't test with subprocess git command 164 | # because git will move up directory tree until it 165 | # finds the parent repo git dir! 166 | stat.sync_state = ExternalStatus.UNKNOWN 167 | else: 168 | self._check_sync_logic(stat, repo_dir_path) 169 | 170 | def _check_sync_logic(self, stat, repo_dir_path): 171 | """Compare the underlying hashes of the currently checkout ref and the 172 | expected ref. 173 | 174 | Output: sets the sync_state as well as the current and 175 | expected ref in the input status object. 176 | 177 | """ 178 | def compare_refs(current_ref, expected_ref): 179 | """Compare the current and expected ref. 180 | 181 | """ 182 | if current_ref == expected_ref: 183 | status = ExternalStatus.STATUS_OK 184 | else: 185 | status = ExternalStatus.MODEL_MODIFIED 186 | return status 187 | 188 | cwd = os.getcwd() 189 | os.chdir(repo_dir_path) 190 | 191 | # get the full hash of the current commit 192 | _, current_ref = self._git_current_hash() 193 | 194 | if self._branch: 195 | if self._url == LOCAL_PATH_INDICATOR: 196 | expected_ref = self._branch 197 | else: 198 | remote_name = self._determine_remote_name() 199 | if not remote_name: 200 | # git doesn't know about this remote. by definition 201 | # this is a modified state. 202 | expected_ref = "unknown_remote/{0}".format(self._branch) 203 | else: 204 | expected_ref = "{0}/{1}".format(remote_name, self._branch) 205 | elif self._hash: 206 | expected_ref = self._hash 207 | elif self._tag: 208 | expected_ref = self._tag 209 | else: 210 | msg = 'In repo "{0}": none of branch, hash or tag are set'.format( 211 | self._name) 212 | fatal_error(msg) 213 | 214 | # record the *names* of the current and expected branches 215 | stat.current_version = self._current_ref() 216 | stat.expected_version = copy.deepcopy(expected_ref) 217 | 218 | if current_ref == EMPTY_STR: 219 | stat.sync_state = ExternalStatus.UNKNOWN 220 | else: 221 | # get the underlying hash of the expected ref 222 | revparse_status, expected_ref_hash = self._git_revparse_commit( 223 | expected_ref) 224 | if revparse_status: 225 | # We failed to get the hash associated with 226 | # expected_ref. Maybe we should assign this to some special 227 | # status, but for now we're just calling this out-of-sync to 228 | # remain consistent with how this worked before. 229 | stat.sync_state = ExternalStatus.MODEL_MODIFIED 230 | else: 231 | # compare the underlying hashes 232 | stat.sync_state = compare_refs(current_ref, expected_ref_hash) 233 | 234 | os.chdir(cwd) 235 | 236 | def _determine_remote_name(self): 237 | """Return the remote name. 238 | 239 | Note that this is for the *future* repo url and branch, not 240 | the current working copy! 241 | 242 | """ 243 | git_output = self._git_remote_verbose() 244 | git_output = git_output.splitlines() 245 | remote_name = '' 246 | for line in git_output: 247 | data = line.strip() 248 | if not data: 249 | continue 250 | data = data.split() 251 | name = data[0].strip() 252 | url = data[1].strip() 253 | if self._url == url: 254 | remote_name = name 255 | break 256 | return remote_name 257 | 258 | def _create_remote_name(self): 259 | """The url specified in the externals description file was not known 260 | to git. We need to add it, which means adding a unique and 261 | safe name.... 262 | 263 | The assigned name needs to be safe for git to use, e.g. can't 264 | look like a path 'foo/bar' and work with both remote and local paths. 265 | 266 | Remote paths include but are not limited to: git, ssh, https, 267 | github, gitlab, bitbucket, custom server, etc. 268 | 269 | Local paths can be relative or absolute. They may contain 270 | shell variables, e.g. ${REPO_ROOT}/repo_name, or username 271 | expansion, i.e. ~/ or ~someuser/. 272 | 273 | Relative paths must be at least one layer of redirection, i.e. 274 | container/../ext_repo, but may be many layers deep, e.g. 275 | container/../../../../../ext_repo 276 | 277 | NOTE(bja, 2017-11) 278 | 279 | The base name below may not be unique, for example if the 280 | user has local paths like: 281 | 282 | /path/to/my/repos/nice_repo 283 | /path/to/other/repos/nice_repo 284 | 285 | But the current implementation should cover most common 286 | use cases for remotes and still provide usable names. 287 | 288 | """ 289 | url = copy.deepcopy(self._url) 290 | if is_remote_url(url): 291 | url = split_remote_url(url) 292 | else: 293 | url = expand_local_url(url, self._name) 294 | url = url.split('/') 295 | repo_name = url[-1] 296 | base_name = url[-2] 297 | # repo name should nominally already be something that git can 298 | # deal with. We need to remove other possibly troublesome 299 | # punctuation, e.g. /, $, from the base name. 300 | unsafe_characters = '!@#$%^&*()[]{}\\/,;~' 301 | for unsafe in unsafe_characters: 302 | base_name = base_name.replace(unsafe, '') 303 | remote_name = "{0}_{1}".format(base_name, repo_name) 304 | return remote_name 305 | 306 | def _checkout_ref(self, repo_dir, verbosity, submodules): 307 | """Checkout the user supplied reference 308 | if is True, recursively initialize and update 309 | the repo's submodules 310 | """ 311 | # import pdb; pdb.set_trace() 312 | cwd = os.getcwd() 313 | os.chdir(repo_dir) 314 | if self._url.strip() == LOCAL_PATH_INDICATOR: 315 | self._checkout_local_ref(verbosity, submodules) 316 | else: 317 | self._checkout_external_ref(verbosity, submodules) 318 | 319 | os.chdir(cwd) 320 | 321 | def _checkout_local_ref(self, verbosity, submodules): 322 | """Checkout the reference considering the local repo only. Do not 323 | fetch any additional remotes or specify the remote when 324 | checkout out the ref. 325 | if is True, recursively initialize and update 326 | the repo's submodules 327 | """ 328 | if self._tag: 329 | ref = self._tag 330 | elif self._branch: 331 | ref = self._branch 332 | else: 333 | ref = self._hash 334 | 335 | self._check_for_valid_ref(ref) 336 | self._git_checkout_ref(ref, verbosity, submodules) 337 | 338 | def _checkout_external_ref(self, verbosity, submodules): 339 | """Checkout the reference from a remote repository 340 | if is True, recursively initialize and update 341 | the repo's submodules 342 | """ 343 | if self._tag: 344 | ref = self._tag 345 | elif self._branch: 346 | ref = self._branch 347 | else: 348 | ref = self._hash 349 | 350 | remote_name = self._determine_remote_name() 351 | if not remote_name: 352 | remote_name = self._create_remote_name() 353 | self._git_remote_add(remote_name, self._url) 354 | self._git_fetch(remote_name) 355 | 356 | # NOTE(bja, 2018-03) we need to send separate ref and remote 357 | # name to check_for_vaild_ref, but the combined name to 358 | # checkout_ref! 359 | self._check_for_valid_ref(ref, remote_name) 360 | 361 | if self._branch: 362 | ref = '{0}/{1}'.format(remote_name, ref) 363 | self._git_checkout_ref(ref, verbosity, submodules) 364 | 365 | def _check_for_valid_ref(self, ref, remote_name=None): 366 | """Try some basic sanity checks on the user supplied reference so we 367 | can provide a more useful error message than calledprocess 368 | error... 369 | 370 | """ 371 | is_tag = self._ref_is_tag(ref) 372 | is_branch = self._ref_is_branch(ref, remote_name) 373 | is_hash = self._ref_is_hash(ref) 374 | 375 | is_valid = is_tag or is_branch or is_hash 376 | if not is_valid: 377 | msg = ('In repo "{0}": reference "{1}" does not appear to be a ' 378 | 'valid tag, branch or hash! Please verify the reference ' 379 | 'name (e.g. spelling), is available from: {2} '.format( 380 | self._name, ref, self._url)) 381 | fatal_error(msg) 382 | 383 | if is_tag: 384 | is_unique_tag, msg = self._is_unique_tag(ref, remote_name) 385 | if not is_unique_tag: 386 | msg = ('In repo "{0}": tag "{1}" {2}'.format( 387 | self._name, self._tag, msg)) 388 | fatal_error(msg) 389 | 390 | return is_valid 391 | 392 | def _is_unique_tag(self, ref, remote_name): 393 | """Verify that a reference is a valid tag and is unique (not a branch) 394 | 395 | Tags may be tag names, or SHA id's. It is also possible that a 396 | branch and tag have the some name. 397 | 398 | Note: values returned by git_showref_* and git_revparse are 399 | shell return codes, which are zero for success, non-zero for 400 | error! 401 | 402 | """ 403 | is_tag = self._ref_is_tag(ref) 404 | is_branch = self._ref_is_branch(ref, remote_name) 405 | is_hash = self._ref_is_hash(ref) 406 | 407 | msg = '' 408 | is_unique_tag = False 409 | if is_tag and not is_branch: 410 | # unique tag 411 | msg = 'is ok' 412 | is_unique_tag = True 413 | elif is_tag and is_branch: 414 | msg = ('is both a branch and a tag. git may checkout the branch ' 415 | 'instead of the tag depending on your version of git.') 416 | is_unique_tag = False 417 | elif not is_tag and is_branch: 418 | msg = ('is a branch, and not a tag. If you intended to checkout ' 419 | 'a branch, please change the externals description to be ' 420 | 'a branch. If you intended to checkout a tag, it does not ' 421 | 'exist. Please check the name.') 422 | is_unique_tag = False 423 | else: # not is_tag and not is_branch: 424 | if is_hash: 425 | # probably a sha1 or HEAD, etc, we call it a tag 426 | msg = 'is ok' 427 | is_unique_tag = True 428 | else: 429 | # undetermined state. 430 | msg = ('does not appear to be a valid tag, branch or hash! ' 431 | 'Please check the name and repository.') 432 | is_unique_tag = False 433 | 434 | return is_unique_tag, msg 435 | 436 | def _ref_is_tag(self, ref): 437 | """Verify that a reference is a valid tag according to git. 438 | 439 | Note: values returned by git_showref_* and git_revparse are 440 | shell return codes, which are zero for success, non-zero for 441 | error! 442 | """ 443 | is_tag = False 444 | value = self._git_showref_tag(ref) 445 | if value == 0: 446 | is_tag = True 447 | return is_tag 448 | 449 | def _ref_is_branch(self, ref, remote_name=None): 450 | """Verify if a ref is any kind of branch (local, tracked remote, 451 | untracked remote). 452 | 453 | """ 454 | local_branch = False 455 | remote_branch = False 456 | if remote_name: 457 | remote_branch = self._ref_is_remote_branch(ref, remote_name) 458 | local_branch = self._ref_is_local_branch(ref) 459 | 460 | is_branch = False 461 | if local_branch or remote_branch: 462 | is_branch = True 463 | return is_branch 464 | 465 | def _ref_is_local_branch(self, ref): 466 | """Verify that a reference is a valid branch according to git. 467 | 468 | show-ref branch returns local branches that have been 469 | previously checked out. It will not necessarily pick up 470 | untracked remote branches. 471 | 472 | Note: values returned by git_showref_* and git_revparse are 473 | shell return codes, which are zero for success, non-zero for 474 | error! 475 | 476 | """ 477 | is_branch = False 478 | value = self._git_showref_branch(ref) 479 | if value == 0: 480 | is_branch = True 481 | return is_branch 482 | 483 | def _ref_is_remote_branch(self, ref, remote_name): 484 | """Verify that a reference is a valid branch according to git. 485 | 486 | show-ref branch returns local branches that have been 487 | previously checked out. It will not necessarily pick up 488 | untracked remote branches. 489 | 490 | Note: values returned by git_showref_* and git_revparse are 491 | shell return codes, which are zero for success, non-zero for 492 | error! 493 | 494 | """ 495 | is_branch = False 496 | value = self._git_lsremote_branch(ref, remote_name) 497 | if value == 0: 498 | is_branch = True 499 | return is_branch 500 | 501 | def _ref_is_commit(self, ref): 502 | """Verify that a reference is a valid commit according to git. 503 | 504 | This could be a tag, branch, sha1 id, HEAD and potentially others... 505 | 506 | Note: values returned by git_showref_* and git_revparse are 507 | shell return codes, which are zero for success, non-zero for 508 | error! 509 | """ 510 | is_commit = False 511 | value, _ = self._git_revparse_commit(ref) 512 | if value == 0: 513 | is_commit = True 514 | return is_commit 515 | 516 | def _ref_is_hash(self, ref): 517 | """Verify that a reference is a valid hash according to git. 518 | 519 | Git doesn't seem to provide an exact way to determine if user 520 | supplied reference is an actual hash. So we verify that the 521 | ref is a valid commit and return the underlying commit 522 | hash. Then check that the commit hash begins with the user 523 | supplied string. 524 | 525 | Note: values returned by git_showref_* and git_revparse are 526 | shell return codes, which are zero for success, non-zero for 527 | error! 528 | 529 | """ 530 | is_hash = False 531 | status, git_output = self._git_revparse_commit(ref) 532 | if status == 0: 533 | if git_output.strip().startswith(ref): 534 | is_hash = True 535 | return is_hash 536 | 537 | def _status_summary(self, stat, repo_dir_path): 538 | """Determine the clean/dirty status of a git repository 539 | 540 | """ 541 | cwd = os.getcwd() 542 | os.chdir(repo_dir_path) 543 | git_output = self._git_status_porcelain_v1z() 544 | is_dirty = self._status_v1z_is_dirty(git_output) 545 | if is_dirty: 546 | stat.clean_state = ExternalStatus.DIRTY 547 | else: 548 | stat.clean_state = ExternalStatus.STATUS_OK 549 | 550 | # Now save the verbose status output incase the user wants to 551 | # see it. 552 | stat.status_output = self._git_status_verbose() 553 | os.chdir(cwd) 554 | 555 | @staticmethod 556 | def _status_v1z_is_dirty(git_output): 557 | """Parse the git status output from --porcelain=v1 -z and determine if 558 | the repo status is clean or dirty. Dirty means: 559 | 560 | * modified files 561 | * missing files 562 | * added files 563 | * removed 564 | * renamed 565 | * unmerged 566 | 567 | Whether untracked files are considered depends on how the status 568 | command was run (i.e., whether it was run with the '-u' option). 569 | 570 | NOTE: Based on the above definition, the porcelain status 571 | should be an empty string to be considered 'clean'. Of course 572 | this assumes we only get an empty string from an status 573 | command on a clean checkout, and not some error 574 | condition... Could alse use 'git diff --quiet'. 575 | 576 | """ 577 | is_dirty = False 578 | if git_output: 579 | is_dirty = True 580 | return is_dirty 581 | 582 | # ---------------------------------------------------------------- 583 | # 584 | # system call to git for information gathering 585 | # 586 | # ---------------------------------------------------------------- 587 | @staticmethod 588 | def _git_current_hash(): 589 | """Return the full hash of the currently checked-out version. 590 | 591 | Returns a tuple, (hash_found, hash), where hash_found is a 592 | logical specifying whether a hash was found for HEAD (False 593 | could mean we're not in a git repository at all). (If hash_found 594 | is False, then hash is ''.) 595 | """ 596 | status, git_output = GitRepository._git_revparse_commit("HEAD") 597 | hash_found = not status 598 | if not hash_found: 599 | git_output = '' 600 | return hash_found, git_output 601 | 602 | @staticmethod 603 | def _git_current_branch(): 604 | """Determines the name of the current branch. 605 | 606 | Returns a tuple, (branch_found, branch_name), where branch_found 607 | is a logical specifying whether a branch name was found for 608 | HEAD. (If branch_found is False, then branch_name is ''.) 609 | """ 610 | cmd = ['git', 'symbolic-ref', '--short', '-q', 'HEAD'] 611 | status, git_output = execute_subprocess(cmd, 612 | output_to_caller=True, 613 | status_to_caller=True) 614 | branch_found = not status 615 | if branch_found: 616 | git_output = git_output.strip() 617 | else: 618 | git_output = '' 619 | return branch_found, git_output 620 | 621 | @staticmethod 622 | def _git_current_tag(): 623 | """Determines the name tag corresponding to HEAD (if any). 624 | 625 | Returns a tuple, (tag_found, tag_name), where tag_found is a 626 | logical specifying whether we found a tag name corresponding to 627 | HEAD. (If tag_found is False, then tag_name is ''.) 628 | """ 629 | # git describe --exact-match --tags HEAD 630 | cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD'] 631 | status, git_output = execute_subprocess(cmd, 632 | output_to_caller=True, 633 | status_to_caller=True) 634 | tag_found = not status 635 | if tag_found: 636 | git_output = git_output.strip() 637 | else: 638 | git_output = '' 639 | return tag_found, git_output 640 | 641 | @staticmethod 642 | def _git_showref_tag(ref): 643 | """Run git show-ref check if the user supplied ref is a tag. 644 | 645 | could also use git rev-parse --quiet --verify tagname^{tag} 646 | """ 647 | cmd = ['git', 'show-ref', '--quiet', '--verify', 648 | 'refs/tags/{0}'.format(ref), ] 649 | status = execute_subprocess(cmd, status_to_caller=True) 650 | return status 651 | 652 | @staticmethod 653 | def _git_showref_branch(ref): 654 | """Run git show-ref check if the user supplied ref is a local or 655 | tracked remote branch. 656 | 657 | """ 658 | cmd = ['git', 'show-ref', '--quiet', '--verify', 659 | 'refs/heads/{0}'.format(ref), ] 660 | status = execute_subprocess(cmd, status_to_caller=True) 661 | return status 662 | 663 | @staticmethod 664 | def _git_lsremote_branch(ref, remote_name): 665 | """Run git ls-remote to check if the user supplied ref is a remote 666 | branch that is not being tracked 667 | 668 | """ 669 | cmd = ['git', 'ls-remote', '--exit-code', '--heads', 670 | remote_name, ref, ] 671 | status = execute_subprocess(cmd, status_to_caller=True) 672 | return status 673 | 674 | @staticmethod 675 | def _git_revparse_commit(ref): 676 | """Run git rev-parse to detect if a reference is a SHA, HEAD or other 677 | valid commit. 678 | 679 | """ 680 | cmd = ['git', 'rev-parse', '--quiet', '--verify', 681 | '{0}^{1}'.format(ref, '{commit}'), ] 682 | status, git_output = execute_subprocess(cmd, status_to_caller=True, 683 | output_to_caller=True) 684 | git_output = git_output.strip() 685 | return status, git_output 686 | 687 | @staticmethod 688 | def _git_status_porcelain_v1z(): 689 | """Run git status to obtain repository information. 690 | 691 | This is run with '--untracked=no' to ignore untracked files. 692 | 693 | The machine-portable format that is guaranteed not to change 694 | between git versions or *user configuration*. 695 | 696 | """ 697 | cmd = ['git', 'status', '--untracked-files=no', '--porcelain', '-z'] 698 | git_output = execute_subprocess(cmd, output_to_caller=True) 699 | return git_output 700 | 701 | @staticmethod 702 | def _git_status_verbose(): 703 | """Run the git status command to obtain repository information. 704 | """ 705 | cmd = ['git', 'status'] 706 | git_output = execute_subprocess(cmd, output_to_caller=True) 707 | return git_output 708 | 709 | @staticmethod 710 | def _git_remote_verbose(): 711 | """Run the git remote command to obtain repository information. 712 | """ 713 | cmd = ['git', 'remote', '--verbose'] 714 | git_output = execute_subprocess(cmd, output_to_caller=True) 715 | return git_output 716 | 717 | @staticmethod 718 | def has_submodules(repo_dir_path=None): 719 | """Return True iff the repository at (or the current 720 | directory if is None) has a '.gitmodules' file 721 | """ 722 | if repo_dir_path is None: 723 | fname = ExternalsDescription.GIT_SUBMODULES_FILENAME 724 | else: 725 | fname = os.path.join(repo_dir_path, 726 | ExternalsDescription.GIT_SUBMODULES_FILENAME) 727 | 728 | return os.path.exists(fname) 729 | 730 | # ---------------------------------------------------------------- 731 | # 732 | # system call to git for sideffects modifying the working tree 733 | # 734 | # ---------------------------------------------------------------- 735 | @staticmethod 736 | def _git_clone(url, repo_dir_name, verbosity): 737 | """Run git clone for the side effect of creating a repository. 738 | """ 739 | cmd = ['git', 'clone', '--quiet'] 740 | subcmd = None 741 | 742 | cmd.extend([url, repo_dir_name]) 743 | if verbosity >= VERBOSITY_VERBOSE: 744 | printlog(' {0}'.format(' '.join(cmd))) 745 | execute_subprocess(cmd) 746 | if subcmd is not None: 747 | os.chdir(repo_dir_name) 748 | execute_subprocess(subcmd) 749 | 750 | @staticmethod 751 | def _git_remote_add(name, url): 752 | """Run the git remote command for the side effect of adding a remote 753 | """ 754 | cmd = ['git', 'remote', 'add', name, url] 755 | execute_subprocess(cmd) 756 | 757 | @staticmethod 758 | def _git_fetch(remote_name): 759 | """Run the git fetch command for the side effect of updating the repo 760 | """ 761 | cmd = ['git', 'fetch', '--quiet', '--tags', remote_name] 762 | execute_subprocess(cmd) 763 | 764 | @staticmethod 765 | def _git_checkout_ref(ref, verbosity, submodules): 766 | """Run the git checkout command for the side effect of updating the repo 767 | 768 | Param: ref is a reference to a local or remote object in the 769 | form 'origin/my_feature', or 'tag1'. 770 | 771 | """ 772 | cmd = ['git', 'checkout', '--quiet', ref] 773 | if verbosity >= VERBOSITY_VERBOSE: 774 | printlog(' {0}'.format(' '.join(cmd))) 775 | execute_subprocess(cmd) 776 | if submodules: 777 | GitRepository._git_update_submodules(verbosity) 778 | 779 | @staticmethod 780 | def _git_update_submodules(verbosity): 781 | """Run git submodule update for the side effect of updating this 782 | repo's submodules. 783 | """ 784 | # First, verify that we have a .gitmodules file 785 | if os.path.exists(ExternalsDescription.GIT_SUBMODULES_FILENAME): 786 | cmd = ['git', 'submodule', 'update', '--init', '--recursive'] 787 | if verbosity >= VERBOSITY_VERBOSE: 788 | printlog(' {0}'.format(' '.join(cmd))) 789 | 790 | execute_subprocess(cmd) 791 | -------------------------------------------------------------------------------- /manage_externals/manic/externals_description.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Model description 4 | 5 | Model description is the representation of the various externals 6 | included in the model. It processes in input data structure, and 7 | converts it into a standard interface that is used by the rest of the 8 | system. 9 | 10 | To maintain backward compatibility, externals description files should 11 | follow semantic versioning rules, http://semver.org/ 12 | 13 | 14 | 15 | """ 16 | from __future__ import absolute_import 17 | from __future__ import unicode_literals 18 | from __future__ import print_function 19 | 20 | import logging 21 | import os 22 | import os.path 23 | import re 24 | 25 | # ConfigParser in python2 was renamed to configparser in python3. 26 | # In python2, ConfigParser returns byte strings, str, instead of unicode. 27 | # We need unicode to be compatible with xml and json parser and python3. 28 | try: 29 | # python2 30 | from ConfigParser import SafeConfigParser as config_parser 31 | from ConfigParser import MissingSectionHeaderError 32 | from ConfigParser import NoSectionError, NoOptionError 33 | 34 | USE_PYTHON2 = True 35 | 36 | def config_string_cleaner(text): 37 | """convert strings into unicode 38 | """ 39 | return text.decode('utf-8') 40 | except ImportError: 41 | # python3 42 | from configparser import ConfigParser as config_parser 43 | from configparser import MissingSectionHeaderError 44 | from configparser import NoSectionError, NoOptionError 45 | 46 | USE_PYTHON2 = False 47 | 48 | def config_string_cleaner(text): 49 | """Python3 already uses unicode strings, so just return the string 50 | without modification. 51 | 52 | """ 53 | return text 54 | 55 | from .utils import printlog, fatal_error, str_to_bool, expand_local_url 56 | from .utils import execute_subprocess 57 | from .global_constants import EMPTY_STR, PPRINTER, VERSION_SEPERATOR 58 | 59 | # 60 | # Globals 61 | # 62 | DESCRIPTION_SECTION = 'externals_description' 63 | VERSION_ITEM = 'schema_version' 64 | 65 | 66 | def read_externals_description_file(root_dir, file_name): 67 | """Read a file containing an externals description and 68 | create its internal representation. 69 | 70 | """ 71 | root_dir = os.path.abspath(root_dir) 72 | msg = 'In directory : {0}'.format(root_dir) 73 | logging.info(msg) 74 | printlog('Processing externals description file : {0}'.format(file_name)) 75 | 76 | file_path = os.path.join(root_dir, file_name) 77 | if not os.path.exists(file_name): 78 | if file_name.lower() == "none": 79 | msg = ('INTERNAL ERROR: Attempt to read externals file ' 80 | 'from {0} when not configured'.format(file_path)) 81 | else: 82 | msg = ('ERROR: Model description file, "{0}", does not ' 83 | 'exist at path:\n {1}\nDid you run from the root of ' 84 | 'the source tree?'.format(file_name, file_path)) 85 | 86 | fatal_error(msg) 87 | 88 | externals_description = None 89 | if file_name == ExternalsDescription.GIT_SUBMODULES_FILENAME: 90 | externals_description = read_gitmodules_file(root_dir, file_name) 91 | else: 92 | try: 93 | config = config_parser() 94 | config.read(file_path) 95 | externals_description = config 96 | except MissingSectionHeaderError: 97 | # not a cfg file 98 | pass 99 | 100 | if externals_description is None: 101 | msg = 'Unknown file format!' 102 | fatal_error(msg) 103 | 104 | return externals_description 105 | 106 | class LstripReader(object): 107 | "LstripReader formats .gitmodules files to be acceptable for configparser" 108 | def __init__(self, filename): 109 | with open(filename, 'r') as infile: 110 | lines = infile.readlines() 111 | self._lines = list() 112 | self._num_lines = len(lines) 113 | self._index = 0 114 | for line in lines: 115 | self._lines.append(line.lstrip()) 116 | 117 | def readlines(self): 118 | """Return all the lines from this object's file""" 119 | return self._lines 120 | 121 | def readline(self, size=-1): 122 | """Format and return the next line or raise StopIteration""" 123 | try: 124 | line = self.next() 125 | except StopIteration: 126 | line = '' 127 | 128 | if (size > 0) and (len(line) < size): 129 | return line[0:size] 130 | 131 | return line 132 | 133 | def __iter__(self): 134 | """Begin an iteration""" 135 | self._index = 0 136 | return self 137 | 138 | def next(self): 139 | """Return the next line or raise StopIteration""" 140 | if self._index >= self._num_lines: 141 | raise StopIteration 142 | 143 | self._index = self._index + 1 144 | return self._lines[self._index - 1] 145 | 146 | def __next__(self): 147 | return self.next() 148 | 149 | def git_submodule_status(repo_dir): 150 | """Run the git submodule status command to obtain submodule hashes. 151 | """ 152 | # This function is here instead of GitRepository to avoid a dependency loop 153 | cwd = os.getcwd() 154 | os.chdir(repo_dir) 155 | cmd = ['git', 'submodule', 'status'] 156 | git_output = execute_subprocess(cmd, output_to_caller=True) 157 | submodules = {} 158 | submods = git_output.split('\n') 159 | for submod in submods: 160 | if submod: 161 | status = submod[0] 162 | items = submod[1:].split(' ') 163 | if len(items) > 2: 164 | tag = items[2] 165 | else: 166 | tag = None 167 | 168 | submodules[items[1]] = {'hash':items[0], 'status':status, 'tag':tag} 169 | 170 | os.chdir(cwd) 171 | return submodules 172 | 173 | def parse_submodules_desc_section(section_items, file_path): 174 | """Find the path and url for this submodule description""" 175 | path = None 176 | url = None 177 | for item in section_items: 178 | name = item[0].strip().lower() 179 | if name == 'path': 180 | path = item[1].strip() 181 | elif name == 'url': 182 | url = item[1].strip() 183 | else: 184 | msg = 'WARNING: Ignoring unknown {} property, in {}' 185 | msg = msg.format(item[0], file_path) # fool pylint 186 | logging.warning(msg) 187 | 188 | return path, url 189 | 190 | def read_gitmodules_file(root_dir, file_name): 191 | # pylint: disable=deprecated-method 192 | # Disabling this check because the method is only used for python2 193 | """Read a .gitmodules file and convert it to be compatible with an 194 | externals description. 195 | """ 196 | root_dir = os.path.abspath(root_dir) 197 | msg = 'In directory : {0}'.format(root_dir) 198 | logging.info(msg) 199 | printlog('Processing submodules description file : {0}'.format(file_name)) 200 | 201 | file_path = os.path.join(root_dir, file_name) 202 | if not os.path.exists(file_name): 203 | msg = ('ERROR: submodules description file, "{0}", does not ' 204 | 'exist at path:\n {1}'.format(file_name, file_path)) 205 | fatal_error(msg) 206 | 207 | submodules_description = None 208 | externals_description = None 209 | try: 210 | config = config_parser() 211 | if USE_PYTHON2: 212 | config.readfp(LstripReader(file_path), filename=file_name) 213 | else: 214 | config.read_file(LstripReader(file_path), source=file_name) 215 | 216 | submodules_description = config 217 | except MissingSectionHeaderError: 218 | # not a cfg file 219 | pass 220 | 221 | if submodules_description is None: 222 | msg = 'Unknown file format!' 223 | fatal_error(msg) 224 | else: 225 | # Convert the submodules description to an externals description 226 | externals_description = config_parser() 227 | # We need to grab all the commit hashes for this repo 228 | submods = git_submodule_status(root_dir) 229 | for section in submodules_description.sections(): 230 | if section[0:9] == 'submodule': 231 | sec_name = section[9:].strip(' "') 232 | externals_description.add_section(sec_name) 233 | section_items = submodules_description.items(section) 234 | path, url = parse_submodules_desc_section(section_items, 235 | file_path) 236 | 237 | if path is None: 238 | msg = 'Submodule {} missing path'.format(sec_name) 239 | fatal_error(msg) 240 | 241 | if url is None: 242 | msg = 'Submodule {} missing url'.format(sec_name) 243 | fatal_error(msg) 244 | 245 | externals_description.set(sec_name, 246 | ExternalsDescription.PATH, path) 247 | externals_description.set(sec_name, 248 | ExternalsDescription.PROTOCOL, 'git') 249 | externals_description.set(sec_name, 250 | ExternalsDescription.REPO_URL, url) 251 | externals_description.set(sec_name, 252 | ExternalsDescription.REQUIRED, 'True') 253 | git_hash = submods[sec_name]['hash'] 254 | externals_description.set(sec_name, 255 | ExternalsDescription.HASH, git_hash) 256 | 257 | # Required items 258 | externals_description.add_section(DESCRIPTION_SECTION) 259 | externals_description.set(DESCRIPTION_SECTION, VERSION_ITEM, '1.0.0') 260 | 261 | return externals_description 262 | 263 | def create_externals_description( 264 | model_data, model_format='cfg', components=None, parent_repo=None): 265 | """Create the a externals description object from the provided data 266 | """ 267 | externals_description = None 268 | if model_format == 'dict': 269 | externals_description = ExternalsDescriptionDict( 270 | model_data, components=components) 271 | elif model_format == 'cfg': 272 | major, _, _ = get_cfg_schema_version(model_data) 273 | if major == 1: 274 | externals_description = ExternalsDescriptionConfigV1( 275 | model_data, components=components, parent_repo=parent_repo) 276 | else: 277 | msg = ('Externals description file has unsupported schema ' 278 | 'version "{0}".'.format(major)) 279 | fatal_error(msg) 280 | else: 281 | msg = 'Unknown model data format "{0}"'.format(model_format) 282 | fatal_error(msg) 283 | return externals_description 284 | 285 | 286 | def get_cfg_schema_version(model_cfg): 287 | """Extract the major, minor, patch version of the config file schema 288 | 289 | Params: 290 | model_cfg - config parser object containing the externas description data 291 | 292 | Returns: 293 | major = integer major version 294 | minor = integer minor version 295 | patch = integer patch version 296 | """ 297 | semver_str = '' 298 | try: 299 | semver_str = model_cfg.get(DESCRIPTION_SECTION, VERSION_ITEM) 300 | except (NoSectionError, NoOptionError): 301 | msg = ('externals description file must have the required ' 302 | 'section: "{0}" and item "{1}"'.format(DESCRIPTION_SECTION, 303 | VERSION_ITEM)) 304 | fatal_error(msg) 305 | 306 | # NOTE(bja, 2017-11) Assume we don't care about the 307 | # build/pre-release metadata for now! 308 | version_list = re.split(r'[-+]', semver_str) 309 | version_str = version_list[0] 310 | version = version_str.split(VERSION_SEPERATOR) 311 | try: 312 | major = int(version[0].strip()) 313 | minor = int(version[1].strip()) 314 | patch = int(version[2].strip()) 315 | except ValueError: 316 | msg = ('Config file schema version must have integer digits for ' 317 | 'major, minor and patch versions. ' 318 | 'Received "{0}"'.format(version_str)) 319 | fatal_error(msg) 320 | return major, minor, patch 321 | 322 | 323 | class ExternalsDescription(dict): 324 | """Base externals description class that is independent of the user input 325 | format. Different input formats can all be converted to this 326 | representation to provide a consistent represtentation for the 327 | rest of the objects in the system. 328 | 329 | NOTE(bja, 2018-03): do NOT define _schema_major etc at the class 330 | level in the base class. The nested/recursive nature of externals 331 | means different schema versions may be present in a single run! 332 | 333 | All inheriting classes must overwrite: 334 | self._schema_major and self._input_major 335 | self._schema_minor and self._input_minor 336 | self._schema_patch and self._input_patch 337 | 338 | where _schema_x is the supported schema, _input_x is the user 339 | input value. 340 | 341 | """ 342 | # keywords defining the interface into the externals description data 343 | EXTERNALS = 'externals' 344 | BRANCH = 'branch' 345 | SUBMODULE = 'from_submodule' 346 | HASH = 'hash' 347 | NAME = 'name' 348 | PATH = 'local_path' 349 | PROTOCOL = 'protocol' 350 | REPO = 'repo' 351 | REPO_URL = 'repo_url' 352 | REQUIRED = 'required' 353 | TAG = 'tag' 354 | 355 | PROTOCOL_EXTERNALS_ONLY = 'externals_only' 356 | PROTOCOL_GIT = 'git' 357 | PROTOCOL_SVN = 'svn' 358 | GIT_SUBMODULES_FILENAME = '.gitmodules' 359 | KNOWN_PRROTOCOLS = [PROTOCOL_GIT, PROTOCOL_SVN, PROTOCOL_EXTERNALS_ONLY] 360 | 361 | # v1 xml keywords 362 | _V1_TREE_PATH = 'TREE_PATH' 363 | _V1_ROOT = 'ROOT' 364 | _V1_TAG = 'TAG' 365 | _V1_BRANCH = 'BRANCH' 366 | _V1_REQ_SOURCE = 'REQ_SOURCE' 367 | 368 | _source_schema = {REQUIRED: True, 369 | PATH: 'string', 370 | EXTERNALS: 'string', 371 | SUBMODULE : True, 372 | REPO: {PROTOCOL: 'string', 373 | REPO_URL: 'string', 374 | TAG: 'string', 375 | BRANCH: 'string', 376 | HASH: 'string', 377 | } 378 | } 379 | 380 | def __init__(self, parent_repo=None): 381 | """Convert the xml into a standardized dict that can be used to 382 | construct the source objects 383 | 384 | """ 385 | dict.__init__(self) 386 | 387 | self._schema_major = None 388 | self._schema_minor = None 389 | self._schema_patch = None 390 | self._input_major = None 391 | self._input_minor = None 392 | self._input_patch = None 393 | self._parent_repo = parent_repo 394 | 395 | def _verify_schema_version(self): 396 | """Use semantic versioning rules to verify we can process this schema. 397 | 398 | """ 399 | known = '{0}.{1}.{2}'.format(self._schema_major, 400 | self._schema_minor, 401 | self._schema_patch) 402 | received = '{0}.{1}.{2}'.format(self._input_major, 403 | self._input_minor, 404 | self._input_patch) 405 | 406 | if self._input_major != self._schema_major: 407 | # should never get here, the factory should handle this correctly! 408 | msg = ('DEV_ERROR: version "{0}" parser received ' 409 | 'version "{1}" input.'.format(known, received)) 410 | fatal_error(msg) 411 | 412 | if self._input_minor > self._schema_minor: 413 | msg = ('Incompatible schema version:\n' 414 | ' User supplied schema version "{0}" is too new."\n' 415 | ' Can only process version "{1}" files and ' 416 | 'older.'.format(received, known)) 417 | fatal_error(msg) 418 | 419 | if self._input_patch > self._schema_patch: 420 | # NOTE(bja, 2018-03) ignoring for now... Not clear what 421 | # conditions the test is needed. 422 | pass 423 | 424 | def _check_user_input(self): 425 | """Run a series of checks to attempt to validate the user input and 426 | detect errors as soon as possible. 427 | 428 | NOTE(bja, 2018-03) These checks are called *after* the file is 429 | read. That means the schema check can not occur here. 430 | 431 | Note: the order is important. check_optional will create 432 | optional with null data. run check_data first to ensure 433 | required data was provided correctly by the user. 434 | 435 | """ 436 | self._check_data() 437 | self._check_optional() 438 | self._validate() 439 | 440 | def _check_data(self): 441 | # pylint: disable=too-many-branches,too-many-statements 442 | """Check user supplied data is valid where possible. 443 | """ 444 | for ext_name in self.keys(): 445 | if (self[ext_name][self.REPO][self.PROTOCOL] 446 | not in self.KNOWN_PRROTOCOLS): 447 | msg = 'Unknown repository protocol "{0}" in "{1}".'.format( 448 | self[ext_name][self.REPO][self.PROTOCOL], ext_name) 449 | fatal_error(msg) 450 | 451 | if (self[ext_name][self.REPO][self.PROTOCOL] == 452 | self.PROTOCOL_SVN): 453 | if self.HASH in self[ext_name][self.REPO]: 454 | msg = ('In repo description for "{0}". svn repositories ' 455 | 'may not include the "hash" keyword.'.format( 456 | ext_name)) 457 | fatal_error(msg) 458 | 459 | if ((self[ext_name][self.REPO][self.PROTOCOL] != self.PROTOCOL_GIT) 460 | and (self.SUBMODULE in self[ext_name])): 461 | msg = ('self.SUBMODULE is only supported with {0} protocol, ' 462 | '"{1}" is defined as an {2} repository') 463 | fatal_error(msg.format(self.PROTOCOL_GIT, ext_name, 464 | self[ext_name][self.REPO][self.PROTOCOL])) 465 | 466 | if (self[ext_name][self.REPO][self.PROTOCOL] != 467 | self.PROTOCOL_EXTERNALS_ONLY): 468 | ref_count = 0 469 | found_refs = '' 470 | if self.TAG in self[ext_name][self.REPO]: 471 | ref_count += 1 472 | found_refs = '"{0} = {1}", {2}'.format( 473 | self.TAG, self[ext_name][self.REPO][self.TAG], 474 | found_refs) 475 | if self.BRANCH in self[ext_name][self.REPO]: 476 | ref_count += 1 477 | found_refs = '"{0} = {1}", {2}'.format( 478 | self.BRANCH, self[ext_name][self.REPO][self.BRANCH], 479 | found_refs) 480 | if self.HASH in self[ext_name][self.REPO]: 481 | ref_count += 1 482 | found_refs = '"{0} = {1}", {2}'.format( 483 | self.HASH, self[ext_name][self.REPO][self.HASH], 484 | found_refs) 485 | if (self.SUBMODULE in self[ext_name] and 486 | self[ext_name][self.SUBMODULE]): 487 | ref_count += 1 488 | found_refs = '"{0} = {1}", {2}'.format( 489 | self.SUBMODULE, 490 | self[ext_name][self.SUBMODULE], found_refs) 491 | 492 | if ref_count > 1: 493 | msg = 'Model description is over specified! ' 494 | if self.SUBMODULE in self[ext_name]: 495 | msg += ('from_submodule is not compatible with ' 496 | '"tag", "branch", or "hash" ') 497 | else: 498 | msg += (' Only one of "tag", "branch", or "hash" ' 499 | 'may be specified ') 500 | 501 | msg += 'for repo description of "{0}".'.format(ext_name) 502 | msg = '{0}\nFound: {1}'.format(msg, found_refs) 503 | fatal_error(msg) 504 | elif ref_count < 1: 505 | msg = ('Model description is under specified! One of ' 506 | '"tag", "branch", or "hash" must be specified for ' 507 | 'repo description of "{0}"'.format(ext_name)) 508 | fatal_error(msg) 509 | 510 | if (self.REPO_URL not in self[ext_name][self.REPO] and 511 | (self.SUBMODULE not in self[ext_name] or 512 | not self[ext_name][self.SUBMODULE])): 513 | msg = ('Model description is under specified! Must have ' 514 | '"repo_url" in repo ' 515 | 'description for "{0}"'.format(ext_name)) 516 | fatal_error(msg) 517 | 518 | if (self.SUBMODULE in self[ext_name] and 519 | self[ext_name][self.SUBMODULE]): 520 | if self.REPO_URL in self[ext_name][self.REPO]: 521 | msg = ('Model description is over specified! ' 522 | 'from_submodule keyword is not compatible ' 523 | 'with {0} keyword for'.format(self.REPO_URL)) 524 | msg = '{0} repo description of "{1}"'.format(msg, 525 | ext_name) 526 | fatal_error(msg) 527 | 528 | if self.PATH in self[ext_name]: 529 | msg = ('Model description is over specified! ' 530 | 'from_submodule keyword is not compatible with ' 531 | '{0} keyword for'.format(self.PATH)) 532 | msg = '{0} repo description of "{1}"'.format(msg, 533 | ext_name) 534 | fatal_error(msg) 535 | 536 | if self.REPO_URL in self[ext_name][self.REPO]: 537 | url = expand_local_url( 538 | self[ext_name][self.REPO][self.REPO_URL], ext_name) 539 | self[ext_name][self.REPO][self.REPO_URL] = url 540 | 541 | def _check_optional(self): 542 | # pylint: disable=too-many-branches 543 | """Some fields like externals, repo:tag repo:branch are 544 | (conditionally) optional. We don't want the user to be 545 | required to enter them in every externals description file, but 546 | still want to validate the input. Check conditions and add 547 | default values if appropriate. 548 | 549 | """ 550 | submod_desc = None # Only load submodules info once 551 | for field in self: 552 | # truely optional 553 | if self.EXTERNALS not in self[field]: 554 | self[field][self.EXTERNALS] = EMPTY_STR 555 | 556 | # git and svn repos must tags and branches for validation purposes. 557 | if self.TAG not in self[field][self.REPO]: 558 | self[field][self.REPO][self.TAG] = EMPTY_STR 559 | if self.BRANCH not in self[field][self.REPO]: 560 | self[field][self.REPO][self.BRANCH] = EMPTY_STR 561 | if self.HASH not in self[field][self.REPO]: 562 | self[field][self.REPO][self.HASH] = EMPTY_STR 563 | if self.REPO_URL not in self[field][self.REPO]: 564 | self[field][self.REPO][self.REPO_URL] = EMPTY_STR 565 | 566 | # from_submodule has a complex relationship with other fields 567 | if self.SUBMODULE in self[field]: 568 | # User wants to use submodule information, is it available? 569 | if self._parent_repo is None: 570 | # No parent == no submodule information 571 | PPRINTER.pprint(self[field]) 572 | msg = 'No parent submodule for "{0}"'.format(field) 573 | fatal_error(msg) 574 | elif self._parent_repo.protocol() != self.PROTOCOL_GIT: 575 | PPRINTER.pprint(self[field]) 576 | msg = 'Parent protocol, "{0}", does not support submodules' 577 | fatal_error(msg.format(self._parent_repo.protocol())) 578 | else: 579 | args = self._repo_config_from_submodule(field, submod_desc) 580 | repo_url, repo_path, ref_hash, submod_desc = args 581 | 582 | if repo_url is None: 583 | msg = ('Cannot checkout "{0}" as a submodule, ' 584 | 'repo not found in {1} file') 585 | fatal_error(msg.format(field, 586 | self.GIT_SUBMODULES_FILENAME)) 587 | # Fill in submodule fields 588 | self[field][self.REPO][self.REPO_URL] = repo_url 589 | self[field][self.REPO][self.HASH] = ref_hash 590 | self[field][self.PATH] = repo_path 591 | 592 | if self[field][self.SUBMODULE]: 593 | # We should get everything from the parent submodule 594 | # configuration. 595 | pass 596 | # No else (from _submodule = False is the default) 597 | else: 598 | # Add the default value (not using submodule information) 599 | self[field][self.SUBMODULE] = False 600 | 601 | def _repo_config_from_submodule(self, field, submod_desc): 602 | """Find the external config information for a repository from 603 | its submodule configuration information. 604 | """ 605 | if submod_desc is None: 606 | repo_path = os.getcwd() # Is this always correct? 607 | submod_file = self._parent_repo.submodules_file(repo_path=repo_path) 608 | if submod_file is None: 609 | msg = ('Cannot checkout "{0}" from submodule information\n' 610 | ' Parent repo, "{1}" does not have submodules') 611 | fatal_error(msg.format(field, self._parent_repo.name())) 612 | 613 | submod_file = read_gitmodules_file(repo_path, submod_file) 614 | submod_desc = create_externals_description(submod_file) 615 | 616 | # Can we find our external? 617 | repo_url = None 618 | repo_path = None 619 | ref_hash = None 620 | for ext_field in submod_desc: 621 | if field == ext_field: 622 | ext = submod_desc[ext_field] 623 | repo_url = ext[self.REPO][self.REPO_URL] 624 | repo_path = ext[self.PATH] 625 | ref_hash = ext[self.REPO][self.HASH] 626 | break 627 | 628 | return repo_url, repo_path, ref_hash, submod_desc 629 | 630 | def _validate(self): 631 | """Validate that the parsed externals description contains all necessary 632 | fields. 633 | 634 | """ 635 | def print_compare_difference(data_a, data_b, loc_a, loc_b): 636 | """Look through the data structures and print the differences. 637 | 638 | """ 639 | for item in data_a: 640 | if item in data_b: 641 | if not isinstance(data_b[item], type(data_a[item])): 642 | printlog(" {item}: {loc} = {val} ({val_type})".format( 643 | item=item, loc=loc_a, val=data_a[item], 644 | val_type=type(data_a[item]))) 645 | printlog(" {item} {loc} = {val} ({val_type})".format( 646 | item=' ' * len(item), loc=loc_b, val=data_b[item], 647 | val_type=type(data_b[item]))) 648 | else: 649 | printlog(" {item}: {loc} = {val} ({val_type})".format( 650 | item=item, loc=loc_a, val=data_a[item], 651 | val_type=type(data_a[item]))) 652 | printlog(" {item} {loc} missing".format( 653 | item=' ' * len(item), loc=loc_b)) 654 | 655 | def validate_data_struct(schema, data): 656 | """Compare a data structure against a schema and validate all required 657 | fields are present. 658 | 659 | """ 660 | is_valid = False 661 | in_ref = True 662 | valid = True 663 | if isinstance(schema, dict) and isinstance(data, dict): 664 | # Both are dicts, recursively verify that all fields 665 | # in schema are present in the data. 666 | for key in schema: 667 | in_ref = in_ref and (key in data) 668 | if in_ref: 669 | valid = valid and ( 670 | validate_data_struct(schema[key], data[key])) 671 | 672 | is_valid = in_ref and valid 673 | else: 674 | # non-recursive structure. verify data and schema have 675 | # the same type. 676 | is_valid = isinstance(data, type(schema)) 677 | 678 | if not is_valid: 679 | printlog(" Unmatched schema and input:") 680 | if isinstance(schema, dict): 681 | print_compare_difference(schema, data, 'schema', 'input') 682 | print_compare_difference(data, schema, 'input', 'schema') 683 | else: 684 | printlog(" schema = {0} ({1})".format( 685 | schema, type(schema))) 686 | printlog(" input = {0} ({1})".format(data, type(data))) 687 | 688 | return is_valid 689 | 690 | for field in self: 691 | valid = validate_data_struct(self._source_schema, self[field]) 692 | if not valid: 693 | PPRINTER.pprint(self._source_schema) 694 | PPRINTER.pprint(self[field]) 695 | msg = 'ERROR: source for "{0}" did not validate'.format(field) 696 | fatal_error(msg) 697 | 698 | 699 | class ExternalsDescriptionDict(ExternalsDescription): 700 | """Create a externals description object from a dictionary using the API 701 | representations. Primarily used to simplify creating model 702 | description files for unit testing. 703 | 704 | """ 705 | 706 | def __init__(self, model_data, components=None): 707 | """Parse a native dictionary into a externals description. 708 | """ 709 | ExternalsDescription.__init__(self) 710 | self._schema_major = 1 711 | self._schema_minor = 0 712 | self._schema_patch = 0 713 | self._input_major = 1 714 | self._input_minor = 0 715 | self._input_patch = 0 716 | self._verify_schema_version() 717 | if components: 718 | for key in model_data.items(): 719 | if key not in components: 720 | del model_data[key] 721 | 722 | self.update(model_data) 723 | self._check_user_input() 724 | 725 | 726 | class ExternalsDescriptionConfigV1(ExternalsDescription): 727 | """Create a externals description object from a config_parser object, 728 | schema version 1. 729 | 730 | """ 731 | 732 | def __init__(self, model_data, components=None, parent_repo=None): 733 | """Convert the config data into a standardized dict that can be used to 734 | construct the source objects 735 | 736 | """ 737 | ExternalsDescription.__init__(self, parent_repo=parent_repo) 738 | self._schema_major = 1 739 | self._schema_minor = 1 740 | self._schema_patch = 0 741 | self._input_major, self._input_minor, self._input_patch = \ 742 | get_cfg_schema_version(model_data) 743 | self._verify_schema_version() 744 | self._remove_metadata(model_data) 745 | self._parse_cfg(model_data, components=components) 746 | self._check_user_input() 747 | 748 | @staticmethod 749 | def _remove_metadata(model_data): 750 | """Remove the metadata section from the model configuration file so 751 | that it is simpler to look through the file and construct the 752 | externals description. 753 | 754 | """ 755 | model_data.remove_section(DESCRIPTION_SECTION) 756 | 757 | def _parse_cfg(self, cfg_data, components=None): 758 | """Parse a config_parser object into a externals description. 759 | """ 760 | def list_to_dict(input_list, convert_to_lower_case=True): 761 | """Convert a list of key-value pairs into a dictionary. 762 | """ 763 | output_dict = {} 764 | for item in input_list: 765 | key = config_string_cleaner(item[0].strip()) 766 | value = config_string_cleaner(item[1].strip()) 767 | if convert_to_lower_case: 768 | key = key.lower() 769 | output_dict[key] = value 770 | return output_dict 771 | 772 | for section in cfg_data.sections(): 773 | name = config_string_cleaner(section.lower().strip()) 774 | if components and name not in components: 775 | continue 776 | self[name] = {} 777 | self[name].update(list_to_dict(cfg_data.items(section))) 778 | self[name][self.REPO] = {} 779 | loop_keys = self[name].copy().keys() 780 | for item in loop_keys: 781 | if item in self._source_schema: 782 | if isinstance(self._source_schema[item], bool): 783 | self[name][item] = str_to_bool(self[name][item]) 784 | elif item in self._source_schema[self.REPO]: 785 | self[name][self.REPO][item] = self[name][item] 786 | del self[name][item] 787 | else: 788 | msg = ('Invalid input: "{sect}" contains unknown ' 789 | 'item "{item}".'.format(sect=name, item=item)) 790 | fatal_error(msg) 791 | --------------------------------------------------------------------------------