├── .editorconfig ├── .gitignore ├── .p4ignore ├── LICENSE ├── README.md ├── bootstrap.py ├── change-content.py ├── content_checker_config.yaml.dist ├── p4.py └── root_bootstrap.py /.editorconfig: -------------------------------------------------------------------------------- 1 | # Editor-agnostic formatting settings. 2 | # See http://editorconfig.org for more information. 3 | root = true 4 | 5 | [*] 6 | insert_final_newline = true 7 | indent_style = space 8 | indent_size = 4 9 | trim_trailing_whitespace = true 10 | charset = utf-8 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled python bytecode 2 | *.pyc 3 | 4 | # Your actual configuration 5 | content_checker_config.yaml 6 | 7 | # Right now our change-commit script is just some internal 8 | # stuff that's not really usable for other people. 9 | change-commit.py 10 | -------------------------------------------------------------------------------- /.p4ignore: -------------------------------------------------------------------------------- 1 | .git 2 | .editorconfig 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Uber Entertainment, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is a small set of scripts that does two things: 2 | 3 | - Route Perforce triggers to appropriate scripts (to make it easier to add new Perforce triggers with just a `p4 triggers` update and a new script in Perforce.) 4 | - We currently use a version of Perforce that does not support the `%//depot/path/to/script%` syntax in p4 triggers, so that makes this more important. 5 | - Verifies Unity meta-file consistency for configured projects. 6 | 7 | The Unity meta-file verification has three checks: 8 | 9 | - Verify that any new `Asset` has a new .meta file added for it. 10 | - Verify that any deleted `Asset` also has its .meta file deleted. 11 | - Verify that no-one attempts to modify the guid field of an existing .meta file. 12 | 13 | This prevents a lot of common mistakes that happen with Perforce and Unity. 14 | 15 | It does not **yet** support two more checks that would be helpful: 16 | 17 | - Verify that any new `Asset` does not re-use an existing guid. 18 | - Verify that there are no .meta files added without the corresponding `Asset`. 19 | 20 | Here, `Asset` means any file under the `Assets` directory in a project configured in `content_checker_config.yaml`. 21 | 22 | # Setting up 23 | 24 | First, you need to set up a workspace for use by your Perforce server to check out the scripts. This is what ours looks like: 25 | 26 | Client: perforce-admin-tools 27 | 28 | Root: d:\PerforceTriggers\tools\ 29 | 30 | View: 31 | //depot/admin/tools/... //perforce-admin-tools/... 32 | 33 | For us, these scripts live in e.g. `//depot/admin/tools/p4dispatch/bootstrap.py`. 34 | 35 | Copy `root_bootstrap.py` onto your Perforce server to wherever. For us, we use `D:\PerforceTriggers`. You will have to update `root_bootstrap.py` to 36 | have the correct path to `bootstrap.py` in `SECOND_STAGE_RELATIVE_PATH`. You will also have to update it to have the correct name of the workspace in `SECOND_STAGE_WORKSPACE`. 37 | 38 | Install [PyYAML](pyyaml). 39 | 40 | Configure `p4 triggers` to launch `root_bootstrap.py` as follows: 41 | 42 | gatekeeper-content change-content //depot/... "C:\python27\python.exe D:\PerforceTriggers\root_bootstrap.py change-content %changelist%" 43 | 44 | You will have to update the Python path as appropriate. That should be all! 45 | 46 | # Parts 47 | 48 | ## root_bootstrap.py 49 | 50 | This script is just responsible for syncing the `SECOND_STAGE_WORKSPACE` and launching `bootstrap.py` from this repository. 51 | 52 | ## bootstrap.py 53 | 54 | This script checks for a file called `$action.py` where `$action` is the Perforce trigger we're executing for, like change-content, change-commit, etc, 55 | and if it exists it launches it. It sets up some import paths first. 56 | 57 | ## change-content.py 58 | 59 | This script reads `content_checker_config.yaml`, and if any of the files changed in this change match the `paths` in a project, 60 | it launches the check, rejecting a change if it fails the check. The only support check so far is a Unity meta file check. 61 | 62 | 63 | # Questions? 64 | 65 | If you have any questions, file an issue or contact the author at jorgenpt@gmail.com. 66 | 67 | [pyyaml]: http://pyyaml.org/wiki/PyYAML 68 | -------------------------------------------------------------------------------- /bootstrap.py: -------------------------------------------------------------------------------- 1 | # This is the script that is responsible for finding the action handler for 2 | # the current triggger's handler. 3 | 4 | import os 5 | import sys 6 | import site 7 | 8 | if len(sys.argv) < 2: 9 | print >>sys.stderr, "Usage: %s " % (sys.argv[0], ) 10 | sys.exit(0) 11 | 12 | action = sys.argv[1] 13 | 14 | P4DISPATCH_DIR = os.path.realpath(os.path.abspath(os.path.dirname(__file__))) 15 | if P4DISPATCH_DIR not in sys.path: 16 | sys.path.insert(0, P4DISPATCH_DIR) 17 | 18 | # This is used to find the `yaml' Python package, unless you install it in a globally 19 | # accessible location. 20 | site.addsitedir(os.path.join(P4DISPATCH_DIR, '..', 'thirdparty', 'python-packages')) 21 | 22 | current_dir = os.path.dirname(__file__) 23 | action_handler = os.path.join(current_dir, action + '.py') 24 | if os.path.exists(action_handler): 25 | mod_globals = globals() 26 | mod_globals['__file__'] = action_handler 27 | execfile(action_handler, mod_globals) 28 | -------------------------------------------------------------------------------- /change-content.py: -------------------------------------------------------------------------------- 1 | import fnmatch 2 | import os 3 | import sys 4 | import timeit 5 | 6 | # This script requires the `yaml' Python package. 7 | import yaml 8 | 9 | # Our own Perforce wrapper. 10 | import p4 11 | 12 | ## You can include these words in your checkin description, and 13 | ## you'll enable that mode. 14 | DEBUG_CHECKS_KEYWORD = '$DEBUG_CHECKS$' 15 | SKIP_CHECKS_KEYWORD = '$SKIP_CHECKS$' 16 | TIME_CHECKS_KEYWORD = '$TIME_CHECKS$' 17 | 18 | class Timer(object): 19 | show_stats = True 20 | recursion_depth = 0 21 | 22 | def _duration_str(stop): 23 | mins, secs = divmod(int(stop - self._start), 60) 24 | return '{}:{:02}'.format(mins, secs) 25 | 26 | def __init__(self, label): 27 | self._label = label 28 | 29 | def __enter__(self): 30 | self._start = timeit.default_timer() 31 | type(self).recursion_depth += 1 32 | return self 33 | 34 | def __exit__(self, t, v, traceback): 35 | stop = timeit.default_timer() 36 | type(self).recursion_depth -= 1 37 | if type(self).show_stats: 38 | print '{}{} finished in {}'.format(' ' * type(self).recursion_depth, self._label, self._duration_str(stop)) 39 | 40 | def exists_after_change(files, cl, file): 41 | if file in files: 42 | cl = '=' + cl 43 | return p4.fstat('%s@%s' % (file, cl))['exists'] 44 | 45 | def unity_meta_check(changelist, change, matched_files): 46 | with Timer("Verifying added assets"): 47 | metas_correctly_added = verify_metas_added(changelist, change, matched_files) 48 | with Timer("Verifying deleted assets"): 49 | metas_correctly_deleted = verify_no_orphaned_files(changelist, change, matched_files) 50 | with Timer("Modified meta file guids"): 51 | no_meta_guids_modified = verify_no_guid_modifications(changelist, change, matched_files) 52 | 53 | # TODO(jorgenpt): Add a check for duplicate use of GUIDs. This needs a GUID cache (since 54 | # enumerating all of them can be slow,) as well as a reliable way to determine project root. 55 | 56 | return metas_correctly_added and metas_correctly_deleted and no_meta_guids_modified 57 | 58 | CREATE_FILE_ACTIONS = set(['branch', 'add', 'import']) 59 | 60 | def verify_metas_added(cl, change, matched_files): 61 | assets = set(fnmatch.filter(matched_files, '*/Assets/*')) 62 | missing_metas = False 63 | for asset in assets: 64 | if asset.endswith('.meta'): 65 | continue 66 | 67 | # TODO(jorgenpt): Check that no .meta files are added for resources that do not exist. 68 | 69 | if os.path.basename(asset).startswith('.'): 70 | continue 71 | 72 | action = change['files'][asset]['action'] 73 | if action not in CREATE_FILE_ACTIONS: 74 | continue 75 | 76 | # TODO(jorgenpt): Might be able to optimize this by batching the fstat calls. 77 | 78 | file_meta_exists = exists_after_change(matched_files, cl, '%s.meta' % asset) 79 | if not file_meta_exists: 80 | missing_metas = True 81 | print >>sys.stderr, "\nYou need to check in the matching meta file for %s" % asset, 82 | 83 | return not missing_metas 84 | 85 | DELETE_FILE_ACTIONS = set(['move/delete', 'delete', 'purge', 'archive']) 86 | 87 | def verify_no_orphaned_files(cl, change, matched_files): 88 | assets = set(fnmatch.filter(matched_files, '*/Assets/*')) 89 | orphaned_files = False 90 | for asset in assets: 91 | action = change['files'][asset]['action'] 92 | if action not in DELETE_FILE_ACTIONS: 93 | continue 94 | 95 | other_half_of_meta_pair = '%s.meta' % asset 96 | if asset.endswith('.meta'): 97 | other_half_of_meta_pair, ext = os.path.splitext(asset) 98 | 99 | other_half_of_meta_pair_exists = exists_after_change(matched_files, cl, other_half_of_meta_pair) 100 | if other_half_of_meta_pair_exists: 101 | orphaned_files = True 102 | print >>sys.stderr, "\nYou need to delete %s if you're deleting %s" % (other_half_of_meta_pair, asset), 103 | 104 | return not orphaned_files 105 | 106 | def verify_no_guid_modifications(cl, change, matched_files): 107 | metas = fnmatch.filter(matched_files, '*/Assets/*.meta') 108 | mutated_metas = False 109 | for asset in metas: 110 | assetChange = change['files'][asset] 111 | action = assetChange['action'] 112 | if action != 'edit': 113 | continue 114 | 115 | new_meta = yaml.load(p4.printfile('%s@=%s' % (asset, cl))) 116 | old_meta = yaml.load(p4.printfile('%s#%s' % (asset, assetChange['rev']))) 117 | 118 | if 'guid' in new_meta and 'guid' in old_meta: 119 | if new_meta['guid'] != old_meta['guid']: 120 | mutated_metas = True 121 | print >>sys.stderr, "\nYou're not allowed to change the guid of an existing asset %s" % asset, 122 | return not mutated_metas 123 | 124 | # These are the valid types for content_checker_config.yaml 125 | CHECK_TYPES = { 126 | 'unity_meta_check': unity_meta_check 127 | } 128 | 129 | def main(changelist): 130 | change = p4.describe(changelist) 131 | if DEBUG_CHECKS_KEYWORD in change['desc']: 132 | p4.DEBUG = True 133 | # Re-run this to get the debug output. 134 | change = p4.describe(changelist) 135 | 136 | if SKIP_CHECKS_KEYWORD in change['desc']: 137 | return True 138 | 139 | Timer.show_stats = TIME_CHECKS_KEYWORD in change['desc'] 140 | checks_succeeded = True 141 | 142 | config_path = os.path.join(P4DISPATCH_DIR, 'content_checker_config.yaml') 143 | if not os.path.exists(config_path): 144 | print >>sys.stderr, "Unable to find a P4 content checker configuration file at '%s'" % config_path 145 | return True 146 | 147 | try: 148 | checks = yaml.load(open(config_path, 'r')) 149 | 150 | for (name, info) in checks.iteritems(): 151 | check_name = info['check'] 152 | check = CHECK_TYPES.get(check_name) 153 | if not check: 154 | print >>sys.stderr, "Unable to find a check with name %r" % check_name 155 | continue 156 | 157 | matched_files = [] 158 | for path_pattern in info['paths']: 159 | matched_files += fnmatch.filter(change['files'].iterkeys(), path_pattern) 160 | 161 | if matched_files: 162 | with Timer('Checker "%s"' % name): 163 | if not check(changelist, change, matched_files): 164 | checks_succeeded = False 165 | 166 | if not checks_succeeded: 167 | print >>sys.stderr, "\n\nFix the problems above, or if you're intentionally doing this, ask a developer for help." 168 | except Exception as e: 169 | print >>sys.stderr, "Encountered an error trying to validate your change: %r" % e 170 | 171 | 172 | return checks_succeeded 173 | 174 | if __name__ == '__main__': 175 | if main(sys.argv[2]): 176 | sys.exit(0) 177 | else: 178 | sys.exit(1) 179 | -------------------------------------------------------------------------------- /content_checker_config.yaml.dist: -------------------------------------------------------------------------------- 1 | my_project: 2 | paths: 3 | - //depot/proj01/* 4 | check: unity_meta_check 5 | my_other_project: 6 | paths: 7 | - //depot/proj02/dev/* 8 | - //depot/proj02/stable/* 9 | check: unity_meta_check 10 | -------------------------------------------------------------------------------- /p4.py: -------------------------------------------------------------------------------- 1 | import marshal 2 | import os 3 | import subprocess 4 | import tempfile 5 | 6 | DEBUG = False 7 | 8 | class PerforceException(Exception): 9 | pass 10 | 11 | def _invoke(*args): 12 | results = [] 13 | fd, tempfname = tempfile.mkstemp() 14 | try: 15 | os.close(fd) 16 | with open(tempfname, 'w+b') as tempf: 17 | cmd = ['p4.exe', '-G'] + list(args) 18 | if DEBUG: 19 | print '$ %r' % cmd 20 | process = subprocess.Popen(cmd, stdout=tempf) 21 | process.communicate() 22 | tempf.seek(0) 23 | 24 | try: 25 | while 1: 26 | record = marshal.load(tempf) 27 | results.append(record) 28 | except EOFError: 29 | pass 30 | 31 | if process.returncode != 0: 32 | raise PerforceException('command %r return error: %d' % (cmd, process.returncode)) 33 | finally: 34 | os.unlink(tempfname) 35 | 36 | return results 37 | 38 | DESCRIBE_PATH_KEYS = ['depotFile', 'rev', 'action', 'type', 'fileSize', 'digest'] 39 | 40 | def describe(cl): 41 | obj = _invoke('describe', cl)[0] 42 | 43 | fileKeys = filter(lambda k: k.startswith('depotFile'), obj.iterkeys()) 44 | files = {} 45 | 46 | for key in fileKeys: 47 | key_suffix = key[len('depotFile'):] 48 | file_obj = {} 49 | for subkey in DESCRIBE_PATH_KEYS: 50 | specific_subkey = subkey + key_suffix 51 | if specific_subkey in obj: 52 | file_obj[subkey] = obj[specific_subkey] 53 | del obj[specific_subkey] 54 | files[file_obj['depotFile']] = file_obj 55 | 56 | obj['files'] = files 57 | 58 | if DEBUG: 59 | print '> %r' % obj 60 | 61 | return obj 62 | 63 | DELETED_ACTIONS = set(['delete', 'move/delete', 'purge']) 64 | 65 | def fstat(file): 66 | obj = _invoke('fstat', file)[0] 67 | 68 | if obj.get('code') == 'error': 69 | if DEBUG: 70 | print '! %r' % obj 71 | obj = { 'exists': False } 72 | else: 73 | obj['exists'] = obj.get('headAction') not in DELETED_ACTIONS 74 | 75 | if DEBUG: 76 | print '> %r' % obj 77 | 78 | return obj 79 | 80 | def files(pattern): 81 | results = _invoke('files', pattern) 82 | if len(results) == 1 and results[0].get('code') == 'error': 83 | if DEBUG: 84 | print '! %r' % obj 85 | results = [] 86 | 87 | for result in results: 88 | result['exists'] = result.get('action') not in DELETED_ACTIONS 89 | 90 | if DEBUG: 91 | print '> %r' % results 92 | 93 | return results 94 | 95 | def printfile(file): 96 | results = _invoke('print', file) 97 | meta = results[0] 98 | if meta.get('code') == 'error': 99 | if DEBUG: 100 | print '! %r' % meta 101 | print '> None' 102 | return None 103 | 104 | text = results[1]['data'] 105 | if DEBUG: 106 | print '> (... %i byte(s))' % len(text) 107 | return text 108 | -------------------------------------------------------------------------------- /root_bootstrap.py: -------------------------------------------------------------------------------- 1 | # This file is not directly used from Perforce, but it is the 2 | # script on the Perforce daemon that is responsible for updating & executing 3 | # the second-level bootstrap script, in bootstrap.py. 4 | 5 | import os 6 | import subprocess 7 | import sys 8 | 9 | SECOND_STAGE_WORKSPACE = 'perforce-admin-tools' 10 | SECOND_STAGE_RELATIVE_PATH = ("tools", "p4dispatch", "bootstrap.py") 11 | 12 | devnull = open(os.devnull, 'w') 13 | subprocess.check_call(["p4.exe", "-c%s" % SECOND_STAGE_WORKSPACE, "sync"], stdout=devnull, stderr=devnull) 14 | 15 | current_dir = os.path.dirname(__file__) 16 | second_stage = os.path.join(current_dir, *SECOND_STAGE_RELATIVE_PATH) 17 | if os.path.exists(second_stage): 18 | mod_globals = globals() 19 | mod_globals['__file__'] = second_stage 20 | execfile(second_stage, mod_globals) 21 | --------------------------------------------------------------------------------