├── manage_externals ├── test │ ├── doc │ │ ├── .gitignore │ │ ├── index.rst │ │ ├── Makefile │ │ ├── testing.rst │ │ ├── conf.py │ │ └── develop.rst │ ├── repos │ │ ├── container.git │ │ │ ├── HEAD │ │ │ ├── refs │ │ │ │ └── heads │ │ │ │ │ └── master │ │ │ ├── description │ │ │ ├── config │ │ │ ├── objects │ │ │ │ ├── 41 │ │ │ │ │ └── 1de5d96ee418c1c55f3e96e6e6e7c06bb95801 │ │ │ │ ├── 71 │ │ │ │ │ └── 5b8f3e4afe1802a178e1d603af404ba45d59de │ │ │ │ ├── b0 │ │ │ │ │ └── f87705e2b9601cb831878f3d51efa78b910d7b │ │ │ │ └── f9 │ │ │ │ │ └── e08370a737e941de6f6492e3f427c2ef4c1a03 │ │ │ └── info │ │ │ │ └── exclude │ │ ├── mixed-cont-ext.git │ │ │ ├── HEAD │ │ │ ├── refs │ │ │ │ └── heads │ │ │ │ │ ├── master │ │ │ │ │ └── new-feature │ │ │ ├── description │ │ │ ├── config │ │ │ ├── objects │ │ │ │ ├── 14 │ │ │ │ │ └── 368b701616a8c53820b610414a4b9a07540cf6 │ │ │ │ ├── 15 │ │ │ │ │ └── 2b57e1cf23721cd17ff681cb9276e3fb9fc091 │ │ │ │ ├── 37 │ │ │ │ │ └── f0e70b609adc90f4c09ee21d82ed1d79c81d69 │ │ │ │ ├── 38 │ │ │ │ │ └── 9a2b876b8965d3c91a3db8d28a483eaf019d5c │ │ │ │ ├── 41 │ │ │ │ │ └── 1de5d96ee418c1c55f3e96e6e6e7c06bb95801 │ │ │ │ ├── 93 │ │ │ │ │ └── a159deb9175bfeb2820a0006ddd92d78131332 │ │ │ │ ├── 95 │ │ │ │ │ └── 80ecc12f16334ce44e42287d5d46f927bb7b75 │ │ │ │ ├── 00 │ │ │ │ │ └── 437ac2000d5f06fb8a572a01a5bbdae98b17cb │ │ │ │ ├── 01 │ │ │ │ │ └── 97458f2dbe5fcd6bc44fa46983be0a30282379 │ │ │ │ ├── 06 │ │ │ │ │ └── ea30b03ffa2f8574705f8b9583f7ca7e2dccf7 │ │ │ │ ├── 1f │ │ │ │ │ └── 01fa46c17b1f38b37e6259f6e9d041bda3144f │ │ │ │ ├── 6e │ │ │ │ │ └── 9f4baa6e94a0af4e094836c2eb55ccedef5fc4 │ │ │ │ ├── 6f │ │ │ │ │ └── c379457ecb4e576a13c7610ae1fa73f845ee6a │ │ │ │ ├── a9 │ │ │ │ │ └── 288dcd8a719a1f4ed3cba43a2a387ae7cd60fd │ │ │ │ ├── e8 │ │ │ │ │ └── ea32a11d30ee703f6f661ae7c2376f4ab84d38 │ │ │ │ └── fd │ │ │ │ │ └── 15a5ad5204356229c60a831d2a8120a43ac901 │ │ │ └── info │ │ │ │ └── exclude │ │ ├── simple-ext.git │ │ │ ├── HEAD │ │ │ ├── refs │ │ │ │ ├── tags │ │ │ │ │ ├── tag1 │ │ │ │ │ └── tag2 │ │ │ │ └── heads │ │ │ │ │ ├── feature2 │ │ │ │ │ ├── feature3 │ │ │ │ │ └── master │ │ │ ├── description │ │ │ ├── config │ │ │ ├── objects │ │ │ │ ├── 11 │ │ │ │ │ └── a76e3d9a67313dec7ce1230852ab5c86352c5c │ │ │ │ ├── 14 │ │ │ │ │ └── 2711fdbbcb8034d7cad6bae6801887b12fe61d │ │ │ │ ├── 31 │ │ │ │ │ └── dbcd6de441e671a467ef317146539b7ffabb11 │ │ │ │ ├── 36 │ │ │ │ │ └── 418b4e5665956a90725c9a1b5a8e551c5f3d48 │ │ │ │ ├── 41 │ │ │ │ │ └── 1de5d96ee418c1c55f3e96e6e6e7c06bb95801 │ │ │ │ ├── 60 │ │ │ │ │ ├── 7ec299c17dd285c029edc41a0109e49d441380 │ │ │ │ │ └── b1cc1a38d63a4bcaa1e767262bbe23dbf9f5f5 │ │ │ │ ├── 63 │ │ │ │ │ └── a99393d1baff97ccef967af30380659867b139 │ │ │ │ ├── 95 │ │ │ │ │ └── 3256da5612fcd9263590a353bc18c6f224e74f │ │ │ │ ├── 00 │ │ │ │ │ └── fd13e76189f9134b0506b4b8ed3172723b467f │ │ │ │ ├── 09 │ │ │ │ │ └── 0e1034746b2c865f7b0280813dbf4061a700e8 │ │ │ │ ├── 0b │ │ │ │ │ └── 15e8af3d4615b42314216efeae3fff184046a8 │ │ │ │ ├── 9b │ │ │ │ │ └── 75494003deca69527bb64bcaa352e801611dd2 │ │ │ │ ├── a2 │ │ │ │ │ └── 2a5da9119328ea6d693f88861457c07e14ac04 │ │ │ │ ├── b7 │ │ │ │ │ └── 692b6d391899680da7b9b6fd8af4c413f06fe7 │ │ │ │ ├── c5 │ │ │ │ │ └── b315915742133dbdfbeed0753e481b55c1d364 │ │ │ │ ├── d1 │ │ │ │ │ └── 163870d19c3dee34fada3a76b785cfa2a8424b │ │ │ │ ├── d8 │ │ │ │ │ └── ed2f33179d751937f8fde2e33921e4827babf4 │ │ │ │ └── df │ │ │ │ │ └── 312890f93ba4d2c694208599b665c4a08afeff │ │ │ └── info │ │ │ │ └── exclude │ │ ├── simple-ext-fork.git │ │ │ ├── HEAD │ │ │ ├── refs │ │ │ │ ├── heads │ │ │ │ │ └── feature2 │ │ │ │ └── tags │ │ │ │ │ ├── abandoned-feature │ │ │ │ │ └── forked-feature-v1 │ │ │ ├── description │ │ │ ├── objects │ │ │ │ ├── 11 │ │ │ │ │ └── a76e3d9a67313dec7ce1230852ab5c86352c5c │ │ │ │ ├── 16 │ │ │ │ │ └── 5506a7408a482f50493434e13fffeb44af893f │ │ │ │ ├── 24 │ │ │ │ │ └── 4386e788c9bc608613e127a329c742450a60e4 │ │ │ │ ├── 32 │ │ │ │ │ └── 7e97d86e941047d809dba58f2804740c6c30cf │ │ │ │ ├── 36 │ │ │ │ │ └── 418b4e5665956a90725c9a1b5a8e551c5f3d48 │ │ │ │ ├── 41 │ │ │ │ │ └── 1de5d96ee418c1c55f3e96e6e6e7c06bb95801 │ │ │ │ ├── 56 │ │ │ │ │ └── 175e017ad38bf3d33d74b6bd7c74624b28466a │ │ │ │ ├── 67 │ │ │ │ │ └── 136e5ab4d5c1c65d10c8048763b96b0e53c1d6 │ │ │ │ ├── 88 │ │ │ │ │ └── cf20868e0cc445f5642a480ed034c71e0d7e9f │ │ │ │ ├── 00 │ │ │ │ │ └── fd13e76189f9134b0506b4b8ed3172723b467f │ │ │ │ ├── 0b │ │ │ │ │ ├── 15e8af3d4615b42314216efeae3fff184046a8 │ │ │ │ │ └── 67df4e7e8e6e1c6e401542738b352d18744677 │ │ │ │ ├── 3d │ │ │ │ │ ├── 7099c35404ae6c8640ce263b38bef06e98cc26 │ │ │ │ │ └── ec1fdf8e2f5edba28148c5db2fe8d7a842360b │ │ │ │ ├── 4d │ │ │ │ │ └── 837135915ed93eed6fff6b439f284ce317296f │ │ │ │ ├── 5f │ │ │ │ │ └── 1d4786d12e52d7ab28d2f2f1118c1059a9f1ae │ │ │ │ ├── 7b │ │ │ │ │ └── 0bd630ac13865735a1dff3437a137d8ab50663 │ │ │ │ ├── 8d │ │ │ │ │ └── 2b3b35126224c975d23f109aa1e3cbac452989 │ │ │ │ ├── 9b │ │ │ │ │ └── 75494003deca69527bb64bcaa352e801611dd2 │ │ │ │ ├── a2 │ │ │ │ │ └── 2a5da9119328ea6d693f88861457c07e14ac04 │ │ │ │ ├── a4 │ │ │ │ │ └── 2fe9144f5707bc1e9515ce1b44681f7aba6f95 │ │ │ │ ├── b9 │ │ │ │ │ └── 3737be3ea6b19f6255983748a0a0f4d622f936 │ │ │ │ ├── c5 │ │ │ │ │ ├── 32bc8fde96fa63103a52057f0baffcc9f00c6b │ │ │ │ │ └── b315915742133dbdfbeed0753e481b55c1d364 │ │ │ │ └── f2 │ │ │ │ │ └── 68d4e56d067da9bd1d85e55bdc40a8bd2b0bca │ │ │ ├── info │ │ │ │ └── exclude │ │ │ ├── packed-refs │ │ │ └── config │ │ └── error │ │ │ └── readme.txt │ ├── requirements.txt │ ├── .gitignore │ ├── .coveragerc │ ├── README.md │ ├── Makefile │ ├── test_unit_repository.py │ ├── test_sys_repository_git.py │ ├── test_unit_externals_status.py │ ├── test_unit_utils.py │ └── .pylint.rc ├── manic │ ├── utils.pyc │ ├── __init__.pyc │ ├── checkout.pyc │ ├── repository.pyc │ ├── sourcetree.pyc │ ├── repository_git.pyc │ ├── repository_svn.pyc │ ├── externals_status.pyc │ ├── global_constants.pyc │ ├── repository_factory.pyc │ ├── externals_description.pyc │ ├── __init__.py │ ├── global_constants.py │ ├── repository_factory.py │ ├── repository.py │ ├── externals_status.py │ ├── repository_svn.py │ ├── utils.py │ └── sourcetree.py ├── checkout_externals ├── README_FIRST ├── LICENSE.txt └── README.md ├── README.md ├── Externals.cfg ├── .gitignore └── scripts ├── regridding ├── mergeTile.py ├── merge_topography.ncl └── regrid.ncl ├── postproc ├── postproc_mask.py └── postproc_mask_test.py ├── submit_topo_regen_script.sh ├── misc ├── plotTopo30sec.py └── extract_cubed_sphere_face.py ├── setup.sh └── CAM_topo_regen.sh /manage_externals/test/doc/.gitignore: -------------------------------------------------------------------------------- 1 | _build 2 | 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SES 2 | Solid Earth and Sea-level component 3 | -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/HEAD: -------------------------------------------------------------------------------- 1 | ref: refs/heads/master 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/HEAD: -------------------------------------------------------------------------------- 1 | ref: refs/heads/master 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/HEAD: -------------------------------------------------------------------------------- 1 | ref: refs/heads/master 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/HEAD: -------------------------------------------------------------------------------- 1 | ref: refs/heads/master 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/refs/tags/tag1: -------------------------------------------------------------------------------- 1 | 11a76e3d9a67313dec7ce1230852ab5c86352c5c 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/refs/tags/tag2: -------------------------------------------------------------------------------- 1 | b7692b6d391899680da7b9b6fd8af4c413f06fe7 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/refs/heads/master: -------------------------------------------------------------------------------- 1 | 715b8f3e4afe1802a178e1d603af404ba45d59de 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/refs/heads/feature2: -------------------------------------------------------------------------------- 1 | 36418b4e5665956a90725c9a1b5a8e551c5f3d48 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/refs/heads/feature3: -------------------------------------------------------------------------------- 1 | 090e1034746b2c865f7b0280813dbf4061a700e8 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/refs/heads/master: -------------------------------------------------------------------------------- 1 | 607ec299c17dd285c029edc41a0109e49d441380 2 | -------------------------------------------------------------------------------- /manage_externals/manic/utils.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/manic/utils.pyc -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/refs/heads/master: -------------------------------------------------------------------------------- 1 | 6fc379457ecb4e576a13c7610ae1fa73f845ee6a 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/refs/heads/feature2: -------------------------------------------------------------------------------- 1 | f268d4e56d067da9bd1d85e55bdc40a8bd2b0bca 2 | -------------------------------------------------------------------------------- /manage_externals/manic/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/manic/__init__.pyc -------------------------------------------------------------------------------- /manage_externals/manic/checkout.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/manic/checkout.pyc -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/refs/heads/new-feature: -------------------------------------------------------------------------------- 1 | 9580ecc12f16334ce44e42287d5d46f927bb7b75 2 | -------------------------------------------------------------------------------- /manage_externals/manic/repository.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/manic/repository.pyc -------------------------------------------------------------------------------- /manage_externals/manic/sourcetree.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/manic/sourcetree.pyc -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/refs/tags/abandoned-feature: -------------------------------------------------------------------------------- 1 | a42fe9144f5707bc1e9515ce1b44681f7aba6f95 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/refs/tags/forked-feature-v1: -------------------------------------------------------------------------------- 1 | 8d2b3b35126224c975d23f109aa1e3cbac452989 2 | -------------------------------------------------------------------------------- /manage_externals/manic/repository_git.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/manic/repository_git.pyc -------------------------------------------------------------------------------- /manage_externals/manic/repository_svn.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/manic/repository_svn.pyc -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/description: -------------------------------------------------------------------------------- 1 | Unnamed repository; edit this file 'description' to name the repository. 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/description: -------------------------------------------------------------------------------- 1 | Unnamed repository; edit this file 'description' to name the repository. 2 | -------------------------------------------------------------------------------- /manage_externals/manic/externals_status.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/manic/externals_status.pyc -------------------------------------------------------------------------------- /manage_externals/manic/global_constants.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/manic/global_constants.pyc -------------------------------------------------------------------------------- /manage_externals/manic/repository_factory.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/manic/repository_factory.pyc -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/description: -------------------------------------------------------------------------------- 1 | Unnamed repository; edit this file 'description' to name the repository. 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/description: -------------------------------------------------------------------------------- 1 | Unnamed repository; edit this file 'description' to name the repository. 2 | -------------------------------------------------------------------------------- /manage_externals/test/requirements.txt: -------------------------------------------------------------------------------- 1 | pylint>=1.7.0 2 | autopep8>=1.3.0 3 | coverage>=4.4.0 4 | coveralls>=1.2.0 5 | sphinx>=1.6.0 6 | -------------------------------------------------------------------------------- /manage_externals/manic/externals_description.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/manic/externals_description.pyc -------------------------------------------------------------------------------- /manage_externals/test/repos/error/readme.txt: -------------------------------------------------------------------------------- 1 | Invalid or corrupted git repository (.git dir exists, but is empty) for error 2 | testing. 3 | 4 | -------------------------------------------------------------------------------- /manage_externals/test/.gitignore: -------------------------------------------------------------------------------- 1 | # virtual environments 2 | env_python* 3 | 4 | # python code coverage tool output 5 | .coverage 6 | htmlcov 7 | 8 | -------------------------------------------------------------------------------- /manage_externals/test/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | omit = test_unit_*.py 4 | test_sys_*.py 5 | /usr/* 6 | .local/* 7 | */site-packages/* -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/config: -------------------------------------------------------------------------------- 1 | [core] 2 | repositoryformatversion = 0 3 | filemode = true 4 | bare = true 5 | ignorecase = true 6 | precomposeunicode = true 7 | -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/config: -------------------------------------------------------------------------------- 1 | [core] 2 | repositoryformatversion = 0 3 | filemode = true 4 | bare = true 5 | ignorecase = true 6 | precomposeunicode = true 7 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/config: -------------------------------------------------------------------------------- 1 | [core] 2 | repositoryformatversion = 0 3 | filemode = true 4 | bare = true 5 | ignorecase = true 6 | precomposeunicode = true 7 | -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/container.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801 -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/objects/71/5b8f3e4afe1802a178e1d603af404ba45d59de: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/container.git/objects/71/5b8f3e4afe1802a178e1d603af404ba45d59de -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/objects/b0/f87705e2b9601cb831878f3d51efa78b910d7b: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/container.git/objects/b0/f87705e2b9601cb831878f3d51efa78b910d7b -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/objects/f9/e08370a737e941de6f6492e3f427c2ef4c1a03: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/container.git/objects/f9/e08370a737e941de6f6492e3f427c2ef4c1a03 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/00/fd13e76189f9134b0506b4b8ed3172723b467f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/00/fd13e76189f9134b0506b4b8ed3172723b467f -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/09/0e1034746b2c865f7b0280813dbf4061a700e8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/09/0e1034746b2c865f7b0280813dbf4061a700e8 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/0b/15e8af3d4615b42314216efeae3fff184046a8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/0b/15e8af3d4615b42314216efeae3fff184046a8 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/11/a76e3d9a67313dec7ce1230852ab5c86352c5c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/11/a76e3d9a67313dec7ce1230852ab5c86352c5c -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/14/2711fdbbcb8034d7cad6bae6801887b12fe61d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/14/2711fdbbcb8034d7cad6bae6801887b12fe61d -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/31/dbcd6de441e671a467ef317146539b7ffabb11: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/31/dbcd6de441e671a467ef317146539b7ffabb11 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/36/418b4e5665956a90725c9a1b5a8e551c5f3d48: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/36/418b4e5665956a90725c9a1b5a8e551c5f3d48 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/60/7ec299c17dd285c029edc41a0109e49d441380: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/60/7ec299c17dd285c029edc41a0109e49d441380 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/60/b1cc1a38d63a4bcaa1e767262bbe23dbf9f5f5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/60/b1cc1a38d63a4bcaa1e767262bbe23dbf9f5f5 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/63/a99393d1baff97ccef967af30380659867b139: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/63/a99393d1baff97ccef967af30380659867b139 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/95/3256da5612fcd9263590a353bc18c6f224e74f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/95/3256da5612fcd9263590a353bc18c6f224e74f -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/9b/75494003deca69527bb64bcaa352e801611dd2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/9b/75494003deca69527bb64bcaa352e801611dd2 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/a2/2a5da9119328ea6d693f88861457c07e14ac04: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/a2/2a5da9119328ea6d693f88861457c07e14ac04 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/b7/692b6d391899680da7b9b6fd8af4c413f06fe7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/b7/692b6d391899680da7b9b6fd8af4c413f06fe7 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/c5/b315915742133dbdfbeed0753e481b55c1d364: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/c5/b315915742133dbdfbeed0753e481b55c1d364 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/d1/163870d19c3dee34fada3a76b785cfa2a8424b: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/d1/163870d19c3dee34fada3a76b785cfa2a8424b -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/d8/ed2f33179d751937f8fde2e33921e4827babf4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/d8/ed2f33179d751937f8fde2e33921e4827babf4 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/df/312890f93ba4d2c694208599b665c4a08afeff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext.git/objects/df/312890f93ba4d2c694208599b665c4a08afeff -------------------------------------------------------------------------------- /Externals.cfg: -------------------------------------------------------------------------------- 1 | [topo] 2 | local_path = topo 3 | protocol = git 4 | repo_url = https://github.com/NCAR/Topo/ 5 | hash = f75788d8467f350fca9239a 6 | required = True 7 | 8 | [externals_description] 9 | schema_version = 1.0.0 10 | -------------------------------------------------------------------------------- /manage_externals/manic/__init__.py: -------------------------------------------------------------------------------- 1 | """Public API for the manage_externals library 2 | """ 3 | 4 | from manic import checkout 5 | from manic.utils import printlog 6 | 7 | __all__ = [ 8 | 'checkout', 'printlog', 9 | ] 10 | -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/00/437ac2000d5f06fb8a572a01a5bbdae98b17cb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/00/437ac2000d5f06fb8a572a01a5bbdae98b17cb -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/01/97458f2dbe5fcd6bc44fa46983be0a30282379: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/01/97458f2dbe5fcd6bc44fa46983be0a30282379 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/06/ea30b03ffa2f8574705f8b9583f7ca7e2dccf7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/06/ea30b03ffa2f8574705f8b9583f7ca7e2dccf7 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/14/368b701616a8c53820b610414a4b9a07540cf6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/14/368b701616a8c53820b610414a4b9a07540cf6 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/15/2b57e1cf23721cd17ff681cb9276e3fb9fc091: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/15/2b57e1cf23721cd17ff681cb9276e3fb9fc091 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/1f/01fa46c17b1f38b37e6259f6e9d041bda3144f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/1f/01fa46c17b1f38b37e6259f6e9d041bda3144f -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/37/f0e70b609adc90f4c09ee21d82ed1d79c81d69: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/37/f0e70b609adc90f4c09ee21d82ed1d79c81d69 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/38/9a2b876b8965d3c91a3db8d28a483eaf019d5c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/38/9a2b876b8965d3c91a3db8d28a483eaf019d5c -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/6e/9f4baa6e94a0af4e094836c2eb55ccedef5fc4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/6e/9f4baa6e94a0af4e094836c2eb55ccedef5fc4 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/6f/c379457ecb4e576a13c7610ae1fa73f845ee6a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/6f/c379457ecb4e576a13c7610ae1fa73f845ee6a -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/93/a159deb9175bfeb2820a0006ddd92d78131332: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/93/a159deb9175bfeb2820a0006ddd92d78131332 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/95/80ecc12f16334ce44e42287d5d46f927bb7b75: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/95/80ecc12f16334ce44e42287d5d46f927bb7b75 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/a9/288dcd8a719a1f4ed3cba43a2a387ae7cd60fd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/a9/288dcd8a719a1f4ed3cba43a2a387ae7cd60fd -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/e8/ea32a11d30ee703f6f661ae7c2376f4ab84d38: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/e8/ea32a11d30ee703f6f661ae7c2376f4ab84d38 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/fd/15a5ad5204356229c60a831d2a8120a43ac901: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/mixed-cont-ext.git/objects/fd/15a5ad5204356229c60a831d2a8120a43ac901 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/00/fd13e76189f9134b0506b4b8ed3172723b467f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/00/fd13e76189f9134b0506b4b8ed3172723b467f -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/0b/15e8af3d4615b42314216efeae3fff184046a8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/0b/15e8af3d4615b42314216efeae3fff184046a8 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/0b/67df4e7e8e6e1c6e401542738b352d18744677: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/0b/67df4e7e8e6e1c6e401542738b352d18744677 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/11/a76e3d9a67313dec7ce1230852ab5c86352c5c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/11/a76e3d9a67313dec7ce1230852ab5c86352c5c -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/16/5506a7408a482f50493434e13fffeb44af893f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/16/5506a7408a482f50493434e13fffeb44af893f -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/24/4386e788c9bc608613e127a329c742450a60e4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/24/4386e788c9bc608613e127a329c742450a60e4 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/32/7e97d86e941047d809dba58f2804740c6c30cf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/32/7e97d86e941047d809dba58f2804740c6c30cf -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/36/418b4e5665956a90725c9a1b5a8e551c5f3d48: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/36/418b4e5665956a90725c9a1b5a8e551c5f3d48 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/3d/7099c35404ae6c8640ce263b38bef06e98cc26: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/3d/7099c35404ae6c8640ce263b38bef06e98cc26 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/3d/ec1fdf8e2f5edba28148c5db2fe8d7a842360b: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/3d/ec1fdf8e2f5edba28148c5db2fe8d7a842360b -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/4d/837135915ed93eed6fff6b439f284ce317296f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/4d/837135915ed93eed6fff6b439f284ce317296f -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/56/175e017ad38bf3d33d74b6bd7c74624b28466a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/56/175e017ad38bf3d33d74b6bd7c74624b28466a -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/5f/1d4786d12e52d7ab28d2f2f1118c1059a9f1ae: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/5f/1d4786d12e52d7ab28d2f2f1118c1059a9f1ae -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/67/136e5ab4d5c1c65d10c8048763b96b0e53c1d6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/67/136e5ab4d5c1c65d10c8048763b96b0e53c1d6 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/7b/0bd630ac13865735a1dff3437a137d8ab50663: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/7b/0bd630ac13865735a1dff3437a137d8ab50663 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/88/cf20868e0cc445f5642a480ed034c71e0d7e9f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/88/cf20868e0cc445f5642a480ed034c71e0d7e9f -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/8d/2b3b35126224c975d23f109aa1e3cbac452989: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/8d/2b3b35126224c975d23f109aa1e3cbac452989 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/9b/75494003deca69527bb64bcaa352e801611dd2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/9b/75494003deca69527bb64bcaa352e801611dd2 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/a2/2a5da9119328ea6d693f88861457c07e14ac04: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/a2/2a5da9119328ea6d693f88861457c07e14ac04 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/a4/2fe9144f5707bc1e9515ce1b44681f7aba6f95: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/a4/2fe9144f5707bc1e9515ce1b44681f7aba6f95 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/b9/3737be3ea6b19f6255983748a0a0f4d622f936: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/b9/3737be3ea6b19f6255983748a0a0f4d622f936 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/c5/32bc8fde96fa63103a52057f0baffcc9f00c6b: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/c5/32bc8fde96fa63103a52057f0baffcc9f00c6b -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/c5/b315915742133dbdfbeed0753e481b55c1d364: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/c5/b315915742133dbdfbeed0753e481b55c1d364 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/f2/68d4e56d067da9bd1d85e55bdc40a8bd2b0bca: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/SES/master/manage_externals/test/repos/simple-ext-fork.git/objects/f2/68d4e56d067da9bd1d85e55bdc40a8bd2b0bca -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/info/exclude: -------------------------------------------------------------------------------- 1 | # git ls-files --others --exclude-from=.git/info/exclude 2 | # Lines that start with '#' are comments. 3 | # For a project mostly in C, the following would be a good set of 4 | # exclude patterns (uncomment them if you want to use them): 5 | # *.[oa] 6 | # *~ 7 | -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/info/exclude: -------------------------------------------------------------------------------- 1 | # git ls-files --others --exclude-from=.git/info/exclude 2 | # Lines that start with '#' are comments. 3 | # For a project mostly in C, the following would be a good set of 4 | # exclude patterns (uncomment them if you want to use them): 5 | # *.[oa] 6 | # *~ 7 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/info/exclude: -------------------------------------------------------------------------------- 1 | # git ls-files --others --exclude-from=.git/info/exclude 2 | # Lines that start with '#' are comments. 3 | # For a project mostly in C, the following would be a good set of 4 | # exclude patterns (uncomment them if you want to use them): 5 | # *.[oa] 6 | # *~ 7 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/info/exclude: -------------------------------------------------------------------------------- 1 | # git ls-files --others --exclude-from=.git/info/exclude 2 | # Lines that start with '#' are comments. 3 | # For a project mostly in C, the following would be a good set of 4 | # exclude patterns (uncomment them if you want to use them): 5 | # *.[oa] 6 | # *~ 7 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/packed-refs: -------------------------------------------------------------------------------- 1 | # pack-refs with: peeled fully-peeled sorted 2 | 36418b4e5665956a90725c9a1b5a8e551c5f3d48 refs/heads/feature2 3 | 9b75494003deca69527bb64bcaa352e801611dd2 refs/heads/master 4 | 11a76e3d9a67313dec7ce1230852ab5c86352c5c refs/tags/tag1 5 | ^9b75494003deca69527bb64bcaa352e801611dd2 6 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/config: -------------------------------------------------------------------------------- 1 | [core] 2 | repositoryformatversion = 0 3 | filemode = true 4 | bare = true 5 | ignorecase = true 6 | precomposeunicode = true 7 | [remote "origin"] 8 | url = /Users/andreb/projects/ncar/git-conversion/checkout-model-dev/cesm-demo-externals/manage_externals/test/repos/simple-ext.git 9 | -------------------------------------------------------------------------------- /manage_externals/manic/global_constants.py: -------------------------------------------------------------------------------- 1 | """Globals shared across modules 2 | """ 3 | 4 | from __future__ import absolute_import 5 | from __future__ import unicode_literals 6 | from __future__ import print_function 7 | 8 | import pprint 9 | 10 | EMPTY_STR = '' 11 | LOCAL_PATH_INDICATOR = '.' 12 | VERSION_SEPERATOR = '.' 13 | LOG_FILE_NAME = 'manage_externals.log' 14 | PPRINTER = pprint.PrettyPrinter(indent=4) 15 | 16 | VERBOSITY_DEFAULT = 0 17 | VERBOSITY_VERBOSE = 1 18 | VERBOSITY_DUMP = 2 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # directories checked out by manage_externals, and other files created 2 | # by manage_externals 3 | manage_externals.log 4 | topo 5 | 6 | # ignore svn directories 7 | **/.svn/** 8 | .svn/ 9 | 10 | # binary files 11 | *.nc 12 | 13 | # editor files 14 | *.swp 15 | *~ 16 | 17 | # build-namelist files 18 | 19 | # mac files 20 | .DS_Store 21 | 22 | # cmake generated files 23 | build/ 24 | CMakeFiles/ 25 | 26 | # build output 27 | *.o 28 | *.mod 29 | core.* 30 | *.gz 31 | *.log !run.log 32 | 33 | -------------------------------------------------------------------------------- /scripts/regridding/mergeTile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | from netCDF4 import * 5 | 6 | ########################### 7 | 8 | full = Dataset(sys.argv[1],'a') 9 | tile = Dataset(sys.argv[2],'r') 10 | 11 | for Field in ['htopo','landfract']: 12 | print('\nInserting modified %s in high resolution dataset'%Field) 13 | full.variables[Field][16800:,32400:] = tile.variables[Field][:,:] 14 | 15 | full.close() 16 | tile.close() 17 | 18 | print('\n-> High resolution dataset updated successfully\n') 19 | 20 | ## === end of script === ## 21 | -------------------------------------------------------------------------------- /manage_externals/test/doc/index.rst: -------------------------------------------------------------------------------- 1 | .. Manage Externals documentation master file, created by 2 | sphinx-quickstart on Wed Nov 29 10:53:25 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to Manage Externals's documentation! 7 | ============================================ 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | 14 | develop.rst 15 | testing.rst 16 | 17 | Indices and tables 18 | ================== 19 | 20 | * :ref:`genindex` 21 | * :ref:`modindex` 22 | * :ref:`search` 23 | -------------------------------------------------------------------------------- /manage_externals/test/doc/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = ManageExternals 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /manage_externals/checkout_externals: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Main driver wrapper around the manic/checkout utility. 4 | 5 | Tool to assemble external respositories represented in an externals 6 | description file. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | from __future__ import unicode_literals 11 | from __future__ import print_function 12 | 13 | import sys 14 | import traceback 15 | 16 | import manic 17 | 18 | if sys.hexversion < 0x02070000: 19 | print(70 * '*') 20 | print('ERROR: {0} requires python >= 2.7.x. '.format(sys.argv[0])) 21 | print('It appears that you are running python {0}'.format( 22 | '.'.join(str(x) for x in sys.version_info[0:3]))) 23 | print(70 * '*') 24 | sys.exit(1) 25 | 26 | 27 | if __name__ == '__main__': 28 | ARGS = manic.checkout.commandline_arguments() 29 | try: 30 | RET_STATUS, _ = manic.checkout.main(ARGS) 31 | sys.exit(RET_STATUS) 32 | except Exception as error: # pylint: disable=broad-except 33 | manic.printlog(str(error)) 34 | if ARGS.backtrace: 35 | traceback.print_exc() 36 | sys.exit(1) 37 | -------------------------------------------------------------------------------- /manage_externals/manic/repository_factory.py: -------------------------------------------------------------------------------- 1 | """Factory for creating and initializing the appropriate repository class 2 | """ 3 | 4 | from __future__ import absolute_import 5 | from __future__ import unicode_literals 6 | from __future__ import print_function 7 | 8 | from .repository_git import GitRepository 9 | from .repository_svn import SvnRepository 10 | from .externals_description import ExternalsDescription 11 | from .utils import fatal_error 12 | 13 | 14 | def create_repository(component_name, repo_info, svn_ignore_ancestry=False): 15 | """Determine what type of repository we have, i.e. git or svn, and 16 | create the appropriate object. 17 | 18 | """ 19 | protocol = repo_info[ExternalsDescription.PROTOCOL].lower() 20 | if protocol == 'git': 21 | repo = GitRepository(component_name, repo_info) 22 | elif protocol == 'svn': 23 | repo = SvnRepository(component_name, repo_info, ignore_ancestry=svn_ignore_ancestry) 24 | elif protocol == 'externals_only': 25 | repo = None 26 | else: 27 | msg = 'Unknown repo protocol "{0}"'.format(protocol) 28 | fatal_error(msg) 29 | return repo 30 | -------------------------------------------------------------------------------- /scripts/regridding/merge_topography.ncl: -------------------------------------------------------------------------------- 1 | ;Jeremy Fyke 2 | ;03-07-13 3 | 4 | load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" 5 | load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" 6 | load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" 7 | 8 | begin 9 | ;####Open i/o topography files(file names sent by command line arguments) 10 | GIS_topog = addfile(input_file_name,"r") 11 | usgs_topog = addfile(output_file_name,"w") 12 | 13 | ;####Load Greenland tiles of raw USGS topography and land fraction 14 | usgs_topog_tile = usgs_topog->htopo(16800:,32400:) 15 | usgs_landfract_tile = usgs_topog->landfract(16800:,32400:) 16 | 17 | ;#equivalent tile of ISM-derived topography and land fraction 18 | 19 | GIS_topo=doubletoint(GIS_topog->usrf(::-1,:)) 20 | GIS_mask=floattoint(GIS_topog->landmask_float(::-1,:)) 21 | 22 | ;####Merge topography and landmask from ISM-derived values, over island of Greenland 23 | ;ismissing(GIS_mask) returns 'true' outside of GIS regions 24 | ;subsequent ingestion by 'where' sets the landfrac and topography values to the (original) 25 | ;values. Where ismissing(GIS_mask) is false (i.e. the ISM-derived data exists), landfrac 26 | ;and topographies are set to the ISM values. 27 | 28 | usgs_landfract_tile= where( ismissing(GIS_mask), usgs_landfract_tile, GIS_mask ) 29 | usgs_topog_tile= where( ismissing(GIS_mask), usgs_topog_tile, GIS_topo ) 30 | 31 | ;####Insert altered topography and landmask back into full USGS dataset. 32 | usgs_topog->htopo(16800:,32400:)=usgs_topog_tile 33 | usgs_topog->landfract(16800:,32400:)=usgs_landfract_tile 34 | 35 | end 36 | -------------------------------------------------------------------------------- /scripts/postproc/postproc_mask.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | from numpy import * 5 | from netCDF4 import * 6 | 7 | ########################### 8 | # 9 | # Merge fields from cube_to_target 10 | # into one dataset 11 | # 12 | # Author: M. Lofverstrom 13 | # NCAR, Dec 18 2017 14 | # 15 | ########################### 16 | 17 | #outFile = Dataset(sys.argv[1],'a') 18 | camRest = Dataset(sys.argv[1],'r+') 19 | topoDataIn = Dataset(sys.argv[2],'r') 20 | topoDataOut = Dataset(sys.argv[2],'r+') 21 | c2t060 = Dataset(sys.argv[4],'r') 22 | c2t008 = Dataset(sys.argv[5],'r') 23 | 24 | gland_mask = Dataset(sys.argv[3],'r').variables['greenland_mask'][:] 25 | 26 | #### 27 | 28 | c2t060_vars_exclude = ['lat','lon','LANDM_COSLAT','SGH'] 29 | c2t060_vars_camRest = ['PHIS','SGH30'] 30 | c2t008_vars = ['SGH'] 31 | 32 | 33 | phis_060 = c2t060.variables['PHIS'][:] 34 | sgh_060 = c2t060.variables['SGH'][:] 35 | sgh30_008 = c2t008.variables['SGH30'][:] 36 | 37 | 38 | Field = 'SGH30' 39 | topoDataOut.variables[Field][:] = where(gland_mask==1., 40 | sqrt(sgh_060**2+sgh30_008**2), 41 | topoDataIn.variables[Field][:]) 42 | 43 | camRest.variables[Field][:] = where(gland_mask==1., 44 | sqrt(sgh_060**2+sgh30_008**2), 45 | topoDataIn.variables[Field][:]) 46 | 47 | 48 | #### 49 | 50 | Field = 'PHIS' 51 | camRest.variables[Field][:] = where(gland_mask==1., 52 | phis_060, 53 | topoDataIn.variables[Field][:]) 54 | 55 | ### 56 | 57 | camRest.close() 58 | topoDataOut.close() 59 | 60 | 61 | ########################### 62 | 63 | if __name__ == "__main": 64 | pass 65 | 66 | ########################### 67 | ## === end of script === ## 68 | ########################### 69 | -------------------------------------------------------------------------------- /scripts/submit_topo_regen_script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # This script has two main purposes: 4 | # (1) Submit topography regeneration as a separate 5 | # process running in parallel with CESM submission 6 | # 7 | # (2) Merge updated topography with CAM restart 8 | # file if possible 9 | # 10 | # Author: 11 | # M. Lofverstrom 12 | # NCAR, April 2017 13 | # 14 | ########################### 15 | # 16 | # Tailored for CISL Cheyenne 17 | # 18 | ########################### 19 | 20 | module load nco 21 | 22 | if [ $SHELL == /bin/bash ]; then 23 | source /glade/u/apps/ch/opt/Lmod/7.3.14/lmod/7.3.14/init/bash 24 | elif [ $SHELL == /bin/tcsh ]; then 25 | source /glade/u/apps/ch/opt/Lmod/7.3.14/lmod/7.3.14/init/tcsh 26 | fi 27 | 28 | ## Default settings (may be changed from setup.sh) 29 | Project=P93300301 30 | Walltime=00:45:00 31 | Queue=regular 32 | 33 | ScratchRun=/glade/scratch/cmip6/b.e21.B1850G.f09_g17_gl4.CMIP6-ssp585-withism.001/run 34 | ScriptDir=$ScratchRun/dynamic_atm_topog/ 35 | 36 | ### ### ### ### ### ### ### ### 37 | 38 | CAM_Restart_File=`ls -t $ScratchRun/*.cam.r.* | head -n 1` 39 | CISM_Restart_File=`ls -t $ScratchRun/*.cism.r.* | head -n 1` 40 | #Temporary_output_file=$ScratchRun/Temporary_output_file.nc 41 | 42 | ## Update CAM restart file 43 | #if [ -f $Temporary_output_file ]; then 44 | # echo "Temporary file exists!" 45 | # echo "Updating CAM restart file" 46 | # ncks -A -v SGH,SGH30,PHIS $Temporary_output_file $CAM_Restart_File 47 | #else 48 | # echo "Temporary topography file not found!" 49 | # echo "Will not try to update the CAM restart file" 50 | #fi 51 | 52 | ## Submit topography regeneration script to Cheyenne queue 53 | if [ -f $CISM_Restart_File ]; then 54 | echo "Submitting topography regeneration script" 55 | qsub -l select=1:ncpus=1 -l walltime=$Walltime -q $Queue -A $Project $ScriptDir/CAM_topo_regen.sh 56 | 57 | fi 58 | 59 | ## === end of script === ## 60 | -------------------------------------------------------------------------------- /manage_externals/README_FIRST: -------------------------------------------------------------------------------- 1 | CESM is comprised of a number of different components that are 2 | developed and managed independently. Each component may have 3 | additional 'external' dependancies and optional parts that are also 4 | developed and managed independently. 5 | 6 | The checkout_externals.py tool manages retreiving and updating the 7 | components and their externals so you have a complete set of source 8 | files for the model. 9 | 10 | checkout_externals.py relies on a model description file that 11 | describes what components are needed, where to find them and where to 12 | put them in the source tree. The default file is called "CESM.xml" 13 | regardless of whether you are checking out CESM or a standalone 14 | component. 15 | 16 | checkout_externals requires access to git and svn repositories that 17 | require authentication. checkout_externals may pass through 18 | authentication requests, but it will not cache them for you. For the 19 | best and most robust user experience, you should have svn and git 20 | working without password authentication. See: 21 | 22 | https://help.github.com/articles/connecting-to-github-with-ssh/ 23 | 24 | ?svn ref? 25 | 26 | NOTE: checkout_externals.py *MUST* be run from the root of the source 27 | tree it is managing. For example, if you cloned CLM with: 28 | 29 | $ git clone git@github.com/ncar/clm clm-dev 30 | 31 | Then the root of the source tree is /path/to/cesm-dev. If you obtained 32 | CLM via an svn checkout of CESM and you need to checkout the CLM 33 | externals, then the root of the source tree for CLM is: 34 | 35 | /path/to/cesm-dev/components/clm 36 | 37 | The root of the source tree will be referred to as ${SRC_ROOT} below. 38 | 39 | To get started quickly, checkout all required components from the 40 | default model description file: 41 | 42 | $ cd ${SRC_ROOT} 43 | $ ./checkout_cesm/checkout_externals.py 44 | 45 | For additional information about using checkout model, please see: 46 | 47 | ${SRC_ROOT}/checkout_cesm/README 48 | 49 | or run: 50 | 51 | $ cd ${SRC_ROOT} 52 | $ ./checkout_cesm/checkout_externals.py --help 53 | 54 | 55 | -------------------------------------------------------------------------------- /manage_externals/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017-2018, University Corporation for Atmospheric Research (UCAR) 2 | All rights reserved. 3 | 4 | Developed by: 5 | University Corporation for Atmospheric Research - National Center for Atmospheric Research 6 | https://www2.cesm.ucar.edu/working-groups/sewg 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining 9 | a copy of this software and associated documentation files (the "Software"), 10 | to deal with the Software without restriction, including without limitation 11 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 12 | and/or sell copies of the Software, and to permit persons to whom 13 | the Software is furnished to do so, subject to the following conditions: 14 | 15 | - Redistributions of source code must retain the above copyright notice, 16 | this list of conditions and the following disclaimers. 17 | - Redistributions in binary form must reproduce the above copyright notice, 18 | this list of conditions and the following disclaimers in the documentation 19 | and/or other materials provided with the distribution. 20 | - Neither the names of [Name of Development Group, UCAR], 21 | nor the names of its contributors may be used to endorse or promote 22 | products derived from this Software without specific prior written permission. 23 | 24 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 25 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 28 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 | POSSIBILITY OF SUCH DAMAGE. 35 | -------------------------------------------------------------------------------- /scripts/postproc/postproc_mask_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | from numpy import * 5 | from netCDF4 import * 6 | 7 | ########################### 8 | # 9 | # Merge fields from cube_to_target 10 | # into one dataset 11 | # 12 | # Author: M. Lofverstrom 13 | # NCAR, Dec 18 2017 14 | # 15 | ########################### 16 | 17 | #outFile = Dataset(sys.argv[1],'a') 18 | camRest = Dataset(sys.argv[1],'r+') 19 | topoDataIn = Dataset(sys.argv[2],'r') 20 | topoDataOut = Dataset(sys.argv[2],'r+') 21 | c2t060 = Dataset(sys.argv[4],'r') 22 | c2t008 = Dataset(sys.argv[5],'r') 23 | 24 | gland_mask = Dataset(sys.argv[3],'r').variables['greenland_mask'][:] 25 | 26 | #### 27 | 28 | c2t060_vars_exclude = ['lat','lon','LANDM_COSLAT','SGH'] 29 | c2t060_vars_camRest = ['PHIS','SGH30'] 30 | c2t008_vars = ['SGH'] 31 | 32 | 33 | ## Process fields from 60-point smoothing radius 34 | for Field in c2t060.variables.keys(): 35 | if Field not in c2t060_vars_exclude: 36 | topoDataOut.variables[Field][:] = where(gland_mask==1., 37 | c2t060.variables[Field][:], 38 | topoDataIn.variables[Field][:]) 39 | 40 | if Field in c2t060_vars_camRest: 41 | camRest.variables[Field][:] = where(gland_mask==1., 42 | c2t060.variables[Field][:], 43 | topoDataIn.variables[Field][:]) 44 | 45 | 46 | ## Process fields from 8-point smoothing radius 47 | for Field in c2t008_vars: 48 | topoDataOut.variables[Field][:] = where(gland_mask==1., 49 | c2t008.variables[Field][:], 50 | topoDataIn.variables[Field][:]) 51 | 52 | camRest.variables[Field][:] = where(gland_mask==1., 53 | c2t008.variables[Field][:], 54 | topoDataIn.variables[Field][:]) 55 | ### 56 | 57 | camRest.close() 58 | topoDataOut.close() 59 | 60 | 61 | ########################### 62 | 63 | if __name__ == "__main": 64 | pass 65 | 66 | ########################### 67 | ## === end of script === ## 68 | ########################### 69 | -------------------------------------------------------------------------------- /manage_externals/test/README.md: -------------------------------------------------------------------------------- 1 | # Testing for checkout_externals 2 | 3 | NOTE: Python2 is the supported runtime environment. Python3 compatibility is 4 | in progress, complicated by the different proposed input methods 5 | (yaml, xml, cfg/ini, json) and their different handling of strings 6 | (unicode vs byte) in python2. Full python3 compatibility will be 7 | possible once the number of possible input formats has been narrowed. 8 | 9 | ## Setup development environment 10 | 11 | Development environments should be setup for python2 and python3: 12 | 13 | ```SH 14 | cd checkout_externals/test 15 | make python=python2 env 16 | make python=python3 env 17 | ``` 18 | 19 | ## Unit tests 20 | 21 | Tests should be run for both python2 and python3. It is recommended 22 | that you have seperate terminal windows open python2 and python3 23 | testing to avoid errors activating and deactivating environments. 24 | 25 | ```SH 26 | cd checkout_externals/test 27 | . env_python2/bin/activate 28 | make utest 29 | deactivate 30 | ``` 31 | 32 | ```SH 33 | cd checkout_externals/test 34 | . env_python2/bin/activate 35 | make utest 36 | deactivate 37 | ``` 38 | 39 | ## System tests 40 | 41 | Not yet implemented. 42 | 43 | ## Static analysis 44 | 45 | checkout_externals is difficult to test thoroughly because it relies 46 | on git and svn, and svn requires a live network connection and 47 | repository. Static analysis will help catch bugs in code paths that 48 | are not being executed, but it requires conforming to community 49 | standards and best practices. autopep8 and pylint should be run 50 | regularly for automatic code formatting and linting. 51 | 52 | ```SH 53 | cd checkout_externals/test 54 | . env_python2/bin/activate 55 | make lint 56 | deactivate 57 | ``` 58 | 59 | The canonical formatting for the code is whatever autopep8 60 | generates. All issues identified by pylint should be addressed. 61 | 62 | 63 | ## Code coverage 64 | 65 | All changes to the code should include maintaining existing tests and 66 | writing new tests for new or changed functionality. To ensure test 67 | coverage, run the code coverage tool: 68 | 69 | ```SH 70 | cd checkout_externals/test 71 | . env_python2/bin/activate 72 | make coverage 73 | open -a Firefox.app htmlcov/index.html 74 | deactivate 75 | ``` 76 | 77 | 78 | -------------------------------------------------------------------------------- /scripts/misc/plotTopo30sec.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os,sys 4 | from numpy import * 5 | from netCDF4 import * 6 | from scipy import interpolate 7 | from plotObj import * 8 | 9 | ########################### 10 | 11 | 12 | class dataServer: 13 | def __init__(self,dataFile): 14 | self.data = Dataset(dataFile,'r') 15 | 16 | def getVar(self,Field): 17 | return self.data.variables[Field][:] 18 | 19 | def varKeys(self): 20 | print(self.data.variables.keys()) 21 | 22 | def getLatIdx(self,lat,lat0): 23 | return abs(lat-lat0).argmin() 24 | 25 | def getLonIdx(self,lon,lon0): 26 | return abs(lon-lon0).argmin() 27 | 28 | 29 | 30 | def plotData(read=False,topo=None,lat=None,lon=None): 31 | 32 | print('--> plotData()') 33 | 34 | e = 100 35 | 36 | if read is True: 37 | lat = data.getVar('lat')[iy0:iy1:e] 38 | lon = data.getVar('lon')[ix0:ix1:e] 39 | topo = data.getVar('htopo')[iy0:iy1:e,ix0:ix1:e] 40 | 41 | 42 | pp = plotObj(lat,lon) 43 | 44 | # pp.cmapType = 'YlGnBu' 45 | pp.cmapType = 'RdYlGn' 46 | 47 | pp.levels = [250.,500.,750.,1000.,1250.,1500., 48 | 1750.,2000.,2250.,2500.,2750., 49 | 3000.,3250.,3500.,3750.,4000.,4250., 50 | 4500.,4750.,5000.] 51 | 52 | # pp.levels = linspace(250.,5000.,20) 53 | 54 | 55 | 56 | pp.maxValue = pp.levels[-1] 57 | pp.minValue = pp.levels[0] 58 | 59 | pp.overColor = 'k' 60 | pp.underColor = 'w' 61 | 62 | pp.cbarTicks = pp.levels 63 | pp.contourLevs = pp.levels 64 | 65 | # pp.contourf(topo) 66 | pp.customContourfPlot(topo,lon,lat) 67 | 68 | pp.showFigure() 69 | 70 | 71 | 72 | ########################### 73 | 74 | if __name__ == "__main__": 75 | 76 | res30sec = False 77 | resFV1 = False 78 | 79 | res30sec = True 80 | 81 | 82 | 83 | dataPath = '/glade/u/home/marcusl/work/dynamic_atm_topog_implement_Julio/dynamic_atm_topog/regridding' 84 | dataFile = '%s/modified-highRes.nc'%(dataPath) 85 | 86 | 87 | data = dataServer(dataFile) 88 | 89 | 90 | 91 | iy0 = 16800 92 | iy1 = -1 93 | 94 | ix0 = 32400 95 | ix1 = -1 96 | 97 | 98 | plotData(read=True) 99 | 100 | ########################### 101 | print('Game Over!') 102 | ## === end of script === ## 103 | -------------------------------------------------------------------------------- /manage_externals/test/Makefile: -------------------------------------------------------------------------------- 1 | python = not-set 2 | verbose = not-set 3 | debug = not-set 4 | 5 | ifneq ($(python), not-set) 6 | PYTHON=$(python) 7 | else 8 | PYTHON=python 9 | endif 10 | 11 | # we need the python path to point one level up to access the package 12 | # and executables 13 | PYPATH=PYTHONPATH=..: 14 | 15 | # common args for running tests 16 | TEST_ARGS=-m unittest discover 17 | 18 | ifeq ($(debug), not-set) 19 | ifeq ($(verbose), not-set) 20 | # summary only output 21 | TEST_ARGS+=--buffer 22 | else 23 | # show individual test summary 24 | TEST_ARGS+=--buffer --verbose 25 | endif 26 | else 27 | # show detailed test output 28 | TEST_ARGS+=--verbose 29 | endif 30 | 31 | 32 | # auto reformat the code 33 | AUTOPEP8=autopep8 34 | AUTOPEP8_ARGS=--aggressive --in-place 35 | 36 | # run lint 37 | PYLINT=pylint 38 | PYLINT_ARGS=-j 2 --rcfile=.pylint.rc 39 | 40 | # code coverage 41 | COVERAGE=coverage 42 | COVERAGE_ARGS=--rcfile=.coveragerc 43 | 44 | # source files 45 | SRC = \ 46 | ../checkout_externals \ 47 | ../manic/*.py 48 | 49 | CHECKOUT_EXE = ../checkout_externals 50 | 51 | TEST_DIR = . 52 | 53 | README = ../README.md 54 | 55 | # 56 | # testing 57 | # 58 | .PHONY : utest 59 | utest : FORCE 60 | $(PYPATH) $(PYTHON) $(TEST_ARGS) --pattern 'test_unit_*.py' 61 | 62 | .PHONY : stest 63 | stest : FORCE 64 | $(PYPATH) $(PYTHON) $(TEST_ARGS) --pattern 'test_sys_*.py' 65 | 66 | .PHONY : test 67 | test : utest stest 68 | 69 | # 70 | # documentation 71 | # 72 | .PHONY : readme 73 | readme : $(CHECKOUT_EXE) 74 | printf "%s\n\n" "-- AUTOMATICALLY GENERATED FILE. DO NOT EDIT --" > $(README) 75 | printf "%s" '[![Build Status](https://travis-ci.org/ESMCI/manage_externals.svg?branch=master)](https://travis-ci.org/ESMCI/manage_externals)' >> $(README) 76 | printf "%s" '[![Coverage Status](https://coveralls.io/repos/github/ESMCI/manage_externals/badge.svg?branch=master)](https://coveralls.io/github/ESMCI/manage_externals?branch=master)' >> $(README) 77 | printf "\n%s\n" '```' >> $(README) 78 | $(CHECKOUT_EXE) --help >> $(README) 79 | 80 | # 81 | # coding standards 82 | # 83 | .PHONY : style 84 | style : FORCE 85 | $(AUTOPEP8) $(AUTOPEP8_ARGS) --recursive $(SRC) $(TEST_DIR)/test_*.py 86 | 87 | .PHONY : lint 88 | lint : FORCE 89 | $(PYLINT) $(PYLINT_ARGS) $(SRC) $(TEST_DIR)/test_*.py 90 | 91 | .PHONY : stylint 92 | stylint : style lint 93 | 94 | .PHONY : coverage 95 | # Need to use a single coverage run with a single pattern rather than 96 | # using two separate commands with separate patterns for test_unit_*.py 97 | # and test_sys_*.py: The latter clobbers some results from the first 98 | # run, even if we use the --append flag to 'coverage run'. 99 | coverage : FORCE 100 | $(PYPATH) $(COVERAGE) erase 101 | $(PYPATH) $(COVERAGE) run $(COVERAGE_ARGS) $(TEST_ARGS) --pattern 'test_*.py' 102 | $(PYPATH) $(COVERAGE) html 103 | 104 | # 105 | # virtual environment creation 106 | # 107 | .PHONY : env 108 | env : FORCE 109 | $(PYPATH) virtualenv --python $(PYTHON) $@_$(PYTHON) 110 | . $@_$(PYTHON)/bin/activate; pip install -r requirements.txt 111 | 112 | # 113 | # utilites 114 | # 115 | .PHONY : clean 116 | clean : FORCE 117 | -rm -rf *~ *.pyc tmp fake htmlcov 118 | 119 | .PHONY : clobber 120 | clobber : clean 121 | -rm -rf env_* 122 | 123 | FORCE : 124 | 125 | -------------------------------------------------------------------------------- /manage_externals/manic/repository.py: -------------------------------------------------------------------------------- 1 | """Base class representation of a repository 2 | """ 3 | 4 | from .externals_description import ExternalsDescription 5 | from .utils import fatal_error 6 | from .global_constants import EMPTY_STR 7 | 8 | 9 | class Repository(object): 10 | """ 11 | Class to represent and operate on a repository description. 12 | """ 13 | 14 | def __init__(self, component_name, repo): 15 | """ 16 | Parse repo externals description 17 | """ 18 | self._name = component_name 19 | self._protocol = repo[ExternalsDescription.PROTOCOL] 20 | self._tag = repo[ExternalsDescription.TAG] 21 | self._branch = repo[ExternalsDescription.BRANCH] 22 | self._hash = repo[ExternalsDescription.HASH] 23 | self._url = repo[ExternalsDescription.REPO_URL] 24 | self._sparse = repo[ExternalsDescription.SPARSE] 25 | 26 | if self._url is EMPTY_STR: 27 | fatal_error('repo must have a URL') 28 | 29 | if ((self._tag is EMPTY_STR) and (self._branch is EMPTY_STR) and 30 | (self._hash is EMPTY_STR)): 31 | fatal_error('{0} repo must have a branch, tag or hash element') 32 | 33 | ref_count = 0 34 | if self._tag is not EMPTY_STR: 35 | ref_count += 1 36 | if self._branch is not EMPTY_STR: 37 | ref_count += 1 38 | if self._hash is not EMPTY_STR: 39 | ref_count += 1 40 | if ref_count != 1: 41 | fatal_error('repo {0} must have exactly one of ' 42 | 'tag, branch or hash.'.format(self._name)) 43 | 44 | def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument 45 | """ 46 | If the repo destination directory exists, ensure it is correct (from 47 | correct URL, correct branch or tag), and possibly update the source. 48 | If the repo destination directory does not exist, checkout the correce 49 | branch or tag. 50 | NB: is include as an argument for compatibility with 51 | git functionality (repository_git.py) 52 | """ 53 | msg = ('DEV_ERROR: checkout method must be implemented in all ' 54 | 'repository classes! {0}'.format(self.__class__.__name__)) 55 | fatal_error(msg) 56 | 57 | def status(self, stat, repo_dir_path): # pylint: disable=unused-argument 58 | """Report the status of the repo 59 | 60 | """ 61 | msg = ('DEV_ERROR: status method must be implemented in all ' 62 | 'repository classes! {0}'.format(self.__class__.__name__)) 63 | fatal_error(msg) 64 | 65 | def submodules_file(self, repo_path=None): 66 | # pylint: disable=no-self-use,unused-argument 67 | """Stub for use by non-git VC systems""" 68 | return None 69 | 70 | def url(self): 71 | """Public access of repo url. 72 | """ 73 | return self._url 74 | 75 | def tag(self): 76 | """Public access of repo tag 77 | """ 78 | return self._tag 79 | 80 | def branch(self): 81 | """Public access of repo branch. 82 | """ 83 | return self._branch 84 | 85 | def hash(self): 86 | """Public access of repo hash. 87 | """ 88 | return self._hash 89 | 90 | def name(self): 91 | """Public access of repo name. 92 | """ 93 | return self._name 94 | 95 | def protocol(self): 96 | """Public access of repo protocol. 97 | """ 98 | return self._protocol 99 | -------------------------------------------------------------------------------- /scripts/misc/extract_cubed_sphere_face.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | import os 4 | from numpy import * 5 | import numpy.ma as ma 6 | import matplotlib.pyplot as plt 7 | from matplotlib import rcParams 8 | from mpl_toolkits.basemap import Basemap 9 | from netCDF4 import Dataset as NetCDFFile 10 | 11 | ########################### 12 | 13 | class cubedSphereFace(object): 14 | def __init__(self,dataFile): 15 | 16 | self.fpin = NetCDFFile(infile,'r') 17 | self.tlat = self.fpin.variables['lat'][:] 18 | self.tlon = self.fpin.variables['lon'][:] 19 | 20 | ## Reshape 1D array as 3D array (3000,3000,6) 21 | self.nx = 3000 22 | self.ny = 3000 23 | self.nfaces = 6 24 | 25 | self.order = 'F' 26 | self.face = -1 27 | 28 | self.newDims = (self.ny,self.nx,self.nfaces) 29 | 30 | ## Issue with second dimension, hence the ::-1 construction 31 | self.lat = reshape(self.tlat,self.newDims,order=self.order)[:,::-1,self.face] 32 | self.lon = reshape(self.tlon,self.newDims,order=self.order)[:,::-1,self.face] 33 | 34 | 35 | ### 36 | 37 | self.fpin.close() 38 | 39 | 40 | def plotGrid(self,e=50): 41 | 42 | lat = self.lat[::e,::e] 43 | lon = self.lon[::e,::e] 44 | 45 | plt.figure(figsize=(12,12)) 46 | plt.subplot(1,1,1) 47 | 48 | m = Basemap(projection='npstere', 49 | boundinglat=30, 50 | lon_0=270, 51 | resolution='l') 52 | 53 | m.drawcoastlines() 54 | 55 | x, y = m(lon,lat) 56 | im = m.pcolor(x,y, 57 | ma.masked_array(zeros(lat.shape,'f')), 58 | antialiased=True, 59 | cmap=plt.cm.cool) 60 | 61 | plt.title('Cubed sphere gridcells') 62 | plt.show() 63 | 64 | 65 | 66 | def plotField(self,Field='terr',e=20): 67 | 68 | lat = self.lat[::e,::e] 69 | lon = self.lon[::e,::e] 70 | 71 | 72 | if Field == 'terr': 73 | self.tterr = self.fpin.variables['terr'][:] 74 | self.terr = reshape(self.tterr, 75 | self.newDims, 76 | order=self.order)[:,::-1,self.face] 77 | var = self.terr[::e,::e] 78 | 79 | if Field == 'var30': 80 | self.tvar30 = self.fpin.variables['var30'][:] 81 | self.var30 = reshape(self.tvar30, 82 | self.newDims, 83 | order=self.order)[:,::-1,face] 84 | var = self.var30[::e,::e] 85 | 86 | # print(amax(amax(var,axis=0),axis=0)) 87 | 88 | 89 | plt.figure(figsize=(12,12)) 90 | plt.subplot(1,1,1) 91 | 92 | m = Basemap(projection='cyl', 93 | llcrnrlat=-90, 94 | urcrnrlat=90, 95 | llcrnrlon=0, 96 | urcrnrlon=360, 97 | resolution='c') 98 | 99 | boundinglat = 30 100 | if self.face == 6: boundinglat = 30 101 | 102 | 103 | m = Basemap(projection='npstere', 104 | boundinglat=boundinglat, 105 | lon_0=270, 106 | resolution='l') 107 | 108 | x, y = m(lon,lat) 109 | m.drawcoastlines() 110 | 111 | im = m.pcolor(x,y, 112 | var, 113 | antialiased=True, 114 | cmap=plt.cm.jet, 115 | ) 116 | 117 | cbar = plt.colorbar(im) 118 | 119 | plt.show() 120 | 121 | 122 | 123 | ########################### 124 | 125 | if __name__ == "__main__": 126 | 127 | dataPath = os.getcwd() 128 | 129 | infile = '%s/bin_to_cube/ncube3000.nc'%(dataPath) 130 | 131 | 132 | cs = cubedSphereFace(infile) 133 | # cs.plotGrid() 134 | cs.plotField(Field='terr') 135 | 136 | 137 | 138 | ########################### 139 | print('Game Over!') 140 | -------------------------------------------------------------------------------- /manage_externals/test/doc/testing.rst: -------------------------------------------------------------------------------- 1 | Testing 2 | ======= 3 | 4 | The manage_externals package has an automated test suite. All pull 5 | requests are expected to pass 100% of the automated tests, as well as 6 | be pep8 and lint 'clean' and maintain approximately constant (at a 7 | minimum) level of code coverage. 8 | 9 | Quick Start 10 | ----------- 11 | 12 | Do nothing approach 13 | ~~~~~~~~~~~~~~~~~~~ 14 | 15 | When you create a pull request on GitHub, Travis-CI continuous 16 | integration testing will run the test suite in both python2 and 17 | python3. Test results, lint results, and code coverage results are 18 | available online. 19 | 20 | Do something approach 21 | ~~~~~~~~~~~~~~~~~~~~~ 22 | 23 | In the test directory, run: 24 | 25 | .. code-block:: shell 26 | 27 | make env 28 | make lint 29 | make test 30 | make coverage 31 | 32 | 33 | Automated Testing 34 | ----------------- 35 | 36 | The manage_externals manic library and executables are developed to be 37 | python2 and python3 compatible using only the standard library. The 38 | test suites meet the same requirements. But additional tools are 39 | required to provide lint and code coverage metrics and generate 40 | documentation. The requirements are maintained in the requirements.txt 41 | file, and can be automatically installed into an isolated environment 42 | via Makefile. 43 | 44 | Bootstrap requirements: 45 | 46 | * python2 - version 2.7.x or later 47 | 48 | * python3 - version 3.6 tested other versions may work 49 | 50 | * pip and virtualenv for python2 and python3 51 | 52 | Note: all make rules can be of the form ``make python=pythonX rule`` 53 | or ``make rule`` depending if you want to use the default system 54 | python or specify a specific version. 55 | 56 | The Makefile in the test directory has the following rules: 57 | 58 | * ``make python=pythonX env`` - create a python virtual environment 59 | for python2 or python3 and install all required packages. These 60 | packages are required to run lint or coverage. 61 | 62 | * ``make style`` - runs autopep8 63 | 64 | * ``make lint`` - runs autopep8 and pylint 65 | 66 | * ``make test`` - run the full test suite 67 | 68 | * ``make utest`` - run jus the unit tests 69 | 70 | * ``make stest`` - run jus the system integration tests 71 | 72 | * ``make coverage`` - run the full test suite through the code 73 | coverage tool and generate an html report. 74 | 75 | * ``make readme`` - automatically generate the README files. 76 | 77 | * ``make clean`` - remove editor and pyc files 78 | 79 | * ``make clobber`` - remove all generated test files, including 80 | virtual environments, coverage reports, and temporary test 81 | repository directories. 82 | 83 | Unit Tests 84 | ---------- 85 | 86 | Unit tests are probably not 'true unit tests' for the pedantic, but 87 | are pragmatic unit tests. They cover small practicle code blocks: 88 | functions, class methods, and groups of functions and class methods. 89 | 90 | System Integration Tests 91 | ------------------------ 92 | 93 | NOTE(bja, 2017-11) The systems integration tests currently do not include svn repositories. 94 | 95 | The manage_externals package is extremely tedious and error prone to test manually. 96 | 97 | Combinations that must be tested to ensure basic functionality are: 98 | 99 | * container repository pulling in simple externals 100 | 101 | * container repository pulling in mixed externals with sub-externals. 102 | 103 | * mixed repository acting as a container, pulling in simple externals and sub-externals 104 | 105 | Automatic system tests are handled the same way manual testing is done: 106 | 107 | * clone a test repository 108 | 109 | * create an externals description file for the test 110 | 111 | * run the executable with the desired args 112 | 113 | * check the results 114 | 115 | * potentially modify the repo (checkout a different branch) 116 | 117 | * rerun and test 118 | 119 | * etc 120 | 121 | The automated system stores small test repositories in the main repo 122 | by adding them as bare repositories. These repos are cloned via a 123 | subprocess call to git and manipulated during the tests. 124 | -------------------------------------------------------------------------------- /scripts/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ########################### 4 | # 5 | # Configure topography updating 6 | # routines for Yellowstone and 7 | # Cheyenne architecture 8 | # 9 | # Author: 10 | # M. Lofverstrom 11 | # NCAR, 2017 12 | # 13 | ########################### 14 | 15 | while [[ $# -gt 1 ]] 16 | do 17 | key="$1" 18 | 19 | case $key in 20 | -r|--rundir) 21 | ScratchRun="$2" 22 | shift 23 | ;; 24 | -p|--project) 25 | Project="$2" 26 | shift 27 | ;; 28 | -w|--walltime) 29 | Walltime="$2" 30 | shift 31 | ;; 32 | -q|--queue) 33 | Queue="$2" 34 | shift 35 | ;; 36 | *) 37 | ;; 38 | esac 39 | shift 40 | done 41 | 42 | 43 | ScriptDir=$ScratchRun/dynamic_atm_topog 44 | ScriptDefault=${ScriptDir}/defaultScripts 45 | 46 | 47 | ####### 48 | 49 | 50 | if [[ $(hostname) = yslogin* ]]; then 51 | machine=yellowstone 52 | elif [[ $(hostname) = geyser* ]]; then 53 | machine=yellowstone 54 | elif [[ $(hostname) = cheyenne* ]]; then 55 | machine=cheyenne 56 | fi 57 | 58 | 59 | ### ### ### ### ### ### ### ### 60 | 61 | 62 | if [[ $machine == yellowstone ]]; then 63 | 64 | cp ${ScriptDefault}/CAM_topo_regen_y.sh ${ScriptDir}/CAM_topo_regen.sh 65 | cp ${ScriptDefault}/submit_topo_regen_script_y.sh \ 66 | ${ScriptDir}/submit_topo_regen_script.sh 67 | 68 | regenS=${ScriptDir}/CAM_topo_regen.sh 69 | submitS=${ScriptDir}/submit_topo_regen_script.sh 70 | 71 | ### 72 | 73 | # nco=nco/4.4.2 74 | # ncl=ncl/6.3.0 75 | # python=python/2.7.7 76 | 77 | # sed -i "/module load nco/c\module load $nco" $submitS 78 | # sed -i "/module load nco/c\module load $nco" $regenS 79 | # sed -i "/module load ncl/c\module load $ncl" $regenS 80 | # sed -i "/module load python/c\module load $python" $regenS 81 | 82 | ### 83 | 84 | if [[ -n ${ScratchRun} ]]; then 85 | sed -i "/ScratchRun=/c\ScratchRun=${ScratchRun}" $submitS 86 | fi 87 | 88 | if [[ -n ${ScriptDir} ]]; then 89 | sed -i "/ScriptDir=/c\ScriptDir=${ScriptDir}" $submitS 90 | fi 91 | 92 | ## 93 | 94 | if [[ -n ${ScratchRun} ]]; then 95 | sed -i "/ScratchRun=/c\ScratchRun=${ScratchRun}" $regenS 96 | fi 97 | if [[ -n ${Project} ]]; then 98 | sed -i "/#BSUB -P/c\#BSUB -P ${Project}" $regenS 99 | fi 100 | if [[ -n ${Walltime} ]]; then 101 | sed -i "/#BSUB -W/c\#BSUB -W ${Walltime}" $regenS 102 | fi 103 | if [[ -n ${Queue} ]]; then 104 | sed -i "/#BSUB -q/c\#BSUB -q ${Queue}" $regenS 105 | fi 106 | 107 | fi 108 | 109 | 110 | ### ### ### ### ### ### ### ### 111 | 112 | 113 | if [[ $machine == cheyenne ]]; then 114 | 115 | cp ${ScriptDefault}/CAM_topo_regen_c.sh ${ScriptDir}/CAM_topo_regen.sh 116 | cp ${ScriptDefault}/submit_topo_regen_script_c.sh \ 117 | ${ScriptDir}/submit_topo_regen_script.sh 118 | 119 | regenS=${ScriptDir}/CAM_topo_regen.sh 120 | submitS=${ScriptDir}/submit_topo_regen_script.sh 121 | 122 | ### 123 | 124 | # nco=nco/4.6.2 125 | # ncl=ncl/6.4.0 126 | # python=python/2.7.13 127 | 128 | ## Probably not necessay 129 | if [ $SHELL == /bin/bash ]; then 130 | source /glade/u/apps/ch/opt/Lmod/7.3.14/lmod/7.3.14/init/bash 131 | elif [ $SHELL == /bin/tcsh ]; then 132 | source /glade/u/apps/ch/opt/Lmod/7.3.14/lmod/7.3.14/init/tcsh 133 | fi 134 | 135 | 136 | # sed -i "/module load nco/c\module load $nco" $submitS 137 | # sed -i "/module load nco/c\module load $nco" $regenS 138 | # sed -i "/module load ncl/c\module load $ncl" $regenS 139 | # sed -i "/module load python/c\module load $python" $regenS 140 | 141 | ### 142 | 143 | if [[ -n ${ScratchRun} ]]; then 144 | sed -i "/ScratchRun=/c\ScratchRun=${ScratchRun}" $submitS 145 | fi 146 | if [[ -n ${ScriptDir} ]]; then 147 | sed -i "/SciptDir=/c\ScriptDir=${ScriptDir}" $submitS 148 | fi 149 | 150 | if [[ -n ${Project} ]]; then 151 | sed -i "/Project=/c\Project=${Project}" $submitS 152 | fi 153 | if [[ -n ${Walltime} ]]; then 154 | sed -i "/Walltime=/c\Walltime=${Walltime}" $submitS 155 | fi 156 | if [[ -n ${Queue} ]]; then 157 | sed -i "/Queue=/c\Queue=${Queue}" $submitS 158 | fi 159 | 160 | 161 | ## 162 | 163 | if [[ -n ${ScratchRun} ]]; then 164 | sed -i "/ScratchRun=/c\ScratchRun=${ScratchRun}" $regenS 165 | fi 166 | if [[ -n ${Project} ]]; then 167 | sed -i "/#PBS -A/c\#PBS -A ${Project}" $regenS 168 | fi 169 | if [[ -n ${Walltime} ]]; then 170 | sed -i "/#PBS -l w/c\#PBS -l walltime=${Walltime}" $regenS 171 | fi 172 | if [[ -n ${Queue} ]]; then 173 | sed -i "/#PBS -q/c\#PBS -q ${Queue}" $regenS 174 | fi 175 | 176 | fi 177 | 178 | 179 | 180 | ########################### 181 | ## === end of script === ## 182 | ########################### 183 | -------------------------------------------------------------------------------- /scripts/regridding/regrid.ncl: -------------------------------------------------------------------------------- 1 | load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" 2 | load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" 3 | load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" 4 | load "$NCARG_ROOT/lib/ncarg/nclscripts/esmf/ESMF_regridding.ncl" 5 | 6 | begin 7 | ;******************** USER DEFINED VARIABLES*********************** 8 | 9 | ; included as an attribute in the output 10 | input_title="Generated from ISM curvilinear grid" 11 | 12 | ; input data are curvilinear, so lat & lon are specified by 2-d 13 | ; arrays, as given here: 14 | lat_var_name="lat" 15 | lon_var_name="lon" 16 | 17 | ; is the input a regional file? (as opposed to a global file) 18 | input_is_regional = True 19 | 20 | source_large_file = False 21 | 22 | template_file_name="template_out.nc" 23 | 24 | template_var_name="ice_shelf" 25 | 26 | ; is the template a regional file? (as opposed to a global file) 27 | template_is_regional = True 28 | 29 | dest_large_file = True 30 | 31 | wgt_large_file = True 32 | 33 | interp_method = "bilinear" 34 | 35 | ; set the following to true if the necessary files have already been 36 | ; generated 37 | skip_src_grid = True 38 | skip_dst_grid = True 39 | skip_wgt_gen = True 40 | 41 | ;*****************END OF USER DEFINED VARIABLES******************* 42 | 43 | ;************************************************ 44 | ; Get template longitudes and latitudes 45 | ;************************************************ 46 | 47 | template_file = addfile(template_file_name, "r") 48 | template_var = template_file->$template_var_name$ 49 | dsizes = dimsizes(template_var) 50 | rank = dimsizes(dsizes) ; number of dimensions 51 | 52 | template_lon_name = template_var!(rank-1) 53 | template_lat_name = template_var!(rank-2) 54 | template_lon = template_var&$template_lon_name$ 55 | template_lat = template_var&$template_lat_name$ 56 | 57 | ;************************************************ 58 | ; Read in input file (not individual variables, yet) 59 | ;************************************************ 60 | 61 | input_file = addfile(input_file_name, "r") 62 | 63 | ;************************************************ 64 | ; Set regridding options 65 | ;************************************************ 66 | 67 | Opt = True 68 | 69 | Opt@SrcGridLat = input_file->$lat_var_name$ 70 | Opt@SrcGridLon = input_file->$lon_var_name$ 71 | Opt@SrcRegional = input_is_regional 72 | Opt@SrcLargeFile = source_large_file 73 | ; according to the documentation, DstGridType shouldnt be required, 74 | ; but it seems to be 75 | Opt@DstGridType = "rectilinear" 76 | Opt@DstGridLat = template_lat 77 | Opt@DstGridLon = template_lon 78 | Opt@DstRegional = template_is_regional 79 | Opt@DstLargeFile = dest_large_file 80 | Opt@LargeFile = wgt_large_file 81 | 82 | Opt@InterpMethod = interp_method 83 | 84 | Opt@SrcFileName = "source_grid_file.nc" 85 | Opt@DstFileName = "destination_grid_file.nc" 86 | Opt@WgtFileName = "weights_file.nc" 87 | Opt@SkipSrcGrid = skip_src_grid 88 | Opt@SkipDstGrid = skip_dst_grid 89 | Opt@SkipWgtGen = skip_wgt_gen 90 | 91 | Opt@SrcInputFileName=input_file_name 92 | Opt@SrcTitle=input_title 93 | Opt@CopyVarCoords = True 94 | Opt@ForceOverwrite = True 95 | Opt@Debug = True 96 | 97 | ; Main Regridding: 98 | ; Regrid variables, outputting interpolated version file 99 | 100 | GIS_topog = input_file->thk + input_file->topg 101 | GIS_topog = where(GIS_topog.lt.0., 0., GIS_topog) 102 | landmask_float = new(dimsizes(GIS_topog),float,-9999.) 103 | landmask_float = where (GIS_topog.gt.0., 1, landmask_float) 104 | 105 | tmp = doubletoint(ESMF_regrid(GIS_topog,Opt)) 106 | GIS_topog_regridded = tmp(::-1,:) 107 | delete(tmp) 108 | ; after the first variable, we can skip some steps in regridding 109 | ; the other variables; this is done via the following options: 110 | Opt@SkipSrcGrid = True 111 | Opt@SkipDstGrid = True 112 | Opt@SkipWgtGen = True 113 | tmp = floattoint(ESMF_regrid(landmask_float,Opt)) 114 | landmask_float_regridded = tmp(::-1,:) 115 | delete(tmp) 116 | 117 | ; Save regridded tile to an archived reference file 118 | print ("Save regridded tile to an archived reference file") 119 | archive_file = addfile("ISM30sec.archived.nc","c") 120 | archive_file->usrf = GIS_topog_regridded 121 | archive_file->landmask_float = landmask_float_regridded 122 | 123 | 124 | usgs_topog_file = addfile(global_file_name,"w") 125 | usgs_topog_tile = usgs_topog_file->htopo(16800:,32400:) 126 | usgs_landfract_tile = usgs_topog_file->landfract(16800:,32400:) 127 | 128 | ;####Merge topography and landmask from ISM-derived values, over island of Greenland 129 | ;ismissing(GIS_mask) returns 'true' outside of GIS regions 130 | ;subsequent ingestion by 'where' sets the landfrac and topography values to the (original) 131 | ;values. Where ismissing(GIS_mask) is false (i.e. the ISM-derived data exists), landfrac 132 | ;and topographies are set to the ISM values. 133 | 134 | print ("Replace missing values: landfrac") 135 | usgs_landfract_tile= where( ismissing(landmask_float_regridded), usgs_landfract_tile, landmask_float_regridded) 136 | print ("Replace missing values: topography") 137 | usgs_topog_tile= where( ismissing(landmask_float_regridded), usgs_topog_tile, GIS_topog_regridded) 138 | 139 | ;####Insert altered topography and landmask back into full USGS dataset. 140 | ; print ("Insert altered topography back into full USGS dataset") 141 | ; usgs_topog_file->htopo(16800:,32400:)=usgs_topog_tile 142 | ; print ("Insert altered landmask back into full USGS dataset") 143 | ; usgs_topog_file->landfract(16800:,32400:)=usgs_landfract_tile 144 | 145 | 146 | print ("Save regridded tile to file") 147 | archive_file= addfile(output_file_name,"c") 148 | archive_file->htopo = usgs_topog_tile 149 | archive_file->landfract = usgs_landfract_tile 150 | 151 | end 152 | -------------------------------------------------------------------------------- /manage_externals/test/doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Manage Externals documentation build configuration file, created by 4 | # sphinx-quickstart on Wed Nov 29 10:53:25 2017. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | # If extensions (or modules to document with autodoc) are in another directory, 16 | # add these directories to sys.path here. If the directory is relative to the 17 | # documentation root, use os.path.abspath to make it absolute, like shown here. 18 | # 19 | # import os 20 | # import sys 21 | # sys.path.insert(0, os.path.abspath('.')) 22 | 23 | 24 | # -- General configuration ------------------------------------------------ 25 | 26 | # If your documentation needs a minimal Sphinx version, state it here. 27 | # 28 | # needs_sphinx = '1.0' 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = ['sphinx.ext.autodoc', 34 | 'sphinx.ext.todo', 35 | 'sphinx.ext.coverage', 36 | 'sphinx.ext.viewcode', 37 | 'sphinx.ext.githubpages'] 38 | 39 | # Add any paths that contain templates here, relative to this directory. 40 | templates_path = ['_templates'] 41 | 42 | # The suffix(es) of source filenames. 43 | # You can specify multiple suffix as a list of string: 44 | # 45 | # source_suffix = ['.rst', '.md'] 46 | source_suffix = '.rst' 47 | 48 | # The master toctree document. 49 | master_doc = 'index' 50 | 51 | # General information about the project. 52 | project = u'Manage Externals' 53 | copyright = u'2017, CSEG at NCAR' 54 | author = u'CSEG at NCAR' 55 | 56 | # The version info for the project you're documenting, acts as replacement for 57 | # |version| and |release|, also used in various other places throughout the 58 | # built documents. 59 | # 60 | # The short X.Y version. 61 | version = u'1.0.0' 62 | # The full version, including alpha/beta/rc tags. 63 | release = u'1.0.0' 64 | 65 | # The language for content autogenerated by Sphinx. Refer to documentation 66 | # for a list of supported languages. 67 | # 68 | # This is also used if you do content translation via gettext catalogs. 69 | # Usually you set "language" from the command line for these cases. 70 | language = None 71 | 72 | # List of patterns, relative to source directory, that match files and 73 | # directories to ignore when looking for source files. 74 | # This patterns also effect to html_static_path and html_extra_path 75 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 76 | 77 | # The name of the Pygments (syntax highlighting) style to use. 78 | pygments_style = 'sphinx' 79 | 80 | # If true, `todo` and `todoList` produce output, else they produce nothing. 81 | todo_include_todos = True 82 | 83 | 84 | # -- Options for HTML output ---------------------------------------------- 85 | 86 | # The theme to use for HTML and HTML Help pages. See the documentation for 87 | # a list of builtin themes. 88 | # 89 | html_theme = 'alabaster' 90 | 91 | # Theme options are theme-specific and customize the look and feel of a theme 92 | # further. For a list of options available for each theme, see the 93 | # documentation. 94 | # 95 | # html_theme_options = {} 96 | 97 | # Add any paths that contain custom static files (such as style sheets) here, 98 | # relative to this directory. They are copied after the builtin static files, 99 | # so a file named "default.css" will overwrite the builtin "default.css". 100 | html_static_path = ['_static'] 101 | 102 | # Custom sidebar templates, must be a dictionary that maps document names 103 | # to template names. 104 | # 105 | # This is required for the alabaster theme 106 | # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars 107 | html_sidebars = { 108 | '**': [ 109 | 'relations.html', # needs 'show_related': True theme option to display 110 | 'searchbox.html', 111 | ] 112 | } 113 | 114 | 115 | # -- Options for HTMLHelp output ------------------------------------------ 116 | 117 | # Output file base name for HTML help builder. 118 | htmlhelp_basename = 'ManageExternalsdoc' 119 | 120 | 121 | # -- Options for LaTeX output --------------------------------------------- 122 | 123 | latex_elements = { 124 | # The paper size ('letterpaper' or 'a4paper'). 125 | # 126 | # 'papersize': 'letterpaper', 127 | 128 | # The font size ('10pt', '11pt' or '12pt'). 129 | # 130 | # 'pointsize': '10pt', 131 | 132 | # Additional stuff for the LaTeX preamble. 133 | # 134 | # 'preamble': '', 135 | 136 | # Latex figure (float) alignment 137 | # 138 | # 'figure_align': 'htbp', 139 | } 140 | 141 | # Grouping the document tree into LaTeX files. List of tuples 142 | # (source start file, target name, title, 143 | # author, documentclass [howto, manual, or own class]). 144 | latex_documents = [ 145 | (master_doc, 'ManageExternals.tex', u'Manage Externals Documentation', 146 | u'CSEG at NCAR', 'manual'), 147 | ] 148 | 149 | 150 | # -- Options for manual page output --------------------------------------- 151 | 152 | # One entry per manual page. List of tuples 153 | # (source start file, name, description, authors, manual section). 154 | man_pages = [ 155 | (master_doc, 'manageexternals', u'Manage Externals Documentation', 156 | [author], 1) 157 | ] 158 | 159 | 160 | # -- Options for Texinfo output ------------------------------------------- 161 | 162 | # Grouping the document tree into Texinfo files. List of tuples 163 | # (source start file, target name, title, author, 164 | # dir menu entry, description, category) 165 | texinfo_documents = [ 166 | (master_doc, 'ManageExternals', u'Manage Externals Documentation', 167 | author, 'ManageExternals', 'One line description of project.', 168 | 'Miscellaneous'), 169 | ] 170 | 171 | 172 | 173 | -------------------------------------------------------------------------------- /manage_externals/manic/externals_status.py: -------------------------------------------------------------------------------- 1 | """ExternalStatus 2 | 3 | Class to store status and state information about repositories and 4 | create a string representation. 5 | 6 | """ 7 | from __future__ import absolute_import 8 | from __future__ import unicode_literals 9 | from __future__ import print_function 10 | 11 | from .global_constants import EMPTY_STR 12 | from .utils import printlog, indent_string 13 | from .global_constants import VERBOSITY_VERBOSE, VERBOSITY_DUMP 14 | 15 | 16 | class ExternalStatus(object): 17 | """Class to represent the status of a given source repository or tree. 18 | 19 | Individual repositories determine their own status in the 20 | Repository objects. This object is just resposible for storing the 21 | information and passing it up to a higher level for reporting or 22 | global decisions. 23 | 24 | There are two states of concern: 25 | 26 | * If the repository is in-sync with the externals description file. 27 | 28 | * If the repostiory working copy is clean and there are no pending 29 | transactions (e.g. add, remove, rename, untracked files). 30 | 31 | """ 32 | DEFAULT = '-' 33 | UNKNOWN = '?' 34 | EMPTY = 'e' 35 | MODEL_MODIFIED = 's' # a.k.a. out-of-sync 36 | DIRTY = 'M' 37 | 38 | STATUS_OK = ' ' 39 | STATUS_ERROR = '!' 40 | 41 | # source types 42 | OPTIONAL = 'o' 43 | STANDALONE = 's' 44 | MANAGED = ' ' 45 | 46 | def __init__(self): 47 | self.sync_state = self.DEFAULT 48 | self.clean_state = self.DEFAULT 49 | self.source_type = self.DEFAULT 50 | self.path = EMPTY_STR 51 | self.current_version = EMPTY_STR 52 | self.expected_version = EMPTY_STR 53 | self.status_output = EMPTY_STR 54 | 55 | def log_status_message(self, verbosity): 56 | """Write status message to the screen and log file 57 | """ 58 | self._default_status_message() 59 | if verbosity >= VERBOSITY_VERBOSE: 60 | self._verbose_status_message() 61 | if verbosity >= VERBOSITY_DUMP: 62 | self._dump_status_message() 63 | 64 | def _default_status_message(self): 65 | """Return the default terse status message string 66 | """ 67 | msg = '{sync}{clean}{src_type} {path}'.format( 68 | sync=self.sync_state, clean=self.clean_state, 69 | src_type=self.source_type, path=self.path) 70 | printlog(msg) 71 | 72 | def _verbose_status_message(self): 73 | """Return the verbose status message string 74 | """ 75 | clean_str = self.DEFAULT 76 | if self.clean_state == self.STATUS_OK: 77 | clean_str = 'clean sandbox' 78 | elif self.clean_state == self.DIRTY: 79 | clean_str = 'modified sandbox' 80 | 81 | sync_str = 'on {0}'.format(self.current_version) 82 | if self.sync_state != self.STATUS_OK: 83 | sync_str = '{current} --> {expected}'.format( 84 | current=self.current_version, expected=self.expected_version) 85 | msg = ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) 86 | printlog(msg) 87 | 88 | def _dump_status_message(self): 89 | """Return the dump status message string 90 | """ 91 | msg = indent_string(self.status_output, 12) 92 | printlog(msg) 93 | 94 | def safe_to_update(self): 95 | """Report if it is safe to update a repository. Safe is defined as: 96 | 97 | * If a repository is empty, it is safe to update. 98 | 99 | * If a repository exists and has a clean working copy state 100 | with no pending transactions. 101 | 102 | """ 103 | safe_to_update = False 104 | repo_exists = self.exists() 105 | if not repo_exists: 106 | safe_to_update = True 107 | else: 108 | # If the repo exists, it must be in ok or modified 109 | # sync_state. Any other sync_state at this point 110 | # represents a logic error that should have been handled 111 | # before now! 112 | sync_safe = ((self.sync_state == ExternalStatus.STATUS_OK) or 113 | (self.sync_state == ExternalStatus.MODEL_MODIFIED)) 114 | if sync_safe: 115 | # The clean_state must be STATUS_OK to update. Otherwise we 116 | # are dirty or there was a missed error previously. 117 | if self.clean_state == ExternalStatus.STATUS_OK: 118 | safe_to_update = True 119 | return safe_to_update 120 | 121 | def exists(self): 122 | """Determine if the repo exists. This is indicated by: 123 | 124 | * sync_state is not EMPTY 125 | 126 | * if the sync_state is empty, then the valid states for 127 | clean_state are default, empty or unknown. Anything else 128 | and there was probably an internal logic error. 129 | 130 | NOTE(bja, 2017-10) For the moment we are considering a 131 | sync_state of default or unknown to require user intervention, 132 | but we may want to relax this convention. This is probably a 133 | result of a network error or internal logic error but more 134 | testing is needed. 135 | 136 | """ 137 | is_empty = (self.sync_state == ExternalStatus.EMPTY) 138 | clean_valid = ((self.clean_state == ExternalStatus.DEFAULT) or 139 | (self.clean_state == ExternalStatus.EMPTY) or 140 | (self.clean_state == ExternalStatus.UNKNOWN)) 141 | 142 | if is_empty and clean_valid: 143 | exists = False 144 | else: 145 | exists = True 146 | return exists 147 | 148 | 149 | def check_safe_to_update_repos(tree_status): 150 | """Check if *ALL* repositories are in a safe state to update. We don't 151 | want to do a partial update of the repositories then die, leaving 152 | the model in an inconsistent state. 153 | 154 | Note: if there is an update to do, the repositories will by 155 | definiation be out of synce with the externals description, so we 156 | can't use that as criteria for updating. 157 | 158 | """ 159 | safe_to_update = True 160 | for comp in tree_status: 161 | stat = tree_status[comp] 162 | safe_to_update &= stat.safe_to_update() 163 | 164 | return safe_to_update 165 | -------------------------------------------------------------------------------- /manage_externals/test/test_unit_repository.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Unit test driver for checkout_externals 4 | 5 | Note: this script assume the path to the checkout_externals.py module is 6 | already in the python path. 7 | 8 | """ 9 | 10 | from __future__ import absolute_import 11 | from __future__ import unicode_literals 12 | from __future__ import print_function 13 | 14 | import unittest 15 | 16 | from manic.repository_factory import create_repository 17 | from manic.repository_git import GitRepository 18 | from manic.repository_svn import SvnRepository 19 | from manic.repository import Repository 20 | from manic.externals_description import ExternalsDescription 21 | from manic.global_constants import EMPTY_STR 22 | 23 | 24 | class TestCreateRepositoryDict(unittest.TestCase): 25 | """Test the create_repository functionality to ensure it returns the 26 | propper type of repository and errors for unknown repository 27 | types. 28 | 29 | """ 30 | 31 | def setUp(self): 32 | """Common data needed for all tests in this class 33 | """ 34 | self._name = 'test_name' 35 | self._repo = {ExternalsDescription.PROTOCOL: None, 36 | ExternalsDescription.REPO_URL: 'junk_root', 37 | ExternalsDescription.TAG: 'junk_tag', 38 | ExternalsDescription.BRANCH: EMPTY_STR, 39 | ExternalsDescription.HASH: EMPTY_STR, 40 | ExternalsDescription.SPARSE: EMPTY_STR, } 41 | 42 | def test_create_repo_git(self): 43 | """Verify that several possible names for the 'git' protocol 44 | create git repository objects. 45 | 46 | """ 47 | protocols = ['git', 'GIT', 'Git', ] 48 | for protocol in protocols: 49 | self._repo[ExternalsDescription.PROTOCOL] = protocol 50 | repo = create_repository(self._name, self._repo) 51 | self.assertIsInstance(repo, GitRepository) 52 | 53 | def test_create_repo_svn(self): 54 | """Verify that several possible names for the 'svn' protocol 55 | create svn repository objects. 56 | """ 57 | protocols = ['svn', 'SVN', 'Svn', ] 58 | for protocol in protocols: 59 | self._repo[ExternalsDescription.PROTOCOL] = protocol 60 | repo = create_repository(self._name, self._repo) 61 | self.assertIsInstance(repo, SvnRepository) 62 | 63 | def test_create_repo_externals_only(self): 64 | """Verify that an externals only repo returns None. 65 | """ 66 | protocols = ['externals_only', ] 67 | for protocol in protocols: 68 | self._repo[ExternalsDescription.PROTOCOL] = protocol 69 | repo = create_repository(self._name, self._repo) 70 | self.assertEqual(None, repo) 71 | 72 | def test_create_repo_unsupported(self): 73 | """Verify that an unsupported protocol generates a runtime error. 74 | """ 75 | protocols = ['not_a_supported_protocol', ] 76 | for protocol in protocols: 77 | self._repo[ExternalsDescription.PROTOCOL] = protocol 78 | with self.assertRaises(RuntimeError): 79 | create_repository(self._name, self._repo) 80 | 81 | 82 | class TestRepository(unittest.TestCase): 83 | """Test the externals description processing used to create the Repository 84 | base class shared by protocol specific repository classes. 85 | 86 | """ 87 | 88 | def test_tag(self): 89 | """Test creation of a repository object with a tag 90 | """ 91 | name = 'test_repo' 92 | protocol = 'test_protocol' 93 | url = 'test_url' 94 | tag = 'test_tag' 95 | repo_info = {ExternalsDescription.PROTOCOL: protocol, 96 | ExternalsDescription.REPO_URL: url, 97 | ExternalsDescription.TAG: tag, 98 | ExternalsDescription.BRANCH: EMPTY_STR, 99 | ExternalsDescription.HASH: EMPTY_STR, 100 | ExternalsDescription.SPARSE: EMPTY_STR, } 101 | repo = Repository(name, repo_info) 102 | print(repo.__dict__) 103 | self.assertEqual(repo.tag(), tag) 104 | self.assertEqual(repo.url(), url) 105 | 106 | def test_branch(self): 107 | """Test creation of a repository object with a branch 108 | """ 109 | name = 'test_repo' 110 | protocol = 'test_protocol' 111 | url = 'test_url' 112 | branch = 'test_branch' 113 | repo_info = {ExternalsDescription.PROTOCOL: protocol, 114 | ExternalsDescription.REPO_URL: url, 115 | ExternalsDescription.BRANCH: branch, 116 | ExternalsDescription.TAG: EMPTY_STR, 117 | ExternalsDescription.HASH: EMPTY_STR, 118 | ExternalsDescription.SPARSE: EMPTY_STR, } 119 | repo = Repository(name, repo_info) 120 | print(repo.__dict__) 121 | self.assertEqual(repo.branch(), branch) 122 | self.assertEqual(repo.url(), url) 123 | 124 | def test_hash(self): 125 | """Test creation of a repository object with a hash 126 | """ 127 | name = 'test_repo' 128 | protocol = 'test_protocol' 129 | url = 'test_url' 130 | ref = 'deadc0de' 131 | sparse = EMPTY_STR 132 | repo_info = {ExternalsDescription.PROTOCOL: protocol, 133 | ExternalsDescription.REPO_URL: url, 134 | ExternalsDescription.BRANCH: EMPTY_STR, 135 | ExternalsDescription.TAG: EMPTY_STR, 136 | ExternalsDescription.HASH: ref, 137 | ExternalsDescription.SPARSE: sparse, } 138 | repo = Repository(name, repo_info) 139 | print(repo.__dict__) 140 | self.assertEqual(repo.hash(), ref) 141 | self.assertEqual(repo.url(), url) 142 | 143 | def test_tag_branch(self): 144 | """Test creation of a repository object with a tag and branch raises a 145 | runtimer error. 146 | 147 | """ 148 | name = 'test_repo' 149 | protocol = 'test_protocol' 150 | url = 'test_url' 151 | branch = 'test_branch' 152 | tag = 'test_tag' 153 | ref = EMPTY_STR 154 | sparse = EMPTY_STR 155 | repo_info = {ExternalsDescription.PROTOCOL: protocol, 156 | ExternalsDescription.REPO_URL: url, 157 | ExternalsDescription.BRANCH: branch, 158 | ExternalsDescription.TAG: tag, 159 | ExternalsDescription.HASH: ref, 160 | ExternalsDescription.SPARSE: sparse, } 161 | with self.assertRaises(RuntimeError): 162 | Repository(name, repo_info) 163 | 164 | def test_tag_branch_hash(self): 165 | """Test creation of a repository object with a tag, branch and hash raises a 166 | runtimer error. 167 | 168 | """ 169 | name = 'test_repo' 170 | protocol = 'test_protocol' 171 | url = 'test_url' 172 | branch = 'test_branch' 173 | tag = 'test_tag' 174 | ref = 'deadc0de' 175 | sparse = EMPTY_STR 176 | repo_info = {ExternalsDescription.PROTOCOL: protocol, 177 | ExternalsDescription.REPO_URL: url, 178 | ExternalsDescription.BRANCH: branch, 179 | ExternalsDescription.TAG: tag, 180 | ExternalsDescription.HASH: ref, 181 | ExternalsDescription.SPARSE: sparse, } 182 | with self.assertRaises(RuntimeError): 183 | Repository(name, repo_info) 184 | 185 | def test_no_tag_no_branch(self): 186 | """Test creation of a repository object without a tag or branch raises a 187 | runtimer error. 188 | 189 | """ 190 | name = 'test_repo' 191 | protocol = 'test_protocol' 192 | url = 'test_url' 193 | branch = EMPTY_STR 194 | tag = EMPTY_STR 195 | ref = EMPTY_STR 196 | sparse = EMPTY_STR 197 | repo_info = {ExternalsDescription.PROTOCOL: protocol, 198 | ExternalsDescription.REPO_URL: url, 199 | ExternalsDescription.BRANCH: branch, 200 | ExternalsDescription.TAG: tag, 201 | ExternalsDescription.HASH: ref, 202 | ExternalsDescription.SPARSE: sparse, } 203 | with self.assertRaises(RuntimeError): 204 | Repository(name, repo_info) 205 | 206 | 207 | if __name__ == '__main__': 208 | unittest.main() 209 | -------------------------------------------------------------------------------- /manage_externals/test/doc/develop.rst: -------------------------------------------------------------------------------- 1 | Developer Guidelines 2 | ==================== 3 | 4 | The manage externals utilities are a light weight replacement for svn 5 | externals that will work with git repositories pulling in a mixture of 6 | git and svn dependencies. 7 | 8 | Given an externals description and a working copy: 9 | 10 | * *checkout_externals* attempts to make the working copy agree with the 11 | externals description 12 | 13 | * *generate_externals* attempts to make the externals description agree 14 | with the working copy. 15 | 16 | For these operations utilities should: 17 | 18 | * operate consistently across git and svn 19 | 20 | * operate simply with minimal user complexity 21 | 22 | * robustly across a wide range of repository states 23 | 24 | * provide explicit error messages when a problem occurs 25 | 26 | * leave the working copy in a valid state 27 | 28 | The utilities in manage externals are **NOT** generic wrappers around 29 | revision control operations or a replacement for common tasks. Users 30 | are expected to: 31 | 32 | * create branches prior to starting development 33 | 34 | * add remotes and push changes 35 | 36 | * create tags 37 | 38 | * delete branches 39 | 40 | These types of tasks are often highly workflow dependent, e.g. branch 41 | naming conventions may vary between repositories, have the potential 42 | to destroy user data, introduce significant code complexit and 'edge 43 | cases' that are extremely difficult to detect and test, and often 44 | require subtle decision making, especially if a problem occurs. 45 | 46 | Users who want to automate these types are encouraged to create their 47 | own tools. The externals description files are explicitly versioned 48 | and the internal APIs are intended to be stable for these purposes. 49 | 50 | Core Design Principles 51 | ----------------------- 52 | 53 | 1. Users can, and are actively encouraged to, modify the externals 54 | directories using revision control outside of manage_externals 55 | tools. You can't make any assumptions about the state of the 56 | working copy. Examples: adding a remote, creating a branch, 57 | switching to a branch, deleting the directory entirely. 58 | 59 | 2. Give that the user can do anything, the manage externals library 60 | can not preserve state between calls. The only information it can 61 | rely on is what it expectes based on the content of the externals 62 | description file, and what the actual state of the directory tree 63 | is. 64 | 65 | 3. Do *not* do anything that will possibly destroy user data! 66 | 67 | a. Do not remove files from the file system. We are operating on 68 | user supplied input. If you don't call 'rm', you can't 69 | accidentally remove the user's data. Thinking of calling 70 | ``shutil.rmtree(user_input)``? What if the user accidentally 71 | specified user_input such that it resolves to their home 72 | directory.... Yeah. Don't go there. 73 | 74 | b. Rely on git and svn to do their job as much as possible. Don't 75 | duplicate functionality. Examples: 76 | 77 | i. We require the working copies to be 'clean' as reported by 78 | ``git status`` and ``svn status``. What if there are misc 79 | editor files floating around that prevent an update? Use the 80 | git and svn ignore functionality so they are not 81 | reported. Don't try to remove them from manage_externals or 82 | determine if they are 'safe' to ignore. 83 | 84 | ii. Do not use '--force'. Ever. This is a sign you are doing 85 | something dangerous, it may not be what the user 86 | wants. Remember, they are encouraged to modify their repo. 87 | 88 | 4. There are often multiple ways to obtain a particular piece of 89 | information from git. Scraping screen output is brittle and 90 | generally not considered a stable API across different versions of 91 | git. Given a choice between: 92 | 93 | a. a lower level git 'plumbing' command that processes a 94 | specific request and returns a sucess/failure status. 95 | 96 | b. high level git command that produces a bunch of output 97 | that must be processed. 98 | 99 | We always prefer the former. It almost always involves 100 | writing and maintaining less code and is more likely to be 101 | stable. 102 | 103 | 5. Backward compatibility is critical. We have *nested* 104 | repositories. They are trivially easy to change versions. They may 105 | have very different versions of the top level manage_externals. The 106 | ability to read and work with old model description files is 107 | critical to avoid problems for users. We also have automated tools 108 | (testdb) that must generate and read external description 109 | files. Backward compatibility will make staging changes vastly 110 | simpler. 111 | 112 | Model Users 113 | ----------- 114 | 115 | Consider the needs of the following model userswhen developing manage_externals: 116 | 117 | * Users who will checkout the code once, and never change versions. 118 | 119 | * Users who will checkout the code once, then work for several years, 120 | never updating. before trying to update or request integration. 121 | 122 | * Users develope code but do not use revision control beyond the 123 | initial checkout. If they have modified or untracked files in the 124 | repo, they may be irreplacable. Don't destroy user data. 125 | 126 | * Intermediate users who are working with multiple repos or branches 127 | on a regular basis. They may only use manage_externals weekly or 128 | monthly. Keep the user interface and documentation simple and 129 | explicit. The more command line options they have to remember or 130 | look up, the more frustrated they git. 131 | 132 | * Software engineers who use the tools multiple times a day. It should 133 | get out of their way. 134 | 135 | User Interface 136 | -------------- 137 | 138 | Basic operation for the most standard use cases should be kept as 139 | simple as possible. Many users will only rarely run the manage 140 | utilities. Even advanced users don't like reading a lot of help 141 | documentation or struggling to remember commands and piece together 142 | what they need to run. Having many command line options, even if not 143 | needed, is exteremly frustrating and overwhelming for most users. A few 144 | simple, explicitly named commands are better than a single command 145 | with many options. 146 | 147 | How will users get help if something goes wrong? This is a custom, 148 | one-off solution. Searching the internet for manage_externals, will 149 | only return the user doc for this project at best. There isn't likely 150 | to be a stackoverflow question or blog post where someone else already 151 | answered a user's question. And very few people outside this community 152 | will be able to provide help if something goes wrong. The sooner we 153 | kick users out of these utilities and into standard version control 154 | tools, the better off they are going to be if they run into a problem. 155 | 156 | Repositories 157 | ------------ 158 | 159 | There are three basic types of repositories that must be considered: 160 | 161 | * container repositories - repositories that are always top level 162 | repositories, and have a group of externals that must be managed. 163 | 164 | * simple repositories - repositories that are externals to another 165 | repository, and do not have any of their own externals that will be 166 | managed. 167 | 168 | * mixed use repositories - repositories that can act as a top level 169 | container repository or as an external to a top level 170 | container. They may also have their own sub-externals that are 171 | required. They may have different externals needs depening on 172 | whether they are top level or not. 173 | 174 | Repositories must be able to checkout and switch to both branches and 175 | tags. 176 | 177 | Development 178 | =========== 179 | 180 | The functionality to manage externals is broken into a library of core 181 | functionality and applications built with the library. 182 | 183 | The core library is called 'manic', pseduo-homophone of (man)age 184 | (ex)ternals that is: short, pronounceable and spell-checkable. It is 185 | also no more or less meaningful to an unfamiliar user than a random 186 | jumble of letters forming an acronym. 187 | 188 | The core architecture of manic is: 189 | 190 | * externals description - an abstract description on an external, 191 | including of how to obtain it, where to obtain it, where it goes in 192 | the working tree. 193 | 194 | * externals - the software object representing an external. 195 | 196 | * source trees - collection of externals 197 | 198 | * repository wrappers - object oriented wrappers around repository 199 | operations. So the higher level management of the soure tree and 200 | external does not have to be concerned with how a particular 201 | external is obtained and managed. 202 | 203 | -------------------------------------------------------------------------------- /manage_externals/test/test_sys_repository_git.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Tests of some of the functionality in repository_git.py that actually 4 | interacts with git repositories. 5 | 6 | We're calling these "system" tests because we expect them to be a lot 7 | slower than most of the unit tests. 8 | 9 | """ 10 | 11 | from __future__ import absolute_import 12 | from __future__ import unicode_literals 13 | from __future__ import print_function 14 | 15 | import os 16 | import shutil 17 | import tempfile 18 | import unittest 19 | 20 | from manic.repository_git import GitRepository 21 | from manic.externals_description import ExternalsDescription 22 | from manic.externals_description import ExternalsDescriptionDict 23 | from manic.utils import execute_subprocess 24 | 25 | # NOTE(wjs, 2018-04-09) I find a mix of camel case and underscores to be 26 | # more readable for unit test names, so I'm disabling pylint's naming 27 | # convention check 28 | # pylint: disable=C0103 29 | 30 | # Allow access to protected members 31 | # pylint: disable=W0212 32 | 33 | 34 | class GitTestCase(unittest.TestCase): 35 | """Adds some git-specific unit test functionality on top of TestCase""" 36 | 37 | def assertIsHash(self, maybe_hash): 38 | """Assert that the string given by maybe_hash really does look 39 | like a git hash. 40 | """ 41 | 42 | # Ensure it is non-empty 43 | self.assertTrue(maybe_hash, msg="maybe_hash is empty") 44 | 45 | # Ensure it has a single string 46 | self.assertEqual(1, len(maybe_hash.split()), 47 | msg="maybe_hash has multiple strings: {}".format(maybe_hash)) 48 | 49 | # Ensure that the only characters in the string are ones allowed 50 | # in hashes 51 | allowed_chars_set = set('0123456789abcdef') 52 | self.assertTrue(set(maybe_hash) <= allowed_chars_set, 53 | msg="maybe_hash has non-hash characters: {}".format(maybe_hash)) 54 | 55 | 56 | class TestGitTestCase(GitTestCase): 57 | """Tests GitTestCase""" 58 | 59 | def test_assertIsHash_true(self): 60 | """Ensure that assertIsHash passes for something that looks 61 | like a hash""" 62 | self.assertIsHash('abc123') 63 | 64 | def test_assertIsHash_empty(self): 65 | """Ensure that assertIsHash raises an AssertionError for an 66 | empty string""" 67 | with self.assertRaises(AssertionError): 68 | self.assertIsHash('') 69 | 70 | def test_assertIsHash_multipleStrings(self): 71 | """Ensure that assertIsHash raises an AssertionError when 72 | given multiple strings""" 73 | with self.assertRaises(AssertionError): 74 | self.assertIsHash('abc123 def456') 75 | 76 | def test_assertIsHash_badChar(self): 77 | """Ensure that assertIsHash raises an AssertionError when given a 78 | string that has a character that doesn't belong in a hash 79 | """ 80 | with self.assertRaises(AssertionError): 81 | self.assertIsHash('abc123g') 82 | 83 | 84 | class TestGitRepositoryGitCommands(GitTestCase): 85 | """Test some git commands in RepositoryGit 86 | 87 | It's silly that we need to create a repository in order to test 88 | these git commands. Much or all of the git functionality that is 89 | currently in repository_git.py should eventually be moved to a 90 | separate module that is solely responsible for wrapping git 91 | commands; that would allow us to test it independently of this 92 | repository class. 93 | """ 94 | 95 | # ======================================================================== 96 | # Test helper functions 97 | # ======================================================================== 98 | 99 | def setUp(self): 100 | # directory we want to return to after the test system and 101 | # checkout_externals are done cd'ing all over the place. 102 | self._return_dir = os.getcwd() 103 | 104 | self._tmpdir = tempfile.mkdtemp() 105 | os.chdir(self._tmpdir) 106 | 107 | self._name = 'component' 108 | rdata = {ExternalsDescription.PROTOCOL: 'git', 109 | ExternalsDescription.REPO_URL: 110 | '/path/to/local/repo', 111 | ExternalsDescription.TAG: 112 | 'tag1', 113 | } 114 | 115 | data = {self._name: 116 | { 117 | ExternalsDescription.REQUIRED: False, 118 | ExternalsDescription.PATH: 'junk', 119 | ExternalsDescription.EXTERNALS: '', 120 | ExternalsDescription.REPO: rdata, 121 | }, 122 | } 123 | model = ExternalsDescriptionDict(data) 124 | repo = model[self._name][ExternalsDescription.REPO] 125 | self._repo = GitRepository('test', repo) 126 | 127 | def tearDown(self): 128 | # return to our common starting point 129 | os.chdir(self._return_dir) 130 | 131 | shutil.rmtree(self._tmpdir, ignore_errors=True) 132 | 133 | @staticmethod 134 | def make_git_repo(): 135 | """Turn the current directory into an empty git repository""" 136 | execute_subprocess(['git', 'init']) 137 | 138 | @staticmethod 139 | def add_git_commit(): 140 | """Add a git commit in the current directory""" 141 | with open('README', 'a') as myfile: 142 | myfile.write('more info') 143 | execute_subprocess(['git', 'add', 'README']) 144 | execute_subprocess(['git', 'commit', '-m', 'my commit message']) 145 | 146 | @staticmethod 147 | def checkout_git_branch(branchname): 148 | """Checkout a new branch in the current directory""" 149 | execute_subprocess(['git', 'checkout', '-b', branchname]) 150 | 151 | @staticmethod 152 | def make_git_tag(tagname): 153 | """Make a lightweight tag at the current commit""" 154 | execute_subprocess(['git', 'tag', '-m', 'making a tag', tagname]) 155 | 156 | @staticmethod 157 | def checkout_ref(refname): 158 | """Checkout the given refname in the current directory""" 159 | execute_subprocess(['git', 'checkout', refname]) 160 | 161 | # ======================================================================== 162 | # Begin actual tests 163 | # ======================================================================== 164 | 165 | def test_currentHash_returnsHash(self): 166 | """Ensure that the _git_current_hash function returns a hash""" 167 | self.make_git_repo() 168 | self.add_git_commit() 169 | hash_found, myhash = self._repo._git_current_hash() 170 | self.assertTrue(hash_found) 171 | self.assertIsHash(myhash) 172 | 173 | def test_currentHash_outsideGitRepo(self): 174 | """Ensure that the _git_current_hash function returns False when 175 | outside a git repository""" 176 | hash_found, myhash = self._repo._git_current_hash() 177 | self.assertFalse(hash_found) 178 | self.assertEqual('', myhash) 179 | 180 | def test_currentBranch_onBranch(self): 181 | """Ensure that the _git_current_branch function returns the name 182 | of the branch""" 183 | self.make_git_repo() 184 | self.add_git_commit() 185 | self.checkout_git_branch('foo') 186 | branch_found, mybranch = self._repo._git_current_branch() 187 | self.assertTrue(branch_found) 188 | self.assertEqual('foo', mybranch) 189 | 190 | def test_currentBranch_notOnBranch(self): 191 | """Ensure that the _git_current_branch function returns False 192 | when not on a branch""" 193 | self.make_git_repo() 194 | self.add_git_commit() 195 | self.make_git_tag('mytag') 196 | self.checkout_ref('mytag') 197 | branch_found, mybranch = self._repo._git_current_branch() 198 | self.assertFalse(branch_found) 199 | self.assertEqual('', mybranch) 200 | 201 | def test_currentBranch_outsideGitRepo(self): 202 | """Ensure that the _git_current_branch function returns False 203 | when outside a git repository""" 204 | branch_found, mybranch = self._repo._git_current_branch() 205 | self.assertFalse(branch_found) 206 | self.assertEqual('', mybranch) 207 | 208 | def test_currentTag_onTag(self): 209 | """Ensure that the _git_current_tag function returns the name of 210 | the tag""" 211 | self.make_git_repo() 212 | self.add_git_commit() 213 | self.make_git_tag('some_tag') 214 | tag_found, mytag = self._repo._git_current_tag() 215 | self.assertTrue(tag_found) 216 | self.assertEqual('some_tag', mytag) 217 | 218 | def test_currentTag_notOnTag(self): 219 | """Ensure tha the _git_current_tag function returns False when 220 | not on a tag""" 221 | self.make_git_repo() 222 | self.add_git_commit() 223 | self.make_git_tag('some_tag') 224 | self.add_git_commit() 225 | tag_found, mytag = self._repo._git_current_tag() 226 | self.assertFalse(tag_found) 227 | self.assertEqual('', mytag) 228 | 229 | def test_currentTag_outsideGitRepo(self): 230 | """Ensure that the _git_current_tag function returns False when 231 | outside a git repository""" 232 | tag_found, mytag = self._repo._git_current_tag() 233 | self.assertFalse(tag_found) 234 | self.assertEqual('', mytag) 235 | 236 | 237 | if __name__ == '__main__': 238 | unittest.main() 239 | -------------------------------------------------------------------------------- /manage_externals/README.md: -------------------------------------------------------------------------------- 1 | -- AUTOMATICALLY GENERATED FILE. DO NOT EDIT -- 2 | 3 | [![Build Status](https://travis-ci.org/ESMCI/manage_externals.svg?branch=master)](https://travis-ci.org/ESMCI/manage_externals)[![Coverage Status](https://coveralls.io/repos/github/ESMCI/manage_externals/badge.svg?branch=master)](https://coveralls.io/github/ESMCI/manage_externals?branch=master) 4 | ``` 5 | usage: checkout_externals [-h] [-e [EXTERNALS]] [-o] [-S] [-v] [--backtrace] 6 | [-d] [--no-logging] 7 | 8 | checkout_externals manages checking out groups of externals from revision 9 | control based on a externals description file. By default only the 10 | required externals are checkout out. 11 | 12 | Operations performed by manage_externals utilities are explicit and 13 | data driven. checkout_externals will always make the working copy *exactly* 14 | match what is in the externals file when modifying the working copy of 15 | a repository. 16 | 17 | If checkout_externals isn't doing what you expected, double check the contents 18 | of the externals description file. 19 | 20 | Running checkout_externals without the '--status' option will always attempt to 21 | synchronize the working copy to exactly match the externals description. 22 | 23 | optional arguments: 24 | -h, --help show this help message and exit 25 | -e [EXTERNALS], --externals [EXTERNALS] 26 | The externals description filename. Default: 27 | Externals.cfg. 28 | -o, --optional By default only the required externals are checked 29 | out. This flag will also checkout the optional 30 | externals. 31 | -S, --status Output status of the repositories managed by 32 | checkout_externals. By default only summary 33 | information is provided. Use verbose output to see 34 | details. 35 | -v, --verbose Output additional information to the screen and log 36 | file. This flag can be used up to two times, 37 | increasing the verbosity level each time. 38 | --backtrace DEVELOPER: show exception backtraces as extra 39 | debugging output 40 | -d, --debug DEVELOPER: output additional debugging information to 41 | the screen and log file. 42 | --no-logging DEVELOPER: disable logging. 43 | 44 | ``` 45 | NOTE: checkout_externals *MUST* be run from the root of the source tree it 46 | is managing. For example, if you cloned a repository with: 47 | 48 | $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev 49 | 50 | Then the root of the source tree is /path/to/some-project-dev. If you 51 | obtained a sub-project via a checkout of another project: 52 | 53 | $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev 54 | 55 | and you need to checkout the sub-project externals, then the root of the 56 | source tree is /path/to/some-project-dev. Do *NOT* run checkout_externals 57 | from within /path/to/some-project-dev/sub-project 58 | 59 | The root of the source tree will be referred to as `${SRC_ROOT}` below. 60 | 61 | # Supported workflows 62 | 63 | * Checkout all required components from the default externals 64 | description file: 65 | 66 | $ cd ${SRC_ROOT} 67 | $ ./manage_externals/checkout_externals 68 | 69 | * To update all required components to the current values in the 70 | externals description file, re-run checkout_externals: 71 | 72 | $ cd ${SRC_ROOT} 73 | $ ./manage_externals/checkout_externals 74 | 75 | If there are *any* modifications to *any* working copy according 76 | to the git or svn 'status' command, checkout_externals 77 | will not update any external repositories. Modifications 78 | include: modified files, added files, removed files, or missing 79 | files. 80 | 81 | To avoid this safety check, edit the externals description file 82 | and comment out the modified external block. 83 | 84 | * Checkout all required components from a user specified externals 85 | description file: 86 | 87 | $ cd ${SRC_ROOT} 88 | $ ./manage_externals/checkout_externals --externals my-externals.cfg 89 | 90 | * Status summary of the repositories managed by checkout_externals: 91 | 92 | $ cd ${SRC_ROOT} 93 | $ ./manage_externals/checkout_externals --status 94 | 95 | ./cime 96 | s ./components/cism 97 | ./components/mosart 98 | e-o ./components/rtm 99 | M ./src/fates 100 | e-o ./tools/PTCLM 101 | 102 | where: 103 | * column one indicates the status of the repository in relation 104 | to the externals description file. 105 | * column two indicates whether the working copy has modified files. 106 | * column three shows how the repository is managed, optional or required 107 | 108 | Column one will be one of these values: 109 | * s : out-of-sync : repository is checked out at a different commit 110 | compared with the externals description 111 | * e : empty : directory does not exist - checkout_externals has not been run 112 | * ? : unknown : directory exists but .git or .svn directories are missing 113 | 114 | Column two will be one of these values: 115 | * M : Modified : modified, added, deleted or missing files 116 | * : blank / space : clean 117 | * - : dash : no meaningful state, for empty repositories 118 | 119 | Column three will be one of these values: 120 | * o : optional : optionally repository 121 | * : blank / space : required repository 122 | 123 | * Detailed git or svn status of the repositories managed by checkout_externals: 124 | 125 | $ cd ${SRC_ROOT} 126 | $ ./manage_externals/checkout_externals --status --verbose 127 | 128 | # Externals description file 129 | 130 | The externals description contains a list of the external 131 | repositories that are used and their version control locations. The 132 | file format is the standard ini/cfg configuration file format. Each 133 | external is defined by a section containing the component name in 134 | square brackets: 135 | 136 | * name (string) : component name, e.g. [cime], [cism], etc. 137 | 138 | Each section has the following keyword-value pairs: 139 | 140 | * required (boolean) : whether the component is a required checkout, 141 | 'true' or 'false'. 142 | 143 | * local_path (string) : component path *relative* to where 144 | checkout_externals is called. 145 | 146 | * protoctol (string) : version control protocol that is used to 147 | manage the component. Valid values are 'git', 'svn', 148 | 'externals_only'. 149 | 150 | Switching an external between different protocols is not 151 | supported, e.g. from svn to git. To switch protocols, you need to 152 | manually move the old working copy to a new location. 153 | 154 | Note: 'externals_only' will only process the external's own 155 | external description file without trying to manage a repository 156 | for the component. This is used for retreiving externals for 157 | standalone components like cam and clm. If the source root of the 158 | externals_only component is the same as the main source root, then 159 | the local path must be set to '.', the unix current working 160 | directory, e. g. 'local_path = .' 161 | 162 | * repo_url (string) : URL for the repository location, examples: 163 | * https://svn-ccsm-models.cgd.ucar.edu/glc 164 | * git@github.com:esmci/cime.git 165 | * /path/to/local/repository 166 | * . 167 | 168 | NOTE: To operate on only the local clone and and ignore remote 169 | repositories, set the url to '.' (the unix current path), 170 | i.e. 'repo_url = .' . This can be used to checkout a local branch 171 | instead of the upstream branch. 172 | 173 | If a repo url is determined to be a local path (not a network url) 174 | then user expansion, e.g. ~/, and environment variable expansion, 175 | e.g. $HOME or $REPO_ROOT, will be performed. 176 | 177 | Relative paths are difficult to get correct, especially for mixed 178 | use repos. It is advised that local paths expand to absolute paths. 179 | If relative paths are used, they should be relative to one level 180 | above local_path. If local path is 'src/foo', the the relative url 181 | should be relative to 'src'. 182 | 183 | * tag (string) : tag to checkout 184 | 185 | * hash (string) : the git hash to checkout. Only applies to git 186 | repositories. 187 | 188 | * branch (string) : branch to checkout from the specified 189 | repository. Specifying a branch on a remote repository means that 190 | checkout_externals will checkout the version of the branch in the remote, 191 | not the the version in the local repository (if it exists). 192 | 193 | Note: one and only one of tag, branch hash must be supplied. 194 | 195 | * externals (string) : used to make manage_externals aware of 196 | sub-externals required by an external. This is a relative path to 197 | the external's root directory. For example, the main externals 198 | description has an external checkout out at 'src/useful_library'. 199 | useful_library requires additional externals to be complete. 200 | Those additional externals are managed from the source root by the 201 | externals description file pointed 'useful_library/sub-xternals.cfg', 202 | Then the main 'externals' field in the top level repo should point to 203 | 'sub-externals.cfg'. 204 | 205 | * from_submodule (True / False) : used to pull the repo_url, local_path, 206 | and hash properties for this external from the .gitmodules file in 207 | this repository. Note that the section name (the entry in square 208 | brackets) must match the name in the .gitmodules file. 209 | If from_submodule is True, the protocol must be git and no repo_url, 210 | local_path, hash, branch, or tag entries are allowed. 211 | Default: False 212 | 213 | * sparse (string) : used to control a sparse checkout. This optional 214 | entry should point to a filename (path relative to local_path) that 215 | contains instructions on which repository paths to include (or 216 | exclude) from the working tree. 217 | See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree 218 | Default: sparse checkout is disabled 219 | 220 | * Lines begining with '#' or ';' are comments and will be ignored. 221 | 222 | # Obtaining this tool, reporting issues, etc. 223 | 224 | The master repository for manage_externals is 225 | https://github.com/ESMCI/manage_externals. Any issues with this tool 226 | should be reported there. 227 | -------------------------------------------------------------------------------- /manage_externals/test/test_unit_externals_status.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Unit test driver for the manic external status reporting module. 4 | 5 | Note: this script assumes the path to the manic package is already in 6 | the python path. 7 | 8 | """ 9 | 10 | from __future__ import absolute_import 11 | from __future__ import unicode_literals 12 | from __future__ import print_function 13 | 14 | import unittest 15 | 16 | from manic.externals_status import ExternalStatus 17 | 18 | 19 | class TestStatusObject(unittest.TestCase): 20 | """Verify that the Status object behaives as expected. 21 | """ 22 | 23 | def test_exists_empty_all(self): 24 | """If the repository sync-state is empty (doesn't exist), and there is no 25 | clean state, then it is considered not to exist. 26 | 27 | """ 28 | stat = ExternalStatus() 29 | stat.sync_state = ExternalStatus.EMPTY 30 | stat.clean_state = ExternalStatus.DEFAULT 31 | exists = stat.exists() 32 | self.assertFalse(exists) 33 | 34 | stat.clean_state = ExternalStatus.EMPTY 35 | exists = stat.exists() 36 | self.assertFalse(exists) 37 | 38 | stat.clean_state = ExternalStatus.UNKNOWN 39 | exists = stat.exists() 40 | self.assertFalse(exists) 41 | 42 | # this state represtens an internal logic error in how the 43 | # repo status was determined. 44 | stat.clean_state = ExternalStatus.STATUS_OK 45 | exists = stat.exists() 46 | self.assertTrue(exists) 47 | 48 | # this state represtens an internal logic error in how the 49 | # repo status was determined. 50 | stat.clean_state = ExternalStatus.DIRTY 51 | exists = stat.exists() 52 | self.assertTrue(exists) 53 | 54 | def test_exists_default_all(self): 55 | """If the repository sync-state is default, then it is considered to exist 56 | regardless of clean state. 57 | 58 | """ 59 | stat = ExternalStatus() 60 | stat.sync_state = ExternalStatus.DEFAULT 61 | stat.clean_state = ExternalStatus.DEFAULT 62 | exists = stat.exists() 63 | self.assertTrue(exists) 64 | 65 | stat.clean_state = ExternalStatus.EMPTY 66 | exists = stat.exists() 67 | self.assertTrue(exists) 68 | 69 | stat.clean_state = ExternalStatus.UNKNOWN 70 | exists = stat.exists() 71 | self.assertTrue(exists) 72 | 73 | stat.clean_state = ExternalStatus.STATUS_OK 74 | exists = stat.exists() 75 | self.assertTrue(exists) 76 | 77 | stat.clean_state = ExternalStatus.DIRTY 78 | exists = stat.exists() 79 | self.assertTrue(exists) 80 | 81 | def test_exists_unknown_all(self): 82 | """If the repository sync-state is unknown, then it is considered to exist 83 | regardless of clean state. 84 | 85 | """ 86 | stat = ExternalStatus() 87 | stat.sync_state = ExternalStatus.UNKNOWN 88 | stat.clean_state = ExternalStatus.DEFAULT 89 | exists = stat.exists() 90 | self.assertTrue(exists) 91 | 92 | stat.clean_state = ExternalStatus.EMPTY 93 | exists = stat.exists() 94 | self.assertTrue(exists) 95 | 96 | stat.clean_state = ExternalStatus.UNKNOWN 97 | exists = stat.exists() 98 | self.assertTrue(exists) 99 | 100 | stat.clean_state = ExternalStatus.STATUS_OK 101 | exists = stat.exists() 102 | self.assertTrue(exists) 103 | 104 | stat.clean_state = ExternalStatus.DIRTY 105 | exists = stat.exists() 106 | self.assertTrue(exists) 107 | 108 | def test_exists_modified_all(self): 109 | """If the repository sync-state is modified, then it is considered to exist 110 | regardless of clean state. 111 | 112 | """ 113 | stat = ExternalStatus() 114 | stat.sync_state = ExternalStatus.MODEL_MODIFIED 115 | stat.clean_state = ExternalStatus.DEFAULT 116 | exists = stat.exists() 117 | self.assertTrue(exists) 118 | 119 | stat.clean_state = ExternalStatus.EMPTY 120 | exists = stat.exists() 121 | self.assertTrue(exists) 122 | 123 | stat.clean_state = ExternalStatus.UNKNOWN 124 | exists = stat.exists() 125 | self.assertTrue(exists) 126 | 127 | stat.clean_state = ExternalStatus.STATUS_OK 128 | exists = stat.exists() 129 | self.assertTrue(exists) 130 | 131 | stat.clean_state = ExternalStatus.DIRTY 132 | exists = stat.exists() 133 | self.assertTrue(exists) 134 | 135 | def test_exists_ok_all(self): 136 | """If the repository sync-state is ok, then it is considered to exist 137 | regardless of clean state. 138 | 139 | """ 140 | stat = ExternalStatus() 141 | stat.sync_state = ExternalStatus.STATUS_OK 142 | stat.clean_state = ExternalStatus.DEFAULT 143 | exists = stat.exists() 144 | self.assertTrue(exists) 145 | 146 | stat.clean_state = ExternalStatus.EMPTY 147 | exists = stat.exists() 148 | self.assertTrue(exists) 149 | 150 | stat.clean_state = ExternalStatus.UNKNOWN 151 | exists = stat.exists() 152 | self.assertTrue(exists) 153 | 154 | stat.clean_state = ExternalStatus.STATUS_OK 155 | exists = stat.exists() 156 | self.assertTrue(exists) 157 | 158 | stat.clean_state = ExternalStatus.DIRTY 159 | exists = stat.exists() 160 | self.assertTrue(exists) 161 | 162 | def test_update_ok_all(self): 163 | """If the repository in-sync is ok, then it is safe to 164 | update only if clean state is ok 165 | 166 | """ 167 | stat = ExternalStatus() 168 | stat.sync_state = ExternalStatus.STATUS_OK 169 | stat.clean_state = ExternalStatus.DEFAULT 170 | safe_to_update = stat.safe_to_update() 171 | self.assertFalse(safe_to_update) 172 | 173 | stat.clean_state = ExternalStatus.EMPTY 174 | safe_to_update = stat.safe_to_update() 175 | self.assertFalse(safe_to_update) 176 | 177 | stat.clean_state = ExternalStatus.UNKNOWN 178 | safe_to_update = stat.safe_to_update() 179 | self.assertFalse(safe_to_update) 180 | 181 | stat.clean_state = ExternalStatus.STATUS_OK 182 | safe_to_update = stat.safe_to_update() 183 | self.assertTrue(safe_to_update) 184 | 185 | stat.clean_state = ExternalStatus.DIRTY 186 | safe_to_update = stat.safe_to_update() 187 | self.assertFalse(safe_to_update) 188 | 189 | def test_update_modified_all(self): 190 | """If the repository in-sync is modified, then it is safe to 191 | update only if clean state is ok 192 | 193 | """ 194 | stat = ExternalStatus() 195 | stat.sync_state = ExternalStatus.MODEL_MODIFIED 196 | stat.clean_state = ExternalStatus.DEFAULT 197 | safe_to_update = stat.safe_to_update() 198 | self.assertFalse(safe_to_update) 199 | 200 | stat.clean_state = ExternalStatus.EMPTY 201 | safe_to_update = stat.safe_to_update() 202 | self.assertFalse(safe_to_update) 203 | 204 | stat.clean_state = ExternalStatus.UNKNOWN 205 | safe_to_update = stat.safe_to_update() 206 | self.assertFalse(safe_to_update) 207 | 208 | stat.clean_state = ExternalStatus.STATUS_OK 209 | safe_to_update = stat.safe_to_update() 210 | self.assertTrue(safe_to_update) 211 | 212 | stat.clean_state = ExternalStatus.DIRTY 213 | safe_to_update = stat.safe_to_update() 214 | self.assertFalse(safe_to_update) 215 | 216 | def test_update_unknown_all(self): 217 | """If the repository in-sync is unknown, then it is not safe to 218 | update, regardless of the clean state. 219 | 220 | """ 221 | stat = ExternalStatus() 222 | stat.sync_state = ExternalStatus.UNKNOWN 223 | stat.clean_state = ExternalStatus.DEFAULT 224 | safe_to_update = stat.safe_to_update() 225 | self.assertFalse(safe_to_update) 226 | 227 | stat.clean_state = ExternalStatus.EMPTY 228 | safe_to_update = stat.safe_to_update() 229 | self.assertFalse(safe_to_update) 230 | 231 | stat.clean_state = ExternalStatus.UNKNOWN 232 | safe_to_update = stat.safe_to_update() 233 | self.assertFalse(safe_to_update) 234 | 235 | stat.clean_state = ExternalStatus.STATUS_OK 236 | safe_to_update = stat.safe_to_update() 237 | self.assertFalse(safe_to_update) 238 | 239 | stat.clean_state = ExternalStatus.DIRTY 240 | safe_to_update = stat.safe_to_update() 241 | self.assertFalse(safe_to_update) 242 | 243 | def test_update_default_all(self): 244 | """If the repository in-sync is default, then it is not safe to 245 | update, regardless of the clean state. 246 | 247 | """ 248 | stat = ExternalStatus() 249 | stat.sync_state = ExternalStatus.UNKNOWN 250 | stat.clean_state = ExternalStatus.DEFAULT 251 | safe_to_update = stat.safe_to_update() 252 | self.assertFalse(safe_to_update) 253 | 254 | stat.clean_state = ExternalStatus.EMPTY 255 | safe_to_update = stat.safe_to_update() 256 | self.assertFalse(safe_to_update) 257 | 258 | stat.clean_state = ExternalStatus.UNKNOWN 259 | safe_to_update = stat.safe_to_update() 260 | self.assertFalse(safe_to_update) 261 | 262 | stat.clean_state = ExternalStatus.STATUS_OK 263 | safe_to_update = stat.safe_to_update() 264 | self.assertFalse(safe_to_update) 265 | 266 | stat.clean_state = ExternalStatus.DIRTY 267 | safe_to_update = stat.safe_to_update() 268 | self.assertFalse(safe_to_update) 269 | 270 | def test_update_empty_all(self): 271 | """If the repository in-sync is empty, then it is not safe to 272 | update, regardless of the clean state. 273 | 274 | """ 275 | stat = ExternalStatus() 276 | stat.sync_state = ExternalStatus.UNKNOWN 277 | stat.clean_state = ExternalStatus.DEFAULT 278 | safe_to_update = stat.safe_to_update() 279 | self.assertFalse(safe_to_update) 280 | 281 | stat.clean_state = ExternalStatus.EMPTY 282 | safe_to_update = stat.safe_to_update() 283 | self.assertFalse(safe_to_update) 284 | 285 | stat.clean_state = ExternalStatus.UNKNOWN 286 | safe_to_update = stat.safe_to_update() 287 | self.assertFalse(safe_to_update) 288 | 289 | stat.clean_state = ExternalStatus.STATUS_OK 290 | safe_to_update = stat.safe_to_update() 291 | self.assertFalse(safe_to_update) 292 | 293 | stat.clean_state = ExternalStatus.DIRTY 294 | safe_to_update = stat.safe_to_update() 295 | self.assertFalse(safe_to_update) 296 | 297 | 298 | if __name__ == '__main__': 299 | unittest.main() 300 | -------------------------------------------------------------------------------- /manage_externals/manic/repository_svn.py: -------------------------------------------------------------------------------- 1 | """Class for interacting with svn repositories 2 | """ 3 | 4 | from __future__ import absolute_import 5 | from __future__ import unicode_literals 6 | from __future__ import print_function 7 | 8 | import os 9 | import re 10 | import xml.etree.ElementTree as ET 11 | 12 | from .global_constants import EMPTY_STR, VERBOSITY_VERBOSE 13 | from .repository import Repository 14 | from .externals_status import ExternalStatus 15 | from .utils import fatal_error, indent_string, printlog 16 | from .utils import execute_subprocess 17 | 18 | 19 | class SvnRepository(Repository): 20 | """ 21 | Class to represent and operate on a repository description. 22 | 23 | For testing purpose, all system calls to svn should: 24 | 25 | * be isolated in separate functions with no application logic 26 | * of the form: 27 | - cmd = ['svn', ...] 28 | - value = execute_subprocess(cmd, output_to_caller={T|F}, 29 | status_to_caller={T|F}) 30 | - return value 31 | * be static methods (not rely on self) 32 | * name as _svn_subcommand_args(user_args) 33 | 34 | This convention allows easy unit testing of the repository logic 35 | by mocking the specific calls to return predefined results. 36 | 37 | """ 38 | RE_URLLINE = re.compile(r'^URL:') 39 | 40 | def __init__(self, component_name, repo, ignore_ancestry=False): 41 | """ 42 | Parse repo (a XML element). 43 | """ 44 | Repository.__init__(self, component_name, repo) 45 | self._ignore_ancestry = ignore_ancestry 46 | if self._branch: 47 | self._url = os.path.join(self._url, self._branch) 48 | elif self._tag: 49 | self._url = os.path.join(self._url, self._tag) 50 | else: 51 | msg = "DEV_ERROR in svn repository. Shouldn't be here!" 52 | fatal_error(msg) 53 | 54 | # ---------------------------------------------------------------- 55 | # 56 | # Public API, defined by Repository 57 | # 58 | # ---------------------------------------------------------------- 59 | def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument 60 | """Checkout or update the working copy 61 | 62 | If the repo destination directory exists, switch the sandbox to 63 | match the externals description. 64 | 65 | If the repo destination directory does not exist, checkout the 66 | correct branch or tag. 67 | NB: is include as an argument for compatibility with 68 | git functionality (repository_git.py) 69 | 70 | """ 71 | repo_dir_path = os.path.join(base_dir_path, repo_dir_name) 72 | if os.path.exists(repo_dir_path): 73 | cwd = os.getcwd() 74 | os.chdir(repo_dir_path) 75 | self._svn_switch(self._url, self._ignore_ancestry, verbosity) 76 | # svn switch can lead to a conflict state, but it gives a 77 | # return code of 0. So now we need to make sure that we're 78 | # in a clean (non-conflict) state. 79 | self._abort_if_dirty(repo_dir_path, 80 | "Expected clean state following switch") 81 | os.chdir(cwd) 82 | else: 83 | self._svn_checkout(self._url, repo_dir_path, verbosity) 84 | 85 | def status(self, stat, repo_dir_path): 86 | """ 87 | Check and report the status of the repository 88 | """ 89 | self._check_sync(stat, repo_dir_path) 90 | if os.path.exists(repo_dir_path): 91 | self._status_summary(stat, repo_dir_path) 92 | 93 | # ---------------------------------------------------------------- 94 | # 95 | # Internal work functions 96 | # 97 | # ---------------------------------------------------------------- 98 | def _check_sync(self, stat, repo_dir_path): 99 | """Check to see if repository directory exists and is at the expected 100 | url. Return: status object 101 | 102 | """ 103 | if not os.path.exists(repo_dir_path): 104 | # NOTE(bja, 2017-10) this state should have been handled by 105 | # the source object and we never get here! 106 | stat.sync_state = ExternalStatus.STATUS_ERROR 107 | else: 108 | svn_output = self._svn_info(repo_dir_path) 109 | if not svn_output: 110 | # directory exists, but info returned nothing. .svn 111 | # directory removed or incomplete checkout? 112 | stat.sync_state = ExternalStatus.UNKNOWN 113 | else: 114 | stat.sync_state, stat.current_version = \ 115 | self._check_url(svn_output, self._url) 116 | stat.expected_version = '/'.join(self._url.split('/')[3:]) 117 | 118 | def _abort_if_dirty(self, repo_dir_path, message): 119 | """Check if the repo is in a dirty state; if so, abort with a 120 | helpful message. 121 | 122 | """ 123 | 124 | stat = ExternalStatus() 125 | self._status_summary(stat, repo_dir_path) 126 | if stat.clean_state != ExternalStatus.STATUS_OK: 127 | status = self._svn_status_verbose(repo_dir_path) 128 | status = indent_string(status, 4) 129 | errmsg = """In directory 130 | {cwd} 131 | 132 | svn status now shows: 133 | {status} 134 | 135 | ERROR: {message} 136 | 137 | One possible cause of this problem is that there may have been untracked 138 | files in your working directory that had the same name as tracked files 139 | in the new revision. 140 | 141 | To recover: Clean up the above directory (resolving conflicts, etc.), 142 | then rerun checkout_externals. 143 | """.format(cwd=repo_dir_path, message=message, status=status) 144 | 145 | fatal_error(errmsg) 146 | 147 | @staticmethod 148 | def _check_url(svn_output, expected_url): 149 | """Determine the svn url from svn info output and return whether it 150 | matches the expected value. 151 | 152 | """ 153 | url = None 154 | for line in svn_output.splitlines(): 155 | if SvnRepository.RE_URLLINE.match(line): 156 | url = line.split(': ')[1].strip() 157 | break 158 | if not url: 159 | status = ExternalStatus.UNKNOWN 160 | elif url == expected_url: 161 | status = ExternalStatus.STATUS_OK 162 | else: 163 | status = ExternalStatus.MODEL_MODIFIED 164 | 165 | if url: 166 | current_version = '/'.join(url.split('/')[3:]) 167 | else: 168 | current_version = EMPTY_STR 169 | 170 | return status, current_version 171 | 172 | def _status_summary(self, stat, repo_dir_path): 173 | """Report whether the svn repository is in-sync with the model 174 | description and whether the sandbox is clean or dirty. 175 | 176 | """ 177 | svn_output = self._svn_status_xml(repo_dir_path) 178 | is_dirty = self.xml_status_is_dirty(svn_output) 179 | if is_dirty: 180 | stat.clean_state = ExternalStatus.DIRTY 181 | else: 182 | stat.clean_state = ExternalStatus.STATUS_OK 183 | 184 | # Now save the verbose status output incase the user wants to 185 | # see it. 186 | stat.status_output = self._svn_status_verbose(repo_dir_path) 187 | 188 | @staticmethod 189 | def xml_status_is_dirty(svn_output): 190 | """Parse svn status xml output and determine if the working copy is 191 | clean or dirty. Dirty is defined as: 192 | 193 | * modified files 194 | * added files 195 | * deleted files 196 | * missing files 197 | 198 | Unversioned files do not affect the clean/dirty status. 199 | 200 | 'external' is also an acceptable state 201 | 202 | """ 203 | # pylint: disable=invalid-name 204 | SVN_EXTERNAL = 'external' 205 | SVN_UNVERSIONED = 'unversioned' 206 | # pylint: enable=invalid-name 207 | 208 | is_dirty = False 209 | try: 210 | xml_status = ET.fromstring(svn_output) 211 | except BaseException: 212 | fatal_error( 213 | "SVN returned invalid XML message {}".format(svn_output)) 214 | xml_target = xml_status.find('./target') 215 | entries = xml_target.findall('./entry') 216 | for entry in entries: 217 | status = entry.find('./wc-status') 218 | item = status.get('item') 219 | if item == SVN_EXTERNAL: 220 | continue 221 | if item == SVN_UNVERSIONED: 222 | continue 223 | is_dirty = True 224 | break 225 | return is_dirty 226 | 227 | # ---------------------------------------------------------------- 228 | # 229 | # system call to svn for information gathering 230 | # 231 | # ---------------------------------------------------------------- 232 | @staticmethod 233 | def _svn_info(repo_dir_path): 234 | """Return results of svn info command 235 | """ 236 | cmd = ['svn', 'info', repo_dir_path] 237 | output = execute_subprocess(cmd, output_to_caller=True) 238 | return output 239 | 240 | @staticmethod 241 | def _svn_status_verbose(repo_dir_path): 242 | """capture the full svn status output 243 | """ 244 | cmd = ['svn', 'status', repo_dir_path] 245 | svn_output = execute_subprocess(cmd, output_to_caller=True) 246 | return svn_output 247 | 248 | @staticmethod 249 | def _svn_status_xml(repo_dir_path): 250 | """ 251 | Get status of the subversion sandbox in repo_dir 252 | """ 253 | cmd = ['svn', 'status', '--xml', repo_dir_path] 254 | svn_output = execute_subprocess(cmd, output_to_caller=True) 255 | return svn_output 256 | 257 | # ---------------------------------------------------------------- 258 | # 259 | # system call to svn for sideffects modifying the working tree 260 | # 261 | # ---------------------------------------------------------------- 262 | @staticmethod 263 | def _svn_checkout(url, repo_dir_path, verbosity): 264 | """ 265 | Checkout a subversion repository (repo_url) to checkout_dir. 266 | """ 267 | cmd = ['svn', 'checkout', '--quiet', url, repo_dir_path] 268 | if verbosity >= VERBOSITY_VERBOSE: 269 | printlog(' {0}'.format(' '.join(cmd))) 270 | execute_subprocess(cmd) 271 | 272 | @staticmethod 273 | def _svn_switch(url, ignore_ancestry, verbosity): 274 | """ 275 | Switch branches for in an svn sandbox 276 | """ 277 | cmd = ['svn', 'switch', '--quiet'] 278 | if ignore_ancestry: 279 | cmd.append('--ignore-ancestry') 280 | cmd.append(url) 281 | if verbosity >= VERBOSITY_VERBOSE: 282 | printlog(' {0}'.format(' '.join(cmd))) 283 | execute_subprocess(cmd) 284 | -------------------------------------------------------------------------------- /manage_externals/manic/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Common public utilities for manic package 4 | 5 | """ 6 | 7 | from __future__ import absolute_import 8 | from __future__ import unicode_literals 9 | from __future__ import print_function 10 | 11 | import logging 12 | import os 13 | import subprocess 14 | import sys 15 | from threading import Timer 16 | 17 | from .global_constants import LOCAL_PATH_INDICATOR 18 | 19 | # --------------------------------------------------------------------- 20 | # 21 | # screen and logging output and functions to massage text for output 22 | # 23 | # --------------------------------------------------------------------- 24 | 25 | 26 | def log_process_output(output): 27 | """Log each line of process output at debug level so it can be 28 | filtered if necessary. By default, output is a single string, and 29 | logging.debug(output) will only put log info heading on the first 30 | line. This makes it hard to filter with grep. 31 | 32 | """ 33 | output = output.split('\n') 34 | for line in output: 35 | logging.debug(line) 36 | 37 | 38 | def printlog(msg, **kwargs): 39 | """Wrapper script around print to ensure that everything printed to 40 | the screen also gets logged. 41 | 42 | """ 43 | logging.info(msg) 44 | if kwargs: 45 | print(msg, **kwargs) 46 | else: 47 | print(msg) 48 | sys.stdout.flush() 49 | 50 | 51 | def last_n_lines(the_string, n_lines, truncation_message=None): 52 | """Returns the last n lines of the given string 53 | 54 | Args: 55 | the_string: str 56 | n_lines: int 57 | truncation_message: str, optional 58 | 59 | Returns a string containing the last n lines of the_string 60 | 61 | If truncation_message is provided, the returned string begins with 62 | the given message if and only if the string is greater than n lines 63 | to begin with. 64 | """ 65 | 66 | lines = the_string.splitlines(True) 67 | if len(lines) <= n_lines: 68 | return_val = the_string 69 | else: 70 | lines_subset = lines[-n_lines:] 71 | str_truncated = ''.join(lines_subset) 72 | if truncation_message: 73 | str_truncated = truncation_message + '\n' + str_truncated 74 | return_val = str_truncated 75 | 76 | return return_val 77 | 78 | 79 | def indent_string(the_string, indent_level): 80 | """Indents the given string by a given number of spaces 81 | 82 | Args: 83 | the_string: str 84 | indent_level: int 85 | 86 | Returns a new string that is the same as the_string, except that 87 | each line is indented by 'indent_level' spaces. 88 | 89 | In python3, this can be done with textwrap.indent. 90 | """ 91 | 92 | lines = the_string.splitlines(True) 93 | padding = ' ' * indent_level 94 | lines_indented = [padding + line for line in lines] 95 | return ''.join(lines_indented) 96 | 97 | # --------------------------------------------------------------------- 98 | # 99 | # error handling 100 | # 101 | # --------------------------------------------------------------------- 102 | 103 | 104 | def fatal_error(message): 105 | """ 106 | Error output function 107 | """ 108 | logging.error(message) 109 | raise RuntimeError("{0}ERROR: {1}".format(os.linesep, message)) 110 | 111 | 112 | # --------------------------------------------------------------------- 113 | # 114 | # Data conversion / manipulation 115 | # 116 | # --------------------------------------------------------------------- 117 | def str_to_bool(bool_str): 118 | """Convert a sting representation of as boolean into a true boolean. 119 | 120 | Conversion should be case insensitive. 121 | """ 122 | value = None 123 | str_lower = bool_str.lower() 124 | if str_lower in ('true', 't'): 125 | value = True 126 | elif str_lower in ('false', 'f'): 127 | value = False 128 | if value is None: 129 | msg = ('ERROR: invalid boolean string value "{0}". ' 130 | 'Must be "true" or "false"'.format(bool_str)) 131 | fatal_error(msg) 132 | return value 133 | 134 | 135 | REMOTE_PREFIXES = ['http://', 'https://', 'ssh://', 'git@'] 136 | 137 | 138 | def is_remote_url(url): 139 | """check if the user provided a local file path instead of a 140 | remote. If so, it must be expanded to an absolute 141 | path. 142 | 143 | """ 144 | remote_url = False 145 | for prefix in REMOTE_PREFIXES: 146 | if url.startswith(prefix): 147 | remote_url = True 148 | return remote_url 149 | 150 | 151 | def split_remote_url(url): 152 | """check if the user provided a local file path or a 153 | remote. If remote, try to strip off protocol info. 154 | 155 | """ 156 | remote_url = is_remote_url(url) 157 | if not remote_url: 158 | return url 159 | 160 | for prefix in REMOTE_PREFIXES: 161 | url = url.replace(prefix, '') 162 | 163 | if '@' in url: 164 | url = url.split('@')[1] 165 | 166 | if ':' in url: 167 | url = url.split(':')[1] 168 | 169 | return url 170 | 171 | 172 | def expand_local_url(url, field): 173 | """check if the user provided a local file path instead of a 174 | remote. If so, it must be expanded to an absolute 175 | path. 176 | 177 | Note: local paths of LOCAL_PATH_INDICATOR have special meaning and 178 | represent local copy only, don't work with the remotes. 179 | 180 | """ 181 | remote_url = is_remote_url(url) 182 | if not remote_url: 183 | if url.strip() == LOCAL_PATH_INDICATOR: 184 | pass 185 | else: 186 | url = os.path.expandvars(url) 187 | url = os.path.expanduser(url) 188 | if not os.path.isabs(url): 189 | msg = ('WARNING: Externals description for "{0}" contains a ' 190 | 'url that is not remote and does not expand to an ' 191 | 'absolute path. Version control operations may ' 192 | 'fail.\n\nurl={1}'.format(field, url)) 193 | printlog(msg) 194 | else: 195 | url = os.path.normpath(url) 196 | return url 197 | 198 | 199 | # --------------------------------------------------------------------- 200 | # 201 | # subprocess 202 | # 203 | # --------------------------------------------------------------------- 204 | 205 | # Give the user a helpful message if we detect that a command seems to 206 | # be hanging. 207 | _HANGING_SEC = 300 208 | 209 | 210 | def _hanging_msg(working_directory, command): 211 | print(""" 212 | 213 | Command '{command}' 214 | from directory {working_directory} 215 | has taken {hanging_sec} seconds. It may be hanging. 216 | 217 | The command will continue to run, but you may want to abort 218 | manage_externals with ^C and investigate. A possible cause of hangs is 219 | when svn or git require authentication to access a private 220 | repository. On some systems, svn and git requests for authentication 221 | information will not be displayed to the user. In this case, the program 222 | will appear to hang. Ensure you can run svn and git manually and access 223 | all repositories without entering your authentication information. 224 | 225 | """.format(command=command, 226 | working_directory=working_directory, 227 | hanging_sec=_HANGING_SEC)) 228 | 229 | 230 | def execute_subprocess(commands, status_to_caller=False, 231 | output_to_caller=False): 232 | """Wrapper around subprocess.check_output to handle common 233 | exceptions. 234 | 235 | check_output runs a command with arguments and waits 236 | for it to complete. 237 | 238 | check_output raises an exception on a nonzero return code. if 239 | status_to_caller is true, execute_subprocess returns the subprocess 240 | return code, otherwise execute_subprocess treats non-zero return 241 | status as an error and raises an exception. 242 | 243 | """ 244 | cwd = os.getcwd() 245 | msg = 'In directory: {0}\nexecute_subprocess running command:'.format(cwd) 246 | logging.info(msg) 247 | commands_str = ' '.join(commands) 248 | logging.info(commands_str) 249 | return_to_caller = status_to_caller or output_to_caller 250 | status = -1 251 | output = '' 252 | hanging_timer = Timer(_HANGING_SEC, _hanging_msg, 253 | kwargs={"working_directory": cwd, 254 | "command": commands_str}) 255 | hanging_timer.start() 256 | try: 257 | output = subprocess.check_output(commands, stderr=subprocess.STDOUT, 258 | universal_newlines=True) 259 | log_process_output(output) 260 | status = 0 261 | except OSError as error: 262 | msg = failed_command_msg( 263 | 'Command execution failed. Does the executable exist?', 264 | commands) 265 | logging.error(error) 266 | fatal_error(msg) 267 | except ValueError as error: 268 | msg = failed_command_msg( 269 | 'DEV_ERROR: Invalid arguments trying to run subprocess', 270 | commands) 271 | logging.error(error) 272 | fatal_error(msg) 273 | except subprocess.CalledProcessError as error: 274 | # Only report the error if we are NOT returning to the 275 | # caller. If we are returning to the caller, then it may be a 276 | # simple status check. If returning, it is the callers 277 | # responsibility determine if an error occurred and handle it 278 | # appropriately. 279 | if not return_to_caller: 280 | msg_context = ('Process did not run successfully; ' 281 | 'returned status {0}'.format(error.returncode)) 282 | msg = failed_command_msg(msg_context, commands, 283 | output=error.output) 284 | logging.error(error) 285 | logging.error(msg) 286 | log_process_output(error.output) 287 | fatal_error(msg) 288 | status = error.returncode 289 | finally: 290 | hanging_timer.cancel() 291 | 292 | if status_to_caller and output_to_caller: 293 | ret_value = (status, output) 294 | elif status_to_caller: 295 | ret_value = status 296 | elif output_to_caller: 297 | ret_value = output 298 | else: 299 | ret_value = None 300 | 301 | return ret_value 302 | 303 | 304 | def failed_command_msg(msg_context, command, output=None): 305 | """Template for consistent error messages from subprocess calls. 306 | 307 | If 'output' is given, it should provide the output from the failed 308 | command 309 | """ 310 | 311 | if output: 312 | output_truncated = last_n_lines(output, 20, 313 | truncation_message='[... Output truncated for brevity ...]') 314 | errmsg = ('Failed with output:\n' + 315 | indent_string(output_truncated, 4) + 316 | '\nERROR: ') 317 | else: 318 | errmsg = '' 319 | 320 | command_str = ' '.join(command) 321 | errmsg += """In directory 322 | {cwd} 323 | {context}: 324 | {command} 325 | """.format(cwd=os.getcwd(), context=msg_context, command=command_str) 326 | 327 | if output: 328 | errmsg += 'See above for output from failed command.\n' 329 | 330 | return errmsg 331 | -------------------------------------------------------------------------------- /manage_externals/test/test_unit_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Unit test driver for checkout_externals 4 | 5 | Note: this script assume the path to the checkout_externals.py module is 6 | already in the python path. 7 | 8 | """ 9 | 10 | from __future__ import absolute_import 11 | from __future__ import unicode_literals 12 | from __future__ import print_function 13 | 14 | import os 15 | import unittest 16 | 17 | from manic.utils import last_n_lines, indent_string 18 | from manic.utils import str_to_bool, execute_subprocess 19 | from manic.utils import is_remote_url, split_remote_url, expand_local_url 20 | 21 | 22 | class TestExecuteSubprocess(unittest.TestCase): 23 | """Test the application logic of execute_subprocess wrapper 24 | """ 25 | 26 | def test_exesub_return_stat_err(self): 27 | """Test that execute_subprocess returns a status code when caller 28 | requests and the executed subprocess fails. 29 | 30 | """ 31 | cmd = ['false'] 32 | status = execute_subprocess(cmd, status_to_caller=True) 33 | self.assertEqual(status, 1) 34 | 35 | def test_exesub_return_stat_ok(self): 36 | """Test that execute_subprocess returns a status code when caller 37 | requests and the executed subprocess succeeds. 38 | 39 | """ 40 | cmd = ['true'] 41 | status = execute_subprocess(cmd, status_to_caller=True) 42 | self.assertEqual(status, 0) 43 | 44 | def test_exesub_except_stat_err(self): 45 | """Test that execute_subprocess raises an exception on error when 46 | caller doesn't request return code 47 | 48 | """ 49 | cmd = ['false'] 50 | with self.assertRaises(RuntimeError): 51 | execute_subprocess(cmd, status_to_caller=False) 52 | 53 | 54 | class TestLastNLines(unittest.TestCase): 55 | """Test the last_n_lines function. 56 | 57 | """ 58 | 59 | def test_last_n_lines_short(self): 60 | """With a message with <= n lines, result of last_n_lines should 61 | just be the original message. 62 | 63 | """ 64 | mystr = """three 65 | line 66 | string 67 | """ 68 | 69 | mystr_truncated = last_n_lines( 70 | mystr, 3, truncation_message='[truncated]') 71 | self.assertEqual(mystr, mystr_truncated) 72 | 73 | def test_last_n_lines_long(self): 74 | """With a message with > n lines, result of last_n_lines should 75 | be a truncated string. 76 | 77 | """ 78 | mystr = """a 79 | big 80 | five 81 | line 82 | string 83 | """ 84 | expected = """[truncated] 85 | five 86 | line 87 | string 88 | """ 89 | 90 | mystr_truncated = last_n_lines( 91 | mystr, 3, truncation_message='[truncated]') 92 | self.assertEqual(expected, mystr_truncated) 93 | 94 | 95 | class TestIndentStr(unittest.TestCase): 96 | """Test the indent_string function. 97 | 98 | """ 99 | 100 | def test_indent_string_singleline(self): 101 | """Test the indent_string function with a single-line string 102 | 103 | """ 104 | mystr = 'foo' 105 | result = indent_string(mystr, 4) 106 | expected = ' foo' 107 | self.assertEqual(expected, result) 108 | 109 | def test_indent_string_multiline(self): 110 | """Test the indent_string function with a multi-line string 111 | 112 | """ 113 | mystr = """hello 114 | hi 115 | goodbye 116 | """ 117 | result = indent_string(mystr, 2) 118 | expected = """ hello 119 | hi 120 | goodbye 121 | """ 122 | self.assertEqual(expected, result) 123 | 124 | 125 | class TestStrToBool(unittest.TestCase): 126 | """Test the string to boolean conversion routine. 127 | 128 | """ 129 | 130 | def test_case_insensitive_true(self): 131 | """Verify that case insensitive variants of 'true' returns the True 132 | boolean. 133 | 134 | """ 135 | values = ['true', 'TRUE', 'True', 'tRuE', 't', 'T', ] 136 | for value in values: 137 | received = str_to_bool(value) 138 | self.assertTrue(received) 139 | 140 | def test_case_insensitive_false(self): 141 | """Verify that case insensitive variants of 'false' returns the False 142 | boolean. 143 | 144 | """ 145 | values = ['false', 'FALSE', 'False', 'fAlSe', 'f', 'F', ] 146 | for value in values: 147 | received = str_to_bool(value) 148 | self.assertFalse(received) 149 | 150 | def test_invalid_str_error(self): 151 | """Verify that a non-true/false string generates a runtime error. 152 | """ 153 | values = ['not_true_or_false', 'A', '1', '0', 154 | 'false_is_not_true', 'true_is_not_false'] 155 | for value in values: 156 | with self.assertRaises(RuntimeError): 157 | str_to_bool(value) 158 | 159 | 160 | class TestIsRemoteURL(unittest.TestCase): 161 | """Crude url checking to determine if a url is local or remote. 162 | 163 | """ 164 | 165 | def test_url_remote_git(self): 166 | """verify that a remote git url is identified. 167 | """ 168 | url = 'git@somewhere' 169 | is_remote = is_remote_url(url) 170 | self.assertTrue(is_remote) 171 | 172 | def test_url_remote_ssh(self): 173 | """verify that a remote ssh url is identified. 174 | """ 175 | url = 'ssh://user@somewhere' 176 | is_remote = is_remote_url(url) 177 | self.assertTrue(is_remote) 178 | 179 | def test_url_remote_http(self): 180 | """verify that a remote http url is identified. 181 | """ 182 | url = 'http://somewhere' 183 | is_remote = is_remote_url(url) 184 | self.assertTrue(is_remote) 185 | 186 | def test_url_remote_https(self): 187 | """verify that a remote https url is identified. 188 | """ 189 | url = 'https://somewhere' 190 | is_remote = is_remote_url(url) 191 | self.assertTrue(is_remote) 192 | 193 | def test_url_local_user(self): 194 | """verify that a local path with '~/path/to/repo' gets rejected 195 | 196 | """ 197 | url = '~/path/to/repo' 198 | is_remote = is_remote_url(url) 199 | self.assertFalse(is_remote) 200 | 201 | def test_url_local_var_curly(self): 202 | """verify that a local path with env var '${HOME}' gets rejected 203 | """ 204 | url = '${HOME}/path/to/repo' 205 | is_remote = is_remote_url(url) 206 | self.assertFalse(is_remote) 207 | 208 | def test_url_local_var(self): 209 | """verify that a local path with an env var '$HOME' gets rejected 210 | """ 211 | url = '$HOME/path/to/repo' 212 | is_remote = is_remote_url(url) 213 | self.assertFalse(is_remote) 214 | 215 | def test_url_local_abs(self): 216 | """verify that a local abs path gets rejected 217 | """ 218 | url = '/path/to/repo' 219 | is_remote = is_remote_url(url) 220 | self.assertFalse(is_remote) 221 | 222 | def test_url_local_rel(self): 223 | """verify that a local relative path gets rejected 224 | """ 225 | url = '../../path/to/repo' 226 | is_remote = is_remote_url(url) 227 | self.assertFalse(is_remote) 228 | 229 | 230 | class TestSplitRemoteURL(unittest.TestCase): 231 | """Crude url checking to determine if a url is local or remote. 232 | 233 | """ 234 | 235 | def test_url_remote_git(self): 236 | """verify that a remote git url is identified. 237 | """ 238 | url = 'git@somewhere.com:org/repo' 239 | received = split_remote_url(url) 240 | self.assertEqual(received, "org/repo") 241 | 242 | def test_url_remote_ssh(self): 243 | """verify that a remote ssh url is identified. 244 | """ 245 | url = 'ssh://user@somewhere.com/path/to/repo' 246 | received = split_remote_url(url) 247 | self.assertEqual(received, 'somewhere.com/path/to/repo') 248 | 249 | def test_url_remote_http(self): 250 | """verify that a remote http url is identified. 251 | """ 252 | url = 'http://somewhere.org/path/to/repo' 253 | received = split_remote_url(url) 254 | self.assertEqual(received, 'somewhere.org/path/to/repo') 255 | 256 | def test_url_remote_https(self): 257 | """verify that a remote http url is identified. 258 | """ 259 | url = 'http://somewhere.gov/path/to/repo' 260 | received = split_remote_url(url) 261 | self.assertEqual(received, 'somewhere.gov/path/to/repo') 262 | 263 | def test_url_local_url_unchanged(self): 264 | """verify that a local path is unchanged 265 | 266 | """ 267 | url = '/path/to/repo' 268 | received = split_remote_url(url) 269 | self.assertEqual(received, url) 270 | 271 | 272 | class TestExpandLocalURL(unittest.TestCase): 273 | """Crude url checking to determine if a url is local or remote. 274 | 275 | Remote should be unmodified. 276 | 277 | Local, should perform user and variable expansion. 278 | 279 | """ 280 | 281 | def test_url_local_user1(self): 282 | """verify that a local path with '~/path/to/repo' gets expanded to an 283 | absolute path. 284 | 285 | NOTE(bja, 2017-11) we can't test for something like: 286 | '~user/path/to/repo' because the user has to be in the local 287 | machine password directory and we don't know a user name that 288 | is valid on every system....? 289 | 290 | """ 291 | field = 'test' 292 | url = '~/path/to/repo' 293 | received = expand_local_url(url, field) 294 | print(received) 295 | self.assertTrue(os.path.isabs(received)) 296 | 297 | def test_url_local_expand_curly(self): 298 | """verify that a local path with '${HOME}' gets expanded to an absolute path. 299 | """ 300 | field = 'test' 301 | url = '${HOME}/path/to/repo' 302 | received = expand_local_url(url, field) 303 | self.assertTrue(os.path.isabs(received)) 304 | 305 | def test_url_local_expand_var(self): 306 | """verify that a local path with '$HOME' gets expanded to an absolute path. 307 | """ 308 | field = 'test' 309 | url = '$HOME/path/to/repo' 310 | received = expand_local_url(url, field) 311 | self.assertTrue(os.path.isabs(received)) 312 | 313 | def test_url_local_env_missing(self): 314 | """verify that a local path with env var that is missing gets left as-is 315 | 316 | """ 317 | field = 'test' 318 | url = '$TMP_VAR/path/to/repo' 319 | received = expand_local_url(url, field) 320 | print(received) 321 | self.assertEqual(received, url) 322 | 323 | def test_url_local_expand_env(self): 324 | """verify that a local path with another env var gets expanded to an 325 | absolute path. 326 | 327 | """ 328 | field = 'test' 329 | os.environ['TMP_VAR'] = '/some/absolute' 330 | url = '$TMP_VAR/path/to/repo' 331 | received = expand_local_url(url, field) 332 | del os.environ['TMP_VAR'] 333 | print(received) 334 | self.assertTrue(os.path.isabs(received)) 335 | self.assertEqual(received, '/some/absolute/path/to/repo') 336 | 337 | def test_url_local_normalize_rel(self): 338 | """verify that a local path with another env var gets expanded to an 339 | absolute path. 340 | 341 | """ 342 | field = 'test' 343 | url = '/this/is/a/long/../path/to/a/repo' 344 | received = expand_local_url(url, field) 345 | print(received) 346 | self.assertEqual(received, '/this/is/a/path/to/a/repo') 347 | 348 | 349 | if __name__ == '__main__': 350 | unittest.main() 351 | -------------------------------------------------------------------------------- /scripts/CAM_topo_regen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | #PBS -A P93300301 4 | #PBS -l walltime=00:45:00 5 | #PBS -l select=1:ncpus=1 6 | #PBS -j oe 7 | #PBS -q regular 8 | #PBS -N CAM_topo_regen 9 | # 10 | # ------------------------------------------------------------------------- 11 | # J. Fyke: call topography-updating routine for CAM 12 | # 13 | #This script is intended to be called during restarts of CESM, to update 14 | #FV1-degree CAM topography in response to 5-km resolution Greenland CISM geometry 15 | #changes prior to the next runstep. 16 | # 17 | #It can also be used offline to generate CAM grids corresponding to any arbitrary 18 | #ice sheet grid. 19 | # 20 | #This script uses several FORTRAN programs that MUST BE COMPILED AHEAD OF TIME. 21 | # 22 | #This script requires a large memory footprint, and operates on large (GB) files. 23 | # 24 | #This script reads in the bed topography and thickness from a 5 or 4-km resolution 25 | #Greenland CISM restart file, converts this to a surface topography, then merges 26 | #this topography into a high-res global topography. This is then used as input 27 | #to CAM topography and subgrid-roughness-generating routines, to derive new 28 | #FV1-degree PHIS and SGH* fields, which are finally inserted into a CAM restart 29 | #file, overwriting the existing PHIS and SGH* fields. 30 | # 31 | #Required inputs: 32 | #1) ISM_Topo_File: source of ice sheet topography. This will be set by scripts 33 | #during run-time. 34 | #2) CAM_Restart_File: the destination of the updated PHIS and SGH* fields. 35 | #3) Working_Directory: location where this script is being run, and where the sub- 36 | #directories containing the necessary tools preside (these directories include: 37 | # -regridding 38 | # -phis_smoothing (deleted 2017-04-14) 39 | # -bin_to_cube 40 | # -cube_to_target) 41 | #4) Data_Directory: the location of the directory containing a number of input 42 | #datasets that are necessary for the above tools. Where possible, soft-links 43 | #are generated below, to these tools. However, in some cases, files are copied 44 | #into Working_Directory. 45 | # 46 | # ------------------------------------------------------------------------- 47 | # Update 48 | # M. Lofverstrom 49 | # - added support for different resolutions and new GrIS file 50 | # - updated routines: bin_to_cube & cube_to_target (https://github.com/NCAR/Topo/ and special version of cube_to_taget) 51 | # topography smoothing on cubed sphere grid 52 | # - faster routines with special treatment of Greenland 53 | # - postprocessing routine 54 | # ------------------------------------------------------------------------- 55 | 56 | module load nco 57 | module load ncl 58 | module load python 59 | 60 | #module load numpy 61 | #module load netcdf4-python 62 | 63 | ## Load python libraries 64 | source /glade/u/apps/ch/opt/python/2.7.14/intel/17.0.1/bin/ncar_pylib 65 | 66 | ## Load shell 67 | if [ $SHELL == /bin/bash ]; then 68 | source /glade/u/apps/ch/opt/Lmod/7.3.14/lmod/7.3.14/init/bash 69 | elif [ $SHELL == /bin/tcsh ]; then 70 | source /glade/u/apps/ch/opt/Lmod/7.3.14/lmod/7.3.14/init/tcsh 71 | fi 72 | 73 | ##### 74 | 75 | ScratchRun=/glade/scratch/cmip6/b.e21.B1850G.f09_g17_gl4.CMIP6-ssp585-withism.001/run 76 | 77 | ##### 78 | 79 | echo "****Running CAM topography-updating routine...****" 80 | 81 | echo "-> Setting location of necessary input/output files and directories" 82 | 83 | Test_topo_regen=false 84 | 85 | ## Select high resolution topography data set ("gmted2010" (def) or "usgs") 86 | highResDataset=gmted2010 87 | #highResDataset=usgs 88 | 89 | if [ "$Test_topo_regen" == false ]; then 90 | 91 | echo "Defining I/O files for coupled run topography regeneration" 92 | ####Points to most recently modified CISM restart file in $RunDir 93 | export ISM_Topo_File=`ls -t $ScratchRun/*.cism.r.* | head -n 1` 94 | ####Points to most recently modified CAM restart file in $RunDir 95 | export CAM_Restart_File=`ls -t $ScratchRun/*.cam.r.* | head -n 1` 96 | # export Temporary_output_file=$ScratchRun/Temporary_output_file.nc 97 | ####Points to dyn_topog working directory, that is within run directory 98 | export Working_Directory=$ScratchRun/dynamic_atm_topog 99 | 100 | elif [ "$Test_topo_regen" == true ]; then 101 | 102 | echo "WARNING: USING PATHS TO TOPOGRAPHY REGENERATION TEST FILES!" 103 | 104 | ###THESE ARE TEMPORARY 4 km ISM grid TEST PATHS 105 | export ISM_Topo_File=$ScratchRun/temp_io_files/4km_ISM/BG_MG1_CISM2.cism.r.0010-01-01-00000.nc 106 | export CAM_Restart_File=$ScratchRun/temp_io_files/4km_ISM/BG_MG1_CISM2.cam.r.0010-01-01-00000.nc 107 | # export Temporary_output_file=$ScratchRun/Temporary_output_file.nc 108 | export Working_Directory=$ScratchRun 109 | #### 110 | 111 | else 112 | 113 | echo "Error: = Test_topo_regen not defined as 'true' or 'false'" 114 | exit 115 | 116 | fi 117 | 118 | export Data_Directory=/glade/p/cesm/liwg/cam_dyn_topog_data 119 | 120 | echo "Input CISM restart file is $ISM_Topo_File" 121 | echo "CAM restart file (only used for an array size check) is $CAM_Restart_File" 122 | 123 | if [ ! -e $ISM_Topo_File ]; then 124 | echo " Error: Ice sheet topography input file $ISM_Topo_File does not exist" 125 | exit 126 | fi 127 | if [ ! -e $CAM_Restart_File ]; then 128 | echo " Error: CAM restart file $CAM_Restart_File does not exist" 129 | exit 130 | fi 131 | echo " Success: input/output files exist:" 132 | echo " Ice sheet topography input file = $ISM_Topo_File" 133 | echo " CAM restart file = $CAM_Restart_File" 134 | 135 | #Check that incoming CAM and CISM restart files are the right size 136 | 137 | if ncdump -h $ISM_Topo_File | grep -q 'x0 = 300' && ncdump -h $ISM_Topo_File | grep -q 'y0 = 560'; then 138 | echo " Based on dimension size, CISM topography input APPEARS to be on a 5km CISM grid" 139 | # Source SCRIP file 140 | ln -sfv $Data_Directory/CISM1_5km_SCRIP_file.nc $Working_Directory/regridding/source_grid_file.nc 141 | # Weight file between IS and USGS tile grids 142 | ln -sfv $Data_Directory/CISM1_5km_weights_file.nc $Working_Directory/regridding/weights_file.nc 143 | # Ice sheet lat/lon fields 144 | ln -sfv $Data_Directory/CISM1_5km_lat_lon.nc $Working_Directory/regridding/icesheet_lat_lon.nc 145 | 146 | elif ncdump -h $ISM_Topo_File | grep -q 'x0 = 375' && ncdump -h $ISM_Topo_File | grep -q 'y0 = 700'; then 147 | echo " Based on dimension size, CISM topography input APPEARS to be on a 4km CISM grid" 148 | # Source SCRIP file 149 | ln -sfv $Data_Directory/CISM2_4km_SCRIP_file.nc $Working_Directory/regridding/source_grid_file.nc 150 | # Weight file between IS and USGS tile grids 151 | ln -sfv $Data_Directory/CISM2_4km_weights_file.nc $Working_Directory/regridding/weights_file.nc 152 | # Ice sheet lat/lon fields 153 | ln -sfv $Data_Directory/CISM2_4km_lat_lon.nc $Working_Directory/regridding/icesheet_lat_lon.nc 154 | 155 | 156 | elif ncdump -h $ISM_Topo_File | grep -q 'x1 = 393' && ncdump -h $ISM_Topo_File | grep -q 'y1 = 695'; then 157 | ## Note that this dataset only has dims x1 and y1; dims x0 and y0 are missing... 158 | echo " Based on dimension size, CISM topography input APPEARS to be on a 4km CISM grid (version from 2016-12-19)" 159 | # Source SCRIP file 160 | ln -sfv $Data_Directory/CISM2_4km_2016_12_19_SCRIP_file.nc $Working_Directory/regridding/source_grid_file.nc 161 | # Weight file between IS and USGS tile grids 162 | ln -sfv $Data_Directory/CISM2_4km_2016_12_19_weights_file.nc $Working_Directory/regridding/weights_file.nc 163 | # Ice sheet lat/lon fields 164 | ln -sfv $Data_Directory/CISM2_4km_2016_12_19_lat_lon.nc $Working_Directory/regridding/icesheet_lat_lon.nc 165 | 166 | 167 | elif ncdump -h $ISM_Topo_File | grep -q 'x1 = 416' && ncdump -h $ISM_Topo_File | grep -q 'y1 = 704'; then 168 | ## Note that this dataset only has dims x1 and y1; dims x0 and y0 are missing... 169 | echo " Based on dimension size, CISM topography input APPEARS to be on a 4km CISM grid (version from 2017-04-29)" 170 | # Source SCRIP file 171 | ln -sfv $Data_Directory/CISM2_4km_2017_04_29_SCRIP_file.nc $Working_Directory/regridding/source_grid_file.nc 172 | # Weight file between IS and USGS tile grids 173 | ln -sfv $Data_Directory/CISM2_4km_2017_04_29_weights_file.nc $Working_Directory/regridding/weights_file.nc 174 | # Ice sheet lat/lon fields 175 | ln -sfv $Data_Directory/CISM2_4km_2017_08_04_lat_lon.nc $Working_Directory/regridding/icesheet_lat_lon.nc 176 | 177 | else 178 | echo " Error: CISM topography input appears to be of an incompatible resolution with current script" 179 | exit 180 | fi 181 | 182 | 183 | 184 | if ncdump -h $CAM_Restart_File | grep -q 'lon = 288' && ncdump -h $CAM_Restart_File | grep -q 'lat = 192'; then 185 | echo " Based on dimension size, CAM data APPEARS to be on a FV1 CESM grid" 186 | cami=cami_fv1.nc 187 | targetFile=targetFV1.nc 188 | grid=fv_0.9x1.25 189 | gridFile=$grid.nc 190 | glandMaskFile=$Data_Directory/greenland_mask_FV1.nc 191 | topoDatasetDef=$Data_Directory/fv_0.9x1.25_topo_c170415.nc 192 | 193 | elif ncdump -h $CAM_Restart_File | grep -q 'lon = 144' && ncdump -h $CAM_Restart_File | grep -q 'lat = 96'; then 194 | echo " Based on dimension size, CAM data APPEARS to be on a FV2 CESM grid" 195 | cami=cami_fv2.nc 196 | targetFile=targetFV2.nc 197 | grid=fv_1.9x2.5 198 | gridFile=$grid.nc 199 | glandMaskFile=$Data_Directory/greenland_mask_FV2.nc 200 | topoDatasetDef=$Data_Directory/fv_1.9x2.5_topo_c061116.nc 201 | 202 | else 203 | echo " Error: CAM restart appears to be of an incompatible resolution with current script" 204 | exit 205 | fi 206 | 207 | echo " Success: Restart files are correct resolutions" 208 | 209 | if [ ! -d $Working_Directory ]; then 210 | echo " Error: $Working_Directory is not a valid working directory" 211 | exit 212 | fi 213 | if [ ! -d $Data_Directory ]; then 214 | echo " Error: $Data_Directory is not a valid input data directory" 215 | exit 216 | fi 217 | echo " Success: Working and data directories exist" 218 | 219 | cd $Working_Directory 220 | 221 | ######################### 222 | 223 | 224 | echo " " 225 | echo "-> Regridding model topography onto a lat-lon tile, insert into global dataset" 226 | 227 | cd $Working_Directory/regridding 228 | 229 | 230 | if [ "$highResDataset" == gmted2010 ]; then 231 | ln -sfv $Data_Directory/gmted2010_modis-rawdata.nc $Working_Directory/regridding/highRes-rawdata.nc 232 | elif [ "$highResDataset" == usgs ]; then 233 | ln -sfv $Data_Directory/usgs-rawdata.nc $Working_Directory/regridding/highRes-rawdata.nc 234 | fi 235 | 236 | 237 | # Ice-sheet-grid diagnostic 238 | ln -sfv $Data_Directory/template_out.nc $Working_Directory/regridding/template_out.nc 239 | ln -sfv $Data_Directory/destination_grid_file.nc $Working_Directory/regridding/destination_grid_file.nc 240 | 241 | 242 | echo " Creating regridded topography file and merging this into topography" 243 | 244 | # Pre-process input topography file. 245 | ncwa -O -a time -v thk,topg $ISM_Topo_File input_topography_file.nc 246 | ncks -A -v lat,lon icesheet_lat_lon.nc input_topography_file.nc 247 | 248 | # Remove existing archived regridded file and modified global topography files if they exist 249 | if [ -e ISM30sec.archived.nc ]; then 250 | rm ISM30sec.archived.nc 251 | fi 252 | if [ -e modified_usgs.nc ]; then 253 | rm modified-highRes.nc 254 | fi 255 | if [ -e regridded-tile.nc ]; then 256 | rm regridded-tile.nc 257 | fi 258 | 259 | # Copy original global topography dataset into local version, into which CISM topography will be merged 260 | cp -f highRes-rawdata.nc modified-highRes.nc 261 | 262 | # Regrid ice sheet onto 30-sec lat/lon tiled grid 263 | ncl 'input_file_name="input_topography_file.nc"' 'global_file_name="modified-highRes.nc"' 'output_file_name="regridded-tile.nc"' regrid.ncl 264 | 265 | # Merge tile with global high resolution dataset 266 | python mergeTile.py 'modified-highRes.nc' 'regridded-tile.nc' 267 | 268 | ################ 269 | 270 | echo " " 271 | echo "-> Running bin_to_cube generator..." 272 | 273 | cd $Working_Directory/bin_to_cube 274 | 275 | ln -sfv $Data_Directory/landm_coslat.nc landm_coslat.nc 276 | 277 | cat > bin_to_cube.nl < Running cube_to_target SGH/SGH30 generator..." 293 | 294 | cd $Working_Directory/cube_to_target 295 | 296 | cat > cube_to_target.nl < tmp.nl 325 | cp tmp.nl cube_to_target.nl 326 | ./cube_to_target 327 | 328 | #PHASE 1b 329 | # --- find ridges on 60-point deviations 330 | # --- and map to FV model grid 331 | sed '/lread_smooth_topofile/s/.false./.true./' < cube_to_target.nl > tmp2.nl 332 | cp tmp2.nl cube_to_target.nl 333 | #sed '/lfind_ridges/s/.true./.false./' < cube_to_target.nl > tmp2.nl ## may need this 334 | #cp tmp2.nl cube_to_target.nl 335 | ./cube_to_target 336 | 337 | #PHASE 2a 338 | # --- smooth topo w/ 8-point smoother for 339 | # --- Greenland SGH30 adjustment 340 | sed '/lread_smooth_topofile/s/.true./.false./' < cube_to_target.nl > tmp3.nl 341 | cp tmp3.nl cube_to_target.nl 342 | sed '/ncube_sph_smooth_coarse/s/60/8/' < cube_to_target.nl > tmp3.nl 343 | cp tmp3.nl cube_to_target.nl 344 | ./cube_to_target 345 | 346 | #PHASE 2b 347 | # --- skip ridges but map to FV model grid 348 | sed '/lread_smooth_topofile/s/.false./.true./' < cube_to_target.nl > tmp4.nl 349 | cp tmp4.nl cube_to_target.nl 350 | sed '/lfind_ridges/s/.true./.false./' < cube_to_target.nl > tmp4.nl 351 | cp tmp4.nl cube_to_target.nl 352 | ./cube_to_target 353 | 354 | 355 | ################ 356 | 357 | echo "-> Merging datasets..." 358 | 359 | cd $Working_Directory/postproc 360 | 361 | topoDataset=$ScratchRun/topoDataset.nc 362 | cp $topoDatasetDef $topoDataset 363 | 364 | c2tOutputDir=$Working_Directory/cube_to_target/output 365 | c2t060=$c2tOutputDir/${grid}_nc3000_Nsw042_Nrs008_Co060_Fi001.nc 366 | c2t008=$c2tOutputDir/${grid}_nc3000_NoAniso_Co008_Fi001.nc 367 | 368 | python postproc_mask.py $CAM_Restart_File $topoDataset $glandMaskFile $c2t060 $c2t008 369 | 370 | ################ 371 | 372 | echo "" 373 | echo "-> Topography updating finished successfully" 374 | echo "-> Returning to working directory" 375 | cd $Working_Directory 376 | 377 | exit 378 | 379 | ## == end of script == ## 380 | -------------------------------------------------------------------------------- /manage_externals/test/.pylint.rc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # A comma-separated list of package or module names from where C extensions may 4 | # be loaded. Extensions are loading into the active Python interpreter and may 5 | # run arbitrary code 6 | extension-pkg-whitelist= 7 | 8 | # Add files or directories to the blacklist. They should be base names, not 9 | # paths. 10 | ignore=.git,.svn,env2 11 | 12 | # Add files or directories matching the regex patterns to the blacklist. The 13 | # regex matches against base names, not paths. 14 | ignore-patterns= 15 | 16 | # Python code to execute, usually for sys.path manipulation such as 17 | # pygtk.require(). 18 | #init-hook= 19 | 20 | # Use multiple processes to speed up Pylint. 21 | jobs=1 22 | 23 | # List of plugins (as comma separated values of python modules names) to load, 24 | # usually to register additional checkers. 25 | load-plugins= 26 | 27 | # Pickle collected data for later comparisons. 28 | persistent=yes 29 | 30 | # Specify a configuration file. 31 | #rcfile= 32 | 33 | # Allow loading of arbitrary C extensions. Extensions are imported into the 34 | # active Python interpreter and may run arbitrary code. 35 | unsafe-load-any-extension=no 36 | 37 | 38 | [MESSAGES CONTROL] 39 | 40 | # Only show warnings with the listed confidence levels. Leave empty to show 41 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 42 | confidence= 43 | 44 | # Disable the message, report, category or checker with the given id(s). You 45 | # can either give multiple identifiers separated by comma (,) or put this 46 | # option multiple times (only on the command line, not in the configuration 47 | # file where it should appear only once).You can also use "--disable=all" to 48 | # disable everything first and then reenable specific checks. For example, if 49 | # you want to run only the similarities checker, you can use "--disable=all 50 | # --enable=similarities". If you want to run only the classes checker, but have 51 | # no Warning level messages displayed, use"--disable=all --enable=classes 52 | # --disable=W" 53 | disable=bad-continuation,useless-object-inheritance 54 | 55 | 56 | # Enable the message, report, category or checker with the given id(s). You can 57 | # either give multiple identifier separated by comma (,) or put this option 58 | # multiple time (only on the command line, not in the configuration file where 59 | # it should appear only once). See also the "--disable" option for examples. 60 | enable= 61 | 62 | 63 | [REPORTS] 64 | 65 | # Python expression which should return a note less than 10 (10 is the highest 66 | # note). You have access to the variables errors warning, statement which 67 | # respectively contain the number of errors / warnings messages and the total 68 | # number of statements analyzed. This is used by the global evaluation report 69 | # (RP0004). 70 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 71 | 72 | # Template used to display messages. This is a python new-style format string 73 | # used to format the message information. See doc for all details 74 | msg-template={msg_id}:{line:3d},{column:2d}: {msg} ({symbol}) 75 | 76 | # Set the output format. Available formats are text, parseable, colorized, json 77 | # and msvs (visual studio).You can also give a reporter class, eg 78 | # mypackage.mymodule.MyReporterClass. 79 | output-format=text 80 | 81 | # Tells whether to display a full report or only the messages 82 | #reports=yes 83 | 84 | # Activate the evaluation score. 85 | score=yes 86 | 87 | 88 | [REFACTORING] 89 | 90 | # Maximum number of nested blocks for function / method body 91 | max-nested-blocks=5 92 | 93 | 94 | [BASIC] 95 | 96 | # Naming hint for argument names 97 | argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 98 | 99 | # Regular expression matching correct argument names 100 | argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 101 | 102 | # Naming hint for attribute names 103 | attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 104 | 105 | # Regular expression matching correct attribute names 106 | attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 107 | 108 | # Bad variable names which should always be refused, separated by a comma 109 | bad-names=foo,bar,baz,toto,tutu,tata 110 | 111 | # Naming hint for class attribute names 112 | class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 113 | 114 | # Regular expression matching correct class attribute names 115 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 116 | 117 | # Naming hint for class names 118 | class-name-hint=[A-Z_][a-zA-Z0-9]+$ 119 | 120 | # Regular expression matching correct class names 121 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 122 | 123 | # Naming hint for constant names 124 | const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 125 | 126 | # Regular expression matching correct constant names 127 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 128 | 129 | # Minimum line length for functions/classes that require docstrings, shorter 130 | # ones are exempt. 131 | docstring-min-length=-1 132 | 133 | # Naming hint for function names 134 | function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 135 | 136 | # Regular expression matching correct function names 137 | function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 138 | 139 | # Good variable names which should always be accepted, separated by a comma 140 | good-names=i,j,k,ex,Run,_ 141 | 142 | # Include a hint for the correct naming format with invalid-name 143 | include-naming-hint=no 144 | 145 | # Naming hint for inline iteration names 146 | inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ 147 | 148 | # Regular expression matching correct inline iteration names 149 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 150 | 151 | # Naming hint for method names 152 | method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 153 | 154 | # Regular expression matching correct method names 155 | method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 156 | 157 | # Naming hint for module names 158 | module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 159 | 160 | # Regular expression matching correct module names 161 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 162 | 163 | # Colon-delimited sets of names that determine each other's naming style when 164 | # the name regexes allow several styles. 165 | name-group= 166 | 167 | # Regular expression which should only match function or class names that do 168 | # not require a docstring. 169 | no-docstring-rgx=^_ 170 | 171 | # List of decorators that produce properties, such as abc.abstractproperty. Add 172 | # to this list to register other decorators that produce valid properties. 173 | property-classes=abc.abstractproperty 174 | 175 | # Naming hint for variable names 176 | variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 177 | 178 | # Regular expression matching correct variable names 179 | variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 180 | 181 | 182 | [FORMAT] 183 | 184 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 185 | expected-line-ending-format= 186 | 187 | # Regexp for a line that is allowed to be longer than the limit. 188 | ignore-long-lines=^\s*(# )??$ 189 | 190 | # Number of spaces of indent required inside a hanging or continued line. 191 | indent-after-paren=4 192 | 193 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 194 | # tab). 195 | indent-string=' ' 196 | 197 | # Maximum number of characters on a single line. 198 | max-line-length=100 199 | 200 | # Maximum number of lines in a module 201 | max-module-lines=1000 202 | 203 | # List of optional constructs for which whitespace checking is disabled. `dict- 204 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 205 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 206 | # `empty-line` allows space-only lines. 207 | no-space-check=trailing-comma,dict-separator 208 | 209 | # Allow the body of a class to be on the same line as the declaration if body 210 | # contains single statement. 211 | single-line-class-stmt=no 212 | 213 | # Allow the body of an if to be on the same line as the test if there is no 214 | # else. 215 | single-line-if-stmt=no 216 | 217 | 218 | [LOGGING] 219 | 220 | # Logging modules to check that the string format arguments are in logging 221 | # function parameter format 222 | logging-modules=logging 223 | 224 | 225 | [MISCELLANEOUS] 226 | 227 | # List of note tags to take in consideration, separated by a comma. 228 | notes=FIXME,XXX,TODO 229 | 230 | 231 | [SIMILARITIES] 232 | 233 | # Ignore comments when computing similarities. 234 | ignore-comments=yes 235 | 236 | # Ignore docstrings when computing similarities. 237 | ignore-docstrings=yes 238 | 239 | # Ignore imports when computing similarities. 240 | ignore-imports=no 241 | 242 | # Minimum lines number of a similarity. 243 | min-similarity-lines=4 244 | 245 | 246 | [SPELLING] 247 | 248 | # Spelling dictionary name. Available dictionaries: none. To make it working 249 | # install python-enchant package. 250 | spelling-dict= 251 | 252 | # List of comma separated words that should not be checked. 253 | spelling-ignore-words= 254 | 255 | # A path to a file that contains private dictionary; one word per line. 256 | spelling-private-dict-file= 257 | 258 | # Tells whether to store unknown words to indicated private dictionary in 259 | # --spelling-private-dict-file option instead of raising a message. 260 | spelling-store-unknown-words=no 261 | 262 | 263 | [TYPECHECK] 264 | 265 | # List of decorators that produce context managers, such as 266 | # contextlib.contextmanager. Add to this list to register other decorators that 267 | # produce valid context managers. 268 | contextmanager-decorators=contextlib.contextmanager 269 | 270 | # List of members which are set dynamically and missed by pylint inference 271 | # system, and so shouldn't trigger E1101 when accessed. Python regular 272 | # expressions are accepted. 273 | generated-members= 274 | 275 | # Tells whether missing members accessed in mixin class should be ignored. A 276 | # mixin class is detected if its name ends with "mixin" (case insensitive). 277 | ignore-mixin-members=yes 278 | 279 | # This flag controls whether pylint should warn about no-member and similar 280 | # checks whenever an opaque object is returned when inferring. The inference 281 | # can return multiple potential results while evaluating a Python object, but 282 | # some branches might not be evaluated, which results in partial inference. In 283 | # that case, it might be useful to still emit no-member and other checks for 284 | # the rest of the inferred objects. 285 | ignore-on-opaque-inference=yes 286 | 287 | # List of class names for which member attributes should not be checked (useful 288 | # for classes with dynamically set attributes). This supports the use of 289 | # qualified names. 290 | ignored-classes=optparse.Values,thread._local,_thread._local 291 | 292 | # List of module names for which member attributes should not be checked 293 | # (useful for modules/projects where namespaces are manipulated during runtime 294 | # and thus existing member attributes cannot be deduced by static analysis. It 295 | # supports qualified module names, as well as Unix pattern matching. 296 | ignored-modules= 297 | 298 | # Show a hint with possible names when a member name was not found. The aspect 299 | # of finding the hint is based on edit distance. 300 | missing-member-hint=yes 301 | 302 | # The minimum edit distance a name should have in order to be considered a 303 | # similar match for a missing member name. 304 | missing-member-hint-distance=1 305 | 306 | # The total number of similar names that should be taken in consideration when 307 | # showing a hint for a missing member. 308 | missing-member-max-choices=1 309 | 310 | 311 | [VARIABLES] 312 | 313 | # List of additional names supposed to be defined in builtins. Remember that 314 | # you should avoid to define new builtins when possible. 315 | additional-builtins= 316 | 317 | # Tells whether unused global variables should be treated as a violation. 318 | allow-global-unused-variables=yes 319 | 320 | # List of strings which can identify a callback function by name. A callback 321 | # name must start or end with one of those strings. 322 | callbacks=cb_,_cb 323 | 324 | # A regular expression matching the name of dummy variables (i.e. expectedly 325 | # not used). 326 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 327 | 328 | # Argument names that match this expression will be ignored. Default to name 329 | # with leading underscore 330 | ignored-argument-names=_.*|^ignored_|^unused_ 331 | 332 | # Tells whether we should check for unused import in __init__ files. 333 | init-import=no 334 | 335 | # List of qualified module names which can have objects that can redefine 336 | # builtins. 337 | redefining-builtins-modules=six.moves,future.builtins 338 | 339 | 340 | [CLASSES] 341 | 342 | # List of method names used to declare (i.e. assign) instance attributes. 343 | defining-attr-methods=__init__,__new__,setUp 344 | 345 | # List of member names, which should be excluded from the protected access 346 | # warning. 347 | exclude-protected=_asdict,_fields,_replace,_source,_make 348 | 349 | # List of valid names for the first argument in a class method. 350 | valid-classmethod-first-arg=cls 351 | 352 | # List of valid names for the first argument in a metaclass class method. 353 | valid-metaclass-classmethod-first-arg=mcs 354 | 355 | 356 | [DESIGN] 357 | 358 | # Maximum number of arguments for function / method 359 | max-args=5 360 | 361 | # Maximum number of attributes for a class (see R0902). 362 | max-attributes=7 363 | 364 | # Maximum number of boolean expressions in a if statement 365 | max-bool-expr=5 366 | 367 | # Maximum number of branch for function / method body 368 | max-branches=12 369 | 370 | # Maximum number of locals for function / method body 371 | max-locals=15 372 | 373 | # Maximum number of parents for a class (see R0901). 374 | max-parents=7 375 | 376 | # Maximum number of public methods for a class (see R0904). 377 | max-public-methods=20 378 | 379 | # Maximum number of return / yield for function / method body 380 | max-returns=6 381 | 382 | # Maximum number of statements in function / method body 383 | max-statements=50 384 | 385 | # Minimum number of public methods for a class (see R0903). 386 | min-public-methods=2 387 | 388 | 389 | [IMPORTS] 390 | 391 | # Allow wildcard imports from modules that define __all__. 392 | allow-wildcard-with-all=no 393 | 394 | # Analyse import fallback blocks. This can be used to support both Python 2 and 395 | # 3 compatible code, which means that the block might have code that exists 396 | # only in one or another interpreter, leading to false positives when analysed. 397 | analyse-fallback-blocks=no 398 | 399 | # Deprecated modules which should not be used, separated by a comma 400 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 401 | 402 | # Create a graph of external dependencies in the given file (report RP0402 must 403 | # not be disabled) 404 | ext-import-graph= 405 | 406 | # Create a graph of every (i.e. internal and external) dependencies in the 407 | # given file (report RP0402 must not be disabled) 408 | import-graph= 409 | 410 | # Create a graph of internal dependencies in the given file (report RP0402 must 411 | # not be disabled) 412 | int-import-graph= 413 | 414 | # Force import order to recognize a module as part of the standard 415 | # compatibility libraries. 416 | known-standard-library= 417 | 418 | # Force import order to recognize a module as part of a third party library. 419 | known-third-party=enchant 420 | 421 | 422 | [EXCEPTIONS] 423 | 424 | # Exceptions that will emit a warning when being caught. Defaults to 425 | # "Exception" 426 | overgeneral-exceptions=Exception 427 | -------------------------------------------------------------------------------- /manage_externals/manic/sourcetree.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | FIXME(bja, 2017-11) External and SourceTree have a circular dependancy! 4 | """ 5 | 6 | import errno 7 | import logging 8 | import os 9 | 10 | from .externals_description import ExternalsDescription 11 | from .externals_description import read_externals_description_file 12 | from .externals_description import create_externals_description 13 | from .repository_factory import create_repository 14 | from .repository_git import GitRepository 15 | from .externals_status import ExternalStatus 16 | from .utils import fatal_error, printlog 17 | from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR 18 | from .global_constants import VERBOSITY_VERBOSE 19 | 20 | class _External(object): 21 | """ 22 | _External represents an external object inside a SourceTree 23 | """ 24 | 25 | # pylint: disable=R0902 26 | 27 | def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): 28 | """Parse an external description file into a dictionary of externals. 29 | 30 | Input: 31 | 32 | root_dir : string - the root directory path where 33 | 'local_path' is relative to. 34 | 35 | name : string - name of the ext_description object. may or may not 36 | correspond to something in the path. 37 | 38 | ext_description : dict - source ExternalsDescription object 39 | 40 | svn_ignore_ancestry : bool - use --ignore-externals with svn switch 41 | 42 | """ 43 | self._name = name 44 | self._repo = None 45 | self._externals = EMPTY_STR 46 | self._externals_sourcetree = None 47 | self._stat = ExternalStatus() 48 | self._sparse = None 49 | # Parse the sub-elements 50 | 51 | # _path : local path relative to the containing source tree 52 | self._local_path = ext_description[ExternalsDescription.PATH] 53 | # _repo_dir : full repository directory 54 | repo_dir = os.path.join(root_dir, self._local_path) 55 | self._repo_dir_path = os.path.abspath(repo_dir) 56 | # _base_dir : base directory *containing* the repository 57 | self._base_dir_path = os.path.dirname(self._repo_dir_path) 58 | # repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path 59 | self._repo_dir_name = os.path.basename(self._repo_dir_path) 60 | assert(os.path.join(self._base_dir_path, self._repo_dir_name) 61 | == self._repo_dir_path) 62 | 63 | self._required = ext_description[ExternalsDescription.REQUIRED] 64 | self._externals = ext_description[ExternalsDescription.EXTERNALS] 65 | # Treat a .gitmodules file as a backup externals config 66 | if not self._externals: 67 | if GitRepository.has_submodules(self._repo_dir_path): 68 | self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME 69 | 70 | repo = create_repository( 71 | name, ext_description[ExternalsDescription.REPO], 72 | svn_ignore_ancestry=svn_ignore_ancestry) 73 | if repo: 74 | self._repo = repo 75 | 76 | if self._externals and (self._externals.lower() != 'none'): 77 | self._create_externals_sourcetree() 78 | 79 | def get_name(self): 80 | """ 81 | Return the external object's name 82 | """ 83 | return self._name 84 | 85 | def get_local_path(self): 86 | """ 87 | Return the external object's path 88 | """ 89 | return self._local_path 90 | 91 | def status(self): 92 | """ 93 | If the repo destination directory exists, ensure it is correct (from 94 | correct URL, correct branch or tag), and possibly update the external. 95 | If the repo destination directory does not exist, checkout the correce 96 | branch or tag. 97 | If load_all is True, also load all of the the externals sub-externals. 98 | """ 99 | 100 | self._stat.path = self.get_local_path() 101 | if not self._required: 102 | self._stat.source_type = ExternalStatus.OPTIONAL 103 | elif self._local_path == LOCAL_PATH_INDICATOR: 104 | # LOCAL_PATH_INDICATOR, '.' paths, are standalone 105 | # component directories that are not managed by 106 | # checkout_externals. 107 | self._stat.source_type = ExternalStatus.STANDALONE 108 | else: 109 | # managed by checkout_externals 110 | self._stat.source_type = ExternalStatus.MANAGED 111 | 112 | ext_stats = {} 113 | 114 | if not os.path.exists(self._repo_dir_path): 115 | self._stat.sync_state = ExternalStatus.EMPTY 116 | msg = ('status check: repository directory for "{0}" does not ' 117 | 'exist.'.format(self._name)) 118 | logging.info(msg) 119 | self._stat.current_version = 'not checked out' 120 | # NOTE(bja, 2018-01) directory doesn't exist, so we cannot 121 | # use repo to determine the expected version. We just take 122 | # a best-guess based on the assumption that only tag or 123 | # branch should be set, but not both. 124 | if not self._repo: 125 | self._stat.expected_version = 'unknown' 126 | else: 127 | self._stat.expected_version = self._repo.tag() + self._repo.branch() 128 | else: 129 | if self._repo: 130 | self._repo.status(self._stat, self._repo_dir_path) 131 | 132 | if self._externals and self._externals_sourcetree: 133 | # we expect externals and they exist 134 | cwd = os.getcwd() 135 | # SourceTree expects to be called from the correct 136 | # root directory. 137 | os.chdir(self._repo_dir_path) 138 | ext_stats = self._externals_sourcetree.status(self._local_path) 139 | os.chdir(cwd) 140 | 141 | all_stats = {} 142 | # don't add the root component because we don't manage it 143 | # and can't provide useful info about it. 144 | if self._local_path != LOCAL_PATH_INDICATOR: 145 | # store the stats under tha local_path, not comp name so 146 | # it will be sorted correctly 147 | all_stats[self._stat.path] = self._stat 148 | 149 | if ext_stats: 150 | all_stats.update(ext_stats) 151 | 152 | return all_stats 153 | 154 | def checkout(self, verbosity, load_all): 155 | """ 156 | If the repo destination directory exists, ensure it is correct (from 157 | correct URL, correct branch or tag), and possibly update the external. 158 | If the repo destination directory does not exist, checkout the correct 159 | branch or tag. 160 | If load_all is True, also load all of the the externals sub-externals. 161 | """ 162 | if load_all: 163 | pass 164 | # Make sure we are in correct location 165 | 166 | if not os.path.exists(self._repo_dir_path): 167 | # repository directory doesn't exist. Need to check it 168 | # out, and for that we need the base_dir_path to exist 169 | try: 170 | os.makedirs(self._base_dir_path) 171 | except OSError as error: 172 | if error.errno != errno.EEXIST: 173 | msg = 'Could not create directory "{0}"'.format( 174 | self._base_dir_path) 175 | fatal_error(msg) 176 | 177 | if self._stat.source_type != ExternalStatus.STANDALONE: 178 | if verbosity >= VERBOSITY_VERBOSE: 179 | # NOTE(bja, 2018-01) probably do not want to pass 180 | # verbosity in this case, because if (verbosity == 181 | # VERBOSITY_DUMP), then the previous status output would 182 | # also be dumped, adding noise to the output. 183 | self._stat.log_status_message(VERBOSITY_VERBOSE) 184 | 185 | if self._repo: 186 | if self._stat.sync_state == ExternalStatus.STATUS_OK: 187 | # If we're already in sync, avoid showing verbose output 188 | # from the checkout command, unless the verbosity level 189 | # is 2 or more. 190 | checkout_verbosity = verbosity - 1 191 | else: 192 | checkout_verbosity = verbosity 193 | 194 | self._repo.checkout(self._base_dir_path, self._repo_dir_name, 195 | checkout_verbosity, self.clone_recursive()) 196 | 197 | def checkout_externals(self, verbosity, load_all): 198 | """Checkout the sub-externals for this object 199 | """ 200 | if self.load_externals(): 201 | if self._externals_sourcetree: 202 | # NOTE(bja, 2018-02): the subtree externals objects 203 | # were created during initial status check. Updating 204 | # the external may have changed which sub-externals 205 | # are needed. We need to delete those objects and 206 | # re-read the potentially modified externals 207 | # description file. 208 | self._externals_sourcetree = None 209 | self._create_externals_sourcetree() 210 | self._externals_sourcetree.checkout(verbosity, load_all) 211 | 212 | def load_externals(self): 213 | 'Return True iff an externals file should be loaded' 214 | load_ex = False 215 | if os.path.exists(self._repo_dir_path): 216 | if self._externals: 217 | if self._externals.lower() != 'none': 218 | load_ex = os.path.exists(os.path.join(self._repo_dir_path, 219 | self._externals)) 220 | 221 | return load_ex 222 | 223 | def clone_recursive(self): 224 | 'Return True iff any .gitmodules files should be processed' 225 | # Try recursive unless there is an externals entry 226 | recursive = not self._externals 227 | 228 | return recursive 229 | 230 | def _create_externals_sourcetree(self): 231 | """ 232 | """ 233 | if not os.path.exists(self._repo_dir_path): 234 | # NOTE(bja, 2017-10) repository has not been checked out 235 | # yet, can't process the externals file. Assume we are 236 | # checking status before code is checkoud out and this 237 | # will be handled correctly later. 238 | return 239 | 240 | cwd = os.getcwd() 241 | os.chdir(self._repo_dir_path) 242 | if self._externals.lower() == 'none': 243 | msg = ('Internal: Attempt to create source tree for ' 244 | 'externals = none in {}'.format(self._repo_dir_path)) 245 | fatal_error(msg) 246 | 247 | if not os.path.exists(self._externals): 248 | if GitRepository.has_submodules(): 249 | self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME 250 | 251 | if not os.path.exists(self._externals): 252 | # NOTE(bja, 2017-10) this check is redundent with the one 253 | # in read_externals_description_file! 254 | msg = ('External externals description file "{0}" ' 255 | 'does not exist! In directory: {1}'.format( 256 | self._externals, self._repo_dir_path)) 257 | fatal_error(msg) 258 | 259 | externals_root = self._repo_dir_path 260 | model_data = read_externals_description_file(externals_root, 261 | self._externals) 262 | externals = create_externals_description(model_data, 263 | parent_repo=self._repo) 264 | self._externals_sourcetree = SourceTree(externals_root, externals) 265 | os.chdir(cwd) 266 | 267 | class SourceTree(object): 268 | """ 269 | SourceTree represents a group of managed externals 270 | """ 271 | 272 | def __init__(self, root_dir, model, svn_ignore_ancestry=False): 273 | """ 274 | Build a SourceTree object from a model description 275 | """ 276 | self._root_dir = os.path.abspath(root_dir) 277 | self._all_components = {} 278 | self._required_compnames = [] 279 | for comp in model: 280 | src = _External(self._root_dir, comp, model[comp], svn_ignore_ancestry) 281 | self._all_components[comp] = src 282 | if model[comp][ExternalsDescription.REQUIRED]: 283 | self._required_compnames.append(comp) 284 | 285 | def status(self, relative_path_base=LOCAL_PATH_INDICATOR): 286 | """Report the status components 287 | 288 | FIXME(bja, 2017-10) what do we do about situations where the 289 | user checked out the optional components, but didn't add 290 | optional for running status? What do we do where the user 291 | didn't add optional to the checkout but did add it to the 292 | status. -- For now, we run status on all components, and try 293 | to do the right thing based on the results.... 294 | 295 | """ 296 | load_comps = self._all_components.keys() 297 | 298 | summary = {} 299 | for comp in load_comps: 300 | printlog('{0}, '.format(comp), end='') 301 | stat = self._all_components[comp].status() 302 | stat_final = {} 303 | for name in stat.keys(): 304 | # check if we need to append the relative_path_base to 305 | # the path so it will be sorted in the correct order. 306 | if stat[name].path.startswith(relative_path_base): 307 | # use as is, without any changes to path 308 | stat_final[name] = stat[name] 309 | else: 310 | # append relative_path_base to path and store under key = updated path 311 | modified_path = os.path.join(relative_path_base, 312 | stat[name].path) 313 | stat_final[modified_path] = stat[name] 314 | stat_final[modified_path].path = modified_path 315 | summary.update(stat_final) 316 | 317 | return summary 318 | 319 | def checkout(self, verbosity, load_all, load_comp=None): 320 | """ 321 | Checkout or update indicated components into the the configured 322 | subdirs. 323 | 324 | If load_all is True, recursively checkout all externals. 325 | If load_all is False, load_comp is an optional set of components to load. 326 | If load_all is True and load_comp is None, only load the required externals. 327 | """ 328 | if verbosity >= VERBOSITY_VERBOSE: 329 | printlog('Checking out externals: ') 330 | else: 331 | printlog('Checking out externals: ', end='') 332 | 333 | if load_all: 334 | load_comps = self._all_components.keys() 335 | elif load_comp is not None: 336 | load_comps = [load_comp] 337 | else: 338 | load_comps = self._required_compnames 339 | 340 | # checkout the primary externals 341 | for comp in load_comps: 342 | if verbosity < VERBOSITY_VERBOSE: 343 | printlog('{0}, '.format(comp), end='') 344 | else: 345 | # verbose output handled by the _External object, just 346 | # output a newline 347 | printlog(EMPTY_STR) 348 | self._all_components[comp].checkout(verbosity, load_all) 349 | printlog('') 350 | 351 | # now give each external an opportunitity to checkout it's externals. 352 | for comp in load_comps: 353 | self._all_components[comp].checkout_externals(verbosity, load_all) 354 | --------------------------------------------------------------------------------