├── cime_config ├── README.md ├── testmods_dirs │ └── allactive │ │ ├── defaultiomi │ │ ├── include_user_mods │ │ └── shell_commands │ │ ├── cism │ │ └── test_coupling │ │ │ ├── include_user_mods │ │ │ ├── README │ │ │ ├── shell_commands │ │ │ ├── user_nl_cism │ │ │ └── user_nl_clm │ │ ├── defaultio │ │ ├── user_nl_cam │ │ ├── user_nl_clm │ │ ├── user_nl_pop │ │ ├── user_nl_cism │ │ ├── user_nl_rtm │ │ ├── user_nl_mosart │ │ └── user_nl_cice │ │ ├── maxthroughputb │ │ ├── user_nl_cam │ │ └── shell_commands │ │ └── maxthroughputfw │ │ └── shell_commands ├── usermods_dirs │ └── README ├── config_compsets.xml └── testlist_allactive.xml ├── manage_externals ├── test │ ├── doc │ │ ├── .gitignore │ │ ├── index.rst │ │ ├── Makefile │ │ ├── testing.rst │ │ ├── conf.py │ │ └── develop.rst │ ├── repos │ │ ├── container.git │ │ │ ├── HEAD │ │ │ ├── refs │ │ │ │ └── heads │ │ │ │ │ └── master │ │ │ ├── description │ │ │ ├── config │ │ │ ├── objects │ │ │ │ ├── 41 │ │ │ │ │ └── 1de5d96ee418c1c55f3e96e6e6e7c06bb95801 │ │ │ │ ├── 71 │ │ │ │ │ └── 5b8f3e4afe1802a178e1d603af404ba45d59de │ │ │ │ ├── b0 │ │ │ │ │ └── f87705e2b9601cb831878f3d51efa78b910d7b │ │ │ │ └── f9 │ │ │ │ │ └── e08370a737e941de6f6492e3f427c2ef4c1a03 │ │ │ └── info │ │ │ │ └── exclude │ │ ├── mixed-cont-ext.git │ │ │ ├── HEAD │ │ │ ├── refs │ │ │ │ └── heads │ │ │ │ │ ├── master │ │ │ │ │ └── new-feature │ │ │ ├── description │ │ │ ├── config │ │ │ ├── objects │ │ │ │ ├── 14 │ │ │ │ │ └── 368b701616a8c53820b610414a4b9a07540cf6 │ │ │ │ ├── 15 │ │ │ │ │ └── 2b57e1cf23721cd17ff681cb9276e3fb9fc091 │ │ │ │ ├── 37 │ │ │ │ │ └── f0e70b609adc90f4c09ee21d82ed1d79c81d69 │ │ │ │ ├── 38 │ │ │ │ │ └── 9a2b876b8965d3c91a3db8d28a483eaf019d5c │ │ │ │ ├── 41 │ │ │ │ │ └── 1de5d96ee418c1c55f3e96e6e6e7c06bb95801 │ │ │ │ ├── 93 │ │ │ │ │ └── a159deb9175bfeb2820a0006ddd92d78131332 │ │ │ │ ├── 95 │ │ │ │ │ └── 80ecc12f16334ce44e42287d5d46f927bb7b75 │ │ │ │ ├── 00 │ │ │ │ │ └── 437ac2000d5f06fb8a572a01a5bbdae98b17cb │ │ │ │ ├── 01 │ │ │ │ │ └── 97458f2dbe5fcd6bc44fa46983be0a30282379 │ │ │ │ ├── 06 │ │ │ │ │ └── ea30b03ffa2f8574705f8b9583f7ca7e2dccf7 │ │ │ │ ├── 1f │ │ │ │ │ └── 01fa46c17b1f38b37e6259f6e9d041bda3144f │ │ │ │ ├── 6e │ │ │ │ │ └── 9f4baa6e94a0af4e094836c2eb55ccedef5fc4 │ │ │ │ ├── 6f │ │ │ │ │ └── c379457ecb4e576a13c7610ae1fa73f845ee6a │ │ │ │ ├── a9 │ │ │ │ │ └── 288dcd8a719a1f4ed3cba43a2a387ae7cd60fd │ │ │ │ ├── e8 │ │ │ │ │ └── ea32a11d30ee703f6f661ae7c2376f4ab84d38 │ │ │ │ └── fd │ │ │ │ │ └── 15a5ad5204356229c60a831d2a8120a43ac901 │ │ │ └── info │ │ │ │ └── exclude │ │ ├── simple-ext.git │ │ │ ├── HEAD │ │ │ ├── refs │ │ │ │ ├── tags │ │ │ │ │ ├── tag1 │ │ │ │ │ └── tag2 │ │ │ │ └── heads │ │ │ │ │ ├── feature2 │ │ │ │ │ ├── feature3 │ │ │ │ │ └── master │ │ │ ├── description │ │ │ ├── config │ │ │ ├── objects │ │ │ │ ├── 11 │ │ │ │ │ └── a76e3d9a67313dec7ce1230852ab5c86352c5c │ │ │ │ ├── 14 │ │ │ │ │ └── 2711fdbbcb8034d7cad6bae6801887b12fe61d │ │ │ │ ├── 31 │ │ │ │ │ └── dbcd6de441e671a467ef317146539b7ffabb11 │ │ │ │ ├── 36 │ │ │ │ │ └── 418b4e5665956a90725c9a1b5a8e551c5f3d48 │ │ │ │ ├── 41 │ │ │ │ │ └── 1de5d96ee418c1c55f3e96e6e6e7c06bb95801 │ │ │ │ ├── 60 │ │ │ │ │ ├── 7ec299c17dd285c029edc41a0109e49d441380 │ │ │ │ │ └── b1cc1a38d63a4bcaa1e767262bbe23dbf9f5f5 │ │ │ │ ├── 63 │ │ │ │ │ └── a99393d1baff97ccef967af30380659867b139 │ │ │ │ ├── 95 │ │ │ │ │ └── 3256da5612fcd9263590a353bc18c6f224e74f │ │ │ │ ├── 00 │ │ │ │ │ └── fd13e76189f9134b0506b4b8ed3172723b467f │ │ │ │ ├── 09 │ │ │ │ │ └── 0e1034746b2c865f7b0280813dbf4061a700e8 │ │ │ │ ├── 0b │ │ │ │ │ └── 15e8af3d4615b42314216efeae3fff184046a8 │ │ │ │ ├── 9b │ │ │ │ │ └── 75494003deca69527bb64bcaa352e801611dd2 │ │ │ │ ├── a2 │ │ │ │ │ └── 2a5da9119328ea6d693f88861457c07e14ac04 │ │ │ │ ├── b7 │ │ │ │ │ └── 692b6d391899680da7b9b6fd8af4c413f06fe7 │ │ │ │ ├── c5 │ │ │ │ │ └── b315915742133dbdfbeed0753e481b55c1d364 │ │ │ │ ├── d1 │ │ │ │ │ └── 163870d19c3dee34fada3a76b785cfa2a8424b │ │ │ │ ├── d8 │ │ │ │ │ └── ed2f33179d751937f8fde2e33921e4827babf4 │ │ │ │ └── df │ │ │ │ │ └── 312890f93ba4d2c694208599b665c4a08afeff │ │ │ └── info │ │ │ │ └── exclude │ │ ├── simple-ext-fork.git │ │ │ ├── HEAD │ │ │ ├── refs │ │ │ │ ├── heads │ │ │ │ │ └── feature2 │ │ │ │ └── tags │ │ │ │ │ ├── abandoned-feature │ │ │ │ │ └── forked-feature-v1 │ │ │ ├── description │ │ │ ├── objects │ │ │ │ ├── 11 │ │ │ │ │ └── a76e3d9a67313dec7ce1230852ab5c86352c5c │ │ │ │ ├── 16 │ │ │ │ │ └── 5506a7408a482f50493434e13fffeb44af893f │ │ │ │ ├── 24 │ │ │ │ │ └── 4386e788c9bc608613e127a329c742450a60e4 │ │ │ │ ├── 32 │ │ │ │ │ └── 7e97d86e941047d809dba58f2804740c6c30cf │ │ │ │ ├── 36 │ │ │ │ │ └── 418b4e5665956a90725c9a1b5a8e551c5f3d48 │ │ │ │ ├── 41 │ │ │ │ │ └── 1de5d96ee418c1c55f3e96e6e6e7c06bb95801 │ │ │ │ ├── 56 │ │ │ │ │ └── 175e017ad38bf3d33d74b6bd7c74624b28466a │ │ │ │ ├── 67 │ │ │ │ │ └── 136e5ab4d5c1c65d10c8048763b96b0e53c1d6 │ │ │ │ ├── 88 │ │ │ │ │ └── cf20868e0cc445f5642a480ed034c71e0d7e9f │ │ │ │ ├── 00 │ │ │ │ │ └── fd13e76189f9134b0506b4b8ed3172723b467f │ │ │ │ ├── 0b │ │ │ │ │ ├── 15e8af3d4615b42314216efeae3fff184046a8 │ │ │ │ │ └── 67df4e7e8e6e1c6e401542738b352d18744677 │ │ │ │ ├── 3d │ │ │ │ │ ├── 7099c35404ae6c8640ce263b38bef06e98cc26 │ │ │ │ │ └── ec1fdf8e2f5edba28148c5db2fe8d7a842360b │ │ │ │ ├── 4d │ │ │ │ │ └── 837135915ed93eed6fff6b439f284ce317296f │ │ │ │ ├── 5f │ │ │ │ │ └── 1d4786d12e52d7ab28d2f2f1118c1059a9f1ae │ │ │ │ ├── 7b │ │ │ │ │ └── 0bd630ac13865735a1dff3437a137d8ab50663 │ │ │ │ ├── 8d │ │ │ │ │ └── 2b3b35126224c975d23f109aa1e3cbac452989 │ │ │ │ ├── 9b │ │ │ │ │ └── 75494003deca69527bb64bcaa352e801611dd2 │ │ │ │ ├── a2 │ │ │ │ │ └── 2a5da9119328ea6d693f88861457c07e14ac04 │ │ │ │ ├── a4 │ │ │ │ │ └── 2fe9144f5707bc1e9515ce1b44681f7aba6f95 │ │ │ │ ├── b9 │ │ │ │ │ └── 3737be3ea6b19f6255983748a0a0f4d622f936 │ │ │ │ ├── c5 │ │ │ │ │ ├── 32bc8fde96fa63103a52057f0baffcc9f00c6b │ │ │ │ │ └── b315915742133dbdfbeed0753e481b55c1d364 │ │ │ │ └── f2 │ │ │ │ │ └── 68d4e56d067da9bd1d85e55bdc40a8bd2b0bca │ │ │ ├── info │ │ │ │ └── exclude │ │ │ ├── packed-refs │ │ │ └── config │ │ └── error │ │ │ └── readme.txt │ ├── requirements.txt │ ├── .gitignore │ ├── .coveragerc │ ├── README.md │ ├── Makefile │ ├── test_unit_repository.py │ ├── test_sys_repository_git.py │ ├── test_unit_externals_status.py │ ├── test_unit_utils.py │ └── .pylint.rc ├── manic │ ├── __init__.py │ ├── global_constants.py │ ├── repository_factory.py │ ├── repository.py │ ├── externals_status.py │ ├── repository_svn.py │ └── utils.py ├── .gitignore ├── .dir_locals.el ├── .travis.yml ├── .github │ ├── ISSUE_TEMPLATE.md │ └── PULL_REQUEST_TEMPLATE.md ├── checkout_externals ├── README_FIRST ├── LICENSE.txt └── README.md ├── .gitignore ├── README.md ├── tools └── statistical_ensemble_test │ ├── user_nl_cam_LENS │ ├── addmetadata.sh │ ├── README │ └── ensemble.py └── Externals.cfg /cime_config/README.md: -------------------------------------------------------------------------------- 1 | # cime_config 2 | -------------------------------------------------------------------------------- /manage_externals/test/doc/.gitignore: -------------------------------------------------------------------------------- 1 | _build 2 | 3 | -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/HEAD: -------------------------------------------------------------------------------- 1 | ref: refs/heads/master 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/HEAD: -------------------------------------------------------------------------------- 1 | ref: refs/heads/master 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/HEAD: -------------------------------------------------------------------------------- 1 | ref: refs/heads/master 2 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/defaultiomi/include_user_mods: -------------------------------------------------------------------------------- 1 | ../defaultio 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/HEAD: -------------------------------------------------------------------------------- 1 | ref: refs/heads/master 2 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/cism/test_coupling/include_user_mods: -------------------------------------------------------------------------------- 1 | ../../defaultio 2 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/defaultiomi/shell_commands: -------------------------------------------------------------------------------- 1 | ./xmlchange NTASKS_OCN=144 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/refs/tags/tag1: -------------------------------------------------------------------------------- 1 | 11a76e3d9a67313dec7ce1230852ab5c86352c5c 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/refs/tags/tag2: -------------------------------------------------------------------------------- 1 | b7692b6d391899680da7b9b6fd8af4c413f06fe7 2 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/defaultio/user_nl_cam: -------------------------------------------------------------------------------- 1 | nhtfrq = -24 2 | mfilt = 1 3 | ndens = 1 4 | -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/refs/heads/master: -------------------------------------------------------------------------------- 1 | 715b8f3e4afe1802a178e1d603af404ba45d59de 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/refs/heads/feature2: -------------------------------------------------------------------------------- 1 | 36418b4e5665956a90725c9a1b5a8e551c5f3d48 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/refs/heads/feature3: -------------------------------------------------------------------------------- 1 | 090e1034746b2c865f7b0280813dbf4061a700e8 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/refs/heads/master: -------------------------------------------------------------------------------- 1 | 607ec299c17dd285c029edc41a0109e49d441380 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/refs/heads/master: -------------------------------------------------------------------------------- 1 | 6fc379457ecb4e576a13c7610ae1fa73f845ee6a 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/refs/heads/feature2: -------------------------------------------------------------------------------- 1 | f268d4e56d067da9bd1d85e55bdc40a8bd2b0bca 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/refs/heads/new-feature: -------------------------------------------------------------------------------- 1 | 9580ecc12f16334ce44e42287d5d46f927bb7b75 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/refs/tags/abandoned-feature: -------------------------------------------------------------------------------- 1 | a42fe9144f5707bc1e9515ce1b44681f7aba6f95 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/refs/tags/forked-feature-v1: -------------------------------------------------------------------------------- 1 | 8d2b3b35126224c975d23f109aa1e3cbac452989 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/description: -------------------------------------------------------------------------------- 1 | Unnamed repository; edit this file 'description' to name the repository. 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/description: -------------------------------------------------------------------------------- 1 | Unnamed repository; edit this file 'description' to name the repository. 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/description: -------------------------------------------------------------------------------- 1 | Unnamed repository; edit this file 'description' to name the repository. 2 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/description: -------------------------------------------------------------------------------- 1 | Unnamed repository; edit this file 'description' to name the repository. 2 | -------------------------------------------------------------------------------- /manage_externals/test/requirements.txt: -------------------------------------------------------------------------------- 1 | pylint>=1.7.0 2 | autopep8>=1.3.0 3 | coverage>=4.4.0 4 | coveralls>=1.2.0 5 | sphinx>=1.6.0 6 | -------------------------------------------------------------------------------- /manage_externals/test/repos/error/readme.txt: -------------------------------------------------------------------------------- 1 | Invalid or corrupted git repository (.git dir exists, but is empty) for error 2 | testing. 3 | 4 | -------------------------------------------------------------------------------- /manage_externals/test/.gitignore: -------------------------------------------------------------------------------- 1 | # virtual environments 2 | env_python* 3 | 4 | # python code coverage tool output 5 | .coverage 6 | htmlcov 7 | 8 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/defaultio/user_nl_clm: -------------------------------------------------------------------------------- 1 | hist_dov2xy = .true. 2 | hist_ndens = 1 3 | hist_nhtfrq =-24 4 | hist_mfilt = 1 5 | -------------------------------------------------------------------------------- /cime_config/usermods_dirs/README: -------------------------------------------------------------------------------- 1 | Directories here can be included in a case via the --user-mods-dir 2 | argument to create_newcase, or via the CPL_USER_MODS XML variable. 3 | -------------------------------------------------------------------------------- /manage_externals/test/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | omit = test_unit_*.py 4 | test_sys_*.py 5 | /usr/* 6 | .local/* 7 | */site-packages/* -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/config: -------------------------------------------------------------------------------- 1 | [core] 2 | repositoryformatversion = 0 3 | filemode = true 4 | bare = true 5 | ignorecase = true 6 | precomposeunicode = true 7 | -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/config: -------------------------------------------------------------------------------- 1 | [core] 2 | repositoryformatversion = 0 3 | filemode = true 4 | bare = true 5 | ignorecase = true 6 | precomposeunicode = true 7 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/config: -------------------------------------------------------------------------------- 1 | [core] 2 | repositoryformatversion = 0 3 | filemode = true 4 | bare = true 5 | ignorecase = true 6 | precomposeunicode = true 7 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/cism/test_coupling/README: -------------------------------------------------------------------------------- 1 | The purpose of this testmod directory is to enable CISM's dynamics on 2 | the short (multi-day) time scales typical of fully-active tests. 3 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/maxthroughputb/user_nl_cam: -------------------------------------------------------------------------------- 1 | ! Users should add all user specific namelist changes below in the form of 2 | ! namelist_var = new_namelist_value 3 | npr_yz= 64,18,18,64 4 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/defaultio/user_nl_pop: -------------------------------------------------------------------------------- 1 | & tavg_nml 2 | ltavg_ignore_extra_streams = .true. 3 | n_tavg_streams = 1 4 | tavg_freq_opt = 'nday' 5 | tavg_file_freq_opt = 'nday' 6 | / 7 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/defaultio/user_nl_cism: -------------------------------------------------------------------------------- 1 | 2 | ! This is needed to give CISM history output in the (typically short) 3 | ! tests that are done with this testmod directory 4 | history_option = 'coupler' 5 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/cism/test_coupling/shell_commands: -------------------------------------------------------------------------------- 1 | # Since we're doing a daily mass balance time step in CISM, we can't use the default GLC_AVG_PERIOD of yearly 2 | ./xmlchange GLC_AVG_PERIOD=glc_coupling_period 3 | -------------------------------------------------------------------------------- /manage_externals/manic/__init__.py: -------------------------------------------------------------------------------- 1 | """Public API for the manage_externals library 2 | """ 3 | 4 | from manic import checkout 5 | from manic.utils import printlog 6 | 7 | __all__ = [ 8 | 'checkout', 'printlog', 9 | ] 10 | -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/container.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801 -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/objects/71/5b8f3e4afe1802a178e1d603af404ba45d59de: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/container.git/objects/71/5b8f3e4afe1802a178e1d603af404ba45d59de -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/objects/b0/f87705e2b9601cb831878f3d51efa78b910d7b: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/container.git/objects/b0/f87705e2b9601cb831878f3d51efa78b910d7b -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/objects/f9/e08370a737e941de6f6492e3f427c2ef4c1a03: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/container.git/objects/f9/e08370a737e941de6f6492e3f427c2ef4c1a03 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/00/fd13e76189f9134b0506b4b8ed3172723b467f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/00/fd13e76189f9134b0506b4b8ed3172723b467f -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/09/0e1034746b2c865f7b0280813dbf4061a700e8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/09/0e1034746b2c865f7b0280813dbf4061a700e8 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/0b/15e8af3d4615b42314216efeae3fff184046a8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/0b/15e8af3d4615b42314216efeae3fff184046a8 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/11/a76e3d9a67313dec7ce1230852ab5c86352c5c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/11/a76e3d9a67313dec7ce1230852ab5c86352c5c -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/14/2711fdbbcb8034d7cad6bae6801887b12fe61d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/14/2711fdbbcb8034d7cad6bae6801887b12fe61d -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/31/dbcd6de441e671a467ef317146539b7ffabb11: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/31/dbcd6de441e671a467ef317146539b7ffabb11 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/36/418b4e5665956a90725c9a1b5a8e551c5f3d48: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/36/418b4e5665956a90725c9a1b5a8e551c5f3d48 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/60/7ec299c17dd285c029edc41a0109e49d441380: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/60/7ec299c17dd285c029edc41a0109e49d441380 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/60/b1cc1a38d63a4bcaa1e767262bbe23dbf9f5f5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/60/b1cc1a38d63a4bcaa1e767262bbe23dbf9f5f5 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/63/a99393d1baff97ccef967af30380659867b139: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/63/a99393d1baff97ccef967af30380659867b139 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/95/3256da5612fcd9263590a353bc18c6f224e74f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/95/3256da5612fcd9263590a353bc18c6f224e74f -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/9b/75494003deca69527bb64bcaa352e801611dd2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/9b/75494003deca69527bb64bcaa352e801611dd2 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/a2/2a5da9119328ea6d693f88861457c07e14ac04: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/a2/2a5da9119328ea6d693f88861457c07e14ac04 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/b7/692b6d391899680da7b9b6fd8af4c413f06fe7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/b7/692b6d391899680da7b9b6fd8af4c413f06fe7 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/c5/b315915742133dbdfbeed0753e481b55c1d364: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/c5/b315915742133dbdfbeed0753e481b55c1d364 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/d1/163870d19c3dee34fada3a76b785cfa2a8424b: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/d1/163870d19c3dee34fada3a76b785cfa2a8424b -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/d8/ed2f33179d751937f8fde2e33921e4827babf4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/d8/ed2f33179d751937f8fde2e33921e4827babf4 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/objects/df/312890f93ba4d2c694208599b665c4a08afeff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext.git/objects/df/312890f93ba4d2c694208599b665c4a08afeff -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/00/437ac2000d5f06fb8a572a01a5bbdae98b17cb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/00/437ac2000d5f06fb8a572a01a5bbdae98b17cb -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/01/97458f2dbe5fcd6bc44fa46983be0a30282379: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/01/97458f2dbe5fcd6bc44fa46983be0a30282379 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/06/ea30b03ffa2f8574705f8b9583f7ca7e2dccf7: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/06/ea30b03ffa2f8574705f8b9583f7ca7e2dccf7 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/14/368b701616a8c53820b610414a4b9a07540cf6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/14/368b701616a8c53820b610414a4b9a07540cf6 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/15/2b57e1cf23721cd17ff681cb9276e3fb9fc091: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/15/2b57e1cf23721cd17ff681cb9276e3fb9fc091 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/1f/01fa46c17b1f38b37e6259f6e9d041bda3144f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/1f/01fa46c17b1f38b37e6259f6e9d041bda3144f -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/37/f0e70b609adc90f4c09ee21d82ed1d79c81d69: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/37/f0e70b609adc90f4c09ee21d82ed1d79c81d69 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/38/9a2b876b8965d3c91a3db8d28a483eaf019d5c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/38/9a2b876b8965d3c91a3db8d28a483eaf019d5c -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/6e/9f4baa6e94a0af4e094836c2eb55ccedef5fc4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/6e/9f4baa6e94a0af4e094836c2eb55ccedef5fc4 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/6f/c379457ecb4e576a13c7610ae1fa73f845ee6a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/6f/c379457ecb4e576a13c7610ae1fa73f845ee6a -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/93/a159deb9175bfeb2820a0006ddd92d78131332: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/93/a159deb9175bfeb2820a0006ddd92d78131332 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/95/80ecc12f16334ce44e42287d5d46f927bb7b75: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/95/80ecc12f16334ce44e42287d5d46f927bb7b75 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/a9/288dcd8a719a1f4ed3cba43a2a387ae7cd60fd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/a9/288dcd8a719a1f4ed3cba43a2a387ae7cd60fd -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/e8/ea32a11d30ee703f6f661ae7c2376f4ab84d38: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/e8/ea32a11d30ee703f6f661ae7c2376f4ab84d38 -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/objects/fd/15a5ad5204356229c60a831d2a8120a43ac901: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/mixed-cont-ext.git/objects/fd/15a5ad5204356229c60a831d2a8120a43ac901 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/00/fd13e76189f9134b0506b4b8ed3172723b467f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/00/fd13e76189f9134b0506b4b8ed3172723b467f -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/0b/15e8af3d4615b42314216efeae3fff184046a8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/0b/15e8af3d4615b42314216efeae3fff184046a8 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/0b/67df4e7e8e6e1c6e401542738b352d18744677: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/0b/67df4e7e8e6e1c6e401542738b352d18744677 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/11/a76e3d9a67313dec7ce1230852ab5c86352c5c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/11/a76e3d9a67313dec7ce1230852ab5c86352c5c -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/16/5506a7408a482f50493434e13fffeb44af893f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/16/5506a7408a482f50493434e13fffeb44af893f -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/24/4386e788c9bc608613e127a329c742450a60e4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/24/4386e788c9bc608613e127a329c742450a60e4 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/32/7e97d86e941047d809dba58f2804740c6c30cf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/32/7e97d86e941047d809dba58f2804740c6c30cf -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/36/418b4e5665956a90725c9a1b5a8e551c5f3d48: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/36/418b4e5665956a90725c9a1b5a8e551c5f3d48 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/3d/7099c35404ae6c8640ce263b38bef06e98cc26: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/3d/7099c35404ae6c8640ce263b38bef06e98cc26 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/3d/ec1fdf8e2f5edba28148c5db2fe8d7a842360b: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/3d/ec1fdf8e2f5edba28148c5db2fe8d7a842360b -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/41/1de5d96ee418c1c55f3e96e6e6e7c06bb95801 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/4d/837135915ed93eed6fff6b439f284ce317296f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/4d/837135915ed93eed6fff6b439f284ce317296f -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/56/175e017ad38bf3d33d74b6bd7c74624b28466a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/56/175e017ad38bf3d33d74b6bd7c74624b28466a -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/5f/1d4786d12e52d7ab28d2f2f1118c1059a9f1ae: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/5f/1d4786d12e52d7ab28d2f2f1118c1059a9f1ae -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/67/136e5ab4d5c1c65d10c8048763b96b0e53c1d6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/67/136e5ab4d5c1c65d10c8048763b96b0e53c1d6 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/7b/0bd630ac13865735a1dff3437a137d8ab50663: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/7b/0bd630ac13865735a1dff3437a137d8ab50663 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/88/cf20868e0cc445f5642a480ed034c71e0d7e9f: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/88/cf20868e0cc445f5642a480ed034c71e0d7e9f -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/8d/2b3b35126224c975d23f109aa1e3cbac452989: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/8d/2b3b35126224c975d23f109aa1e3cbac452989 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/9b/75494003deca69527bb64bcaa352e801611dd2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/9b/75494003deca69527bb64bcaa352e801611dd2 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/a2/2a5da9119328ea6d693f88861457c07e14ac04: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/a2/2a5da9119328ea6d693f88861457c07e14ac04 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/a4/2fe9144f5707bc1e9515ce1b44681f7aba6f95: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/a4/2fe9144f5707bc1e9515ce1b44681f7aba6f95 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/b9/3737be3ea6b19f6255983748a0a0f4d622f936: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/b9/3737be3ea6b19f6255983748a0a0f4d622f936 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/c5/32bc8fde96fa63103a52057f0baffcc9f00c6b: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/c5/32bc8fde96fa63103a52057f0baffcc9f00c6b -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/c5/b315915742133dbdfbeed0753e481b55c1d364: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/c5/b315915742133dbdfbeed0753e481b55c1d364 -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/objects/f2/68d4e56d067da9bd1d85e55bdc40a8bd2b0bca: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ESCOMP/EarthWorks/master/manage_externals/test/repos/simple-ext-fork.git/objects/f2/68d4e56d067da9bd1d85e55bdc40a8bd2b0bca -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/cism/test_coupling/user_nl_cism: -------------------------------------------------------------------------------- 1 | 2 | ! This option changes the ice sheet dynamics time step to 1 day rather than 1 year 3 | ! Thus, the ice sheet dynamics can be exercised in a few-day run 4 | test_coupling = .true. 5 | -------------------------------------------------------------------------------- /manage_externals/.gitignore: -------------------------------------------------------------------------------- 1 | # directories that are checked out by the tool 2 | cime/ 3 | cime_config/ 4 | components/ 5 | 6 | # generated local files 7 | *.log 8 | 9 | # editor files 10 | *~ 11 | *.bak 12 | 13 | # generated python files 14 | *.pyc 15 | 16 | # test tmp file 17 | test/tmp 18 | -------------------------------------------------------------------------------- /manage_externals/test/repos/container.git/info/exclude: -------------------------------------------------------------------------------- 1 | # git ls-files --others --exclude-from=.git/info/exclude 2 | # Lines that start with '#' are comments. 3 | # For a project mostly in C, the following would be a good set of 4 | # exclude patterns (uncomment them if you want to use them): 5 | # *.[oa] 6 | # *~ 7 | -------------------------------------------------------------------------------- /manage_externals/test/repos/mixed-cont-ext.git/info/exclude: -------------------------------------------------------------------------------- 1 | # git ls-files --others --exclude-from=.git/info/exclude 2 | # Lines that start with '#' are comments. 3 | # For a project mostly in C, the following would be a good set of 4 | # exclude patterns (uncomment them if you want to use them): 5 | # *.[oa] 6 | # *~ 7 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext.git/info/exclude: -------------------------------------------------------------------------------- 1 | # git ls-files --others --exclude-from=.git/info/exclude 2 | # Lines that start with '#' are comments. 3 | # For a project mostly in C, the following would be a good set of 4 | # exclude patterns (uncomment them if you want to use them): 5 | # *.[oa] 6 | # *~ 7 | -------------------------------------------------------------------------------- /manage_externals/.dir_locals.el: -------------------------------------------------------------------------------- 1 | ; -*- mode: Lisp -*- 2 | 3 | ((python-mode 4 | . ( 5 | ;; fill the paragraph to 80 columns when using M-q 6 | (fill-column . 80) 7 | 8 | ;; Use 4 spaces to indent in Python 9 | (python-indent-offset . 4) 10 | (indent-tabs-mode . nil) 11 | ))) 12 | 13 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/info/exclude: -------------------------------------------------------------------------------- 1 | # git ls-files --others --exclude-from=.git/info/exclude 2 | # Lines that start with '#' are comments. 3 | # For a project mostly in C, the following would be a good set of 4 | # exclude patterns (uncomment them if you want to use them): 5 | # *.[oa] 6 | # *~ 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # managed directories that are checked out by the externals tool 2 | cime/ 3 | components/ 4 | libraries/ 5 | share/ 6 | ccs_config/ 7 | tools/statistical_ensemble_test/ 8 | 9 | # generated local files 10 | *.log 11 | 12 | # editor files 13 | *~ 14 | *.bak 15 | 16 | # generated python files 17 | *.pyc 18 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/packed-refs: -------------------------------------------------------------------------------- 1 | # pack-refs with: peeled fully-peeled sorted 2 | 36418b4e5665956a90725c9a1b5a8e551c5f3d48 refs/heads/feature2 3 | 9b75494003deca69527bb64bcaa352e801611dd2 refs/heads/master 4 | 11a76e3d9a67313dec7ce1230852ab5c86352c5c refs/tags/tag1 5 | ^9b75494003deca69527bb64bcaa352e801611dd2 6 | -------------------------------------------------------------------------------- /manage_externals/test/repos/simple-ext-fork.git/config: -------------------------------------------------------------------------------- 1 | [core] 2 | repositoryformatversion = 0 3 | filemode = true 4 | bare = true 5 | ignorecase = true 6 | precomposeunicode = true 7 | [remote "origin"] 8 | url = /Users/andreb/projects/ncar/git-conversion/checkout-model-dev/cesm-demo-externals/manage_externals/test/repos/simple-ext.git 9 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/maxthroughputfw/shell_commands: -------------------------------------------------------------------------------- 1 | # copy the spmd_mod.F90 2 | # This performance test is only expected to work on yellowstone 3 | cp /glade/u/home/jedwards/b1850/SourceMods/src.cam/* SourceMods/src.cam 4 | ./xmlchange NTASKS=2048 5 | ./xmlchange NTHRDS=4 6 | ./xmlchange --append CAM_CONFIG_OPTS="-pcols 8" 7 | 8 | 9 | -------------------------------------------------------------------------------- /manage_externals/.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | os: linux 3 | python: 4 | - "3.4" 5 | - "3.5" 6 | - "3.6" 7 | - "3.7" 8 | - "3.8" 9 | install: 10 | - pip install -r test/requirements.txt 11 | before_script: 12 | - git --version 13 | script: 14 | - cd test; make test 15 | - cd test; make lint 16 | after_success: 17 | - cd test; make coverage 18 | - cd test; coveralls 19 | -------------------------------------------------------------------------------- /manage_externals/.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Summary of Issue: 2 | ### Expected behavior and actual behavior: 3 | ### Steps to reproduce the problem (should include model description file(s) or link to publi c repository): 4 | ### What is the changeset ID of the code, and the machine you are using: 5 | ### have you modified the code? If so, it must be committed and available for testing: 6 | ### Screen output or log file showing the error message and context: 7 | -------------------------------------------------------------------------------- /manage_externals/manic/global_constants.py: -------------------------------------------------------------------------------- 1 | """Globals shared across modules 2 | """ 3 | 4 | from __future__ import absolute_import 5 | from __future__ import unicode_literals 6 | from __future__ import print_function 7 | 8 | import pprint 9 | 10 | EMPTY_STR = '' 11 | LOCAL_PATH_INDICATOR = '.' 12 | VERSION_SEPERATOR = '.' 13 | LOG_FILE_NAME = 'manage_externals.log' 14 | PPRINTER = pprint.PrettyPrinter(indent=4) 15 | 16 | VERBOSITY_DEFAULT = 0 17 | VERBOSITY_VERBOSE = 1 18 | VERBOSITY_DUMP = 2 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | trackgit-views 3 | 4 | # earthworks 5 | Start the earthworks repository based on cesm2_3_beta08 6 | 7 | The Externals.cfg file is updated for forks from various components. 8 | 9 | Checkout the earthworks code by doing the following: 10 | 11 | git clone https://github.com/dazlich/earthworks.git 12 | 13 | cd earthworks 14 | 15 | ./manage_externals/checkout_externals -v 16 | -------------------------------------------------------------------------------- /manage_externals/.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | [ 50 character, one line summary ] 2 | 3 | [ Description of the changes in this commit. It should be enough 4 | information for someone not following this development to understand. 5 | Lines should be wrapped at about 72 characters. ] 6 | 7 | User interface changes?: [ No/Yes ] 8 | [ If yes, describe what changed, and steps taken to ensure backward compatibilty ] 9 | 10 | Fixes: [Github issue #s] And brief description of each issue. 11 | 12 | Testing: 13 | test removed: 14 | unit tests: 15 | system tests: 16 | manual testing: 17 | 18 | -------------------------------------------------------------------------------- /manage_externals/test/doc/index.rst: -------------------------------------------------------------------------------- 1 | .. Manage Externals documentation master file, created by 2 | sphinx-quickstart on Wed Nov 29 10:53:25 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to Manage Externals's documentation! 7 | ============================================ 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | 14 | develop.rst 15 | testing.rst 16 | 17 | Indices and tables 18 | ================== 19 | 20 | * :ref:`genindex` 21 | * :ref:`modindex` 22 | * :ref:`search` 23 | -------------------------------------------------------------------------------- /manage_externals/test/doc/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = ManageExternals 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/defaultio/user_nl_rtm: -------------------------------------------------------------------------------- 1 | !---------------------------------------------------------------------------------- 2 | ! Users should add all user specific namelist changes below in the form of 3 | ! namelist_var = new_namelist_value 4 | ! NOTE: namelist variable rtm_tstep CAN ONLY be changed by modifying the value 5 | ! of the xml variable ROF_NCPL in env_run.xml 6 | ! NOTE: if the xml variable RTM_MODE in env_run.xml is set to 'NULL', then 7 | ! then rtm will set rtm_present to .false. - and will ignore everything else 8 | !---------------------------------------------------------------------------------- 9 | rtmhist_ndens = 1 10 | rtmhist_nhtfrq =-24 11 | rtmhist_mfilt = 1 12 | 13 | 14 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/defaultio/user_nl_mosart: -------------------------------------------------------------------------------- 1 | !---------------------------------------------------------------------------------- 2 | ! Users should add all user specific namelist changes below in the form of 3 | ! namelist_var = new_namelist_value 4 | ! NOTE: namelist variable rtm_tstep CAN ONLY be changed by modifying the value 5 | ! of the xml variable ROF_NCPL in env_run.xml 6 | ! NOTE: if the xml variable RTM_MODE in env_run.xml is set to 'NULL', then 7 | ! then rtm will set rtm_present to .false. - and will ignore everything else 8 | !---------------------------------------------------------------------------------- 9 | rtmhist_ndens = 1 10 | rtmhist_nhtfrq =-24 11 | rtmhist_mfilt = 1 12 | 13 | 14 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/cism/test_coupling/user_nl_clm: -------------------------------------------------------------------------------- 1 | 2 | ! This is needed to tell CLM to allow the non-annual-boundary glacier changes that arise 3 | ! with this testmod. 4 | for_testing_allow_non_annual_changes = .true. 5 | 6 | ! When we have daily rather than annual glacier dynamics (as we do in this testmod, due 7 | ! to having test_coupling in user_nl_cism), CLM applies the dynbal adjustments in a 8 | ! single time step rather than spreading them throughout the year. This can cause 9 | ! sensible heat fluxes of thousands of W m-2, which causes CAM's PBL scheme to blow up. 10 | ! So force these fluxes to zero for this testmod; this breaks water and energy 11 | ! conservation in CLM, but should allow the test to pass. 12 | for_testing_zero_dynbal_fluxes = .true. 13 | -------------------------------------------------------------------------------- /manage_externals/checkout_externals: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Main driver wrapper around the manic/checkout utility. 4 | 5 | Tool to assemble external respositories represented in an externals 6 | description file. 7 | 8 | """ 9 | from __future__ import absolute_import 10 | from __future__ import unicode_literals 11 | from __future__ import print_function 12 | 13 | import sys 14 | import traceback 15 | 16 | import manic 17 | 18 | if sys.hexversion < 0x02070000: 19 | print(70 * '*') 20 | print('ERROR: {0} requires python >= 2.7.x. '.format(sys.argv[0])) 21 | print('It appears that you are running python {0}'.format( 22 | '.'.join(str(x) for x in sys.version_info[0:3]))) 23 | print(70 * '*') 24 | sys.exit(1) 25 | 26 | 27 | if __name__ == '__main__': 28 | ARGS = manic.checkout.commandline_arguments() 29 | try: 30 | RET_STATUS, _ = manic.checkout.main(ARGS) 31 | sys.exit(RET_STATUS) 32 | except Exception as error: # pylint: disable=broad-except 33 | manic.printlog(str(error)) 34 | if ARGS.backtrace: 35 | traceback.print_exc() 36 | sys.exit(1) 37 | -------------------------------------------------------------------------------- /manage_externals/manic/repository_factory.py: -------------------------------------------------------------------------------- 1 | """Factory for creating and initializing the appropriate repository class 2 | """ 3 | 4 | from __future__ import absolute_import 5 | from __future__ import unicode_literals 6 | from __future__ import print_function 7 | 8 | from .repository_git import GitRepository 9 | from .repository_svn import SvnRepository 10 | from .externals_description import ExternalsDescription 11 | from .utils import fatal_error 12 | 13 | 14 | def create_repository(component_name, repo_info, svn_ignore_ancestry=False): 15 | """Determine what type of repository we have, i.e. git or svn, and 16 | create the appropriate object. 17 | 18 | """ 19 | protocol = repo_info[ExternalsDescription.PROTOCOL].lower() 20 | if protocol == 'git': 21 | repo = GitRepository(component_name, repo_info) 22 | elif protocol == 'svn': 23 | repo = SvnRepository(component_name, repo_info, ignore_ancestry=svn_ignore_ancestry) 24 | elif protocol == 'externals_only': 25 | repo = None 26 | else: 27 | msg = 'Unknown repo protocol "{0}"'.format(protocol) 28 | fatal_error(msg) 29 | return repo 30 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/maxthroughputb/shell_commands: -------------------------------------------------------------------------------- 1 | # copy the spmd_mod.F90 2 | # This performance test is only expected to work on yellowstone 3 | cp /glade/u/home/jedwards/b1850/SourceMods/src.cam/* SourceMods/src.cam 4 | NTASKS_ATM=1920 5 | NTASKS_CPL=608 6 | NTASKS_LND=1184 7 | NTASKS_ICE=608 8 | NTASKS_WAV=128 9 | NTASKS_OCN=192 10 | NTHRDS=2 11 | 12 | ROOTPE_WAV=$(($NTASKS_LND + $NTASKS_ICE)) 13 | #ROOTPE_CPL=$(($ROOTPE_WAV + $NTASKS_WAV)) 14 | ROOTPE_CPL=$NTASKS_LND 15 | COST_PES=$(($NTHRDS*($NTASKS_ATM + $NTASKS_OCN))) 16 | ROOTPE_OCN=$(($NTASKS_ATM)) 17 | 18 | ./xmlchange MAX_MPITASKS_PER_NODE=16 19 | ./xmlchange MAX_TASKS_PER_NODE=16 20 | ./xmlchange NTASKS_ATM=$NTASKS_ATM 21 | ./xmlchange NTASKS_CPL=$NTASKS_CPL 22 | ./xmlchange NTASKS_GLC=$NTASKS_ATM 23 | ./xmlchange NTASKS_LND=$NTASKS_LND 24 | ./xmlchange NTASKS_ROF=$NTASKS_LND 25 | ./xmlchange NTASKS_ICE=$NTASKS_ICE 26 | ./xmlchange NTASKS_WAV=$NTASKS_WAV 27 | ./xmlchange NTASKS_OCN=$NTASKS_OCN 28 | ./xmlchange ROOTPE_OCN=$ROOTPE_OCN 29 | ./xmlchange ROOTPE_ICE=$NTASKS_LND 30 | ./xmlchange ROOTPE_WAV=$ROOTPE_WAV 31 | ./xmlchange ROOTPE_CPL=$ROOTPE_CPL 32 | ./xmlchange NTHRDS=$NTHRDS 33 | 34 | -------------------------------------------------------------------------------- /tools/statistical_ensemble_test/user_nl_cam_LENS: -------------------------------------------------------------------------------- 1 | ! Users should add all user specific namelist changes below in the form of 2 | ! namelist_var = new_namelist_value 3 | ! These are specific changes to CAM for the LENS experiment 4 | cldfrc_rhminl = 0.8925D0 5 | empty_htapes=.true. 6 | fincl1='ABSORB:A','ANRAIN:A','ANSNOW:A','AODDUST1:A','AODDUST2:A','AODDUST3:A', 7 | 'AODVIS:A','AQRAIN:A','AQSNOW:A','AREI:A','AREL:A','AWNC:A','AWNI:A','CDNUMC:A', 8 | 'CLDHGH:A','CLDICE:A','CLDLIQ:A','CLDLOW:A','CLDMED:A','CLDTOT:A','CLOUD:A', 9 | 'DCQ:A','DTCOND:A','DTV:A','FICE:A','FLDS:A','FLNS:A','FLNSC:A','FLNT:A', 10 | 'FLNTC:A','FLUT:A','FLUTC:A','FREQI:A','FREQL:A','FREQR:A','FREQS:A','FSDS:A', 11 | 'FSDSC:A','FSNS:A','FSNSC:A','FSNT:A','FSNTC:A','FSNTOA:A','FSNTOAC:A', 12 | 'ICEFRAC:A','ICIMR:A','ICWMR:A','IWC:A','LANDFRAC:A','LHFLX:A','LWCF:A', 13 | 'NUMICE:A','NUMLIQ:A','OCNFRAC:A','OMEGA:A','OMEGAT:A','PBLH:A','PRECC:A', 14 | 'PRECL:A','PRECSC:A','PRECSL:A','PS:A','PSL:A','Q:A','QFLX:A','QRL:A','QRS:A', 15 | 'RELHUM:A','SHFLX:A','SNOWHICE:A','SNOWHLND:A','SOLIN:A','SRFRAD:A','SWCF:A', 16 | 'T:A','TAUX:A','TAUY:A','TGCLDIWP:A','TGCLDLWP:A','TMQ:A','TREFHT:A','TS:A', 17 | 'U:A','U10:A','UU:A','V:A','VD01:A','VQ:A','VT:A','VU:A','VV:A','WSUB:A','Z3:A', 18 | 'CCN3:A','UQ:A','WGUSTD:X','WSPDSRFMX:A','TSMX:X','TSMN:M','TREFHTMX:X','TREFHTMN:M', 19 | 'bc_a1_SRF:A','dst_a1_SRF:A','dst_a3_SRF:A','pom_a1_SRF:A','so4_a1_SRF:A', 20 | 'so4_a2_SRF:A','so4_a3_SRF:A','soa_a1_SRF:A','soa_a2_SRF:A','BURDENSO4:A', 21 | 'BURDENBC:A','BURDENPOM:A','BURDENSOA:A','BURDENDUST:A','BURDENSEASALT:A', 22 | 'AODABS:A','EXTINCT:A','PHIS:A','TROP_P:A','TROP_T:A','TOT_CLD_VISTAU:A', 23 | 'ICLDIWP:A','ICLDTWP:A','CO2:A','CO2_LND:A','CO2_OCN:A','SFCO2:A','SFCO2_LND:A', 24 | 'SFCO2_OCN:A','TMCO2:A','TMCO2_LND:A','TMCO2_OCN:A''CO2_FFF:A', 'SFCO2_FFF:A', 'TMCO2_FFF:A' 25 | -------------------------------------------------------------------------------- /tools/statistical_ensemble_test/addmetadata.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Adds metadata to netcdf statistical ensemble test files. 4 | # 5 | Args=("$@") 6 | i=0 7 | while [ $i -le ${#Args[@]} ]; do 8 | case ${Args[$i]} in 9 | --caseroot ) 10 | i=$((i+1)) 11 | caseroot=${Args[$i]} 12 | if [ ! -d ${caseroot} ]; then 13 | echo "ERROR: caseroot not found: $caseroot" 14 | exit 2 15 | fi 16 | if [ ! -f ${caseroot}/xmlquery ]; then 17 | echo "ERROR: Directory $caseroot does not appear to be a cesm case directory" 18 | exit 3 19 | fi 20 | ;; 21 | --histfile ) 22 | i=$((i+1)) 23 | histfile=${Args[$i]} 24 | if [ ! -f ${histfile} ]; then 25 | echo "ERROR: file not found $histfile" 26 | exit 4 27 | fi 28 | ;; 29 | --help ) 30 | echo "usage: addmetadata --histroot CASEROOT --histfile HISTFILE [--help] 31 | " 32 | echo "Script to add metadata to validation files. 33 | " 34 | echo "Optional arguments: 35 | --help show this help message and exit 36 | --caseroot Full pathname to the CASE directory. 37 | --histfile Full filename of the history file to add the metadata." 38 | exit 39 | ;; 40 | esac 41 | i=$((i+1)) 42 | done 43 | 44 | if [ "$caseroot" = "" ] || [ "$histfile" = "" ]; then 45 | echo "Please run ./addmetadata.sh --help for correct usage." 46 | exit 47 | fi 48 | 49 | cd $caseroot 50 | 51 | stop_option=`./xmlquery --value STOP_OPTION` 52 | test_type="UF-ECT" 53 | if [ "$stop_option" = "nmonths" ]; then 54 | test_type="ECT" 55 | elif [ "$stop_option" = "nyears" ]; then 56 | test_type="POP-ECT" 57 | fi 58 | 59 | if hash ncks 2>/dev/null; then 60 | ncks --glb compset=`./xmlquery --value COMPSET` --glb grid=`./xmlquery --value GRID` --glb testtype="$test_type" --glb compiler=`./xmlquery --value COMPILER` --glb machineid=`./xmlquery --value MACH` --glb model_version=`./xmlquery --value MODEL_VERSION` $histfile $histfile.tmp 61 | mv $histfile.tmp $histfile 62 | else 63 | echo "This script requires the ncks tool" 64 | fi 65 | -------------------------------------------------------------------------------- /manage_externals/README_FIRST: -------------------------------------------------------------------------------- 1 | CESM is comprised of a number of different components that are 2 | developed and managed independently. Each component may have 3 | additional 'external' dependancies and optional parts that are also 4 | developed and managed independently. 5 | 6 | The checkout_externals.py tool manages retreiving and updating the 7 | components and their externals so you have a complete set of source 8 | files for the model. 9 | 10 | checkout_externals.py relies on a model description file that 11 | describes what components are needed, where to find them and where to 12 | put them in the source tree. The default file is called "CESM.xml" 13 | regardless of whether you are checking out CESM or a standalone 14 | component. 15 | 16 | checkout_externals requires access to git and svn repositories that 17 | require authentication. checkout_externals may pass through 18 | authentication requests, but it will not cache them for you. For the 19 | best and most robust user experience, you should have svn and git 20 | working without password authentication. See: 21 | 22 | https://help.github.com/articles/connecting-to-github-with-ssh/ 23 | 24 | ?svn ref? 25 | 26 | NOTE: checkout_externals.py *MUST* be run from the root of the source 27 | tree it is managing. For example, if you cloned CLM with: 28 | 29 | $ git clone git@github.com/ncar/clm clm-dev 30 | 31 | Then the root of the source tree is /path/to/cesm-dev. If you obtained 32 | CLM via an svn checkout of CESM and you need to checkout the CLM 33 | externals, then the root of the source tree for CLM is: 34 | 35 | /path/to/cesm-dev/components/clm 36 | 37 | The root of the source tree will be referred to as ${SRC_ROOT} below. 38 | 39 | To get started quickly, checkout all required components from the 40 | default model description file: 41 | 42 | $ cd ${SRC_ROOT} 43 | $ ./checkout_cesm/checkout_externals.py 44 | 45 | For additional information about using checkout model, please see: 46 | 47 | ${SRC_ROOT}/checkout_cesm/README 48 | 49 | or run: 50 | 51 | $ cd ${SRC_ROOT} 52 | $ ./checkout_cesm/checkout_externals.py --help 53 | 54 | 55 | -------------------------------------------------------------------------------- /manage_externals/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017-2018, University Corporation for Atmospheric Research (UCAR) 2 | All rights reserved. 3 | 4 | Developed by: 5 | University Corporation for Atmospheric Research - National Center for Atmospheric Research 6 | https://www2.cesm.ucar.edu/working-groups/sewg 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining 9 | a copy of this software and associated documentation files (the "Software"), 10 | to deal with the Software without restriction, including without limitation 11 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 12 | and/or sell copies of the Software, and to permit persons to whom 13 | the Software is furnished to do so, subject to the following conditions: 14 | 15 | - Redistributions of source code must retain the above copyright notice, 16 | this list of conditions and the following disclaimers. 17 | - Redistributions in binary form must reproduce the above copyright notice, 18 | this list of conditions and the following disclaimers in the documentation 19 | and/or other materials provided with the distribution. 20 | - Neither the names of [Name of Development Group, UCAR], 21 | nor the names of its contributors may be used to endorse or promote 22 | products derived from this Software without specific prior written permission. 23 | 24 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 25 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 28 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 | POSSIBILITY OF SUCH DAMAGE. 35 | -------------------------------------------------------------------------------- /manage_externals/test/README.md: -------------------------------------------------------------------------------- 1 | # Testing for checkout_externals 2 | 3 | NOTE: Python2 is the supported runtime environment. Python3 compatibility is 4 | in progress, complicated by the different proposed input methods 5 | (yaml, xml, cfg/ini, json) and their different handling of strings 6 | (unicode vs byte) in python2. Full python3 compatibility will be 7 | possible once the number of possible input formats has been narrowed. 8 | 9 | ## Setup development environment 10 | 11 | Development environments should be setup for python2 and python3: 12 | 13 | ```SH 14 | cd checkout_externals/test 15 | make python=python2 env 16 | make python=python3 env 17 | ``` 18 | 19 | ## Unit tests 20 | 21 | Tests should be run for both python2 and python3. It is recommended 22 | that you have seperate terminal windows open python2 and python3 23 | testing to avoid errors activating and deactivating environments. 24 | 25 | ```SH 26 | cd checkout_externals/test 27 | . env_python2/bin/activate 28 | make utest 29 | deactivate 30 | ``` 31 | 32 | ```SH 33 | cd checkout_externals/test 34 | . env_python2/bin/activate 35 | make utest 36 | deactivate 37 | ``` 38 | 39 | ## System tests 40 | 41 | Not yet implemented. 42 | 43 | ## Static analysis 44 | 45 | checkout_externals is difficult to test thoroughly because it relies 46 | on git and svn, and svn requires a live network connection and 47 | repository. Static analysis will help catch bugs in code paths that 48 | are not being executed, but it requires conforming to community 49 | standards and best practices. autopep8 and pylint should be run 50 | regularly for automatic code formatting and linting. 51 | 52 | ```SH 53 | cd checkout_externals/test 54 | . env_python2/bin/activate 55 | make lint 56 | deactivate 57 | ``` 58 | 59 | The canonical formatting for the code is whatever autopep8 60 | generates. All issues identified by pylint should be addressed. 61 | 62 | 63 | ## Code coverage 64 | 65 | All changes to the code should include maintaining existing tests and 66 | writing new tests for new or changed functionality. To ensure test 67 | coverage, run the code coverage tool: 68 | 69 | ```SH 70 | cd checkout_externals/test 71 | . env_python2/bin/activate 72 | make coverage 73 | open -a Firefox.app htmlcov/index.html 74 | deactivate 75 | ``` 76 | 77 | 78 | -------------------------------------------------------------------------------- /cime_config/testmods_dirs/allactive/defaultio/user_nl_cice: -------------------------------------------------------------------------------- 1 | !---------------------------------------------------------------------------------- 2 | ! Have commented out the following due to known problems on restart 3 | ! f_albice = 'dxxxx' 4 | ! f_albsno = 'dxxxx' 5 | ! f_albpnd = 'dxxxx' 6 | !---------------------------------------------------------------------------------- 7 | histfreq = 'd','x','x','x','x' 8 | histfreq_n = 1,1,1,1,1 9 | f_hi = 'dxxxx' 10 | f_hs = 'dxxxx' 11 | f_Tsfc = 'dxxxx' 12 | f_aice = 'dxxxx' 13 | f_uvel = 'dxxxx' 14 | f_vvel = 'dxxxx' 15 | f_fswdn = 'dxxxx' 16 | f_flwdn = 'dxxxx' 17 | f_snow = 'dxxxx' 18 | f_snow_ai = 'dxxxx' 19 | f_rain = 'dxxxx' 20 | f_rain_ai = 'dxxxx' 21 | f_faero_atm = 'dxxxx' 22 | f_faero_ocn = 'dxxxx' 23 | f_sst = 'dxxxx' 24 | f_sss = 'dxxxx' 25 | f_uocn = 'dxxxx' 26 | f_vocn = 'dxxxx' 27 | f_frzmlt = 'dxxxx' 28 | f_fswfac = 'dxxxx' 29 | f_fswabs = 'dxxxx' 30 | f_fswabs_ai = 'dxxxx' 31 | f_alidr = 'dxxxx' 32 | f_alidf = 'dxxxx' 33 | f_alvdr = 'dxxxx' 34 | f_alvdf = 'dxxxx' 35 | f_coszen = 'dxxxx' 36 | f_flat = 'dxxxx' 37 | f_flat_ai = 'dxxxx' 38 | f_fsens = 'dxxxx' 39 | f_fsens_ai = 'dxxxx' 40 | f_flwup = 'dxxxx' 41 | f_flwup_ai = 'dxxxx' 42 | f_evap = 'dxxxx' 43 | f_evap_ai = 'dxxxx' 44 | f_Tair = 'dxxxx' 45 | f_Tref = 'dxxxx' 46 | f_Qref = 'dxxxx' 47 | f_congel = 'dxxxx' 48 | f_frazil = 'dxxxx' 49 | f_snoice = 'dxxxx' 50 | f_meltt = 'dxxxx' 51 | f_melts = 'dxxxx' 52 | f_meltb = 'dxxxx' 53 | f_meltl = 'dxxxx' 54 | f_fresh = 'dxxxx' 55 | f_fresh_ai = 'dxxxx' 56 | f_fsalt = 'dxxxx' 57 | f_fsalt_ai = 'dxxxx' 58 | f_fhocn = 'dxxxx' 59 | f_fhocn_ai = 'dxxxx' 60 | f_fswthru = 'dxxxx' 61 | f_fswthru_ai = 'dxxxx' 62 | f_strairx = 'dxxxx' 63 | f_strairy = 'dxxxx' 64 | f_strtltx = 'dxxxx' 65 | f_strtlty = 'dxxxx' 66 | f_strcorx = 'dxxxx' 67 | f_strcory = 'dxxxx' 68 | f_strocnx = 'dxxxx' 69 | f_strocny = 'dxxxx' 70 | f_strintx = 'dxxxx' 71 | f_strinty = 'dxxxx' 72 | f_strength = 'dxxxx' 73 | f_opening = 'dxxxx' 74 | f_divu = 'dxxxx' 75 | f_shear = 'dxxxx' 76 | f_sig1 = 'dxxxx' 77 | f_sig2 = 'dxxxx' 78 | f_dvidtt = 'dxxxx' 79 | f_dvidtd = 'dxxxx' 80 | f_daidtt = 'dxxxx' 81 | f_daidtd = 'dxxxx' 82 | f_dardg1dt = 'dxxxx' 83 | f_dardg2dt = 'dxxxx' 84 | f_dvirdgdt = 'dxxxx' 85 | f_iage = 'dxxxx' 86 | f_ardg = 'dxxxx' 87 | f_vrdg = 'dxxxx' 88 | f_alvl = 'dxxxx' 89 | f_vlvl = 'dxxxx' 90 | f_FY = 'dxxxx' 91 | f_aisnap = 'dxxxx' 92 | f_hisnap = 'dxxxx' 93 | f_aicen = 'dxxxx' 94 | f_vicen = 'dxxxx' 95 | f_trsig = 'dxxxx' 96 | f_icepresent = 'dxxxx' 97 | f_fsurf_ai = 'dxxxx' 98 | f_fcondtop_ai = 'dxxxx' 99 | -------------------------------------------------------------------------------- /manage_externals/test/Makefile: -------------------------------------------------------------------------------- 1 | python = not-set 2 | verbose = not-set 3 | debug = not-set 4 | 5 | ifneq ($(python), not-set) 6 | PYTHON=$(python) 7 | else 8 | PYTHON=python 9 | endif 10 | 11 | # we need the python path to point one level up to access the package 12 | # and executables 13 | PYPATH=PYTHONPATH=..: 14 | 15 | # common args for running tests 16 | TEST_ARGS=-m unittest discover 17 | 18 | ifeq ($(debug), not-set) 19 | ifeq ($(verbose), not-set) 20 | # summary only output 21 | TEST_ARGS+=--buffer 22 | else 23 | # show individual test summary 24 | TEST_ARGS+=--buffer --verbose 25 | endif 26 | else 27 | # show detailed test output 28 | TEST_ARGS+=--verbose 29 | endif 30 | 31 | 32 | # auto reformat the code 33 | AUTOPEP8=autopep8 34 | AUTOPEP8_ARGS=--aggressive --in-place 35 | 36 | # run lint 37 | PYLINT=pylint 38 | PYLINT_ARGS=-j 2 --rcfile=.pylint.rc 39 | 40 | # code coverage 41 | COVERAGE=coverage 42 | COVERAGE_ARGS=--rcfile=.coveragerc 43 | 44 | # source files 45 | SRC = \ 46 | ../checkout_externals \ 47 | ../manic/*.py 48 | 49 | CHECKOUT_EXE = ../checkout_externals 50 | 51 | TEST_DIR = . 52 | 53 | README = ../README.md 54 | 55 | # 56 | # testing 57 | # 58 | .PHONY : utest 59 | utest : FORCE 60 | $(PYPATH) $(PYTHON) $(TEST_ARGS) --pattern 'test_unit_*.py' 61 | 62 | .PHONY : stest 63 | stest : FORCE 64 | $(PYPATH) $(PYTHON) $(TEST_ARGS) --pattern 'test_sys_*.py' 65 | 66 | .PHONY : test 67 | test : utest stest 68 | 69 | # 70 | # documentation 71 | # 72 | .PHONY : readme 73 | readme : $(CHECKOUT_EXE) 74 | printf "%s\n\n" "-- AUTOMATICALLY GENERATED FILE. DO NOT EDIT --" > $(README) 75 | printf "%s" '[![Build Status](https://travis-ci.org/ESMCI/manage_externals.svg?branch=master)](https://travis-ci.org/ESMCI/manage_externals)' >> $(README) 76 | printf "%s" '[![Coverage Status](https://coveralls.io/repos/github/ESMCI/manage_externals/badge.svg?branch=master)](https://coveralls.io/github/ESMCI/manage_externals?branch=master)' >> $(README) 77 | printf "\n%s\n" '```' >> $(README) 78 | $(CHECKOUT_EXE) --help >> $(README) 79 | 80 | # 81 | # coding standards 82 | # 83 | .PHONY : style 84 | style : FORCE 85 | $(AUTOPEP8) $(AUTOPEP8_ARGS) --recursive $(SRC) $(TEST_DIR)/test_*.py 86 | 87 | .PHONY : lint 88 | lint : FORCE 89 | $(PYLINT) $(PYLINT_ARGS) $(SRC) $(TEST_DIR)/test_*.py 90 | 91 | .PHONY : stylint 92 | stylint : style lint 93 | 94 | .PHONY : coverage 95 | # Need to use a single coverage run with a single pattern rather than 96 | # using two separate commands with separate patterns for test_unit_*.py 97 | # and test_sys_*.py: The latter clobbers some results from the first 98 | # run, even if we use the --append flag to 'coverage run'. 99 | coverage : FORCE 100 | $(PYPATH) $(COVERAGE) erase 101 | $(PYPATH) $(COVERAGE) run $(COVERAGE_ARGS) $(TEST_ARGS) --pattern 'test_*.py' 102 | $(PYPATH) $(COVERAGE) html 103 | 104 | # 105 | # virtual environment creation 106 | # 107 | .PHONY : env 108 | env : FORCE 109 | $(PYPATH) virtualenv --python $(PYTHON) $@_$(PYTHON) 110 | . $@_$(PYTHON)/bin/activate; pip install -r requirements.txt 111 | 112 | # 113 | # utilites 114 | # 115 | .PHONY : clean 116 | clean : FORCE 117 | -rm -rf *~ *.pyc tmp fake htmlcov 118 | 119 | .PHONY : clobber 120 | clobber : clean 121 | -rm -rf env_* 122 | 123 | FORCE : 124 | 125 | -------------------------------------------------------------------------------- /manage_externals/manic/repository.py: -------------------------------------------------------------------------------- 1 | """Base class representation of a repository 2 | """ 3 | 4 | from .externals_description import ExternalsDescription 5 | from .utils import fatal_error 6 | from .global_constants import EMPTY_STR 7 | 8 | 9 | class Repository(object): 10 | """ 11 | Class to represent and operate on a repository description. 12 | """ 13 | 14 | def __init__(self, component_name, repo): 15 | """ 16 | Parse repo externals description 17 | """ 18 | self._name = component_name 19 | self._protocol = repo[ExternalsDescription.PROTOCOL] 20 | self._tag = repo[ExternalsDescription.TAG] 21 | self._branch = repo[ExternalsDescription.BRANCH] 22 | self._hash = repo[ExternalsDescription.HASH] 23 | self._url = repo[ExternalsDescription.REPO_URL] 24 | self._sparse = repo[ExternalsDescription.SPARSE] 25 | 26 | if self._url is EMPTY_STR: 27 | fatal_error('repo must have a URL') 28 | 29 | if ((self._tag is EMPTY_STR) and (self._branch is EMPTY_STR) and 30 | (self._hash is EMPTY_STR)): 31 | fatal_error('{0} repo must have a branch, tag or hash element') 32 | 33 | ref_count = 0 34 | if self._tag is not EMPTY_STR: 35 | ref_count += 1 36 | if self._branch is not EMPTY_STR: 37 | ref_count += 1 38 | if self._hash is not EMPTY_STR: 39 | ref_count += 1 40 | if ref_count != 1: 41 | fatal_error('repo {0} must have exactly one of ' 42 | 'tag, branch or hash.'.format(self._name)) 43 | 44 | def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument 45 | """ 46 | If the repo destination directory exists, ensure it is correct (from 47 | correct URL, correct branch or tag), and possibly update the source. 48 | If the repo destination directory does not exist, checkout the correce 49 | branch or tag. 50 | NB: is include as an argument for compatibility with 51 | git functionality (repository_git.py) 52 | """ 53 | msg = ('DEV_ERROR: checkout method must be implemented in all ' 54 | 'repository classes! {0}'.format(self.__class__.__name__)) 55 | fatal_error(msg) 56 | 57 | def status(self, stat, repo_dir_path): # pylint: disable=unused-argument 58 | """Report the status of the repo 59 | 60 | """ 61 | msg = ('DEV_ERROR: status method must be implemented in all ' 62 | 'repository classes! {0}'.format(self.__class__.__name__)) 63 | fatal_error(msg) 64 | 65 | def submodules_file(self, repo_path=None): 66 | # pylint: disable=no-self-use,unused-argument 67 | """Stub for use by non-git VC systems""" 68 | return None 69 | 70 | def url(self): 71 | """Public access of repo url. 72 | """ 73 | return self._url 74 | 75 | def tag(self): 76 | """Public access of repo tag 77 | """ 78 | return self._tag 79 | 80 | def branch(self): 81 | """Public access of repo branch. 82 | """ 83 | return self._branch 84 | 85 | def hash(self): 86 | """Public access of repo hash. 87 | """ 88 | return self._hash 89 | 90 | def name(self): 91 | """Public access of repo name. 92 | """ 93 | return self._name 94 | 95 | def protocol(self): 96 | """Public access of repo protocol. 97 | """ 98 | return self._protocol 99 | -------------------------------------------------------------------------------- /cime_config/config_compsets.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ========================================= 7 | compset naming convention 8 | ========================================= 9 | The compset longname below has the specified order 10 | atm, lnd, ice, ocn, river, glc wave cesm-options 11 | 12 | The notation for the compset longname is 13 | TIME_ATM[%phys]_LND[%phys]_ICE[%phys]_OCN[%phys]_ROF[%phys]_GLC[%phys]_WAV[%phys][_ESP%phys][_BGC%phys] 14 | Where for the specific compsets below the following is supported 15 | TIME = Time period (e.g. 2000, HIST, RCP8...) 16 | ATM = [CAM40, CAM50, CAM60] 17 | LND = [CLM45, CLM50, SLND] 18 | ICE = [CICE, DICE, SICE] 19 | OCN = [DOCN, ,AQUAP, SOCN] 20 | ROF = [RTM, MOSART, SROF] 21 | GLC = [CISM2, SGLC] 22 | WAV = [WW3, DWAV, XWAV, SWAV] 23 | ESP = [SESP] 24 | BGC = optional BGC scenario 25 | 26 | The OPTIONAL %phys attributes specify submodes of the given system 27 | For example DOCN%DOM is the data ocean model for DOCN 28 | ALL the possible %phys choices for each component are listed. 29 | ALL data models must have a %phys option that corresponds to the data model mode 30 | 31 | Each compset node is associated with the following elements 32 | - lname 33 | - alias 34 | - support (optional description of the support level for this compset) 35 | Each compset node can also have the following attributes 36 | - grid (optional regular expression match for grid to work with the compset) 37 | 38 | 39 | 40 | 41 | 42 | B1850 43 | 1850_CAM60_CLM50%BGC-CROP_CICE_POP2%ECO_MOSART_CISM2%GRIS-NOEVOLVE_WW3_BGC%BDRD 44 | 45 | 46 | 47 | BW1850 48 | 1850_CAM60%WCTS_CLM50%BGC-CROP_CICE_POP2%ECO%NDEP_MOSART_CISM2%GRIS-NOEVOLVE_WW3 49 | 50 | 51 | BWma1850 52 | 1850_CAM60%WCCM_CLM50%BGC-CROP_CICE_POP2%ECO%NDEP_MOSART_CISM2%GRIS-NOEVOLVE_WW3 53 | 54 | 55 | 56 | BHIST 57 | HIST_CAM60_CLM50%BGC-CROP_CICE_POP2%ECO_MOSART_CISM2%GRIS-NOEVOLVE_WW3_BGC%BDRD 58 | 59 | 60 | 61 | 62 | 63 | B1850G 64 | 1850_CAM60_CLM50%BGC-CROP_CICE_POP2%ECO_MOSART_CISM2%GRIS-EVOLVE_WW3_BGC%BDRD 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | ETEST 80 | 2000_CAM60_CLM50%SP_CICE_DOCN%SOM_MOSART_SGLC_SWAV_TEST 81 | 82 | 83 | 84 | E1850TEST 85 | 1850_CAM60_CLM50%SP_CICE_DOCN%SOM_MOSART_SGLC_SWAV_TEST 86 | 87 | 88 | 89 | 90 | 92 | B1850MOM 93 | 1850_CAM60_CLM50%BGC-CROP_CICE_MOM6_MOSART_CISM2%GRIS-NOEVOLVE_SWAV_BGC%BDRD 94 | 95 | 96 | 98 | 99 | 100 | J1850G 101 | 1850_DATM%CRUv7_CLM50%BGC-CROP_CICE_POP2_MOSART_CISM2%GRIS-EVOLVE_SWAV 102 | 103 | 104 | 105 | 106 | 107 | 0001-01-01 108 | 0001-01-01 109 | 1850-01-01 110 | 1955-01-01 111 | 2005-01-01 112 | 2013-01-01 113 | 114 | 115 | 116 | 117 | 118 | -------------------------------------------------------------------------------- /manage_externals/test/doc/testing.rst: -------------------------------------------------------------------------------- 1 | Testing 2 | ======= 3 | 4 | The manage_externals package has an automated test suite. All pull 5 | requests are expected to pass 100% of the automated tests, as well as 6 | be pep8 and lint 'clean' and maintain approximately constant (at a 7 | minimum) level of code coverage. 8 | 9 | Quick Start 10 | ----------- 11 | 12 | Do nothing approach 13 | ~~~~~~~~~~~~~~~~~~~ 14 | 15 | When you create a pull request on GitHub, Travis-CI continuous 16 | integration testing will run the test suite in both python2 and 17 | python3. Test results, lint results, and code coverage results are 18 | available online. 19 | 20 | Do something approach 21 | ~~~~~~~~~~~~~~~~~~~~~ 22 | 23 | In the test directory, run: 24 | 25 | .. code-block:: shell 26 | 27 | make env 28 | make lint 29 | make test 30 | make coverage 31 | 32 | 33 | Automated Testing 34 | ----------------- 35 | 36 | The manage_externals manic library and executables are developed to be 37 | python2 and python3 compatible using only the standard library. The 38 | test suites meet the same requirements. But additional tools are 39 | required to provide lint and code coverage metrics and generate 40 | documentation. The requirements are maintained in the requirements.txt 41 | file, and can be automatically installed into an isolated environment 42 | via Makefile. 43 | 44 | Bootstrap requirements: 45 | 46 | * python2 - version 2.7.x or later 47 | 48 | * python3 - version 3.6 tested other versions may work 49 | 50 | * pip and virtualenv for python2 and python3 51 | 52 | Note: all make rules can be of the form ``make python=pythonX rule`` 53 | or ``make rule`` depending if you want to use the default system 54 | python or specify a specific version. 55 | 56 | The Makefile in the test directory has the following rules: 57 | 58 | * ``make python=pythonX env`` - create a python virtual environment 59 | for python2 or python3 and install all required packages. These 60 | packages are required to run lint or coverage. 61 | 62 | * ``make style`` - runs autopep8 63 | 64 | * ``make lint`` - runs autopep8 and pylint 65 | 66 | * ``make test`` - run the full test suite 67 | 68 | * ``make utest`` - run jus the unit tests 69 | 70 | * ``make stest`` - run jus the system integration tests 71 | 72 | * ``make coverage`` - run the full test suite through the code 73 | coverage tool and generate an html report. 74 | 75 | * ``make readme`` - automatically generate the README files. 76 | 77 | * ``make clean`` - remove editor and pyc files 78 | 79 | * ``make clobber`` - remove all generated test files, including 80 | virtual environments, coverage reports, and temporary test 81 | repository directories. 82 | 83 | Unit Tests 84 | ---------- 85 | 86 | Unit tests are probably not 'true unit tests' for the pedantic, but 87 | are pragmatic unit tests. They cover small practicle code blocks: 88 | functions, class methods, and groups of functions and class methods. 89 | 90 | System Integration Tests 91 | ------------------------ 92 | 93 | NOTE(bja, 2017-11) The systems integration tests currently do not include svn repositories. 94 | 95 | The manage_externals package is extremely tedious and error prone to test manually. 96 | 97 | Combinations that must be tested to ensure basic functionality are: 98 | 99 | * container repository pulling in simple externals 100 | 101 | * container repository pulling in mixed externals with sub-externals. 102 | 103 | * mixed repository acting as a container, pulling in simple externals and sub-externals 104 | 105 | Automatic system tests are handled the same way manual testing is done: 106 | 107 | * clone a test repository 108 | 109 | * create an externals description file for the test 110 | 111 | * run the executable with the desired args 112 | 113 | * check the results 114 | 115 | * potentially modify the repo (checkout a different branch) 116 | 117 | * rerun and test 118 | 119 | * etc 120 | 121 | The automated system stores small test repositories in the main repo 122 | by adding them as bare repositories. These repos are cloned via a 123 | subprocess call to git and manipulated during the tests. 124 | -------------------------------------------------------------------------------- /Externals.cfg: -------------------------------------------------------------------------------- 1 | [ccs_config] 2 | branch = earthworks 3 | protocol = git 4 | repo_url = https://github.com/gdicker1/ccs_config_cesm.git 5 | local_path = ccs_config 6 | required = True 7 | 8 | [cam] 9 | branch = earthworks 10 | protocol = git 11 | repo_url = https://github.com/gdicker1/CAM 12 | local_path = components/cam 13 | externals = Externals_CAM.cfg 14 | required = True 15 | 16 | [cice5] 17 | tag = cice5_20220204 18 | protocol = git 19 | repo_url = https://github.com/ESCOMP/CESM_CICE5 20 | local_path = components/cice5 21 | required = True 22 | 23 | [cice6] 24 | tag = cesm_cice6_2_0_22 25 | protocol = git 26 | repo_url = https://github.com/ESCOMP/CESM_CICE 27 | local_path = components/cice 28 | externals = Externals.cfg 29 | required = True 30 | 31 | [cmeps] 32 | branch = earthworks 33 | protocol = git 34 | repo_url = https://github.com/gdicker1/CMEPS.git 35 | local_path = components/cmeps 36 | required = True 37 | 38 | [cdeps] 39 | tag = cdeps0.12.54 40 | protocol = git 41 | repo_url = https://github.com/ESCOMP/CDEPS.git 42 | local_path = components/cdeps 43 | externals = Externals_CDEPS.cfg 44 | required = True 45 | 46 | [cpl7] 47 | tag = cpl7.0.13 48 | protocol = git 49 | repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps 50 | local_path = components/cpl7 51 | required = True 52 | 53 | [share] 54 | tag = share1.0.12 55 | protocol = git 56 | repo_url = https://github.com/ESCOMP/CESM_share 57 | local_path = share 58 | required = True 59 | 60 | [mct] 61 | tag = MCT_2.11.0 62 | protocol = git 63 | repo_url = https://github.com/MCSclimate/MCT 64 | local_path = libraries/mct 65 | required = True 66 | 67 | [parallelio] 68 | tag = pio2_5_7 69 | protocol = git 70 | repo_url = https://github.com/NCAR/ParallelIO 71 | local_path = libraries/parallelio 72 | required = True 73 | 74 | [cime] 75 | branch = earthworks 76 | protocol = git 77 | repo_url = https://github.com/gdicker1/cime 78 | local_path = cime 79 | required = True 80 | 81 | [cism] 82 | tag = cismwrap_2_1_95 83 | protocol = git 84 | repo_url = https://github.com/ESCOMP/CISM-wrapper 85 | local_path = components/cism 86 | externals = Externals_CISM.cfg 87 | required = True 88 | 89 | [clm] 90 | tag = ctsm5.1.dev082 91 | protocol = git 92 | repo_url = https://github.com/ESCOMP/CTSM 93 | local_path = components/clm 94 | externals = Externals_CLM.cfg 95 | required = True 96 | 97 | [mpas-ocean] 98 | branch = main 99 | protocol = git 100 | repo_url = https://github.com/dazlich/mpas-ocean.git 101 | local_path = components/mpas-ocean 102 | required = True 103 | 104 | [mpas-seaice] 105 | branch = main 106 | protocol = git 107 | repo_url = https://github.com/dazlich/mpas-seaice.git 108 | local_path = components/mpas-seaice 109 | required = True 110 | 111 | [mpas-framework] 112 | branch = main 113 | protocol = git 114 | repo_url = https://github.com/dazlich/mpas-framework.git 115 | local_path = components/mpas-framework 116 | required = True 117 | 118 | [fms] 119 | tag = fi_20220425 120 | protocol = git 121 | repo_url = https://github.com/ESCOMP/FMS_interface 122 | local_path = libraries/FMS 123 | externals = Externals_FMS.cfg 124 | required = False 125 | 126 | [mom] 127 | tag = mi_220624 128 | protocol = git 129 | repo_url = https://github.com/ESCOMP/MOM_interface 130 | local_path = components/mom 131 | externals = Externals.cfg 132 | required = False 133 | 134 | [mosart] 135 | tag = mosart1_0_45 136 | protocol = git 137 | repo_url = https://github.com/ESCOMP/MOSART 138 | local_path = components/mosart 139 | required = True 140 | 141 | [pop] 142 | tag = cesm_pop_2_1_20220322 143 | protocol = git 144 | repo_url = https://github.com/ESCOMP/POP2-CESM 145 | local_path = components/pop 146 | externals = Externals_POP.cfg 147 | required = True 148 | 149 | [pycect] 150 | tag = 3.2.2 151 | protocol = git 152 | repo_url = https://github.com/NCAR/PyCECT 153 | local_path = tools/statistical_ensemble_test/pyCECT 154 | required = False 155 | 156 | [rtm] 157 | tag = rtm1_0_78 158 | protocol = git 159 | repo_url = https://github.com/ESCOMP/RTM 160 | local_path = components/rtm 161 | required = True 162 | 163 | [ww3] 164 | tag = ww3_220322 165 | protocol = git 166 | repo_url = https://github.com/ESCOMP/WW3-CESM 167 | local_path = components/ww3 168 | required = True 169 | 170 | [ww3dev] 171 | tag = main_0.0.2 172 | protocol = git 173 | repo_url = https://github.com/ESCOMP/WW3_interface 174 | local_path = components/ww3dev 175 | externals = Externals.cfg 176 | required = False 177 | 178 | [externals_description] 179 | schema_version = 1.0.01 180 | -------------------------------------------------------------------------------- /tools/statistical_ensemble_test/README: -------------------------------------------------------------------------------- 1 | ------------------------------------------ 2 | CESM-ECT (CESM Ensemble Consistency Test: 3 | ------------------------------------------ 4 | 5 | CESM-ECT is a suite of tests to determine whether a new 6 | simulation set up (new machine, compiler, etc.) is statistically 7 | distinguishable from an accepted ensemble. The verification tools in 8 | the CESM-ECT suite are: 9 | 10 | CAM-ECT - detects issues in CAM and CLM (12 month runs) 11 | UF-CAM-ECT - detects issues in CAM and CLM (9 time step runs) 12 | POP-ECT - detects issues in POP and CICE (12 month runs) 13 | 14 | The ECT process involves comparing runs generated with 15 | the new scenario ( 3 for CAM-ECT and UF-CAM-ECT, and 1 for POP-ECT) 16 | to an ensemble built on a trusted machine (currently 17 | cheyenne). The python ECT tools are located in the pyCECT 18 | subdirectory or https://github.com/NCAR/PyCECT/releases. 19 | 20 | -OR- 21 | 22 | We now provide a web server for CAM-ECT and UF-CAM-ECT, where 23 | you can upload the (3) generated runs for comparison to our ensemble. 24 | Please see the webpage at http://www.cesm.ucar.edu/models/cesm2/verification/ 25 | for further instructions. 26 | 27 | ----------------------------------- 28 | Creating or obtaining a summary file: 29 | ----------------------------------- 30 | 31 | Before the test can be run, a summary file is needed of the ensemble 32 | runs to which the comparison will be made. Ensemble summary files 33 | (NetCDF) for existing tags for CAM-ECT, UF-CAM-ECT, and POP-ECT that 34 | were created by CSEG are located (respectively) in the CESM input data 35 | directories: 36 | 37 | $CESMDATAROOT/inputdata/validation/ensembles 38 | $CESMDATAROOT/inputdata/validation/uf_ensembles 39 | $CESMDATAROOT/inputdata/validation/pop_ensembles 40 | 41 | If none of our ensembles are suitable for your needs, then you may create 42 | your own ensemble (and summary file) using the following instructions: 43 | 44 | (1) To create a new ensemble, use the ensemble.py script in this directory. 45 | This script creates and compiles a case, then creates clones of the 46 | original case, where the initial temperature perturbation is slightly modified 47 | for each ensemble member. At this time, cime includes functionality 48 | to create ensembles for CAM-ECT, UF-CAM-ECT, and POP-ECT. 49 | 50 | (2) Use --ect to specify whether ensemble is for CAM or POP. 51 | (See 'python ensemble.py -h' for additional details). 52 | 53 | (3) Use --ensemble to specify the ensemble size. 54 | Recommended ensemble sizes: 55 | CAM-ECT: 151 56 | UF-CAM-ECT: 350 57 | POP-ECT 40 58 | 59 | (4) Examples: 60 | 61 | CAM-ECT: 62 | 63 | python ensemble.py --case /glade/scratch/cesm_user/cesm_tag/ensemble/ensemble.cesm_tag.000 --mach cheyenne --ensemble 151 --ect cam --project P99999999 64 | 65 | 66 | UF-CAM-ECT: 67 | 68 | python ensemble.py --case /glade/scratch/cesm_user/cesm_tag/uf_ensemble/ensemble.cesm_tag.uf.000 --mach cheyenne --ensemble 350 --uf --ect cam --project P99999999 69 | 70 | POP-ECT: 71 | 72 | python ensemble.py --case /glade/scratch/cesm_user/cesm_tag/uf_ensemble/ensemble.cesm_tag.000 --mach cheyenne --ensemble 40 --ect pop --project P99999999 73 | 74 | Notes: 75 | (a) ensemble.py accepts (most of) the argumenets of create_newcase 76 | 77 | (b) case name must end in ".000" and include the full path 78 | 79 | (c) ensemble size must be specified, and suggested defaults are listed 80 | above. Note that for CAM-ECT and UF-CAM-ECT, the ensemble size 81 | needs to be larger than the number of variables that ECT will evaluate. 82 | 83 | 84 | (5) Once all ensemble simulations have run successfully, copy every cam history 85 | file (*.cam.h0.*) for CAM-ECT and UF-CAM-ECT) or monthly pop history file 86 | (*.pop.h.*) for POP-ECT from each ensemble run directory into a separate directory. 87 | Next create the ensemble summary using the pyCECT tool pyEnsSum.py (for CAM-ECT and 88 | UF-CAM-ECT) or pyEnsSumPop.py (for POP-ECT). For details see README_pyEnsSum.rst 89 | and README_pyEnsSumPop.rst with the pyCECT tools. 90 | 91 | ------------------- 92 | Creating test runs: 93 | ------------------- 94 | 95 | (1) Once an ensemble summary file has been created or chosen to 96 | use from $CESMDATAROOT/inputdata/validation, the simulation 97 | run(s) to be verified by ECT must be created via script ensemble.py. 98 | 99 | NOTE: It is important that the **same** resolution and compset be used in the 100 | individual runs as in the ensemble. The NetCDF ensemble summary file global 101 | attributes give this information. 102 | 103 | (2) For example, for CAM-ECT: 104 | 105 | python ensemble.py --case /glade/scratch/cesm_user/cesm_tag/camcase.cesm_tag.000 --ect cam --mach cheyenne --project P99999999 106 | --compset F2000climo --res f19_f19_mg17 107 | For example, for UF-CAM-ECT: 108 | 109 | python ensemble.py --case /glade/scratch/cesm_user/cesm_tag/uf.camcase.cesm_tag.000 --ect cam --uf --mach cheyenne --project P99999999 --compset F2000climo --res f19_f19_mg17 110 | 111 | For example, for POP-ECT: 112 | 113 | python ensemble.py --case /glade/scratch/cesm_user/cesm_tag/popcase.cesm_tag.000 --ect pop --mach cheyenne --project P99999999 --compset G --res T62_g17 114 | 115 | (3) Next verify the new simulation(s) with the pyCECT tool pyCECT.py (see 116 | README_pyCECT.rst with the pyCECT tools). 117 | -------------------------------------------------------------------------------- /manage_externals/test/doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Manage Externals documentation build configuration file, created by 4 | # sphinx-quickstart on Wed Nov 29 10:53:25 2017. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | # If extensions (or modules to document with autodoc) are in another directory, 16 | # add these directories to sys.path here. If the directory is relative to the 17 | # documentation root, use os.path.abspath to make it absolute, like shown here. 18 | # 19 | # import os 20 | # import sys 21 | # sys.path.insert(0, os.path.abspath('.')) 22 | 23 | 24 | # -- General configuration ------------------------------------------------ 25 | 26 | # If your documentation needs a minimal Sphinx version, state it here. 27 | # 28 | # needs_sphinx = '1.0' 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = ['sphinx.ext.autodoc', 34 | 'sphinx.ext.todo', 35 | 'sphinx.ext.coverage', 36 | 'sphinx.ext.viewcode', 37 | 'sphinx.ext.githubpages'] 38 | 39 | # Add any paths that contain templates here, relative to this directory. 40 | templates_path = ['_templates'] 41 | 42 | # The suffix(es) of source filenames. 43 | # You can specify multiple suffix as a list of string: 44 | # 45 | # source_suffix = ['.rst', '.md'] 46 | source_suffix = '.rst' 47 | 48 | # The master toctree document. 49 | master_doc = 'index' 50 | 51 | # General information about the project. 52 | project = u'Manage Externals' 53 | copyright = u'2017, CSEG at NCAR' 54 | author = u'CSEG at NCAR' 55 | 56 | # The version info for the project you're documenting, acts as replacement for 57 | # |version| and |release|, also used in various other places throughout the 58 | # built documents. 59 | # 60 | # The short X.Y version. 61 | version = u'1.0.0' 62 | # The full version, including alpha/beta/rc tags. 63 | release = u'1.0.0' 64 | 65 | # The language for content autogenerated by Sphinx. Refer to documentation 66 | # for a list of supported languages. 67 | # 68 | # This is also used if you do content translation via gettext catalogs. 69 | # Usually you set "language" from the command line for these cases. 70 | language = None 71 | 72 | # List of patterns, relative to source directory, that match files and 73 | # directories to ignore when looking for source files. 74 | # This patterns also effect to html_static_path and html_extra_path 75 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 76 | 77 | # The name of the Pygments (syntax highlighting) style to use. 78 | pygments_style = 'sphinx' 79 | 80 | # If true, `todo` and `todoList` produce output, else they produce nothing. 81 | todo_include_todos = True 82 | 83 | 84 | # -- Options for HTML output ---------------------------------------------- 85 | 86 | # The theme to use for HTML and HTML Help pages. See the documentation for 87 | # a list of builtin themes. 88 | # 89 | html_theme = 'alabaster' 90 | 91 | # Theme options are theme-specific and customize the look and feel of a theme 92 | # further. For a list of options available for each theme, see the 93 | # documentation. 94 | # 95 | # html_theme_options = {} 96 | 97 | # Add any paths that contain custom static files (such as style sheets) here, 98 | # relative to this directory. They are copied after the builtin static files, 99 | # so a file named "default.css" will overwrite the builtin "default.css". 100 | html_static_path = ['_static'] 101 | 102 | # Custom sidebar templates, must be a dictionary that maps document names 103 | # to template names. 104 | # 105 | # This is required for the alabaster theme 106 | # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars 107 | html_sidebars = { 108 | '**': [ 109 | 'relations.html', # needs 'show_related': True theme option to display 110 | 'searchbox.html', 111 | ] 112 | } 113 | 114 | 115 | # -- Options for HTMLHelp output ------------------------------------------ 116 | 117 | # Output file base name for HTML help builder. 118 | htmlhelp_basename = 'ManageExternalsdoc' 119 | 120 | 121 | # -- Options for LaTeX output --------------------------------------------- 122 | 123 | latex_elements = { 124 | # The paper size ('letterpaper' or 'a4paper'). 125 | # 126 | # 'papersize': 'letterpaper', 127 | 128 | # The font size ('10pt', '11pt' or '12pt'). 129 | # 130 | # 'pointsize': '10pt', 131 | 132 | # Additional stuff for the LaTeX preamble. 133 | # 134 | # 'preamble': '', 135 | 136 | # Latex figure (float) alignment 137 | # 138 | # 'figure_align': 'htbp', 139 | } 140 | 141 | # Grouping the document tree into LaTeX files. List of tuples 142 | # (source start file, target name, title, 143 | # author, documentclass [howto, manual, or own class]). 144 | latex_documents = [ 145 | (master_doc, 'ManageExternals.tex', u'Manage Externals Documentation', 146 | u'CSEG at NCAR', 'manual'), 147 | ] 148 | 149 | 150 | # -- Options for manual page output --------------------------------------- 151 | 152 | # One entry per manual page. List of tuples 153 | # (source start file, name, description, authors, manual section). 154 | man_pages = [ 155 | (master_doc, 'manageexternals', u'Manage Externals Documentation', 156 | [author], 1) 157 | ] 158 | 159 | 160 | # -- Options for Texinfo output ------------------------------------------- 161 | 162 | # Grouping the document tree into Texinfo files. List of tuples 163 | # (source start file, target name, title, author, 164 | # dir menu entry, description, category) 165 | texinfo_documents = [ 166 | (master_doc, 'ManageExternals', u'Manage Externals Documentation', 167 | author, 'ManageExternals', 'One line description of project.', 168 | 'Miscellaneous'), 169 | ] 170 | 171 | 172 | 173 | -------------------------------------------------------------------------------- /manage_externals/manic/externals_status.py: -------------------------------------------------------------------------------- 1 | """ExternalStatus 2 | 3 | Class to store status and state information about repositories and 4 | create a string representation. 5 | 6 | """ 7 | from __future__ import absolute_import 8 | from __future__ import unicode_literals 9 | from __future__ import print_function 10 | 11 | from .global_constants import EMPTY_STR 12 | from .utils import printlog, indent_string 13 | from .global_constants import VERBOSITY_VERBOSE, VERBOSITY_DUMP 14 | 15 | 16 | class ExternalStatus(object): 17 | """Class to represent the status of a given source repository or tree. 18 | 19 | Individual repositories determine their own status in the 20 | Repository objects. This object is just resposible for storing the 21 | information and passing it up to a higher level for reporting or 22 | global decisions. 23 | 24 | There are two states of concern: 25 | 26 | * If the repository is in-sync with the externals description file. 27 | 28 | * If the repostiory working copy is clean and there are no pending 29 | transactions (e.g. add, remove, rename, untracked files). 30 | 31 | """ 32 | DEFAULT = '-' 33 | UNKNOWN = '?' 34 | EMPTY = 'e' 35 | MODEL_MODIFIED = 's' # a.k.a. out-of-sync 36 | DIRTY = 'M' 37 | 38 | STATUS_OK = ' ' 39 | STATUS_ERROR = '!' 40 | 41 | # source types 42 | OPTIONAL = 'o' 43 | STANDALONE = 's' 44 | MANAGED = ' ' 45 | 46 | def __init__(self): 47 | self.sync_state = self.DEFAULT 48 | self.clean_state = self.DEFAULT 49 | self.source_type = self.DEFAULT 50 | self.path = EMPTY_STR 51 | self.current_version = EMPTY_STR 52 | self.expected_version = EMPTY_STR 53 | self.status_output = EMPTY_STR 54 | 55 | def log_status_message(self, verbosity): 56 | """Write status message to the screen and log file 57 | """ 58 | self._default_status_message() 59 | if verbosity >= VERBOSITY_VERBOSE: 60 | self._verbose_status_message() 61 | if verbosity >= VERBOSITY_DUMP: 62 | self._dump_status_message() 63 | 64 | def _default_status_message(self): 65 | """Return the default terse status message string 66 | """ 67 | msg = '{sync}{clean}{src_type} {path}'.format( 68 | sync=self.sync_state, clean=self.clean_state, 69 | src_type=self.source_type, path=self.path) 70 | printlog(msg) 71 | 72 | def _verbose_status_message(self): 73 | """Return the verbose status message string 74 | """ 75 | clean_str = self.DEFAULT 76 | if self.clean_state == self.STATUS_OK: 77 | clean_str = 'clean sandbox' 78 | elif self.clean_state == self.DIRTY: 79 | clean_str = 'modified sandbox' 80 | 81 | sync_str = 'on {0}'.format(self.current_version) 82 | if self.sync_state != self.STATUS_OK: 83 | sync_str = '{current} --> {expected}'.format( 84 | current=self.current_version, expected=self.expected_version) 85 | msg = ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) 86 | printlog(msg) 87 | 88 | def _dump_status_message(self): 89 | """Return the dump status message string 90 | """ 91 | msg = indent_string(self.status_output, 12) 92 | printlog(msg) 93 | 94 | def safe_to_update(self): 95 | """Report if it is safe to update a repository. Safe is defined as: 96 | 97 | * If a repository is empty, it is safe to update. 98 | 99 | * If a repository exists and has a clean working copy state 100 | with no pending transactions. 101 | 102 | """ 103 | safe_to_update = False 104 | repo_exists = self.exists() 105 | if not repo_exists: 106 | safe_to_update = True 107 | else: 108 | # If the repo exists, it must be in ok or modified 109 | # sync_state. Any other sync_state at this point 110 | # represents a logic error that should have been handled 111 | # before now! 112 | sync_safe = ((self.sync_state == ExternalStatus.STATUS_OK) or 113 | (self.sync_state == ExternalStatus.MODEL_MODIFIED)) 114 | if sync_safe: 115 | # The clean_state must be STATUS_OK to update. Otherwise we 116 | # are dirty or there was a missed error previously. 117 | if self.clean_state == ExternalStatus.STATUS_OK: 118 | safe_to_update = True 119 | return safe_to_update 120 | 121 | def exists(self): 122 | """Determine if the repo exists. This is indicated by: 123 | 124 | * sync_state is not EMPTY 125 | 126 | * if the sync_state is empty, then the valid states for 127 | clean_state are default, empty or unknown. Anything else 128 | and there was probably an internal logic error. 129 | 130 | NOTE(bja, 2017-10) For the moment we are considering a 131 | sync_state of default or unknown to require user intervention, 132 | but we may want to relax this convention. This is probably a 133 | result of a network error or internal logic error but more 134 | testing is needed. 135 | 136 | """ 137 | is_empty = (self.sync_state == ExternalStatus.EMPTY) 138 | clean_valid = ((self.clean_state == ExternalStatus.DEFAULT) or 139 | (self.clean_state == ExternalStatus.EMPTY) or 140 | (self.clean_state == ExternalStatus.UNKNOWN)) 141 | 142 | if is_empty and clean_valid: 143 | exists = False 144 | else: 145 | exists = True 146 | return exists 147 | 148 | 149 | def check_safe_to_update_repos(tree_status): 150 | """Check if *ALL* repositories are in a safe state to update. We don't 151 | want to do a partial update of the repositories then die, leaving 152 | the model in an inconsistent state. 153 | 154 | Note: if there is an update to do, the repositories will by 155 | definiation be out of synce with the externals description, so we 156 | can't use that as criteria for updating. 157 | 158 | """ 159 | safe_to_update = True 160 | for comp in tree_status: 161 | stat = tree_status[comp] 162 | safe_to_update &= stat.safe_to_update() 163 | 164 | return safe_to_update 165 | -------------------------------------------------------------------------------- /tools/statistical_ensemble_test/ensemble.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | from __future__ import print_function 3 | import os, sys, getopt 4 | import random 5 | from single_run import process_args_dict, single_case 6 | 7 | # ============================================================================== 8 | # set up and submit 12-month (original) or 9-time step (uf) run. then create 9 | # clones for a complete ensemble or a set of (3) test cases 10 | # ============================================================================== 11 | 12 | # generate positive random integers in [0, end-1] 13 | # can't have any duplicates 14 | def random_pick(num_pick, end): 15 | ar = range(0, end) 16 | rand_list = random.sample(ar, num_pick) 17 | # for i in rand_list: 18 | # print i 19 | return rand_list 20 | 21 | 22 | # get the pertlim corressponding to the random int 23 | def get_pertlim_uf(rand_num): 24 | i = rand_num 25 | if i == 0: 26 | ptlim = 0 27 | else: 28 | j = 2 * int((i - 1) / 100) + 101 29 | k = (i - 1) % 100 30 | if i % 2 != 0: 31 | ll = j + int(k / 2) * 18 32 | ippt = str(ll).zfill(3) 33 | ptlim = "0." + ippt + "d-13" 34 | else: 35 | ll = j + int((k - 1) / 2) * 18 36 | ippt = str(ll).zfill(3) 37 | ptlim = "-0." + ippt + "d-13" 38 | return ptlim 39 | 40 | 41 | def main(argv): 42 | 43 | caller = "ensemble.py" 44 | 45 | # directory with single_run.py and ensemble.py 46 | stat_dir = os.path.dirname(os.path.realpath(__file__)) 47 | print("STATUS: stat_dir = " + stat_dir) 48 | 49 | opts_dict, case_flags = process_args_dict(caller, argv) 50 | 51 | # default is verification mode (3 runs) 52 | run_type = "verify" 53 | if opts_dict["ect"] == "pop": 54 | clone_count = 0 55 | else: 56 | clone_count = 2 57 | 58 | uf = opts_dict["uf"] 59 | 60 | # check for run_type change (i.e., if doing ensemble instead of verify) 61 | ens_size = opts_dict["ensemble"] 62 | if ens_size > 0: 63 | run_type = "ensemble" 64 | clone_count = ens_size - 1 65 | if ens_size > 999: 66 | print("Error: cannot have an ensemble size greater than 999.") 67 | sys.exit() 68 | print("STATUS: ensemble size = " + str(ens_size)) 69 | 70 | # generate random pertlim(s) for verify 71 | if run_type == "verify": 72 | if opts_dict["ect"] == "pop": 73 | rand_ints = random_pick(1, 40) 74 | else: # cam 75 | if uf: 76 | end_range = 350 77 | else: 78 | end_range = 150 79 | rand_ints = random_pick(3, end_range) 80 | 81 | # now create cases 82 | print("STATUS: creating first case ...") 83 | 84 | # create first case - then clone 85 | if run_type == "verify": 86 | opts_dict["pertlim"] = get_pertlim_uf(rand_ints[0]) 87 | else: # full ensemble 88 | opts_dict["pertlim"] = "0" 89 | 90 | # first case 91 | single_case(opts_dict, case_flags, stat_dir) 92 | 93 | # clone? 94 | if clone_count > 0: 95 | 96 | # now clone 97 | print("STATUS: cloning additional cases ...") 98 | 99 | # scripts dir 100 | print("STATUS: stat_dir = " + stat_dir) 101 | ret = os.chdir(stat_dir) 102 | ret = os.chdir("../../cime/scripts") 103 | scripts_dir = os.getcwd() 104 | print("STATUS: scripts dir = " + scripts_dir) 105 | 106 | # we know case name ends in '.000' (already checked) 107 | clone_case = opts_dict["case"] 108 | case_pfx = clone_case[:-4] 109 | 110 | for i in range(1, clone_count + 1): # 1: clone_count 111 | if run_type == "verify": 112 | this_pertlim = get_pertlim_uf(rand_ints[i]) 113 | else: # full ensemble 114 | this_pertlim = get_pertlim_uf(i) 115 | 116 | iens = "{0:03d}".format(i) 117 | new_case = case_pfx + "." + iens 118 | 119 | os.chdir(scripts_dir) 120 | print("STATUS: creating new cloned case: " + new_case) 121 | 122 | clone_args = " --keepexe --case " + new_case + " --clone " + clone_case 123 | print(" with args: " + clone_args) 124 | 125 | command = scripts_dir + "/create_clone" + clone_args 126 | ret = os.system(command) 127 | 128 | print("STATUS: running setup for new cloned case: " + new_case) 129 | os.chdir(new_case) 130 | command = "./case.setup" 131 | ret = os.system(command) 132 | 133 | # adjust perturbation 134 | if opts_dict["ect"] == "pop": 135 | if run_type == "verify": # remove old init_ts_perturb 136 | f = open("user_nl_pop", "r+") 137 | all_lines = f.readlines() 138 | f.seek(0) 139 | for line in all_lines: 140 | if line.find("init_ts_perturb") == -1: 141 | f.write(line) 142 | f.truncate() 143 | f.close() 144 | text = "init_ts_perturb = " + this_pertlim 145 | else: 146 | text = "\ninit_ts_perturb = " + this_pertlim 147 | 148 | # now append new pertlim 149 | with open("user_nl_pop", "a") as f: 150 | f.write(text) 151 | 152 | else: 153 | if run_type == "verify": # remove old pertlim first 154 | f = open("user_nl_cam", "r+") 155 | all_lines = f.readlines() 156 | f.seek(0) 157 | for line in all_lines: 158 | if line.find("pertlim") == -1: 159 | f.write(line) 160 | f.truncate() 161 | f.close() 162 | text = "pertlim = " + this_pertlim 163 | else: 164 | text = "\npertlim = " + this_pertlim 165 | 166 | # now append new pertlim 167 | with open("user_nl_cam", "a") as f: 168 | f.write(text) 169 | 170 | # preview namelists 171 | command = "./preview_namelists" 172 | ret = os.system(command) 173 | 174 | # submit? 175 | if opts_dict["ns"] == False: 176 | command = "./case.submit" 177 | ret = os.system(command) 178 | 179 | # Final output 180 | if run_type == "verify": 181 | if opts_dict["ect"] == "pop": 182 | print("STATUS: ---POP-ECT VERIFICATION CASE COMPLETE---") 183 | print("Set up one case using the following init_ts_perturb value:") 184 | print(get_pertlim_uf(rand_ints[0])) 185 | else: 186 | print("STATUS: ---CAM-ECT VERIFICATION CASES COMPLETE---") 187 | print("Set up three cases using the following pertlim values:") 188 | print( 189 | get_pertlim_uf(rand_ints[0]) 190 | + " " 191 | + get_pertlim_uf(rand_ints[1]) 192 | + " " 193 | + get_pertlim_uf(rand_ints[2]) 194 | ) 195 | else: 196 | print("STATUS: --ENSEMBLE CASES COMPLETE---") 197 | 198 | 199 | if __name__ == "__main__": 200 | main(sys.argv[1:]) 201 | -------------------------------------------------------------------------------- /manage_externals/test/test_unit_repository.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Unit test driver for checkout_externals 4 | 5 | Note: this script assume the path to the checkout_externals.py module is 6 | already in the python path. 7 | 8 | """ 9 | 10 | from __future__ import absolute_import 11 | from __future__ import unicode_literals 12 | from __future__ import print_function 13 | 14 | import unittest 15 | 16 | from manic.repository_factory import create_repository 17 | from manic.repository_git import GitRepository 18 | from manic.repository_svn import SvnRepository 19 | from manic.repository import Repository 20 | from manic.externals_description import ExternalsDescription 21 | from manic.global_constants import EMPTY_STR 22 | 23 | 24 | class TestCreateRepositoryDict(unittest.TestCase): 25 | """Test the create_repository functionality to ensure it returns the 26 | propper type of repository and errors for unknown repository 27 | types. 28 | 29 | """ 30 | 31 | def setUp(self): 32 | """Common data needed for all tests in this class 33 | """ 34 | self._name = 'test_name' 35 | self._repo = {ExternalsDescription.PROTOCOL: None, 36 | ExternalsDescription.REPO_URL: 'junk_root', 37 | ExternalsDescription.TAG: 'junk_tag', 38 | ExternalsDescription.BRANCH: EMPTY_STR, 39 | ExternalsDescription.HASH: EMPTY_STR, 40 | ExternalsDescription.SPARSE: EMPTY_STR, } 41 | 42 | def test_create_repo_git(self): 43 | """Verify that several possible names for the 'git' protocol 44 | create git repository objects. 45 | 46 | """ 47 | protocols = ['git', 'GIT', 'Git', ] 48 | for protocol in protocols: 49 | self._repo[ExternalsDescription.PROTOCOL] = protocol 50 | repo = create_repository(self._name, self._repo) 51 | self.assertIsInstance(repo, GitRepository) 52 | 53 | def test_create_repo_svn(self): 54 | """Verify that several possible names for the 'svn' protocol 55 | create svn repository objects. 56 | """ 57 | protocols = ['svn', 'SVN', 'Svn', ] 58 | for protocol in protocols: 59 | self._repo[ExternalsDescription.PROTOCOL] = protocol 60 | repo = create_repository(self._name, self._repo) 61 | self.assertIsInstance(repo, SvnRepository) 62 | 63 | def test_create_repo_externals_only(self): 64 | """Verify that an externals only repo returns None. 65 | """ 66 | protocols = ['externals_only', ] 67 | for protocol in protocols: 68 | self._repo[ExternalsDescription.PROTOCOL] = protocol 69 | repo = create_repository(self._name, self._repo) 70 | self.assertEqual(None, repo) 71 | 72 | def test_create_repo_unsupported(self): 73 | """Verify that an unsupported protocol generates a runtime error. 74 | """ 75 | protocols = ['not_a_supported_protocol', ] 76 | for protocol in protocols: 77 | self._repo[ExternalsDescription.PROTOCOL] = protocol 78 | with self.assertRaises(RuntimeError): 79 | create_repository(self._name, self._repo) 80 | 81 | 82 | class TestRepository(unittest.TestCase): 83 | """Test the externals description processing used to create the Repository 84 | base class shared by protocol specific repository classes. 85 | 86 | """ 87 | 88 | def test_tag(self): 89 | """Test creation of a repository object with a tag 90 | """ 91 | name = 'test_repo' 92 | protocol = 'test_protocol' 93 | url = 'test_url' 94 | tag = 'test_tag' 95 | repo_info = {ExternalsDescription.PROTOCOL: protocol, 96 | ExternalsDescription.REPO_URL: url, 97 | ExternalsDescription.TAG: tag, 98 | ExternalsDescription.BRANCH: EMPTY_STR, 99 | ExternalsDescription.HASH: EMPTY_STR, 100 | ExternalsDescription.SPARSE: EMPTY_STR, } 101 | repo = Repository(name, repo_info) 102 | print(repo.__dict__) 103 | self.assertEqual(repo.tag(), tag) 104 | self.assertEqual(repo.url(), url) 105 | 106 | def test_branch(self): 107 | """Test creation of a repository object with a branch 108 | """ 109 | name = 'test_repo' 110 | protocol = 'test_protocol' 111 | url = 'test_url' 112 | branch = 'test_branch' 113 | repo_info = {ExternalsDescription.PROTOCOL: protocol, 114 | ExternalsDescription.REPO_URL: url, 115 | ExternalsDescription.BRANCH: branch, 116 | ExternalsDescription.TAG: EMPTY_STR, 117 | ExternalsDescription.HASH: EMPTY_STR, 118 | ExternalsDescription.SPARSE: EMPTY_STR, } 119 | repo = Repository(name, repo_info) 120 | print(repo.__dict__) 121 | self.assertEqual(repo.branch(), branch) 122 | self.assertEqual(repo.url(), url) 123 | 124 | def test_hash(self): 125 | """Test creation of a repository object with a hash 126 | """ 127 | name = 'test_repo' 128 | protocol = 'test_protocol' 129 | url = 'test_url' 130 | ref = 'deadc0de' 131 | sparse = EMPTY_STR 132 | repo_info = {ExternalsDescription.PROTOCOL: protocol, 133 | ExternalsDescription.REPO_URL: url, 134 | ExternalsDescription.BRANCH: EMPTY_STR, 135 | ExternalsDescription.TAG: EMPTY_STR, 136 | ExternalsDescription.HASH: ref, 137 | ExternalsDescription.SPARSE: sparse, } 138 | repo = Repository(name, repo_info) 139 | print(repo.__dict__) 140 | self.assertEqual(repo.hash(), ref) 141 | self.assertEqual(repo.url(), url) 142 | 143 | def test_tag_branch(self): 144 | """Test creation of a repository object with a tag and branch raises a 145 | runtimer error. 146 | 147 | """ 148 | name = 'test_repo' 149 | protocol = 'test_protocol' 150 | url = 'test_url' 151 | branch = 'test_branch' 152 | tag = 'test_tag' 153 | ref = EMPTY_STR 154 | sparse = EMPTY_STR 155 | repo_info = {ExternalsDescription.PROTOCOL: protocol, 156 | ExternalsDescription.REPO_URL: url, 157 | ExternalsDescription.BRANCH: branch, 158 | ExternalsDescription.TAG: tag, 159 | ExternalsDescription.HASH: ref, 160 | ExternalsDescription.SPARSE: sparse, } 161 | with self.assertRaises(RuntimeError): 162 | Repository(name, repo_info) 163 | 164 | def test_tag_branch_hash(self): 165 | """Test creation of a repository object with a tag, branch and hash raises a 166 | runtimer error. 167 | 168 | """ 169 | name = 'test_repo' 170 | protocol = 'test_protocol' 171 | url = 'test_url' 172 | branch = 'test_branch' 173 | tag = 'test_tag' 174 | ref = 'deadc0de' 175 | sparse = EMPTY_STR 176 | repo_info = {ExternalsDescription.PROTOCOL: protocol, 177 | ExternalsDescription.REPO_URL: url, 178 | ExternalsDescription.BRANCH: branch, 179 | ExternalsDescription.TAG: tag, 180 | ExternalsDescription.HASH: ref, 181 | ExternalsDescription.SPARSE: sparse, } 182 | with self.assertRaises(RuntimeError): 183 | Repository(name, repo_info) 184 | 185 | def test_no_tag_no_branch(self): 186 | """Test creation of a repository object without a tag or branch raises a 187 | runtimer error. 188 | 189 | """ 190 | name = 'test_repo' 191 | protocol = 'test_protocol' 192 | url = 'test_url' 193 | branch = EMPTY_STR 194 | tag = EMPTY_STR 195 | ref = EMPTY_STR 196 | sparse = EMPTY_STR 197 | repo_info = {ExternalsDescription.PROTOCOL: protocol, 198 | ExternalsDescription.REPO_URL: url, 199 | ExternalsDescription.BRANCH: branch, 200 | ExternalsDescription.TAG: tag, 201 | ExternalsDescription.HASH: ref, 202 | ExternalsDescription.SPARSE: sparse, } 203 | with self.assertRaises(RuntimeError): 204 | Repository(name, repo_info) 205 | 206 | 207 | if __name__ == '__main__': 208 | unittest.main() 209 | -------------------------------------------------------------------------------- /manage_externals/test/doc/develop.rst: -------------------------------------------------------------------------------- 1 | Developer Guidelines 2 | ==================== 3 | 4 | The manage externals utilities are a light weight replacement for svn 5 | externals that will work with git repositories pulling in a mixture of 6 | git and svn dependencies. 7 | 8 | Given an externals description and a working copy: 9 | 10 | * *checkout_externals* attempts to make the working copy agree with the 11 | externals description 12 | 13 | * *generate_externals* attempts to make the externals description agree 14 | with the working copy. 15 | 16 | For these operations utilities should: 17 | 18 | * operate consistently across git and svn 19 | 20 | * operate simply with minimal user complexity 21 | 22 | * robustly across a wide range of repository states 23 | 24 | * provide explicit error messages when a problem occurs 25 | 26 | * leave the working copy in a valid state 27 | 28 | The utilities in manage externals are **NOT** generic wrappers around 29 | revision control operations or a replacement for common tasks. Users 30 | are expected to: 31 | 32 | * create branches prior to starting development 33 | 34 | * add remotes and push changes 35 | 36 | * create tags 37 | 38 | * delete branches 39 | 40 | These types of tasks are often highly workflow dependent, e.g. branch 41 | naming conventions may vary between repositories, have the potential 42 | to destroy user data, introduce significant code complexit and 'edge 43 | cases' that are extremely difficult to detect and test, and often 44 | require subtle decision making, especially if a problem occurs. 45 | 46 | Users who want to automate these types are encouraged to create their 47 | own tools. The externals description files are explicitly versioned 48 | and the internal APIs are intended to be stable for these purposes. 49 | 50 | Core Design Principles 51 | ----------------------- 52 | 53 | 1. Users can, and are actively encouraged to, modify the externals 54 | directories using revision control outside of manage_externals 55 | tools. You can't make any assumptions about the state of the 56 | working copy. Examples: adding a remote, creating a branch, 57 | switching to a branch, deleting the directory entirely. 58 | 59 | 2. Give that the user can do anything, the manage externals library 60 | can not preserve state between calls. The only information it can 61 | rely on is what it expectes based on the content of the externals 62 | description file, and what the actual state of the directory tree 63 | is. 64 | 65 | 3. Do *not* do anything that will possibly destroy user data! 66 | 67 | a. Do not remove files from the file system. We are operating on 68 | user supplied input. If you don't call 'rm', you can't 69 | accidentally remove the user's data. Thinking of calling 70 | ``shutil.rmtree(user_input)``? What if the user accidentally 71 | specified user_input such that it resolves to their home 72 | directory.... Yeah. Don't go there. 73 | 74 | b. Rely on git and svn to do their job as much as possible. Don't 75 | duplicate functionality. Examples: 76 | 77 | i. We require the working copies to be 'clean' as reported by 78 | ``git status`` and ``svn status``. What if there are misc 79 | editor files floating around that prevent an update? Use the 80 | git and svn ignore functionality so they are not 81 | reported. Don't try to remove them from manage_externals or 82 | determine if they are 'safe' to ignore. 83 | 84 | ii. Do not use '--force'. Ever. This is a sign you are doing 85 | something dangerous, it may not be what the user 86 | wants. Remember, they are encouraged to modify their repo. 87 | 88 | 4. There are often multiple ways to obtain a particular piece of 89 | information from git. Scraping screen output is brittle and 90 | generally not considered a stable API across different versions of 91 | git. Given a choice between: 92 | 93 | a. a lower level git 'plumbing' command that processes a 94 | specific request and returns a sucess/failure status. 95 | 96 | b. high level git command that produces a bunch of output 97 | that must be processed. 98 | 99 | We always prefer the former. It almost always involves 100 | writing and maintaining less code and is more likely to be 101 | stable. 102 | 103 | 5. Backward compatibility is critical. We have *nested* 104 | repositories. They are trivially easy to change versions. They may 105 | have very different versions of the top level manage_externals. The 106 | ability to read and work with old model description files is 107 | critical to avoid problems for users. We also have automated tools 108 | (testdb) that must generate and read external description 109 | files. Backward compatibility will make staging changes vastly 110 | simpler. 111 | 112 | Model Users 113 | ----------- 114 | 115 | Consider the needs of the following model userswhen developing manage_externals: 116 | 117 | * Users who will checkout the code once, and never change versions. 118 | 119 | * Users who will checkout the code once, then work for several years, 120 | never updating. before trying to update or request integration. 121 | 122 | * Users develope code but do not use revision control beyond the 123 | initial checkout. If they have modified or untracked files in the 124 | repo, they may be irreplacable. Don't destroy user data. 125 | 126 | * Intermediate users who are working with multiple repos or branches 127 | on a regular basis. They may only use manage_externals weekly or 128 | monthly. Keep the user interface and documentation simple and 129 | explicit. The more command line options they have to remember or 130 | look up, the more frustrated they git. 131 | 132 | * Software engineers who use the tools multiple times a day. It should 133 | get out of their way. 134 | 135 | User Interface 136 | -------------- 137 | 138 | Basic operation for the most standard use cases should be kept as 139 | simple as possible. Many users will only rarely run the manage 140 | utilities. Even advanced users don't like reading a lot of help 141 | documentation or struggling to remember commands and piece together 142 | what they need to run. Having many command line options, even if not 143 | needed, is exteremly frustrating and overwhelming for most users. A few 144 | simple, explicitly named commands are better than a single command 145 | with many options. 146 | 147 | How will users get help if something goes wrong? This is a custom, 148 | one-off solution. Searching the internet for manage_externals, will 149 | only return the user doc for this project at best. There isn't likely 150 | to be a stackoverflow question or blog post where someone else already 151 | answered a user's question. And very few people outside this community 152 | will be able to provide help if something goes wrong. The sooner we 153 | kick users out of these utilities and into standard version control 154 | tools, the better off they are going to be if they run into a problem. 155 | 156 | Repositories 157 | ------------ 158 | 159 | There are three basic types of repositories that must be considered: 160 | 161 | * container repositories - repositories that are always top level 162 | repositories, and have a group of externals that must be managed. 163 | 164 | * simple repositories - repositories that are externals to another 165 | repository, and do not have any of their own externals that will be 166 | managed. 167 | 168 | * mixed use repositories - repositories that can act as a top level 169 | container repository or as an external to a top level 170 | container. They may also have their own sub-externals that are 171 | required. They may have different externals needs depening on 172 | whether they are top level or not. 173 | 174 | Repositories must be able to checkout and switch to both branches and 175 | tags. 176 | 177 | Development 178 | =========== 179 | 180 | The functionality to manage externals is broken into a library of core 181 | functionality and applications built with the library. 182 | 183 | The core library is called 'manic', pseduo-homophone of (man)age 184 | (ex)ternals that is: short, pronounceable and spell-checkable. It is 185 | also no more or less meaningful to an unfamiliar user than a random 186 | jumble of letters forming an acronym. 187 | 188 | The core architecture of manic is: 189 | 190 | * externals description - an abstract description on an external, 191 | including of how to obtain it, where to obtain it, where it goes in 192 | the working tree. 193 | 194 | * externals - the software object representing an external. 195 | 196 | * source trees - collection of externals 197 | 198 | * repository wrappers - object oriented wrappers around repository 199 | operations. So the higher level management of the soure tree and 200 | external does not have to be concerned with how a particular 201 | external is obtained and managed. 202 | 203 | -------------------------------------------------------------------------------- /cime_config/testlist_allactive.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | -------------------------------------------------------------------------------- /manage_externals/test/test_sys_repository_git.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Tests of some of the functionality in repository_git.py that actually 4 | interacts with git repositories. 5 | 6 | We're calling these "system" tests because we expect them to be a lot 7 | slower than most of the unit tests. 8 | 9 | """ 10 | 11 | from __future__ import absolute_import 12 | from __future__ import unicode_literals 13 | from __future__ import print_function 14 | 15 | import os 16 | import shutil 17 | import tempfile 18 | import unittest 19 | 20 | from manic.repository_git import GitRepository 21 | from manic.externals_description import ExternalsDescription 22 | from manic.externals_description import ExternalsDescriptionDict 23 | from manic.utils import execute_subprocess 24 | 25 | # NOTE(wjs, 2018-04-09) I find a mix of camel case and underscores to be 26 | # more readable for unit test names, so I'm disabling pylint's naming 27 | # convention check 28 | # pylint: disable=C0103 29 | 30 | # Allow access to protected members 31 | # pylint: disable=W0212 32 | 33 | 34 | class GitTestCase(unittest.TestCase): 35 | """Adds some git-specific unit test functionality on top of TestCase""" 36 | 37 | def assertIsHash(self, maybe_hash): 38 | """Assert that the string given by maybe_hash really does look 39 | like a git hash. 40 | """ 41 | 42 | # Ensure it is non-empty 43 | self.assertTrue(maybe_hash, msg="maybe_hash is empty") 44 | 45 | # Ensure it has a single string 46 | self.assertEqual(1, len(maybe_hash.split()), 47 | msg="maybe_hash has multiple strings: {}".format(maybe_hash)) 48 | 49 | # Ensure that the only characters in the string are ones allowed 50 | # in hashes 51 | allowed_chars_set = set('0123456789abcdef') 52 | self.assertTrue(set(maybe_hash) <= allowed_chars_set, 53 | msg="maybe_hash has non-hash characters: {}".format(maybe_hash)) 54 | 55 | 56 | class TestGitTestCase(GitTestCase): 57 | """Tests GitTestCase""" 58 | 59 | def test_assertIsHash_true(self): 60 | """Ensure that assertIsHash passes for something that looks 61 | like a hash""" 62 | self.assertIsHash('abc123') 63 | 64 | def test_assertIsHash_empty(self): 65 | """Ensure that assertIsHash raises an AssertionError for an 66 | empty string""" 67 | with self.assertRaises(AssertionError): 68 | self.assertIsHash('') 69 | 70 | def test_assertIsHash_multipleStrings(self): 71 | """Ensure that assertIsHash raises an AssertionError when 72 | given multiple strings""" 73 | with self.assertRaises(AssertionError): 74 | self.assertIsHash('abc123 def456') 75 | 76 | def test_assertIsHash_badChar(self): 77 | """Ensure that assertIsHash raises an AssertionError when given a 78 | string that has a character that doesn't belong in a hash 79 | """ 80 | with self.assertRaises(AssertionError): 81 | self.assertIsHash('abc123g') 82 | 83 | 84 | class TestGitRepositoryGitCommands(GitTestCase): 85 | """Test some git commands in RepositoryGit 86 | 87 | It's silly that we need to create a repository in order to test 88 | these git commands. Much or all of the git functionality that is 89 | currently in repository_git.py should eventually be moved to a 90 | separate module that is solely responsible for wrapping git 91 | commands; that would allow us to test it independently of this 92 | repository class. 93 | """ 94 | 95 | # ======================================================================== 96 | # Test helper functions 97 | # ======================================================================== 98 | 99 | def setUp(self): 100 | # directory we want to return to after the test system and 101 | # checkout_externals are done cd'ing all over the place. 102 | self._return_dir = os.getcwd() 103 | 104 | self._tmpdir = tempfile.mkdtemp() 105 | os.chdir(self._tmpdir) 106 | 107 | self._name = 'component' 108 | rdata = {ExternalsDescription.PROTOCOL: 'git', 109 | ExternalsDescription.REPO_URL: 110 | '/path/to/local/repo', 111 | ExternalsDescription.TAG: 112 | 'tag1', 113 | } 114 | 115 | data = {self._name: 116 | { 117 | ExternalsDescription.REQUIRED: False, 118 | ExternalsDescription.PATH: 'junk', 119 | ExternalsDescription.EXTERNALS: '', 120 | ExternalsDescription.REPO: rdata, 121 | }, 122 | } 123 | model = ExternalsDescriptionDict(data) 124 | repo = model[self._name][ExternalsDescription.REPO] 125 | self._repo = GitRepository('test', repo) 126 | 127 | def tearDown(self): 128 | # return to our common starting point 129 | os.chdir(self._return_dir) 130 | 131 | shutil.rmtree(self._tmpdir, ignore_errors=True) 132 | 133 | @staticmethod 134 | def make_git_repo(): 135 | """Turn the current directory into an empty git repository""" 136 | execute_subprocess(['git', 'init']) 137 | 138 | @staticmethod 139 | def add_git_commit(): 140 | """Add a git commit in the current directory""" 141 | with open('README', 'a') as myfile: 142 | myfile.write('more info') 143 | execute_subprocess(['git', 'add', 'README']) 144 | execute_subprocess(['git', 'commit', '-m', 'my commit message']) 145 | 146 | @staticmethod 147 | def checkout_git_branch(branchname): 148 | """Checkout a new branch in the current directory""" 149 | execute_subprocess(['git', 'checkout', '-b', branchname]) 150 | 151 | @staticmethod 152 | def make_git_tag(tagname): 153 | """Make a lightweight tag at the current commit""" 154 | execute_subprocess(['git', 'tag', '-m', 'making a tag', tagname]) 155 | 156 | @staticmethod 157 | def checkout_ref(refname): 158 | """Checkout the given refname in the current directory""" 159 | execute_subprocess(['git', 'checkout', refname]) 160 | 161 | # ======================================================================== 162 | # Begin actual tests 163 | # ======================================================================== 164 | 165 | def test_currentHash_returnsHash(self): 166 | """Ensure that the _git_current_hash function returns a hash""" 167 | self.make_git_repo() 168 | self.add_git_commit() 169 | hash_found, myhash = self._repo._git_current_hash() 170 | self.assertTrue(hash_found) 171 | self.assertIsHash(myhash) 172 | 173 | def test_currentHash_outsideGitRepo(self): 174 | """Ensure that the _git_current_hash function returns False when 175 | outside a git repository""" 176 | hash_found, myhash = self._repo._git_current_hash() 177 | self.assertFalse(hash_found) 178 | self.assertEqual('', myhash) 179 | 180 | def test_currentBranch_onBranch(self): 181 | """Ensure that the _git_current_branch function returns the name 182 | of the branch""" 183 | self.make_git_repo() 184 | self.add_git_commit() 185 | self.checkout_git_branch('foo') 186 | branch_found, mybranch = self._repo._git_current_branch() 187 | self.assertTrue(branch_found) 188 | self.assertEqual('foo', mybranch) 189 | 190 | def test_currentBranch_notOnBranch(self): 191 | """Ensure that the _git_current_branch function returns False 192 | when not on a branch""" 193 | self.make_git_repo() 194 | self.add_git_commit() 195 | self.make_git_tag('mytag') 196 | self.checkout_ref('mytag') 197 | branch_found, mybranch = self._repo._git_current_branch() 198 | self.assertFalse(branch_found) 199 | self.assertEqual('', mybranch) 200 | 201 | def test_currentBranch_outsideGitRepo(self): 202 | """Ensure that the _git_current_branch function returns False 203 | when outside a git repository""" 204 | branch_found, mybranch = self._repo._git_current_branch() 205 | self.assertFalse(branch_found) 206 | self.assertEqual('', mybranch) 207 | 208 | def test_currentTag_onTag(self): 209 | """Ensure that the _git_current_tag function returns the name of 210 | the tag""" 211 | self.make_git_repo() 212 | self.add_git_commit() 213 | self.make_git_tag('some_tag') 214 | tag_found, mytag = self._repo._git_current_tag() 215 | self.assertTrue(tag_found) 216 | self.assertEqual('some_tag', mytag) 217 | 218 | def test_currentTag_notOnTag(self): 219 | """Ensure tha the _git_current_tag function returns False when 220 | not on a tag""" 221 | self.make_git_repo() 222 | self.add_git_commit() 223 | self.make_git_tag('some_tag') 224 | self.add_git_commit() 225 | tag_found, mytag = self._repo._git_current_tag() 226 | self.assertFalse(tag_found) 227 | self.assertEqual('', mytag) 228 | 229 | def test_currentTag_outsideGitRepo(self): 230 | """Ensure that the _git_current_tag function returns False when 231 | outside a git repository""" 232 | tag_found, mytag = self._repo._git_current_tag() 233 | self.assertFalse(tag_found) 234 | self.assertEqual('', mytag) 235 | 236 | 237 | if __name__ == '__main__': 238 | unittest.main() 239 | -------------------------------------------------------------------------------- /manage_externals/test/test_unit_externals_status.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Unit test driver for the manic external status reporting module. 4 | 5 | Note: this script assumes the path to the manic package is already in 6 | the python path. 7 | 8 | """ 9 | 10 | from __future__ import absolute_import 11 | from __future__ import unicode_literals 12 | from __future__ import print_function 13 | 14 | import unittest 15 | 16 | from manic.externals_status import ExternalStatus 17 | 18 | 19 | class TestStatusObject(unittest.TestCase): 20 | """Verify that the Status object behaives as expected. 21 | """ 22 | 23 | def test_exists_empty_all(self): 24 | """If the repository sync-state is empty (doesn't exist), and there is no 25 | clean state, then it is considered not to exist. 26 | 27 | """ 28 | stat = ExternalStatus() 29 | stat.sync_state = ExternalStatus.EMPTY 30 | stat.clean_state = ExternalStatus.DEFAULT 31 | exists = stat.exists() 32 | self.assertFalse(exists) 33 | 34 | stat.clean_state = ExternalStatus.EMPTY 35 | exists = stat.exists() 36 | self.assertFalse(exists) 37 | 38 | stat.clean_state = ExternalStatus.UNKNOWN 39 | exists = stat.exists() 40 | self.assertFalse(exists) 41 | 42 | # this state represtens an internal logic error in how the 43 | # repo status was determined. 44 | stat.clean_state = ExternalStatus.STATUS_OK 45 | exists = stat.exists() 46 | self.assertTrue(exists) 47 | 48 | # this state represtens an internal logic error in how the 49 | # repo status was determined. 50 | stat.clean_state = ExternalStatus.DIRTY 51 | exists = stat.exists() 52 | self.assertTrue(exists) 53 | 54 | def test_exists_default_all(self): 55 | """If the repository sync-state is default, then it is considered to exist 56 | regardless of clean state. 57 | 58 | """ 59 | stat = ExternalStatus() 60 | stat.sync_state = ExternalStatus.DEFAULT 61 | stat.clean_state = ExternalStatus.DEFAULT 62 | exists = stat.exists() 63 | self.assertTrue(exists) 64 | 65 | stat.clean_state = ExternalStatus.EMPTY 66 | exists = stat.exists() 67 | self.assertTrue(exists) 68 | 69 | stat.clean_state = ExternalStatus.UNKNOWN 70 | exists = stat.exists() 71 | self.assertTrue(exists) 72 | 73 | stat.clean_state = ExternalStatus.STATUS_OK 74 | exists = stat.exists() 75 | self.assertTrue(exists) 76 | 77 | stat.clean_state = ExternalStatus.DIRTY 78 | exists = stat.exists() 79 | self.assertTrue(exists) 80 | 81 | def test_exists_unknown_all(self): 82 | """If the repository sync-state is unknown, then it is considered to exist 83 | regardless of clean state. 84 | 85 | """ 86 | stat = ExternalStatus() 87 | stat.sync_state = ExternalStatus.UNKNOWN 88 | stat.clean_state = ExternalStatus.DEFAULT 89 | exists = stat.exists() 90 | self.assertTrue(exists) 91 | 92 | stat.clean_state = ExternalStatus.EMPTY 93 | exists = stat.exists() 94 | self.assertTrue(exists) 95 | 96 | stat.clean_state = ExternalStatus.UNKNOWN 97 | exists = stat.exists() 98 | self.assertTrue(exists) 99 | 100 | stat.clean_state = ExternalStatus.STATUS_OK 101 | exists = stat.exists() 102 | self.assertTrue(exists) 103 | 104 | stat.clean_state = ExternalStatus.DIRTY 105 | exists = stat.exists() 106 | self.assertTrue(exists) 107 | 108 | def test_exists_modified_all(self): 109 | """If the repository sync-state is modified, then it is considered to exist 110 | regardless of clean state. 111 | 112 | """ 113 | stat = ExternalStatus() 114 | stat.sync_state = ExternalStatus.MODEL_MODIFIED 115 | stat.clean_state = ExternalStatus.DEFAULT 116 | exists = stat.exists() 117 | self.assertTrue(exists) 118 | 119 | stat.clean_state = ExternalStatus.EMPTY 120 | exists = stat.exists() 121 | self.assertTrue(exists) 122 | 123 | stat.clean_state = ExternalStatus.UNKNOWN 124 | exists = stat.exists() 125 | self.assertTrue(exists) 126 | 127 | stat.clean_state = ExternalStatus.STATUS_OK 128 | exists = stat.exists() 129 | self.assertTrue(exists) 130 | 131 | stat.clean_state = ExternalStatus.DIRTY 132 | exists = stat.exists() 133 | self.assertTrue(exists) 134 | 135 | def test_exists_ok_all(self): 136 | """If the repository sync-state is ok, then it is considered to exist 137 | regardless of clean state. 138 | 139 | """ 140 | stat = ExternalStatus() 141 | stat.sync_state = ExternalStatus.STATUS_OK 142 | stat.clean_state = ExternalStatus.DEFAULT 143 | exists = stat.exists() 144 | self.assertTrue(exists) 145 | 146 | stat.clean_state = ExternalStatus.EMPTY 147 | exists = stat.exists() 148 | self.assertTrue(exists) 149 | 150 | stat.clean_state = ExternalStatus.UNKNOWN 151 | exists = stat.exists() 152 | self.assertTrue(exists) 153 | 154 | stat.clean_state = ExternalStatus.STATUS_OK 155 | exists = stat.exists() 156 | self.assertTrue(exists) 157 | 158 | stat.clean_state = ExternalStatus.DIRTY 159 | exists = stat.exists() 160 | self.assertTrue(exists) 161 | 162 | def test_update_ok_all(self): 163 | """If the repository in-sync is ok, then it is safe to 164 | update only if clean state is ok 165 | 166 | """ 167 | stat = ExternalStatus() 168 | stat.sync_state = ExternalStatus.STATUS_OK 169 | stat.clean_state = ExternalStatus.DEFAULT 170 | safe_to_update = stat.safe_to_update() 171 | self.assertFalse(safe_to_update) 172 | 173 | stat.clean_state = ExternalStatus.EMPTY 174 | safe_to_update = stat.safe_to_update() 175 | self.assertFalse(safe_to_update) 176 | 177 | stat.clean_state = ExternalStatus.UNKNOWN 178 | safe_to_update = stat.safe_to_update() 179 | self.assertFalse(safe_to_update) 180 | 181 | stat.clean_state = ExternalStatus.STATUS_OK 182 | safe_to_update = stat.safe_to_update() 183 | self.assertTrue(safe_to_update) 184 | 185 | stat.clean_state = ExternalStatus.DIRTY 186 | safe_to_update = stat.safe_to_update() 187 | self.assertFalse(safe_to_update) 188 | 189 | def test_update_modified_all(self): 190 | """If the repository in-sync is modified, then it is safe to 191 | update only if clean state is ok 192 | 193 | """ 194 | stat = ExternalStatus() 195 | stat.sync_state = ExternalStatus.MODEL_MODIFIED 196 | stat.clean_state = ExternalStatus.DEFAULT 197 | safe_to_update = stat.safe_to_update() 198 | self.assertFalse(safe_to_update) 199 | 200 | stat.clean_state = ExternalStatus.EMPTY 201 | safe_to_update = stat.safe_to_update() 202 | self.assertFalse(safe_to_update) 203 | 204 | stat.clean_state = ExternalStatus.UNKNOWN 205 | safe_to_update = stat.safe_to_update() 206 | self.assertFalse(safe_to_update) 207 | 208 | stat.clean_state = ExternalStatus.STATUS_OK 209 | safe_to_update = stat.safe_to_update() 210 | self.assertTrue(safe_to_update) 211 | 212 | stat.clean_state = ExternalStatus.DIRTY 213 | safe_to_update = stat.safe_to_update() 214 | self.assertFalse(safe_to_update) 215 | 216 | def test_update_unknown_all(self): 217 | """If the repository in-sync is unknown, then it is not safe to 218 | update, regardless of the clean state. 219 | 220 | """ 221 | stat = ExternalStatus() 222 | stat.sync_state = ExternalStatus.UNKNOWN 223 | stat.clean_state = ExternalStatus.DEFAULT 224 | safe_to_update = stat.safe_to_update() 225 | self.assertFalse(safe_to_update) 226 | 227 | stat.clean_state = ExternalStatus.EMPTY 228 | safe_to_update = stat.safe_to_update() 229 | self.assertFalse(safe_to_update) 230 | 231 | stat.clean_state = ExternalStatus.UNKNOWN 232 | safe_to_update = stat.safe_to_update() 233 | self.assertFalse(safe_to_update) 234 | 235 | stat.clean_state = ExternalStatus.STATUS_OK 236 | safe_to_update = stat.safe_to_update() 237 | self.assertFalse(safe_to_update) 238 | 239 | stat.clean_state = ExternalStatus.DIRTY 240 | safe_to_update = stat.safe_to_update() 241 | self.assertFalse(safe_to_update) 242 | 243 | def test_update_default_all(self): 244 | """If the repository in-sync is default, then it is not safe to 245 | update, regardless of the clean state. 246 | 247 | """ 248 | stat = ExternalStatus() 249 | stat.sync_state = ExternalStatus.UNKNOWN 250 | stat.clean_state = ExternalStatus.DEFAULT 251 | safe_to_update = stat.safe_to_update() 252 | self.assertFalse(safe_to_update) 253 | 254 | stat.clean_state = ExternalStatus.EMPTY 255 | safe_to_update = stat.safe_to_update() 256 | self.assertFalse(safe_to_update) 257 | 258 | stat.clean_state = ExternalStatus.UNKNOWN 259 | safe_to_update = stat.safe_to_update() 260 | self.assertFalse(safe_to_update) 261 | 262 | stat.clean_state = ExternalStatus.STATUS_OK 263 | safe_to_update = stat.safe_to_update() 264 | self.assertFalse(safe_to_update) 265 | 266 | stat.clean_state = ExternalStatus.DIRTY 267 | safe_to_update = stat.safe_to_update() 268 | self.assertFalse(safe_to_update) 269 | 270 | def test_update_empty_all(self): 271 | """If the repository in-sync is empty, then it is not safe to 272 | update, regardless of the clean state. 273 | 274 | """ 275 | stat = ExternalStatus() 276 | stat.sync_state = ExternalStatus.UNKNOWN 277 | stat.clean_state = ExternalStatus.DEFAULT 278 | safe_to_update = stat.safe_to_update() 279 | self.assertFalse(safe_to_update) 280 | 281 | stat.clean_state = ExternalStatus.EMPTY 282 | safe_to_update = stat.safe_to_update() 283 | self.assertFalse(safe_to_update) 284 | 285 | stat.clean_state = ExternalStatus.UNKNOWN 286 | safe_to_update = stat.safe_to_update() 287 | self.assertFalse(safe_to_update) 288 | 289 | stat.clean_state = ExternalStatus.STATUS_OK 290 | safe_to_update = stat.safe_to_update() 291 | self.assertFalse(safe_to_update) 292 | 293 | stat.clean_state = ExternalStatus.DIRTY 294 | safe_to_update = stat.safe_to_update() 295 | self.assertFalse(safe_to_update) 296 | 297 | 298 | if __name__ == '__main__': 299 | unittest.main() 300 | -------------------------------------------------------------------------------- /manage_externals/README.md: -------------------------------------------------------------------------------- 1 | -- AUTOMATICALLY GENERATED FILE. DO NOT EDIT -- 2 | 3 | [![Build Status](https://travis-ci.org/ESMCI/manage_externals.svg?branch=master)](https://travis-ci.org/ESMCI/manage_externals)[![Coverage Status](https://coveralls.io/repos/github/ESMCI/manage_externals/badge.svg?branch=master)](https://coveralls.io/github/ESMCI/manage_externals?branch=master) 4 | ``` 5 | usage: checkout_externals [-h] [-e [EXTERNALS]] [-o] [-S] [-v] [--backtrace] 6 | [-d] [--no-logging] 7 | 8 | checkout_externals manages checking out groups of externals from revision 9 | control based on a externals description file. By default only the 10 | required externals are checkout out. 11 | 12 | Operations performed by manage_externals utilities are explicit and 13 | data driven. checkout_externals will always make the working copy *exactly* 14 | match what is in the externals file when modifying the working copy of 15 | a repository. 16 | 17 | If checkout_externals isn't doing what you expected, double check the contents 18 | of the externals description file. 19 | 20 | Running checkout_externals without the '--status' option will always attempt to 21 | synchronize the working copy to exactly match the externals description. 22 | 23 | optional arguments: 24 | -h, --help show this help message and exit 25 | -e [EXTERNALS], --externals [EXTERNALS] 26 | The externals description filename. Default: 27 | Externals.cfg. 28 | -o, --optional By default only the required externals are checked 29 | out. This flag will also checkout the optional 30 | externals. 31 | -S, --status Output status of the repositories managed by 32 | checkout_externals. By default only summary 33 | information is provided. Use verbose output to see 34 | details. 35 | -v, --verbose Output additional information to the screen and log 36 | file. This flag can be used up to two times, 37 | increasing the verbosity level each time. 38 | --backtrace DEVELOPER: show exception backtraces as extra 39 | debugging output 40 | -d, --debug DEVELOPER: output additional debugging information to 41 | the screen and log file. 42 | --no-logging DEVELOPER: disable logging. 43 | 44 | ``` 45 | NOTE: checkout_externals *MUST* be run from the root of the source tree it 46 | is managing. For example, if you cloned a repository with: 47 | 48 | $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev 49 | 50 | Then the root of the source tree is /path/to/some-project-dev. If you 51 | obtained a sub-project via a checkout of another project: 52 | 53 | $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev 54 | 55 | and you need to checkout the sub-project externals, then the root of the 56 | source tree is /path/to/some-project-dev. Do *NOT* run checkout_externals 57 | from within /path/to/some-project-dev/sub-project 58 | 59 | The root of the source tree will be referred to as `${SRC_ROOT}` below. 60 | 61 | # Supported workflows 62 | 63 | * Checkout all required components from the default externals 64 | description file: 65 | 66 | $ cd ${SRC_ROOT} 67 | $ ./manage_externals/checkout_externals 68 | 69 | * To update all required components to the current values in the 70 | externals description file, re-run checkout_externals: 71 | 72 | $ cd ${SRC_ROOT} 73 | $ ./manage_externals/checkout_externals 74 | 75 | If there are *any* modifications to *any* working copy according 76 | to the git or svn 'status' command, checkout_externals 77 | will not update any external repositories. Modifications 78 | include: modified files, added files, removed files, or missing 79 | files. 80 | 81 | To avoid this safety check, edit the externals description file 82 | and comment out the modified external block. 83 | 84 | * Checkout all required components from a user specified externals 85 | description file: 86 | 87 | $ cd ${SRC_ROOT} 88 | $ ./manage_externals/checkout_externals --externals my-externals.cfg 89 | 90 | * Status summary of the repositories managed by checkout_externals: 91 | 92 | $ cd ${SRC_ROOT} 93 | $ ./manage_externals/checkout_externals --status 94 | 95 | ./cime 96 | s ./components/cism 97 | ./components/mosart 98 | e-o ./components/rtm 99 | M ./src/fates 100 | e-o ./tools/PTCLM 101 | 102 | where: 103 | * column one indicates the status of the repository in relation 104 | to the externals description file. 105 | * column two indicates whether the working copy has modified files. 106 | * column three shows how the repository is managed, optional or required 107 | 108 | Column one will be one of these values: 109 | * s : out-of-sync : repository is checked out at a different commit 110 | compared with the externals description 111 | * e : empty : directory does not exist - checkout_externals has not been run 112 | * ? : unknown : directory exists but .git or .svn directories are missing 113 | 114 | Column two will be one of these values: 115 | * M : Modified : modified, added, deleted or missing files 116 | * : blank / space : clean 117 | * - : dash : no meaningful state, for empty repositories 118 | 119 | Column three will be one of these values: 120 | * o : optional : optionally repository 121 | * : blank / space : required repository 122 | 123 | * Detailed git or svn status of the repositories managed by checkout_externals: 124 | 125 | $ cd ${SRC_ROOT} 126 | $ ./manage_externals/checkout_externals --status --verbose 127 | 128 | # Externals description file 129 | 130 | The externals description contains a list of the external 131 | repositories that are used and their version control locations. The 132 | file format is the standard ini/cfg configuration file format. Each 133 | external is defined by a section containing the component name in 134 | square brackets: 135 | 136 | * name (string) : component name, e.g. [cime], [cism], etc. 137 | 138 | Each section has the following keyword-value pairs: 139 | 140 | * required (boolean) : whether the component is a required checkout, 141 | 'true' or 'false'. 142 | 143 | * local_path (string) : component path *relative* to where 144 | checkout_externals is called. 145 | 146 | * protoctol (string) : version control protocol that is used to 147 | manage the component. Valid values are 'git', 'svn', 148 | 'externals_only'. 149 | 150 | Switching an external between different protocols is not 151 | supported, e.g. from svn to git. To switch protocols, you need to 152 | manually move the old working copy to a new location. 153 | 154 | Note: 'externals_only' will only process the external's own 155 | external description file without trying to manage a repository 156 | for the component. This is used for retreiving externals for 157 | standalone components like cam and clm. If the source root of the 158 | externals_only component is the same as the main source root, then 159 | the local path must be set to '.', the unix current working 160 | directory, e. g. 'local_path = .' 161 | 162 | * repo_url (string) : URL for the repository location, examples: 163 | * https://svn-ccsm-models.cgd.ucar.edu/glc 164 | * git@github.com:esmci/cime.git 165 | * /path/to/local/repository 166 | * . 167 | 168 | NOTE: To operate on only the local clone and and ignore remote 169 | repositories, set the url to '.' (the unix current path), 170 | i.e. 'repo_url = .' . This can be used to checkout a local branch 171 | instead of the upstream branch. 172 | 173 | If a repo url is determined to be a local path (not a network url) 174 | then user expansion, e.g. ~/, and environment variable expansion, 175 | e.g. $HOME or $REPO_ROOT, will be performed. 176 | 177 | Relative paths are difficult to get correct, especially for mixed 178 | use repos. It is advised that local paths expand to absolute paths. 179 | If relative paths are used, they should be relative to one level 180 | above local_path. If local path is 'src/foo', the the relative url 181 | should be relative to 'src'. 182 | 183 | * tag (string) : tag to checkout 184 | 185 | * hash (string) : the git hash to checkout. Only applies to git 186 | repositories. 187 | 188 | * branch (string) : branch to checkout from the specified 189 | repository. Specifying a branch on a remote repository means that 190 | checkout_externals will checkout the version of the branch in the remote, 191 | not the the version in the local repository (if it exists). 192 | 193 | Note: one and only one of tag, branch hash must be supplied. 194 | 195 | * externals (string) : used to make manage_externals aware of 196 | sub-externals required by an external. This is a relative path to 197 | the external's root directory. For example, the main externals 198 | description has an external checkout out at 'src/useful_library'. 199 | useful_library requires additional externals to be complete. 200 | Those additional externals are managed from the source root by the 201 | externals description file pointed 'useful_library/sub-xternals.cfg', 202 | Then the main 'externals' field in the top level repo should point to 203 | 'sub-externals.cfg'. 204 | Note that by default, `checkout_externals` will clone an external's 205 | submodules. As a special case, the entry, `externals = None`, will 206 | prevent this behavior. For more control over which externals are 207 | checked out, create an externals file (and see the `from_submodule` 208 | configuration entry below). 209 | 210 | * from_submodule (True / False) : used to pull the repo_url, local_path, 211 | and hash properties for this external from the .gitmodules file in 212 | this repository. Note that the section name (the entry in square 213 | brackets) must match the name in the .gitmodules file. 214 | If from_submodule is True, the protocol must be git and no repo_url, 215 | local_path, hash, branch, or tag entries are allowed. 216 | Default: False 217 | 218 | * sparse (string) : used to control a sparse checkout. This optional 219 | entry should point to a filename (path relative to local_path) that 220 | contains instructions on which repository paths to include (or 221 | exclude) from the working tree. 222 | See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree 223 | Default: sparse checkout is disabled 224 | 225 | * Lines begining with '#' or ';' are comments and will be ignored. 226 | 227 | # Obtaining this tool, reporting issues, etc. 228 | 229 | The master repository for manage_externals is 230 | https://github.com/ESMCI/manage_externals. Any issues with this tool 231 | should be reported there. 232 | -------------------------------------------------------------------------------- /manage_externals/manic/repository_svn.py: -------------------------------------------------------------------------------- 1 | """Class for interacting with svn repositories 2 | """ 3 | 4 | from __future__ import absolute_import 5 | from __future__ import unicode_literals 6 | from __future__ import print_function 7 | 8 | import os 9 | import re 10 | import xml.etree.ElementTree as ET 11 | 12 | from .global_constants import EMPTY_STR, VERBOSITY_VERBOSE 13 | from .repository import Repository 14 | from .externals_status import ExternalStatus 15 | from .utils import fatal_error, indent_string, printlog 16 | from .utils import execute_subprocess 17 | 18 | 19 | class SvnRepository(Repository): 20 | """ 21 | Class to represent and operate on a repository description. 22 | 23 | For testing purpose, all system calls to svn should: 24 | 25 | * be isolated in separate functions with no application logic 26 | * of the form: 27 | - cmd = ['svn', ...] 28 | - value = execute_subprocess(cmd, output_to_caller={T|F}, 29 | status_to_caller={T|F}) 30 | - return value 31 | * be static methods (not rely on self) 32 | * name as _svn_subcommand_args(user_args) 33 | 34 | This convention allows easy unit testing of the repository logic 35 | by mocking the specific calls to return predefined results. 36 | 37 | """ 38 | RE_URLLINE = re.compile(r'^URL:') 39 | 40 | def __init__(self, component_name, repo, ignore_ancestry=False): 41 | """ 42 | Parse repo (a XML element). 43 | """ 44 | Repository.__init__(self, component_name, repo) 45 | self._ignore_ancestry = ignore_ancestry 46 | if self._url.endswith('/'): 47 | # there is already a '/' separator in the URL; no need to add another 48 | url_sep = '' 49 | else: 50 | url_sep = '/' 51 | if self._branch: 52 | self._url = self._url + url_sep + self._branch 53 | elif self._tag: 54 | self._url = self._url + url_sep + self._tag 55 | else: 56 | msg = "DEV_ERROR in svn repository. Shouldn't be here!" 57 | fatal_error(msg) 58 | 59 | # ---------------------------------------------------------------- 60 | # 61 | # Public API, defined by Repository 62 | # 63 | # ---------------------------------------------------------------- 64 | def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument 65 | """Checkout or update the working copy 66 | 67 | If the repo destination directory exists, switch the sandbox to 68 | match the externals description. 69 | 70 | If the repo destination directory does not exist, checkout the 71 | correct branch or tag. 72 | NB: is include as an argument for compatibility with 73 | git functionality (repository_git.py) 74 | 75 | """ 76 | repo_dir_path = os.path.join(base_dir_path, repo_dir_name) 77 | if os.path.exists(repo_dir_path): 78 | cwd = os.getcwd() 79 | os.chdir(repo_dir_path) 80 | self._svn_switch(self._url, self._ignore_ancestry, verbosity) 81 | # svn switch can lead to a conflict state, but it gives a 82 | # return code of 0. So now we need to make sure that we're 83 | # in a clean (non-conflict) state. 84 | self._abort_if_dirty(repo_dir_path, 85 | "Expected clean state following switch") 86 | os.chdir(cwd) 87 | else: 88 | self._svn_checkout(self._url, repo_dir_path, verbosity) 89 | 90 | def status(self, stat, repo_dir_path): 91 | """ 92 | Check and report the status of the repository 93 | """ 94 | self._check_sync(stat, repo_dir_path) 95 | if os.path.exists(repo_dir_path): 96 | self._status_summary(stat, repo_dir_path) 97 | 98 | # ---------------------------------------------------------------- 99 | # 100 | # Internal work functions 101 | # 102 | # ---------------------------------------------------------------- 103 | def _check_sync(self, stat, repo_dir_path): 104 | """Check to see if repository directory exists and is at the expected 105 | url. Return: status object 106 | 107 | """ 108 | if not os.path.exists(repo_dir_path): 109 | # NOTE(bja, 2017-10) this state should have been handled by 110 | # the source object and we never get here! 111 | stat.sync_state = ExternalStatus.STATUS_ERROR 112 | else: 113 | svn_output = self._svn_info(repo_dir_path) 114 | if not svn_output: 115 | # directory exists, but info returned nothing. .svn 116 | # directory removed or incomplete checkout? 117 | stat.sync_state = ExternalStatus.UNKNOWN 118 | else: 119 | stat.sync_state, stat.current_version = \ 120 | self._check_url(svn_output, self._url) 121 | stat.expected_version = '/'.join(self._url.split('/')[3:]) 122 | 123 | def _abort_if_dirty(self, repo_dir_path, message): 124 | """Check if the repo is in a dirty state; if so, abort with a 125 | helpful message. 126 | 127 | """ 128 | 129 | stat = ExternalStatus() 130 | self._status_summary(stat, repo_dir_path) 131 | if stat.clean_state != ExternalStatus.STATUS_OK: 132 | status = self._svn_status_verbose(repo_dir_path) 133 | status = indent_string(status, 4) 134 | errmsg = """In directory 135 | {cwd} 136 | 137 | svn status now shows: 138 | {status} 139 | 140 | ERROR: {message} 141 | 142 | One possible cause of this problem is that there may have been untracked 143 | files in your working directory that had the same name as tracked files 144 | in the new revision. 145 | 146 | To recover: Clean up the above directory (resolving conflicts, etc.), 147 | then rerun checkout_externals. 148 | """.format(cwd=repo_dir_path, message=message, status=status) 149 | 150 | fatal_error(errmsg) 151 | 152 | @staticmethod 153 | def _check_url(svn_output, expected_url): 154 | """Determine the svn url from svn info output and return whether it 155 | matches the expected value. 156 | 157 | """ 158 | url = None 159 | for line in svn_output.splitlines(): 160 | if SvnRepository.RE_URLLINE.match(line): 161 | url = line.split(': ')[1].strip() 162 | break 163 | if not url: 164 | status = ExternalStatus.UNKNOWN 165 | elif url == expected_url: 166 | status = ExternalStatus.STATUS_OK 167 | else: 168 | status = ExternalStatus.MODEL_MODIFIED 169 | 170 | if url: 171 | current_version = '/'.join(url.split('/')[3:]) 172 | else: 173 | current_version = EMPTY_STR 174 | 175 | return status, current_version 176 | 177 | def _status_summary(self, stat, repo_dir_path): 178 | """Report whether the svn repository is in-sync with the model 179 | description and whether the sandbox is clean or dirty. 180 | 181 | """ 182 | svn_output = self._svn_status_xml(repo_dir_path) 183 | is_dirty = self.xml_status_is_dirty(svn_output) 184 | if is_dirty: 185 | stat.clean_state = ExternalStatus.DIRTY 186 | else: 187 | stat.clean_state = ExternalStatus.STATUS_OK 188 | 189 | # Now save the verbose status output incase the user wants to 190 | # see it. 191 | stat.status_output = self._svn_status_verbose(repo_dir_path) 192 | 193 | @staticmethod 194 | def xml_status_is_dirty(svn_output): 195 | """Parse svn status xml output and determine if the working copy is 196 | clean or dirty. Dirty is defined as: 197 | 198 | * modified files 199 | * added files 200 | * deleted files 201 | * missing files 202 | 203 | Unversioned files do not affect the clean/dirty status. 204 | 205 | 'external' is also an acceptable state 206 | 207 | """ 208 | # pylint: disable=invalid-name 209 | SVN_EXTERNAL = 'external' 210 | SVN_UNVERSIONED = 'unversioned' 211 | # pylint: enable=invalid-name 212 | 213 | is_dirty = False 214 | try: 215 | xml_status = ET.fromstring(svn_output) 216 | except BaseException: 217 | fatal_error( 218 | "SVN returned invalid XML message {}".format(svn_output)) 219 | xml_target = xml_status.find('./target') 220 | entries = xml_target.findall('./entry') 221 | for entry in entries: 222 | status = entry.find('./wc-status') 223 | item = status.get('item') 224 | if item == SVN_EXTERNAL: 225 | continue 226 | if item == SVN_UNVERSIONED: 227 | continue 228 | is_dirty = True 229 | break 230 | return is_dirty 231 | 232 | # ---------------------------------------------------------------- 233 | # 234 | # system call to svn for information gathering 235 | # 236 | # ---------------------------------------------------------------- 237 | @staticmethod 238 | def _svn_info(repo_dir_path): 239 | """Return results of svn info command 240 | """ 241 | cmd = ['svn', 'info', repo_dir_path] 242 | output = execute_subprocess(cmd, output_to_caller=True) 243 | return output 244 | 245 | @staticmethod 246 | def _svn_status_verbose(repo_dir_path): 247 | """capture the full svn status output 248 | """ 249 | cmd = ['svn', 'status', repo_dir_path] 250 | svn_output = execute_subprocess(cmd, output_to_caller=True) 251 | return svn_output 252 | 253 | @staticmethod 254 | def _svn_status_xml(repo_dir_path): 255 | """ 256 | Get status of the subversion sandbox in repo_dir 257 | """ 258 | cmd = ['svn', 'status', '--xml', repo_dir_path] 259 | svn_output = execute_subprocess(cmd, output_to_caller=True) 260 | return svn_output 261 | 262 | # ---------------------------------------------------------------- 263 | # 264 | # system call to svn for sideffects modifying the working tree 265 | # 266 | # ---------------------------------------------------------------- 267 | @staticmethod 268 | def _svn_checkout(url, repo_dir_path, verbosity): 269 | """ 270 | Checkout a subversion repository (repo_url) to checkout_dir. 271 | """ 272 | cmd = ['svn', 'checkout', '--quiet', url, repo_dir_path] 273 | if verbosity >= VERBOSITY_VERBOSE: 274 | printlog(' {0}'.format(' '.join(cmd))) 275 | execute_subprocess(cmd) 276 | 277 | @staticmethod 278 | def _svn_switch(url, ignore_ancestry, verbosity): 279 | """ 280 | Switch branches for in an svn sandbox 281 | """ 282 | cmd = ['svn', 'switch', '--quiet'] 283 | if ignore_ancestry: 284 | cmd.append('--ignore-ancestry') 285 | cmd.append(url) 286 | if verbosity >= VERBOSITY_VERBOSE: 287 | printlog(' {0}'.format(' '.join(cmd))) 288 | execute_subprocess(cmd) 289 | -------------------------------------------------------------------------------- /manage_externals/manic/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Common public utilities for manic package 4 | 5 | """ 6 | 7 | from __future__ import absolute_import 8 | from __future__ import unicode_literals 9 | from __future__ import print_function 10 | 11 | import logging 12 | import os 13 | import subprocess 14 | import sys 15 | from threading import Timer 16 | 17 | from .global_constants import LOCAL_PATH_INDICATOR 18 | 19 | # --------------------------------------------------------------------- 20 | # 21 | # screen and logging output and functions to massage text for output 22 | # 23 | # --------------------------------------------------------------------- 24 | 25 | 26 | def log_process_output(output): 27 | """Log each line of process output at debug level so it can be 28 | filtered if necessary. By default, output is a single string, and 29 | logging.debug(output) will only put log info heading on the first 30 | line. This makes it hard to filter with grep. 31 | 32 | """ 33 | output = output.split('\n') 34 | for line in output: 35 | logging.debug(line) 36 | 37 | 38 | def printlog(msg, **kwargs): 39 | """Wrapper script around print to ensure that everything printed to 40 | the screen also gets logged. 41 | 42 | """ 43 | logging.info(msg) 44 | if kwargs: 45 | print(msg, **kwargs) 46 | else: 47 | print(msg) 48 | sys.stdout.flush() 49 | 50 | 51 | def last_n_lines(the_string, n_lines, truncation_message=None): 52 | """Returns the last n lines of the given string 53 | 54 | Args: 55 | the_string: str 56 | n_lines: int 57 | truncation_message: str, optional 58 | 59 | Returns a string containing the last n lines of the_string 60 | 61 | If truncation_message is provided, the returned string begins with 62 | the given message if and only if the string is greater than n lines 63 | to begin with. 64 | """ 65 | 66 | lines = the_string.splitlines(True) 67 | if len(lines) <= n_lines: 68 | return_val = the_string 69 | else: 70 | lines_subset = lines[-n_lines:] 71 | str_truncated = ''.join(lines_subset) 72 | if truncation_message: 73 | str_truncated = truncation_message + '\n' + str_truncated 74 | return_val = str_truncated 75 | 76 | return return_val 77 | 78 | 79 | def indent_string(the_string, indent_level): 80 | """Indents the given string by a given number of spaces 81 | 82 | Args: 83 | the_string: str 84 | indent_level: int 85 | 86 | Returns a new string that is the same as the_string, except that 87 | each line is indented by 'indent_level' spaces. 88 | 89 | In python3, this can be done with textwrap.indent. 90 | """ 91 | 92 | lines = the_string.splitlines(True) 93 | padding = ' ' * indent_level 94 | lines_indented = [padding + line for line in lines] 95 | return ''.join(lines_indented) 96 | 97 | # --------------------------------------------------------------------- 98 | # 99 | # error handling 100 | # 101 | # --------------------------------------------------------------------- 102 | 103 | 104 | def fatal_error(message): 105 | """ 106 | Error output function 107 | """ 108 | logging.error(message) 109 | raise RuntimeError("{0}ERROR: {1}".format(os.linesep, message)) 110 | 111 | 112 | # --------------------------------------------------------------------- 113 | # 114 | # Data conversion / manipulation 115 | # 116 | # --------------------------------------------------------------------- 117 | def str_to_bool(bool_str): 118 | """Convert a sting representation of as boolean into a true boolean. 119 | 120 | Conversion should be case insensitive. 121 | """ 122 | value = None 123 | str_lower = bool_str.lower() 124 | if str_lower in ('true', 't'): 125 | value = True 126 | elif str_lower in ('false', 'f'): 127 | value = False 128 | if value is None: 129 | msg = ('ERROR: invalid boolean string value "{0}". ' 130 | 'Must be "true" or "false"'.format(bool_str)) 131 | fatal_error(msg) 132 | return value 133 | 134 | 135 | REMOTE_PREFIXES = ['http://', 'https://', 'ssh://', 'git@'] 136 | 137 | 138 | def is_remote_url(url): 139 | """check if the user provided a local file path instead of a 140 | remote. If so, it must be expanded to an absolute 141 | path. 142 | 143 | """ 144 | remote_url = False 145 | for prefix in REMOTE_PREFIXES: 146 | if url.startswith(prefix): 147 | remote_url = True 148 | return remote_url 149 | 150 | 151 | def split_remote_url(url): 152 | """check if the user provided a local file path or a 153 | remote. If remote, try to strip off protocol info. 154 | 155 | """ 156 | remote_url = is_remote_url(url) 157 | if not remote_url: 158 | return url 159 | 160 | for prefix in REMOTE_PREFIXES: 161 | url = url.replace(prefix, '') 162 | 163 | if '@' in url: 164 | url = url.split('@')[1] 165 | 166 | if ':' in url: 167 | url = url.split(':')[1] 168 | 169 | return url 170 | 171 | 172 | def expand_local_url(url, field): 173 | """check if the user provided a local file path instead of a 174 | remote. If so, it must be expanded to an absolute 175 | path. 176 | 177 | Note: local paths of LOCAL_PATH_INDICATOR have special meaning and 178 | represent local copy only, don't work with the remotes. 179 | 180 | """ 181 | remote_url = is_remote_url(url) 182 | if not remote_url: 183 | if url.strip() == LOCAL_PATH_INDICATOR: 184 | pass 185 | else: 186 | url = os.path.expandvars(url) 187 | url = os.path.expanduser(url) 188 | if not os.path.isabs(url): 189 | msg = ('WARNING: Externals description for "{0}" contains a ' 190 | 'url that is not remote and does not expand to an ' 191 | 'absolute path. Version control operations may ' 192 | 'fail.\n\nurl={1}'.format(field, url)) 193 | printlog(msg) 194 | else: 195 | url = os.path.normpath(url) 196 | return url 197 | 198 | 199 | # --------------------------------------------------------------------- 200 | # 201 | # subprocess 202 | # 203 | # --------------------------------------------------------------------- 204 | 205 | # Give the user a helpful message if we detect that a command seems to 206 | # be hanging. 207 | _HANGING_SEC = 300 208 | 209 | 210 | def _hanging_msg(working_directory, command): 211 | print(""" 212 | 213 | Command '{command}' 214 | from directory {working_directory} 215 | has taken {hanging_sec} seconds. It may be hanging. 216 | 217 | The command will continue to run, but you may want to abort 218 | manage_externals with ^C and investigate. A possible cause of hangs is 219 | when svn or git require authentication to access a private 220 | repository. On some systems, svn and git requests for authentication 221 | information will not be displayed to the user. In this case, the program 222 | will appear to hang. Ensure you can run svn and git manually and access 223 | all repositories without entering your authentication information. 224 | 225 | """.format(command=command, 226 | working_directory=working_directory, 227 | hanging_sec=_HANGING_SEC)) 228 | 229 | 230 | def execute_subprocess(commands, status_to_caller=False, 231 | output_to_caller=False): 232 | """Wrapper around subprocess.check_output to handle common 233 | exceptions. 234 | 235 | check_output runs a command with arguments and waits 236 | for it to complete. 237 | 238 | check_output raises an exception on a nonzero return code. if 239 | status_to_caller is true, execute_subprocess returns the subprocess 240 | return code, otherwise execute_subprocess treats non-zero return 241 | status as an error and raises an exception. 242 | 243 | """ 244 | cwd = os.getcwd() 245 | msg = 'In directory: {0}\nexecute_subprocess running command:'.format(cwd) 246 | logging.info(msg) 247 | commands_str = ' '.join(commands) 248 | logging.info(commands_str) 249 | return_to_caller = status_to_caller or output_to_caller 250 | status = -1 251 | output = '' 252 | hanging_timer = Timer(_HANGING_SEC, _hanging_msg, 253 | kwargs={"working_directory": cwd, 254 | "command": commands_str}) 255 | hanging_timer.start() 256 | try: 257 | output = subprocess.check_output(commands, stderr=subprocess.STDOUT, 258 | universal_newlines=True) 259 | log_process_output(output) 260 | status = 0 261 | except OSError as error: 262 | msg = failed_command_msg( 263 | 'Command execution failed. Does the executable exist?', 264 | commands) 265 | logging.error(error) 266 | fatal_error(msg) 267 | except ValueError as error: 268 | msg = failed_command_msg( 269 | 'DEV_ERROR: Invalid arguments trying to run subprocess', 270 | commands) 271 | logging.error(error) 272 | fatal_error(msg) 273 | except subprocess.CalledProcessError as error: 274 | # Only report the error if we are NOT returning to the 275 | # caller. If we are returning to the caller, then it may be a 276 | # simple status check. If returning, it is the callers 277 | # responsibility determine if an error occurred and handle it 278 | # appropriately. 279 | if not return_to_caller: 280 | msg_context = ('Process did not run successfully; ' 281 | 'returned status {0}'.format(error.returncode)) 282 | msg = failed_command_msg(msg_context, commands, 283 | output=error.output) 284 | logging.error(error) 285 | logging.error(msg) 286 | log_process_output(error.output) 287 | fatal_error(msg) 288 | status = error.returncode 289 | finally: 290 | hanging_timer.cancel() 291 | 292 | if status_to_caller and output_to_caller: 293 | ret_value = (status, output) 294 | elif status_to_caller: 295 | ret_value = status 296 | elif output_to_caller: 297 | ret_value = output 298 | else: 299 | ret_value = None 300 | 301 | return ret_value 302 | 303 | 304 | def failed_command_msg(msg_context, command, output=None): 305 | """Template for consistent error messages from subprocess calls. 306 | 307 | If 'output' is given, it should provide the output from the failed 308 | command 309 | """ 310 | 311 | if output: 312 | output_truncated = last_n_lines(output, 20, 313 | truncation_message='[... Output truncated for brevity ...]') 314 | errmsg = ('Failed with output:\n' + 315 | indent_string(output_truncated, 4) + 316 | '\nERROR: ') 317 | else: 318 | errmsg = '' 319 | 320 | command_str = ' '.join(command) 321 | errmsg += """In directory 322 | {cwd} 323 | {context}: 324 | {command} 325 | """.format(cwd=os.getcwd(), context=msg_context, command=command_str) 326 | 327 | if output: 328 | errmsg += 'See above for output from failed command.\n' 329 | 330 | return errmsg 331 | -------------------------------------------------------------------------------- /manage_externals/test/test_unit_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Unit test driver for checkout_externals 4 | 5 | Note: this script assume the path to the checkout_externals.py module is 6 | already in the python path. 7 | 8 | """ 9 | 10 | from __future__ import absolute_import 11 | from __future__ import unicode_literals 12 | from __future__ import print_function 13 | 14 | import os 15 | import unittest 16 | 17 | from manic.utils import last_n_lines, indent_string 18 | from manic.utils import str_to_bool, execute_subprocess 19 | from manic.utils import is_remote_url, split_remote_url, expand_local_url 20 | 21 | 22 | class TestExecuteSubprocess(unittest.TestCase): 23 | """Test the application logic of execute_subprocess wrapper 24 | """ 25 | 26 | def test_exesub_return_stat_err(self): 27 | """Test that execute_subprocess returns a status code when caller 28 | requests and the executed subprocess fails. 29 | 30 | """ 31 | cmd = ['false'] 32 | status = execute_subprocess(cmd, status_to_caller=True) 33 | self.assertEqual(status, 1) 34 | 35 | def test_exesub_return_stat_ok(self): 36 | """Test that execute_subprocess returns a status code when caller 37 | requests and the executed subprocess succeeds. 38 | 39 | """ 40 | cmd = ['true'] 41 | status = execute_subprocess(cmd, status_to_caller=True) 42 | self.assertEqual(status, 0) 43 | 44 | def test_exesub_except_stat_err(self): 45 | """Test that execute_subprocess raises an exception on error when 46 | caller doesn't request return code 47 | 48 | """ 49 | cmd = ['false'] 50 | with self.assertRaises(RuntimeError): 51 | execute_subprocess(cmd, status_to_caller=False) 52 | 53 | 54 | class TestLastNLines(unittest.TestCase): 55 | """Test the last_n_lines function. 56 | 57 | """ 58 | 59 | def test_last_n_lines_short(self): 60 | """With a message with <= n lines, result of last_n_lines should 61 | just be the original message. 62 | 63 | """ 64 | mystr = """three 65 | line 66 | string 67 | """ 68 | 69 | mystr_truncated = last_n_lines( 70 | mystr, 3, truncation_message='[truncated]') 71 | self.assertEqual(mystr, mystr_truncated) 72 | 73 | def test_last_n_lines_long(self): 74 | """With a message with > n lines, result of last_n_lines should 75 | be a truncated string. 76 | 77 | """ 78 | mystr = """a 79 | big 80 | five 81 | line 82 | string 83 | """ 84 | expected = """[truncated] 85 | five 86 | line 87 | string 88 | """ 89 | 90 | mystr_truncated = last_n_lines( 91 | mystr, 3, truncation_message='[truncated]') 92 | self.assertEqual(expected, mystr_truncated) 93 | 94 | 95 | class TestIndentStr(unittest.TestCase): 96 | """Test the indent_string function. 97 | 98 | """ 99 | 100 | def test_indent_string_singleline(self): 101 | """Test the indent_string function with a single-line string 102 | 103 | """ 104 | mystr = 'foo' 105 | result = indent_string(mystr, 4) 106 | expected = ' foo' 107 | self.assertEqual(expected, result) 108 | 109 | def test_indent_string_multiline(self): 110 | """Test the indent_string function with a multi-line string 111 | 112 | """ 113 | mystr = """hello 114 | hi 115 | goodbye 116 | """ 117 | result = indent_string(mystr, 2) 118 | expected = """ hello 119 | hi 120 | goodbye 121 | """ 122 | self.assertEqual(expected, result) 123 | 124 | 125 | class TestStrToBool(unittest.TestCase): 126 | """Test the string to boolean conversion routine. 127 | 128 | """ 129 | 130 | def test_case_insensitive_true(self): 131 | """Verify that case insensitive variants of 'true' returns the True 132 | boolean. 133 | 134 | """ 135 | values = ['true', 'TRUE', 'True', 'tRuE', 't', 'T', ] 136 | for value in values: 137 | received = str_to_bool(value) 138 | self.assertTrue(received) 139 | 140 | def test_case_insensitive_false(self): 141 | """Verify that case insensitive variants of 'false' returns the False 142 | boolean. 143 | 144 | """ 145 | values = ['false', 'FALSE', 'False', 'fAlSe', 'f', 'F', ] 146 | for value in values: 147 | received = str_to_bool(value) 148 | self.assertFalse(received) 149 | 150 | def test_invalid_str_error(self): 151 | """Verify that a non-true/false string generates a runtime error. 152 | """ 153 | values = ['not_true_or_false', 'A', '1', '0', 154 | 'false_is_not_true', 'true_is_not_false'] 155 | for value in values: 156 | with self.assertRaises(RuntimeError): 157 | str_to_bool(value) 158 | 159 | 160 | class TestIsRemoteURL(unittest.TestCase): 161 | """Crude url checking to determine if a url is local or remote. 162 | 163 | """ 164 | 165 | def test_url_remote_git(self): 166 | """verify that a remote git url is identified. 167 | """ 168 | url = 'git@somewhere' 169 | is_remote = is_remote_url(url) 170 | self.assertTrue(is_remote) 171 | 172 | def test_url_remote_ssh(self): 173 | """verify that a remote ssh url is identified. 174 | """ 175 | url = 'ssh://user@somewhere' 176 | is_remote = is_remote_url(url) 177 | self.assertTrue(is_remote) 178 | 179 | def test_url_remote_http(self): 180 | """verify that a remote http url is identified. 181 | """ 182 | url = 'http://somewhere' 183 | is_remote = is_remote_url(url) 184 | self.assertTrue(is_remote) 185 | 186 | def test_url_remote_https(self): 187 | """verify that a remote https url is identified. 188 | """ 189 | url = 'https://somewhere' 190 | is_remote = is_remote_url(url) 191 | self.assertTrue(is_remote) 192 | 193 | def test_url_local_user(self): 194 | """verify that a local path with '~/path/to/repo' gets rejected 195 | 196 | """ 197 | url = '~/path/to/repo' 198 | is_remote = is_remote_url(url) 199 | self.assertFalse(is_remote) 200 | 201 | def test_url_local_var_curly(self): 202 | """verify that a local path with env var '${HOME}' gets rejected 203 | """ 204 | url = '${HOME}/path/to/repo' 205 | is_remote = is_remote_url(url) 206 | self.assertFalse(is_remote) 207 | 208 | def test_url_local_var(self): 209 | """verify that a local path with an env var '$HOME' gets rejected 210 | """ 211 | url = '$HOME/path/to/repo' 212 | is_remote = is_remote_url(url) 213 | self.assertFalse(is_remote) 214 | 215 | def test_url_local_abs(self): 216 | """verify that a local abs path gets rejected 217 | """ 218 | url = '/path/to/repo' 219 | is_remote = is_remote_url(url) 220 | self.assertFalse(is_remote) 221 | 222 | def test_url_local_rel(self): 223 | """verify that a local relative path gets rejected 224 | """ 225 | url = '../../path/to/repo' 226 | is_remote = is_remote_url(url) 227 | self.assertFalse(is_remote) 228 | 229 | 230 | class TestSplitRemoteURL(unittest.TestCase): 231 | """Crude url checking to determine if a url is local or remote. 232 | 233 | """ 234 | 235 | def test_url_remote_git(self): 236 | """verify that a remote git url is identified. 237 | """ 238 | url = 'git@somewhere.com:org/repo' 239 | received = split_remote_url(url) 240 | self.assertEqual(received, "org/repo") 241 | 242 | def test_url_remote_ssh(self): 243 | """verify that a remote ssh url is identified. 244 | """ 245 | url = 'ssh://user@somewhere.com/path/to/repo' 246 | received = split_remote_url(url) 247 | self.assertEqual(received, 'somewhere.com/path/to/repo') 248 | 249 | def test_url_remote_http(self): 250 | """verify that a remote http url is identified. 251 | """ 252 | url = 'http://somewhere.org/path/to/repo' 253 | received = split_remote_url(url) 254 | self.assertEqual(received, 'somewhere.org/path/to/repo') 255 | 256 | def test_url_remote_https(self): 257 | """verify that a remote http url is identified. 258 | """ 259 | url = 'http://somewhere.gov/path/to/repo' 260 | received = split_remote_url(url) 261 | self.assertEqual(received, 'somewhere.gov/path/to/repo') 262 | 263 | def test_url_local_url_unchanged(self): 264 | """verify that a local path is unchanged 265 | 266 | """ 267 | url = '/path/to/repo' 268 | received = split_remote_url(url) 269 | self.assertEqual(received, url) 270 | 271 | 272 | class TestExpandLocalURL(unittest.TestCase): 273 | """Crude url checking to determine if a url is local or remote. 274 | 275 | Remote should be unmodified. 276 | 277 | Local, should perform user and variable expansion. 278 | 279 | """ 280 | 281 | def test_url_local_user1(self): 282 | """verify that a local path with '~/path/to/repo' gets expanded to an 283 | absolute path. 284 | 285 | NOTE(bja, 2017-11) we can't test for something like: 286 | '~user/path/to/repo' because the user has to be in the local 287 | machine password directory and we don't know a user name that 288 | is valid on every system....? 289 | 290 | """ 291 | field = 'test' 292 | url = '~/path/to/repo' 293 | received = expand_local_url(url, field) 294 | print(received) 295 | self.assertTrue(os.path.isabs(received)) 296 | 297 | def test_url_local_expand_curly(self): 298 | """verify that a local path with '${HOME}' gets expanded to an absolute path. 299 | """ 300 | field = 'test' 301 | url = '${HOME}/path/to/repo' 302 | received = expand_local_url(url, field) 303 | self.assertTrue(os.path.isabs(received)) 304 | 305 | def test_url_local_expand_var(self): 306 | """verify that a local path with '$HOME' gets expanded to an absolute path. 307 | """ 308 | field = 'test' 309 | url = '$HOME/path/to/repo' 310 | received = expand_local_url(url, field) 311 | self.assertTrue(os.path.isabs(received)) 312 | 313 | def test_url_local_env_missing(self): 314 | """verify that a local path with env var that is missing gets left as-is 315 | 316 | """ 317 | field = 'test' 318 | url = '$TMP_VAR/path/to/repo' 319 | received = expand_local_url(url, field) 320 | print(received) 321 | self.assertEqual(received, url) 322 | 323 | def test_url_local_expand_env(self): 324 | """verify that a local path with another env var gets expanded to an 325 | absolute path. 326 | 327 | """ 328 | field = 'test' 329 | os.environ['TMP_VAR'] = '/some/absolute' 330 | url = '$TMP_VAR/path/to/repo' 331 | received = expand_local_url(url, field) 332 | del os.environ['TMP_VAR'] 333 | print(received) 334 | self.assertTrue(os.path.isabs(received)) 335 | self.assertEqual(received, '/some/absolute/path/to/repo') 336 | 337 | def test_url_local_normalize_rel(self): 338 | """verify that a local path with another env var gets expanded to an 339 | absolute path. 340 | 341 | """ 342 | field = 'test' 343 | url = '/this/is/a/long/../path/to/a/repo' 344 | received = expand_local_url(url, field) 345 | print(received) 346 | self.assertEqual(received, '/this/is/a/path/to/a/repo') 347 | 348 | 349 | if __name__ == '__main__': 350 | unittest.main() 351 | -------------------------------------------------------------------------------- /manage_externals/test/.pylint.rc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # A comma-separated list of package or module names from where C extensions may 4 | # be loaded. Extensions are loading into the active Python interpreter and may 5 | # run arbitrary code 6 | extension-pkg-whitelist= 7 | 8 | # Add files or directories to the blacklist. They should be base names, not 9 | # paths. 10 | ignore=.git,.svn,env2 11 | 12 | # Add files or directories matching the regex patterns to the blacklist. The 13 | # regex matches against base names, not paths. 14 | ignore-patterns= 15 | 16 | # Python code to execute, usually for sys.path manipulation such as 17 | # pygtk.require(). 18 | #init-hook= 19 | 20 | # Use multiple processes to speed up Pylint. 21 | jobs=1 22 | 23 | # List of plugins (as comma separated values of python modules names) to load, 24 | # usually to register additional checkers. 25 | load-plugins= 26 | 27 | # Pickle collected data for later comparisons. 28 | persistent=yes 29 | 30 | # Specify a configuration file. 31 | #rcfile= 32 | 33 | # Allow loading of arbitrary C extensions. Extensions are imported into the 34 | # active Python interpreter and may run arbitrary code. 35 | unsafe-load-any-extension=no 36 | 37 | 38 | [MESSAGES CONTROL] 39 | 40 | # Only show warnings with the listed confidence levels. Leave empty to show 41 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 42 | confidence= 43 | 44 | # Disable the message, report, category or checker with the given id(s). You 45 | # can either give multiple identifiers separated by comma (,) or put this 46 | # option multiple times (only on the command line, not in the configuration 47 | # file where it should appear only once).You can also use "--disable=all" to 48 | # disable everything first and then reenable specific checks. For example, if 49 | # you want to run only the similarities checker, you can use "--disable=all 50 | # --enable=similarities". If you want to run only the classes checker, but have 51 | # no Warning level messages displayed, use"--disable=all --enable=classes 52 | # --disable=W" 53 | disable=bad-continuation,useless-object-inheritance 54 | 55 | 56 | # Enable the message, report, category or checker with the given id(s). You can 57 | # either give multiple identifier separated by comma (,) or put this option 58 | # multiple time (only on the command line, not in the configuration file where 59 | # it should appear only once). See also the "--disable" option for examples. 60 | enable= 61 | 62 | 63 | [REPORTS] 64 | 65 | # Python expression which should return a note less than 10 (10 is the highest 66 | # note). You have access to the variables errors warning, statement which 67 | # respectively contain the number of errors / warnings messages and the total 68 | # number of statements analyzed. This is used by the global evaluation report 69 | # (RP0004). 70 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 71 | 72 | # Template used to display messages. This is a python new-style format string 73 | # used to format the message information. See doc for all details 74 | msg-template={msg_id}:{line:3d},{column:2d}: {msg} ({symbol}) 75 | 76 | # Set the output format. Available formats are text, parseable, colorized, json 77 | # and msvs (visual studio).You can also give a reporter class, eg 78 | # mypackage.mymodule.MyReporterClass. 79 | output-format=text 80 | 81 | # Tells whether to display a full report or only the messages 82 | #reports=yes 83 | 84 | # Activate the evaluation score. 85 | score=yes 86 | 87 | 88 | [REFACTORING] 89 | 90 | # Maximum number of nested blocks for function / method body 91 | max-nested-blocks=5 92 | 93 | 94 | [BASIC] 95 | 96 | # Naming hint for argument names 97 | argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 98 | 99 | # Regular expression matching correct argument names 100 | argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 101 | 102 | # Naming hint for attribute names 103 | attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 104 | 105 | # Regular expression matching correct attribute names 106 | attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 107 | 108 | # Bad variable names which should always be refused, separated by a comma 109 | bad-names=foo,bar,baz,toto,tutu,tata 110 | 111 | # Naming hint for class attribute names 112 | class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 113 | 114 | # Regular expression matching correct class attribute names 115 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 116 | 117 | # Naming hint for class names 118 | class-name-hint=[A-Z_][a-zA-Z0-9]+$ 119 | 120 | # Regular expression matching correct class names 121 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 122 | 123 | # Naming hint for constant names 124 | const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 125 | 126 | # Regular expression matching correct constant names 127 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 128 | 129 | # Minimum line length for functions/classes that require docstrings, shorter 130 | # ones are exempt. 131 | docstring-min-length=-1 132 | 133 | # Naming hint for function names 134 | function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 135 | 136 | # Regular expression matching correct function names 137 | function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 138 | 139 | # Good variable names which should always be accepted, separated by a comma 140 | good-names=i,j,k,ex,Run,_ 141 | 142 | # Include a hint for the correct naming format with invalid-name 143 | include-naming-hint=no 144 | 145 | # Naming hint for inline iteration names 146 | inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ 147 | 148 | # Regular expression matching correct inline iteration names 149 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 150 | 151 | # Naming hint for method names 152 | method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 153 | 154 | # Regular expression matching correct method names 155 | method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 156 | 157 | # Naming hint for module names 158 | module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 159 | 160 | # Regular expression matching correct module names 161 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 162 | 163 | # Colon-delimited sets of names that determine each other's naming style when 164 | # the name regexes allow several styles. 165 | name-group= 166 | 167 | # Regular expression which should only match function or class names that do 168 | # not require a docstring. 169 | no-docstring-rgx=^_ 170 | 171 | # List of decorators that produce properties, such as abc.abstractproperty. Add 172 | # to this list to register other decorators that produce valid properties. 173 | property-classes=abc.abstractproperty 174 | 175 | # Naming hint for variable names 176 | variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 177 | 178 | # Regular expression matching correct variable names 179 | variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ 180 | 181 | 182 | [FORMAT] 183 | 184 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 185 | expected-line-ending-format= 186 | 187 | # Regexp for a line that is allowed to be longer than the limit. 188 | ignore-long-lines=^\s*(# )??$ 189 | 190 | # Number of spaces of indent required inside a hanging or continued line. 191 | indent-after-paren=4 192 | 193 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 194 | # tab). 195 | indent-string=' ' 196 | 197 | # Maximum number of characters on a single line. 198 | max-line-length=100 199 | 200 | # Maximum number of lines in a module 201 | max-module-lines=1000 202 | 203 | # List of optional constructs for which whitespace checking is disabled. `dict- 204 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 205 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 206 | # `empty-line` allows space-only lines. 207 | no-space-check=trailing-comma,dict-separator 208 | 209 | # Allow the body of a class to be on the same line as the declaration if body 210 | # contains single statement. 211 | single-line-class-stmt=no 212 | 213 | # Allow the body of an if to be on the same line as the test if there is no 214 | # else. 215 | single-line-if-stmt=no 216 | 217 | 218 | [LOGGING] 219 | 220 | # Logging modules to check that the string format arguments are in logging 221 | # function parameter format 222 | logging-modules=logging 223 | 224 | 225 | [MISCELLANEOUS] 226 | 227 | # List of note tags to take in consideration, separated by a comma. 228 | notes=FIXME,XXX,TODO 229 | 230 | 231 | [SIMILARITIES] 232 | 233 | # Ignore comments when computing similarities. 234 | ignore-comments=yes 235 | 236 | # Ignore docstrings when computing similarities. 237 | ignore-docstrings=yes 238 | 239 | # Ignore imports when computing similarities. 240 | ignore-imports=no 241 | 242 | # Minimum lines number of a similarity. 243 | min-similarity-lines=4 244 | 245 | 246 | [SPELLING] 247 | 248 | # Spelling dictionary name. Available dictionaries: none. To make it working 249 | # install python-enchant package. 250 | spelling-dict= 251 | 252 | # List of comma separated words that should not be checked. 253 | spelling-ignore-words= 254 | 255 | # A path to a file that contains private dictionary; one word per line. 256 | spelling-private-dict-file= 257 | 258 | # Tells whether to store unknown words to indicated private dictionary in 259 | # --spelling-private-dict-file option instead of raising a message. 260 | spelling-store-unknown-words=no 261 | 262 | 263 | [TYPECHECK] 264 | 265 | # List of decorators that produce context managers, such as 266 | # contextlib.contextmanager. Add to this list to register other decorators that 267 | # produce valid context managers. 268 | contextmanager-decorators=contextlib.contextmanager 269 | 270 | # List of members which are set dynamically and missed by pylint inference 271 | # system, and so shouldn't trigger E1101 when accessed. Python regular 272 | # expressions are accepted. 273 | generated-members= 274 | 275 | # Tells whether missing members accessed in mixin class should be ignored. A 276 | # mixin class is detected if its name ends with "mixin" (case insensitive). 277 | ignore-mixin-members=yes 278 | 279 | # This flag controls whether pylint should warn about no-member and similar 280 | # checks whenever an opaque object is returned when inferring. The inference 281 | # can return multiple potential results while evaluating a Python object, but 282 | # some branches might not be evaluated, which results in partial inference. In 283 | # that case, it might be useful to still emit no-member and other checks for 284 | # the rest of the inferred objects. 285 | ignore-on-opaque-inference=yes 286 | 287 | # List of class names for which member attributes should not be checked (useful 288 | # for classes with dynamically set attributes). This supports the use of 289 | # qualified names. 290 | ignored-classes=optparse.Values,thread._local,_thread._local 291 | 292 | # List of module names for which member attributes should not be checked 293 | # (useful for modules/projects where namespaces are manipulated during runtime 294 | # and thus existing member attributes cannot be deduced by static analysis. It 295 | # supports qualified module names, as well as Unix pattern matching. 296 | ignored-modules= 297 | 298 | # Show a hint with possible names when a member name was not found. The aspect 299 | # of finding the hint is based on edit distance. 300 | missing-member-hint=yes 301 | 302 | # The minimum edit distance a name should have in order to be considered a 303 | # similar match for a missing member name. 304 | missing-member-hint-distance=1 305 | 306 | # The total number of similar names that should be taken in consideration when 307 | # showing a hint for a missing member. 308 | missing-member-max-choices=1 309 | 310 | 311 | [VARIABLES] 312 | 313 | # List of additional names supposed to be defined in builtins. Remember that 314 | # you should avoid to define new builtins when possible. 315 | additional-builtins= 316 | 317 | # Tells whether unused global variables should be treated as a violation. 318 | allow-global-unused-variables=yes 319 | 320 | # List of strings which can identify a callback function by name. A callback 321 | # name must start or end with one of those strings. 322 | callbacks=cb_,_cb 323 | 324 | # A regular expression matching the name of dummy variables (i.e. expectedly 325 | # not used). 326 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 327 | 328 | # Argument names that match this expression will be ignored. Default to name 329 | # with leading underscore 330 | ignored-argument-names=_.*|^ignored_|^unused_ 331 | 332 | # Tells whether we should check for unused import in __init__ files. 333 | init-import=no 334 | 335 | # List of qualified module names which can have objects that can redefine 336 | # builtins. 337 | redefining-builtins-modules=six.moves,future.builtins 338 | 339 | 340 | [CLASSES] 341 | 342 | # List of method names used to declare (i.e. assign) instance attributes. 343 | defining-attr-methods=__init__,__new__,setUp 344 | 345 | # List of member names, which should be excluded from the protected access 346 | # warning. 347 | exclude-protected=_asdict,_fields,_replace,_source,_make 348 | 349 | # List of valid names for the first argument in a class method. 350 | valid-classmethod-first-arg=cls 351 | 352 | # List of valid names for the first argument in a metaclass class method. 353 | valid-metaclass-classmethod-first-arg=mcs 354 | 355 | 356 | [DESIGN] 357 | 358 | # Maximum number of arguments for function / method 359 | max-args=5 360 | 361 | # Maximum number of attributes for a class (see R0902). 362 | max-attributes=7 363 | 364 | # Maximum number of boolean expressions in a if statement 365 | max-bool-expr=5 366 | 367 | # Maximum number of branch for function / method body 368 | max-branches=12 369 | 370 | # Maximum number of locals for function / method body 371 | max-locals=15 372 | 373 | # Maximum number of parents for a class (see R0901). 374 | max-parents=7 375 | 376 | # Maximum number of public methods for a class (see R0904). 377 | max-public-methods=20 378 | 379 | # Maximum number of return / yield for function / method body 380 | max-returns=6 381 | 382 | # Maximum number of statements in function / method body 383 | max-statements=50 384 | 385 | # Minimum number of public methods for a class (see R0903). 386 | min-public-methods=2 387 | 388 | 389 | [IMPORTS] 390 | 391 | # Allow wildcard imports from modules that define __all__. 392 | allow-wildcard-with-all=no 393 | 394 | # Analyse import fallback blocks. This can be used to support both Python 2 and 395 | # 3 compatible code, which means that the block might have code that exists 396 | # only in one or another interpreter, leading to false positives when analysed. 397 | analyse-fallback-blocks=no 398 | 399 | # Deprecated modules which should not be used, separated by a comma 400 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 401 | 402 | # Create a graph of external dependencies in the given file (report RP0402 must 403 | # not be disabled) 404 | ext-import-graph= 405 | 406 | # Create a graph of every (i.e. internal and external) dependencies in the 407 | # given file (report RP0402 must not be disabled) 408 | import-graph= 409 | 410 | # Create a graph of internal dependencies in the given file (report RP0402 must 411 | # not be disabled) 412 | int-import-graph= 413 | 414 | # Force import order to recognize a module as part of the standard 415 | # compatibility libraries. 416 | known-standard-library= 417 | 418 | # Force import order to recognize a module as part of a third party library. 419 | known-third-party=enchant 420 | 421 | 422 | [EXCEPTIONS] 423 | 424 | # Exceptions that will emit a warning when being caught. Defaults to 425 | # "Exception" 426 | overgeneral-exceptions=Exception 427 | --------------------------------------------------------------------------------