├── .lib
    └── git-fleximod
    │   ├── git_fleximod
    │       ├── __init__.py
    │       ├── lstripreader.py
    │       ├── gitmodules.py
    │       ├── cli.py
    │       ├── gitinterface.py
    │       ├── metoflexi.py
    │       └── utils.py
    │   ├── tests
    │       ├── __init__.py
    │       ├── test_a_import.py
    │       ├── utils_for_tests.py
    │       ├── test_c_required.py
    │       ├── test_d_complex.py
    │       ├── test_e_complex_update.py
    │       ├── test_b_update.py
    │       └── conftest.py
    │   ├── .github
    │       └── workflows
    │       │   ├── pre-commit
    │       │   └── pytest.yaml
    │   ├── doc
    │       ├── index.rst
    │       ├── Makefile
    │       ├── make.bat
    │       └── conf.py
    │   ├── .pre-commit-config.yaml
    │   ├── escomp_install
    │   ├── tbump.toml
    │   ├── License
    │   ├── pyproject.toml
    │   ├── README.md
    │   └── CODE_OF_CONDUCT.md
├── cime_config
    ├── README.md
    ├── testmods_dirs
    │   └── allactive
    │   │   ├── aoflux_agrid
    │   │       ├── include_user_mods
    │   │       └── user_nl_cpl
    │   │   ├── aoflux_ogrid
    │   │       ├── include_user_mods
    │   │       └── user_nl_cpl
    │   │   ├── crossleap
    │   │       ├── include_user_mods
    │   │       └── shell_commands
    │   │   ├── decstart
    │   │       ├── include_user_mods
    │   │       └── shell_commands
    │   │   ├── defaultiomi
    │   │       ├── include_user_mods
    │   │       └── shell_commands
    │   │   ├── cism
    │   │       └── test_coupling
    │   │       │   ├── include_user_mods
    │   │       │   ├── README
    │   │       │   ├── shell_commands
    │   │       │   ├── user_nl_cism
    │   │       │   └── user_nl_clm
    │   │   ├── crossleap_gregorian
    │   │       ├── include_user_mods
    │   │       └── shell_commands
    │   │   ├── defaultio
    │   │       ├── user_nl_cam
    │   │       ├── user_nl_clm
    │   │       ├── user_nl_cism
    │   │       ├── user_nl_mosart
    │   │       ├── user_nl_rtm
    │   │       └── user_nl_cice
    │   │   ├── maxthroughputb
    │   │       ├── user_nl_cam
    │   │       └── shell_commands
    │   │   └── maxthroughputfw
    │   │       └── shell_commands
    ├── usermods_dirs
    │   └── README
    ├── config_tests.xml
    ├── SystemTests
    │   └── funitshare.py
    ├── testfiles
    │   └── ExpectedTestFails.xml
    ├── config_compsets.xml
    └── testlist_allactive.xml
├── doc
    ├── source
    │   ├── _templates
    │   │   ├── layout.html
    │   │   └── footer.html
    │   ├── index.rst
    │   ├── _static
    │   │   └── pop_ver.js
    │   ├── introduction.rst
    │   ├── conf.py
    │   ├── downloading_cesm.rst
    │   └── quickstart.rst
    ├── Makefile
    └── README.md
├── .github
    ├── ISSUE_TEMPLATE
    │   ├── 02_enhancement.md
    │   ├── 01_bug_report.md
    │   └── config.yml
    ├── PULL_REQUEST_TEMPLATE.md
    └── workflows
    │   ├── derecho.yaml
    │   ├── fleximod_test.yaml
    │   ├── build.yaml
    │   └── preview_namelist.yaml
├── .gitignore
├── bin
    └── git-fleximod
├── tools
    └── statistical_ensemble_test
    │   ├── user_nl_cam_LENS
    │   ├── addmetadata.sh
    │   ├── README
    │   └── ensemble.py
├── describe_version
├── ChangeLog_template
├── LICENSE.txt
├── .gitmodules
└── README.rst


/.lib/git-fleximod/git_fleximod/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/cime_config/README.md:
--------------------------------------------------------------------------------
1 | # cime_config
2 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/aoflux_agrid/include_user_mods:
--------------------------------------------------------------------------------
1 | ../defaultio
2 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/aoflux_ogrid/include_user_mods:
--------------------------------------------------------------------------------
1 | ../defaultio
2 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/crossleap/include_user_mods:
--------------------------------------------------------------------------------
1 | ../defaultio
2 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/decstart/include_user_mods:
--------------------------------------------------------------------------------
1 | ../defaultio
2 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/defaultiomi/include_user_mods:
--------------------------------------------------------------------------------
1 | ../defaultio
2 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/aoflux_agrid/user_nl_cpl:
--------------------------------------------------------------------------------
1 | aoflux_grid = "agrid"
2 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/aoflux_ogrid/user_nl_cpl:
--------------------------------------------------------------------------------
1 | aoflux_grid = "ogrid"
2 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/cism/test_coupling/include_user_mods:
--------------------------------------------------------------------------------
1 | ../../defaultio
2 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/crossleap_gregorian/include_user_mods:
--------------------------------------------------------------------------------
1 | ../crossleap
2 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/defaultiomi/shell_commands:
--------------------------------------------------------------------------------
1 | ./xmlchange NTASKS_OCN=-2
2 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/crossleap_gregorian/shell_commands:
--------------------------------------------------------------------------------
1 | ./xmlchange CALENDAR=GREGORIAN
2 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/defaultio/user_nl_cam:
--------------------------------------------------------------------------------
1 | nhtfrq = -24
2 | mfilt  =  1
3 | ndens  =  1
4 | 


--------------------------------------------------------------------------------
/doc/source/_templates/layout.html:
--------------------------------------------------------------------------------
1 | {% extends "!layout.html" %}
2 | 
3 | {% set script_files = script_files + ["_static/pop_ver.js"] %}


--------------------------------------------------------------------------------
/.lib/git-fleximod/tests/__init__.py:
--------------------------------------------------------------------------------
1 | import sys, os
2 | 
3 | sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir, "src"))
4 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/decstart/shell_commands:
--------------------------------------------------------------------------------
1 | ./xmlchange RUN_STARTDATE=2001-12-30
2 | ./xmlchange CLM_BLDNML_OPTS=-ignore_warnings --append
3 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/crossleap/shell_commands:
--------------------------------------------------------------------------------
1 | ./xmlchange RUN_STARTDATE=2004-02-28
2 | ./xmlchange CLM_BLDNML_OPTS=-ignore_warnings --append
3 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/defaultio/user_nl_clm:
--------------------------------------------------------------------------------
1 |  hist_dov2xy    = .true.
2 |  hist_ndens     = 1
3 |  hist_nhtfrq    =-24
4 |  hist_mfilt     = 1
5 | 


--------------------------------------------------------------------------------
/cime_config/usermods_dirs/README:
--------------------------------------------------------------------------------
1 | Directories here can be included in a case via the --user-mods-dir
2 | argument to create_newcase, or via the CPL_USER_MODS XML variable.
3 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/cism/test_coupling/README:
--------------------------------------------------------------------------------
1 | The purpose of this testmod directory is to enable CISM's dynamics on
2 | the short (multi-day) time scales typical of fully-active tests.
3 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/maxthroughputb/user_nl_cam:
--------------------------------------------------------------------------------
1 | ! Users should add all user specific namelist changes below in the form of 
2 | ! namelist_var = new_namelist_value 
3 | npr_yz= 64,18,18,64 
4 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/defaultio/user_nl_cism:
--------------------------------------------------------------------------------
1 | 
2 | ! This is needed to give CISM history output in the (typically short)
3 | ! tests that are done with this testmod directory
4 | history_option = 'coupler'
5 | 


--------------------------------------------------------------------------------
/doc/source/_templates/footer.html:
--------------------------------------------------------------------------------
1 | {% extends "!footer.html" %}
2 | {% block extrafooter %}
3 |     {{ super() }}
4 | <script>var version_json_loc = "{{ pathto('../../versions.json', 1) }}";</script>
5 | {% endblock %}
6 | 


--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/02_enhancement.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Enhancement request, CESM-wide or unknown component
3 | about: Suggest a CESM-wide enhancement; in many cases, one of the links below is more appropriate
4 | 
5 | ---
6 | 
7 | 
8 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/cism/test_coupling/shell_commands:
--------------------------------------------------------------------------------
1 | # Since we're doing a daily mass balance time step in CISM, we can't use the default GLC_AVG_PERIOD of yearly
2 | ./xmlchange GLC_AVG_PERIOD=glc_coupling_period
3 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # generated local files
 2 | *.log
 3 | 
 4 | # editor files
 5 | *~
 6 | *.bak
 7 | 
 8 | # generated python files
 9 | *.pyc
10 | __pycache__
11 | 
12 | # ignore for the merge to master and orphaning of the gh-pages branch
13 | doc/build
14 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/cism/test_coupling/user_nl_cism:
--------------------------------------------------------------------------------
1 | 
2 | ! This option changes the ice sheet dynamics time step to 1 day rather than 1 year
3 | ! Thus, the ice sheet dynamics can be exercised in a few-day run
4 | test_coupling = .true.
5 | 


--------------------------------------------------------------------------------
/bin/git-fleximod:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | import sys
3 | import os
4 | sys.path.insert(0,os.path.abspath(os.path.join(os.path.dirname(__file__),"..",".lib","git-fleximod")))
5 | from git_fleximod.git_fleximod import main
6 | 
7 | if __name__ == '__main__':
8 |     sys.exit(main())
9 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/tests/test_a_import.py:
--------------------------------------------------------------------------------
 1 | # pylint: disable=unused-import
 2 | from git_fleximod import cli
 3 | from git_fleximod import utils
 4 | from git_fleximod.gitinterface import GitInterface
 5 | from git_fleximod.gitmodules import GitModules
 6 | 
 7 | 
 8 | def test_import():
 9 |     print("here")
10 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/.github/workflows/pre-commit:
--------------------------------------------------------------------------------
 1 | name: pre-commit
 2 | on:
 3 |   pull_request:
 4 |   push:
 5 |     branches: [main]
 6 | 
 7 | jobs:
 8 |   pre-commit:
 9 |     runs-on: ubuntu-latest
10 |     steps:
11 |     - uses: actions/checkout@v3
12 |     - uses: actions/setup-python@v3
13 |     - uses: pre-commit/action@v3.0.0
14 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/maxthroughputfw/shell_commands:
--------------------------------------------------------------------------------
1 | # copy the spmd_mod.F90
2 | # This performance test is only expected to work on yellowstone
3 | cp /glade/u/home/jedwards/b1850/SourceMods/src.cam/* SourceMods/src.cam
4 | ./xmlchange NTASKS=2048
5 | ./xmlchange NTHRDS=4
6 | ./xmlchange --append CAM_CONFIG_OPTS="-pcols 8"
7 | 
8 | 
9 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/tests/utils_for_tests.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Helper functions that can be used in tests
 3 | """
 4 | 
 5 | 
 6 | def normalize_whitespace(text):
 7 |     """
 8 |     Normalize whitespace for flexible string comparisons in tests.
 9 | 
10 |     This removes leading and trailing whitespace and collapses other whitespace down to a
11 |     single space.
12 |     """
13 |     return " ".join(text.split())
14 | 


--------------------------------------------------------------------------------
/.github/PULL_REQUEST_TEMPLATE.md:
--------------------------------------------------------------------------------
 1 | ### Description of changes
 2 | 
 3 | ### Specific notes
 4 | 
 5 | Contributors other than yourself, if any:
 6 | 
 7 | Fixes: [Github issue #s] And brief description of each issue.
 8 | 
 9 | User interface changes?: [ No/Yes ]
10 | [ If yes, describe what changed, and steps taken to ensure backward compatibility ]
11 | 
12 | Testing performed (automated tests and/or manual tests):
13 | 
14 | 


--------------------------------------------------------------------------------
/cime_config/config_tests.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0"?>
 2 | 
 3 | <!--
 4 | This defines various CESM-specific system tests
 5 | -->
 6 | 
 7 | <config_test>
 8 | 
 9 |   <test NAME="FUNITSHARE">
10 |     <DESC>Run the share code's Fortran unit tests. Grid and compset (and most case settings) are ignored.</DESC>
11 |     <INFO_DBUG>1</INFO_DBUG>
12 |     <STOP_OPTION>ndays</STOP_OPTION>
13 |     <STOP_N>11</STOP_N>
14 |     <CHECK_TIMING>FALSE</CHECK_TIMING>
15 |     <DOUT_S>FALSE</DOUT_S>
16 |   </test>
17 | 
18 | </config_test>
19 | 


--------------------------------------------------------------------------------
/doc/Makefile:
--------------------------------------------------------------------------------
 1 | # Minimal makefile for Sphinx documentation
 2 | #
 3 | 
 4 | # You can set these variables from the command line.
 5 | SPHINXOPTS    =
 6 | SPHINXBUILD   = sphinx-build
 7 | SPHINXPROJ    = cesm2_quickstart
 8 | SOURCEDIR     = source
 9 | BUILDDIR      = build
10 | 
11 | # Put it first so that "make" without argument is like "make help".
12 | help:
13 | 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14 | 
15 | .PHONY: help Makefile
16 | 
17 | # Catch-all target: route all unknown targets to Sphinx using the new
18 | # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
19 | %: Makefile
20 | 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/defaultio/user_nl_mosart:
--------------------------------------------------------------------------------
 1 | !----------------------------------------------------------------------------------
 2 | ! Users should add all user specific namelist changes below in the form of
 3 | ! namelist_var = new_namelist_value
 4 | ! NOTE: namelist variable tstep CAN ONLY be changed by modifying the value
 5 | !       of the xml variable ROF_NCPL in env_run.xml
 6 | ! NOTE: if the xml variable RTM_MODE in env_run.xml is set to 'NULL', then
 7 | !        then rtm will set rtm_present to .false. - and will ignore everything else
 8 | !----------------------------------------------------------------------------------
 9 | ndens     = 1
10 | nhtfrq    =-24
11 | mfilt     = 1
12 | 
13 | 
14 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/doc/index.rst:
--------------------------------------------------------------------------------
 1 | .. git-fleximod documentation master file, created by
 2 |    sphinx-quickstart on Sat Feb  3 12:02:22 2024.
 3 |    You can adapt this file completely to your liking, but it should at least
 4 |    contain the root `toctree` directive.
 5 | 
 6 | Welcome to git-fleximod's documentation!
 7 | ========================================
 8 | 
 9 | .. toctree::
10 |    :maxdepth: 2
11 |    :caption: Contents:
12 | .. module:: sphinxcontrib.autoprogram
13 | .. sphinx_argparse_cli::
14 |    :module: git_fleximod.cli
15 |    :func: get_parser
16 |    :prog: git-fleximod
17 | 
18 | 
19 | Indices and tables
20 | ==================
21 | 
22 | * :ref:`genindex`
23 | * :ref:`modindex`
24 | * :ref:`search`
25 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/defaultio/user_nl_rtm:
--------------------------------------------------------------------------------
 1 | !----------------------------------------------------------------------------------
 2 | ! Users should add all user specific namelist changes below in the form of
 3 | ! namelist_var = new_namelist_value
 4 | ! NOTE: namelist variable rtm_tstep CAN ONLY be changed by modifying the value
 5 | !       of the xml variable ROF_NCPL in env_run.xml
 6 | ! NOTE: if the xml variable RTM_MODE in env_run.xml is set to 'NULL', then
 7 | !        then rtm will set rtm_present to .false. - and will ignore everything else
 8 | !----------------------------------------------------------------------------------
 9 | rtmhist_ndens     = 1
10 | rtmhist_nhtfrq    =-24
11 | rtmhist_mfilt     = 1
12 | 
13 | 
14 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/doc/Makefile:
--------------------------------------------------------------------------------
 1 | # Minimal makefile for Sphinx documentation
 2 | #
 3 | 
 4 | # You can set these variables from the command line, and also
 5 | # from the environment for the first two.
 6 | SPHINXOPTS    ?=
 7 | SPHINXBUILD   ?= sphinx-build
 8 | SOURCEDIR     = .
 9 | BUILDDIR      = _build
10 | 
11 | # Put it first so that "make" without argument is like "make help".
12 | help:
13 | 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14 | 
15 | .PHONY: help Makefile
16 | 
17 | # Catch-all target: route all unknown targets to Sphinx using the new
18 | # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
19 | %: Makefile
20 | 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
21 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/.pre-commit-config.yaml:
--------------------------------------------------------------------------------
 1 | exclude: ^utils/.*$
 2 | 
 3 | repos:
 4 |   - repo: https://github.com/pre-commit/pre-commit-hooks
 5 |     rev: v6.0.0
 6 |     hooks:
 7 |       - id: end-of-file-fixer
 8 |       - id: trailing-whitespace
 9 |   - repo: https://github.com/psf/black
10 |     rev: 22.3.0
11 |     hooks:
12 |       - id: black
13 |   - repo: https://github.com/PyCQA/pylint
14 |     rev: v2.11.1
15 |     hooks:
16 |       - id: pylint
17 |         args:
18 |           - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error,no-member
19 | 


--------------------------------------------------------------------------------
/doc/source/index.rst:
--------------------------------------------------------------------------------
 1 | .. CESM2 Quickstart Guide master file, created by
 2 |    sphinx-quickstart on Wed Mar 29 12:57:57 2017.
 3 |    You can adapt this file completely to your liking, but it should at least
 4 |    contain the root `toctree` directive.
 5 | 
 6 | CESM Quickstart Guide (|version|)
 7 | =================================
 8 | 
 9 | .. important::
10 |    **There are separate versions of this documentation for each maintained CESM
11 |    release. Use the menu at the top left to select the version of CESM you are using.**
12 | 
13 | Table of contents
14 | -----------------
15 | 
16 | .. toctree::
17 |    :maxdepth: 2
18 | 
19 |    introduction.rst
20 |    downloading_cesm.rst
21 |    cesm_configurations.rst
22 |    quickstart.rst
23 | 
24 | 
25 | 
26 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/cism/test_coupling/user_nl_clm:
--------------------------------------------------------------------------------
 1 | 
 2 |  ! This is needed to tell CLM to allow the non-annual-boundary glacier changes that arise
 3 |  ! with this testmod.
 4 |  for_testing_allow_non_annual_changes = .true.
 5 | 
 6 |  ! When we have daily rather than annual glacier dynamics (as we do in this testmod, due
 7 |  ! to having test_coupling in user_nl_cism), CLM applies the dynbal adjustments in a
 8 |  ! single time step rather than spreading them throughout the year. This can cause
 9 |  ! sensible heat fluxes of thousands of W m-2, which causes CAM's PBL scheme to blow up.
10 |  ! So force these fluxes to zero for this testmod; this breaks water and energy
11 |  ! conservation in CLM, but should allow the test to pass.
12 |  for_testing_zero_dynbal_fluxes = .true.
13 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/doc/make.bat:
--------------------------------------------------------------------------------
 1 | @ECHO OFF
 2 | 
 3 | pushd %~dp0
 4 | 
 5 | REM Command file for Sphinx documentation
 6 | 
 7 | if "%SPHINXBUILD%" == "" (
 8 | 	set SPHINXBUILD=sphinx-build
 9 | )
10 | set SOURCEDIR=.
11 | set BUILDDIR=_build
12 | 
13 | %SPHINXBUILD% >NUL 2>NUL
14 | if errorlevel 9009 (
15 | 	echo.
16 | 	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
17 | 	echo.installed, then set the SPHINXBUILD environment variable to point
18 | 	echo.to the full path of the 'sphinx-build' executable. Alternatively you
19 | 	echo.may add the Sphinx directory to PATH.
20 | 	echo.
21 | 	echo.If you don't have Sphinx installed, grab it from
22 | 	echo.https://www.sphinx-doc.org/
23 | 	exit /b 1
24 | )
25 | 
26 | if "%1" == "" goto help
27 | 
28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
29 | goto end
30 | 
31 | :help
32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
33 | 
34 | :end
35 | popd
36 | 


--------------------------------------------------------------------------------
/.github/workflows/derecho.yaml:
--------------------------------------------------------------------------------
 1 | name: cesm-derecho-hosted
 2 | run-name: ${{ github.actor }} is testing the project
 3 | 
 4 | on:
 5 |   pull_request:
 6 |     branches: [ cesm3.0-alphabranch ]
 7 |     types: [ labeled ]
 8 |     
 9 | jobs:
10 | 
11 |   setup:
12 |     # only run this workflow if the label run-pr-tests is applied
13 |     if: ${{ github.event.label.name == 'run-pr-tests' }}
14 |     # currently hpc-runner is derecho.hpc.ucar.edu
15 |     runs-on: hpc-runner
16 |    
17 |     steps:
18 |     - name: Checkout code
19 |       uses: actions/checkout@v4
20 |     - name: Run git-fleximod
21 |       run: |
22 |         pwd
23 |         module list
24 |         ./bin/git-fleximod update
25 | 
26 |   build:
27 |     strategy:
28 |       # run all matrix members even if one fails
29 |       fail-fast: false
30 |       matrix:
31 |         compiler: [gnu, intel, intel-oneapi, nvhpc]
32 |     uses: ./.github/workflows/build.yaml
33 |     with:
34 |       compiler: ${{ matrix.compiler }}
35 | 


--------------------------------------------------------------------------------
/.github/workflows/fleximod_test.yaml:
--------------------------------------------------------------------------------
 1 | on:
 2 |   push:
 3 |     branches: [ cesm3.0-alphabranch ]
 4 |   pull_request:
 5 |     branches: [ cesm3.0-alphabranch ]
 6 | jobs:
 7 |   fleximod-test:
 8 |     runs-on: ubuntu-latest
 9 |     strategy:
10 |       fail-fast: false
11 |       matrix:
12 |         # oldest supported and latest supported
13 |         python-version: ["3.7", "3.x"]
14 |     steps:
15 |       - id: checkout-CESM
16 |         uses: actions/checkout@v4
17 |       - id: run-fleximod
18 |         run: |
19 |           $GITHUB_WORKSPACE/bin/git-fleximod update -o
20 |           echo
21 |           echo "Update complete, checking status"
22 |           echo
23 |           $GITHUB_WORKSPACE/bin/git-fleximod test
24 |       - id: check-cleanliness
25 |         run: |
26 |           echo
27 |           echo "Checking if git fleximod matches expected externals"
28 |           echo
29 |           git add . && git diff --exit-code && git diff --cached --exit-code
30 | #      - name: Setup tmate session
31 | #        if: ${{ failure() }}
32 | #        uses: mxschmitt/action-tmate@v3
33 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/doc/conf.py:
--------------------------------------------------------------------------------
 1 | # Configuration file for the Sphinx documentation builder.
 2 | #
 3 | # For the full list of built-in configuration values, see the documentation:
 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html
 5 | 
 6 | # -- Project information -----------------------------------------------------
 7 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 8 | 
 9 | project = "git-fleximod"
10 | author = "Jim Edwards <jedwards@ucar.edu>"
11 | release = "0.4.0"
12 | 
13 | # -- General configuration ---------------------------------------------------
14 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
15 | 
16 | extensions = ["sphinx_argparse_cli"]
17 | 
18 | templates_path = ["_templates"]
19 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
20 | 
21 | 
22 | # -- Options for HTML output -------------------------------------------------
23 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
24 | 
25 | html_theme = "alabaster"
26 | html_static_path = ["_static"]
27 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/escomp_install:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # updates git-fleximod in an ESCOMP model
 3 | # this script should be run from the model root directory, it expects
 4 | # git-fleximod to already be installed with the script in bin
 5 | # and the classes in lib/python/site-packages
 6 | import sys
 7 | import shutil
 8 | import os
 9 | 
10 | from glob import iglob
11 | 
12 | fleximod_root = sys.argv[1]
13 | fleximod_path = os.path.join(fleximod_root,"src","git-fleximod")
14 | if os.path.isfile(fleximod_path):
15 |    with open(fleximod_path,"r") as f:
16 |         fleximod = f.readlines()
17 |    with open(os.path.join(".","bin","git-fleximod"),"w") as f:
18 |         for line in fleximod:
19 |             f.write(line)
20 |             if "import argparse" in line:
21 |                f.write('\nsys.path.append(os.path.join(os.path.dirname(__file__),"..","lib","python","site-packages"))\n\n')
22 | 
23 |    for file in iglob(os.path.join(fleximod_root, "src", "fleximod", "*.py")):
24 |        shutil.copy(file,
25 |         os.path.join("lib","python","site-packages","fleximod",os.path.basename(file)))
26 | 


--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/01_bug_report.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | name: Bug report, CESM-wide or unknown component
 3 | about: Report a problem with CESM for an unknown component; in many cases, one of the links below is more appropriate
 4 | 
 5 | ---
 6 | 
 7 | ### Brief summary of bug
 8 | 
 9 | [Give a one or two sentence summary. This could be the same as the issue title if you feel that is a sufficient summary.]
10 | 
11 | ### General bug information
12 | 
13 | **CESM version you are using:** [Output of `git describe`]
14 | 
15 | **Machine you are using:** [Fill this in]
16 | 
17 | **Have you modified the code? (If so, please point us to you changes.):** [Yes / No]
18 | 
19 | ### Details of bug
20 | 
21 | [Fill in details here.]
22 | 
23 | ### Steps to reproduce the problem. Include externals description file(s) and link to public repository
24 | 
25 | [Fill this in with anything relevant that you haven't already noted; if there is nothing to add, delete this section.]
26 | 
27 | ### Screen output or log file showing the error message and context
28 | 
29 | [Fill this in with anything relevant that you haven't already noted; if there is nothing to add, delete this section.]
30 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/tbump.toml:
--------------------------------------------------------------------------------
 1 | # Uncomment this if your project is hosted on GitHub:
 2 | github_url = "https://github.com/jedwards4b/git-fleximod/"
 3 | 
 4 | [version]
 5 | current = "1.1.1"
 6 | 
 7 | # Example of a semver regexp.
 8 | # Make sure this matches current_version before
 9 | # using tbump
10 | regex = '''
11 |   (?P<major>\d+)
12 |   \.
13 |   (?P<minor>\d+)
14 |   \.
15 |   (?P<patch>\d+)
16 |   '''
17 | 
18 | [git]
19 | message_template = "Bump to {new_version}"
20 | tag_template = "v{new_version}"
21 | 
22 | # For each file to patch, add a [[file]] config
23 | # section containing the path of the file, relative to the
24 | # tbump.toml location.
25 | [[file]]
26 | src = "git_fleximod/cli.py"
27 | 
28 | [[file]]
29 | src = "pyproject.toml"
30 | 
31 | # You can specify a list of commands to
32 | # run after the files have been patched
33 | # and before the git commit is made
34 | 
35 | #  [[before_commit]]
36 | #  name = "check changelog"
37 | #  cmd = "grep -q {new_version} Changelog.rst"
38 | 
39 | # Or run some commands after the git tag and the branch
40 | # have been pushed:
41 | #  [[after_push]]
42 | #  name = "publish"
43 | #  cmd = "./publish.sh"
44 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/maxthroughputb/shell_commands:
--------------------------------------------------------------------------------
 1 | # copy the spmd_mod.F90
 2 | # This performance test is only expected to work on yellowstone
 3 | cp /glade/u/home/jedwards/b1850/SourceMods/src.cam/* SourceMods/src.cam
 4 | NTASKS_ATM=1920
 5 | NTASKS_CPL=608
 6 | NTASKS_LND=1184
 7 | NTASKS_ICE=608
 8 | NTASKS_WAV=128
 9 | NTASKS_OCN=192
10 | NTHRDS=2
11 | 
12 | ROOTPE_WAV=$(($NTASKS_LND + $NTASKS_ICE))
13 | #ROOTPE_CPL=$(($ROOTPE_WAV + $NTASKS_WAV))
14 | ROOTPE_CPL=$NTASKS_LND
15 | COST_PES=$(($NTHRDS*($NTASKS_ATM + $NTASKS_OCN)))
16 | ROOTPE_OCN=$(($NTASKS_ATM))
17 | 
18 | ./xmlchange MAX_MPITASKS_PER_NODE=16
19 | ./xmlchange MAX_TASKS_PER_NODE=16
20 | ./xmlchange NTASKS_ATM=$NTASKS_ATM
21 | ./xmlchange NTASKS_CPL=$NTASKS_CPL
22 | ./xmlchange NTASKS_GLC=$NTASKS_ATM
23 | ./xmlchange NTASKS_LND=$NTASKS_LND
24 | ./xmlchange NTASKS_ROF=$NTASKS_LND
25 | ./xmlchange NTASKS_ICE=$NTASKS_ICE
26 | ./xmlchange NTASKS_WAV=$NTASKS_WAV
27 | ./xmlchange NTASKS_OCN=$NTASKS_OCN
28 | ./xmlchange ROOTPE_OCN=$ROOTPE_OCN
29 | ./xmlchange ROOTPE_ICE=$NTASKS_LND
30 | ./xmlchange ROOTPE_WAV=$ROOTPE_WAV
31 | ./xmlchange ROOTPE_CPL=$ROOTPE_CPL
32 | ./xmlchange NTHRDS=$NTHRDS
33 | 
34 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/License:
--------------------------------------------------------------------------------
 1 | Copyright 2024 NSF National Center for Atmospheric Sciences (NCAR)
 2 | 
 3 | Permission is hereby granted, free of charge, to any person obtaining
 4 | a copy of this software and associated documentation files (the
 5 | “Software”), to deal in the Software without restriction, including
 6 | without limitation the rights to use, copy, modify, merge, publish,
 7 | distribute, sublicense, and/or sell copies of the Software, and to
 8 | permit persons to whom the Software is furnished to do so, subject to
 9 | the following conditions:
10 | 
11 | The above copyright notice and this permission notice shall be
12 | included in all copies or substantial portions of the Software.
13 | 
14 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/pyproject.toml:
--------------------------------------------------------------------------------
 1 | [tool.poetry]
 2 | name = "git-fleximod"
 3 | version = "1.1.1"
 4 | description = "Extended support for git-submodule and git-sparse-checkout"
 5 | authors = ["Jim Edwards <jedwards@ucar.edu>"]
 6 | maintainers = ["Jim Edwards <jedwards@ucar.edu>"]
 7 | license = "MIT"
 8 | readme = "README.md"
 9 | homepage = "https://github.com/jedwards4b/git-fleximod"
10 | keywords = ["git", "submodule", "sparse-checkout"]
11 | packages = [
12 | { include = "git_fleximod"},
13 | { include = "doc"},
14 | { include = "README.md"},
15 | ]
16 | 
17 | [tool.poetry.scripts]
18 | git-fleximod = "git_fleximod.git_fleximod:main"
19 | me2flexi = "git_fleximod.metoflexi:_main"
20 | fsspec = "fsspec.fuse:main"
21 | 
22 | [tool.poetry.dependencies]
23 | python = ">=3.9"
24 | GitPython = "^3.1.0"
25 | sphinx = "^5.0.0"
26 | fsspec = "^2023.12.2"
27 | wheel = "^0.42.0"
28 | pytest = "^8.0.0"
29 | pyfakefs = "^5.3.5"
30 | pre-commit = "4.3.0"
31 | wrapt = ">=2.0.0"
32 | 
33 | [tool.poetry.urls]
34 | "Bug Tracker" = "https://github.com/jedwards4b/git-fleximod/issues"
35 | 
36 | [tool.pytest.ini_options]
37 | markers = [
38 |         "skip_after_first: only run on first iteration"
39 | ]
40 | 
41 | [build-system]
42 | requires = ["poetry-core"]
43 | build-backend = "poetry.core.masonry.api"
44 | 


--------------------------------------------------------------------------------
/cime_config/SystemTests/funitshare.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Implementation of the share code FUNIT test.
 3 | 
 4 | This "system" test runs the share code's Fortran unit tests. We're abusing the system test
 5 | infrastructure to run these, so that a run of the test suite can result in the unit tests
 6 | being run as well.
 7 | 
 8 | Grid and compset are irrelevant for this test type.
 9 | """
10 | 
11 | import os
12 | from CIME.SystemTests.funit import FUNIT
13 | from CIME.XML.standard_module_setup import *
14 | 
15 | logger = logging.getLogger(__name__)
16 | 
17 | 
18 | class FUNITSHARE(FUNIT):
19 |     def __init__(self, case):
20 |         FUNIT.__init__(self, case)
21 | 
22 |         # The CMake build for the share unit tests uses the MPISERIAL env var to find the
23 |         # mpi-serial library. On derecho, MPISERIAL is not set even if the mpi-serial
24 |         # module is loaded; instead, there is an NCAR_ROOT_MPI_SERIAL variable. Set the
25 |         # needed MPISERIAL env var for this test.
26 |         if "NCAR_ROOT_MPI_SERIAL" in os.environ:
27 |             os.environ["MPISERIAL"] = os.environ["NCAR_ROOT_MPI_SERIAL"]
28 | 
29 |     def get_test_spec_dir(self):
30 |         return os.path.join(self._case.get_value("SRCROOT"), "share")
31 | 
32 |     def get_extra_run_tests_args(self):
33 |         return '--cmake-args " -DUNITTESTS=ON -DUSE_CIME_MACROS=ON"'
34 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/git_fleximod/lstripreader.py:
--------------------------------------------------------------------------------
 1 | class LstripReader(object):
 2 |     "LstripReader formats .gitmodules files to be acceptable for configparser"
 3 | 
 4 |     def __init__(self, filename):
 5 |         with open(filename, "r") as infile:
 6 |             lines = infile.readlines()
 7 |         self._lines = list()
 8 |         self._num_lines = len(lines)
 9 |         self._index = 0
10 |         for line in lines:
11 |             self._lines.append(line.lstrip())
12 | 
13 |     def readlines(self):
14 |         """Return all the lines from this object's file"""
15 |         return self._lines
16 | 
17 |     def readline(self, size=-1):
18 |         """Format and return the next line or raise StopIteration"""
19 |         try:
20 |             line = self.next()
21 |         except StopIteration:
22 |             line = ""
23 | 
24 |         if (size > 0) and (len(line) < size):
25 |             return line[0:size]
26 | 
27 |         return line
28 | 
29 |     def __iter__(self):
30 |         """Begin an iteration"""
31 |         self._index = 0
32 |         return self
33 | 
34 |     def next(self):
35 |         """Return the next line or raise StopIteration"""
36 |         if self._index >= self._num_lines:
37 |             raise StopIteration
38 | 
39 |         self._index = self._index + 1
40 |         return self._lines[self._index - 1]
41 | 
42 |     def __next__(self):
43 |         return self.next()
44 | 


--------------------------------------------------------------------------------
/tools/statistical_ensemble_test/user_nl_cam_LENS:
--------------------------------------------------------------------------------
 1 | ! Users should add all user specific namelist changes below in the form of
 2 | ! namelist_var = new_namelist_value
 3 | ! These are specific changes to CAM for the LENS experiment
 4 |  cldfrc_rhminl = 0.8925D0
 5 |  empty_htapes=.true.
 6 |  fincl1='ABSORB:A','ANRAIN:A','ANSNOW:A','AODDUST1:A','AODDUST2:A','AODDUST3:A',
 7 |   'AODVIS:A','AQRAIN:A','AQSNOW:A','AREI:A','AREL:A','AWNC:A','AWNI:A','CDNUMC:A',
 8 |   'CLDHGH:A','CLDICE:A','CLDLIQ:A','CLDLOW:A','CLDMED:A','CLDTOT:A','CLOUD:A',
 9 |   'DCQ:A','DTCOND:A','DTV:A','FICE:A','FLDS:A','FLNS:A','FLNSC:A','FLNT:A',
10 |   'FLNTC:A','FLUT:A','FLUTC:A','FREQI:A','FREQL:A','FREQR:A','FREQS:A','FSDS:A',
11 |   'FSDSC:A','FSNS:A','FSNSC:A','FSNT:A','FSNTC:A','FSNTOA:A','FSNTOAC:A',
12 |   'ICEFRAC:A','ICIMR:A','ICWMR:A','IWC:A','LANDFRAC:A','LHFLX:A','LWCF:A',
13 |   'NUMICE:A','NUMLIQ:A','OCNFRAC:A','OMEGA:A','OMEGAT:A','PBLH:A','PRECC:A',
14 |   'PRECL:A','PRECSC:A','PRECSL:A','PS:A','PSL:A','Q:A','QFLX:A','QRL:A','QRS:A',
15 |   'RELHUM:A','SHFLX:A','SNOWHICE:A','SNOWHLND:A','SOLIN:A','SRFRAD:A','SWCF:A',
16 |   'T:A','TAUX:A','TAUY:A','TGCLDIWP:A','TGCLDLWP:A','TMQ:A','TREFHT:A','TS:A',
17 |   'U:A','U10:A','UU:A','V:A','VD01:A','VQ:A','VT:A','VU:A','VV:A','WSUB:A','Z3:A',
18 |   'CCN3:A','UQ:A','WGUSTD:X','WSPDSRFMX:A','TSMX:X','TSMN:M','TREFHTMX:X','TREFHTMN:M',
19 |   'bc_a1_SRF:A','dst_a1_SRF:A','dst_a3_SRF:A','pom_a1_SRF:A','so4_a1_SRF:A',
20 |   'so4_a2_SRF:A','so4_a3_SRF:A','soa_a1_SRF:A','soa_a2_SRF:A','BURDENSO4:A',
21 |   'BURDENBC:A','BURDENPOM:A','BURDENSOA:A','BURDENDUST:A','BURDENSEASALT:A',
22 |   'AODABS:A','EXTINCT:A','PHIS:A','TROP_P:A','TROP_T:A','TOT_CLD_VISTAU:A',
23 |   'ICLDIWP:A','ICLDTWP:A','CO2:A','CO2_LND:A','CO2_OCN:A','SFCO2:A','SFCO2_LND:A',
24 |   'SFCO2_OCN:A','TMCO2:A','TMCO2_LND:A','TMCO2_OCN:A''CO2_FFF:A', 'SFCO2_FFF:A', 'TMCO2_FFF:A'
25 | 


--------------------------------------------------------------------------------
/.github/workflows/build.yaml:
--------------------------------------------------------------------------------
 1 | on:
 2 |   workflow_call:
 3 |     inputs:
 4 |       compiler:
 5 |         type: string
 6 |         required: true
 7 | 
 8 | jobs:
 9 |   test:
10 |     # This statement avoids failures in another instance (another compiler) to cancel this run
11 |     # it also prevents canceling the run using the cancel button on the github website
12 |     # another form (if: success() || failure()) should allow the cancel button to work but tests with that form didnt act as expected    
13 |     if: ${{ ! cancelled() }}
14 |     env:
15 |       CC: mpicc
16 |       FC: mpifort
17 |       CXX: mpicxx
18 |       CIME_MODEL: cesm
19 |       CIME_DRIVER: nuopc
20 |     runs-on: hpc-runner
21 |     defaults:
22 |       run:
23 |         shell: bash {0}
24 |     steps:
25 |       - name: Run ${{ inputs.compiler }} tests
26 |         # Allows the check status step to run even if this one fails 
27 |         continue-on-error: true
28 |         run: |
29 |           pwd
30 |           cd cime/scripts
31 |           module load cmake
32 |           # Need to use main queue to avoid memory overflow errors 
33 |           qcmd -q main -v PROJECT=P93300606 -A P93300606 -l walltime=02:00:00 -- ./create_test --xml-machine derecho\
34 |               --xml-category github --no-run --compiler ${{ inputs.compiler }} --test-id ${GITHUB_RUN_ID}${{ inputs.compiler }}
35 |       - name: check status
36 |         run: |
37 |           cd $SCRATCH
38 |           # if no fails and no passes in expected fails then cleanup and exit 0 otherwise exit 1
39 |           ./cs.status.${GITHUB_RUN_ID}${{ inputs.compiler }} --fails-only --expected-fails-file $GITHUB_WORKSPACE/cime_config/testfiles/ExpectedTestFails.xml | grep FAIL | grep -v FAILURE
40 |           retcode=$?
41 |           if [ $retcode -eq 0 ]; then
42 |             exit 1
43 |           fi
44 |           # tests pass, clean up
45 |           rm -fr *.${GITHUB_RUN_ID}${{ inputs.compiler }}
46 |           exit 0
47 | 


--------------------------------------------------------------------------------
/doc/source/_static/pop_ver.js:
--------------------------------------------------------------------------------
 1 | $(document).ready(function() {
 2 |     /* For a URL that looks like
 3 |     https://blah.github.io/versions/VERSIONFOO/html/bar/index.html, set cur_version_dir to
 4 |     'VERSIONFOO' (i.e., the portion of the path following 'versions').
 5 |     */
 6 |     var proj_end = document.baseURI.indexOf("versions") + 9;
 7 |     var end = document.baseURI.indexOf("/", proj_end);
 8 |     var cur_version_dir = document.baseURI.substring(proj_end, end);
 9 |     var mylist = $("#version-list");
10 |     mylist.empty();
11 |     $.getJSON(version_json_loc, function(data) {
12 |         if (data.hasOwnProperty(cur_version_dir)) {
13 |             /* First add the current version so that it appears first in the drop-down
14 |                menu and starts as the selected element of the menu. If you click on the
15 |                current version, you should stay at the current page.
16 | 
17 |                The conditional around this block should generally be true, but we check it
18 |                just in case the current version is missing from the versions.json file for
19 |                some reason.
20 |             */
21 |             cur_version_name = data[cur_version_dir];
22 |             mylist.append($("<option>", {value: document.baseURI, text: cur_version_name}));
23 |         }
24 |         // Now add the other versions
25 |         $.each(data, function(version_dir, version_name) {
26 |             if (version_dir != cur_version_dir) {
27 |                 /* If you click on a different version, you should go to the root of the
28 |                    documentation for that version. This assumes paths like
29 |                    https://blah.github.io/versions/VERSIONFOO/html/bar/index.html. So we
30 |                    need to go up two levels from the URL_ROOT (html) to then go back down
31 |                    into the appropriate version's html directory.
32 |                 */
33 |                 mylist.append($("<option>", {value: DOCUMENTATION_OPTIONS.URL_ROOT + '../../' + version_dir + '/html', text: version_name}));
34 |             }
35 |         });
36 |     });
37 | });
38 | 


--------------------------------------------------------------------------------
/tools/statistical_ensemble_test/addmetadata.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | #
 3 | # Adds metadata to netcdf statistical ensemble test files.
 4 | #
 5 | Args=("$@")
 6 | i=0
 7 | while [ $i -le ${#Args[@]} ]; do
 8 |   case ${Args[$i]} in
 9 |     --caseroot )
10 |       i=$((i+1))
11 |       caseroot=${Args[$i]}
12 |       if [ ! -d ${caseroot} ]; then
13 |         echo "ERROR: caseroot not found: $caseroot"
14 |         exit 2
15 |       fi
16 |       if [ ! -f ${caseroot}/xmlquery ]; then
17 | 	echo "ERROR: Directory $caseroot does not appear to be a cesm case directory"
18 | 	exit 3
19 |       fi
20 |     ;;
21 |     --histfile )
22 |       i=$((i+1))
23 |       histfile=${Args[$i]}
24 |       if [ ! -f ${histfile} ]; then
25 | 	echo "ERROR: file not found $histfile"
26 |         exit 4
27 |       fi
28 |     ;;
29 |     --help )
30 |       echo "usage: addmetadata --histroot CASEROOT --histfile HISTFILE [--help]
31 |       "
32 |       echo "Script to add metadata to validation files.
33 |       "
34 |        echo "Optional arguments:
35 |      --help           show this help message and exit
36 |      --caseroot       Full pathname to the CASE directory.
37 |      --histfile       Full filename of the history file to add the metadata."
38 |       exit
39 |     ;;
40 |   esac
41 |   i=$((i+1))
42 | done
43 | 
44 | if [ "$caseroot" = "" ] || [ "$histfile" = "" ]; then
45 |   echo "Please run ./addmetadata.sh --help for correct usage."
46 |   exit
47 | fi
48 | 
49 | cd $caseroot
50 | 
51 | stop_option=`./xmlquery --value STOP_OPTION`
52 | test_type="UF-ECT"
53 | if [ "$stop_option" = "nmonths" ]; then
54 |     test_type="ECT"
55 | elif [ "$stop_option" = "nyears" ]; then
56 |     test_type="POP-ECT"
57 | fi
58 | 
59 | if hash ncks 2>/dev/null; then
60 |  ncks --glb compset=`./xmlquery --value COMPSET` --glb grid=`./xmlquery --value GRID` --glb testtype="$test_type" --glb compiler=`./xmlquery --value COMPILER` --glb machineid=`./xmlquery --value MACH`  --glb model_version=`./xmlquery --value MODEL_VERSION`   $histfile $histfile.tmp
61 |   mv $histfile.tmp $histfile
62 | else
63 |   echo "This script requires the ncks tool"
64 | fi
65 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/tests/test_c_required.py:
--------------------------------------------------------------------------------
 1 | # tests/test_c_required.py
 2 | import re
 3 | 
 4 | from tests.utils_for_tests import normalize_whitespace
 5 | 
 6 | 
 7 | def test_required(git_fleximod, test_repo, shared_repos):
 8 |     file_path = test_repo / ".gitmodules"
 9 |     gm = shared_repos["gitmodules_content"]
10 |     repo_name = shared_repos["submodule_name"]
11 |     if file_path.exists():
12 |         with file_path.open("r") as f:
13 |             gitmodules_content = f.read()
14 |             # add the entry if it does not exist
15 |             if repo_name not in gitmodules_content:
16 |                 file_path.write_text(gitmodules_content + gm)
17 |             # or if it is incomplete
18 |             elif gm not in gitmodules_content:
19 |                 file_path.write_text(gm)
20 |     else:
21 |         file_path.write_text(gm)
22 |     result = git_fleximod(test_repo, "update")
23 |     assert result.returncode == 0
24 |     status = git_fleximod(test_repo, f"status {repo_name}")
25 |     assert shared_repos["status3"] in normalize_whitespace(status.stdout)
26 |     status = git_fleximod(test_repo, "update --optional")
27 |     assert result.returncode == 0
28 |     status = git_fleximod(test_repo, f"status {repo_name}")
29 |     assert shared_repos["status4"] in normalize_whitespace(status.stdout)
30 |     status = git_fleximod(test_repo, f"update {repo_name}")
31 |     assert result.returncode == 0
32 |     status = git_fleximod(test_repo, f"status {repo_name}")
33 |     assert shared_repos["status4"] in normalize_whitespace(status.stdout)
34 | 
35 |     text = file_path.read_text()
36 |     new_value = "somethingelse"
37 |     pattern = r"(^\s*fxtag\s*=\s*).*$"
38 |     replacement = r"\1" + new_value
39 |     new_text = re.sub(pattern, replacement, text, flags=re.MULTILINE)
40 | 
41 |     # Write updated content back to file
42 |     file_path.write_text(new_text)
43 | 
44 |     result = git_fleximod(test_repo, f"update {repo_name}")
45 |     assert (
46 |         "fatal: couldn't find remote ref" in result.stderr
47 |         or "error: pathspec 'somethingelse' did not match any file(s) known to git"
48 |         in result.stderr
49 |     )
50 | 


--------------------------------------------------------------------------------
/doc/README.md:
--------------------------------------------------------------------------------
 1 | The documentation source is stored with the CESM master code base. However, 
 2 | the built html files are stored separately in the orphan gh-pages branch
 3 | and can be viewed from a browser at URL:
 4 | 
 5 | http://escomp.github.io/cesm
 6 | 
 7 | The easiest way to build the documentation is to use a Docker image that
 8 | contains the necessary dependencies together with the doc-builder helper
 9 | tool.
10 | 
11 | To set this up initially on a given machine:
12 | 1. Install Docker
13 | 2. Launch Docker's desktop application and run: `docker pull escomp/base` (you should also redo this step if you haven't updated this docker image in a while)
14 | 3. Obtain the doc-builder helper tool: `git clone git@github.com:ESMCI/doc-builder.git`
15 | 4. Ensure that the `build_docs` command from the `doc-builder` repository is in your path (by adding the directory to your path or creating a symbolic link to it somewhere in your path
16 | 
17 | Then, to build the documentation:
18 | 1. Clone the CESM repository and checkout the gh-pages branch: `git clone -b gh-pages git@github.com:ESCOMP/CESM.git /PATH/TO/cesm-gh-pages` (replacing `/PATH/TO` with the desired path). **Note that both your main CESM clone and this cesm-gh-pages clone must reside somewhere within your home directory for the Docker-based workflow to work properly.**
19 | 2. From this `doc` directory, run a command like the following: `build_docs -d -c -r /PATH/TO/cesm-gh-pages -v cesm2.1`.
20 |    - The `-d` argument tells `build_docs` to use Docker
21 |    - The `-c` argument first runs `make clean` (which is often unnecessary, but doesn't hurt for this quick documentation build)
22 |    - The `-r` argument gives the path to the root of the documentation repository; replace this with the path to your `cesm-gh-pages` clone
23 |    - The `-v` argument gives the version you are building; replace this with one of the versions in the `versions` directory of the `cesm-gh-pages` clone
24 | 3. From the `cesm-gh-pages` clone, run `git add .`, `git commit` and `git push` to publish the new documentation. This will automatically update the documentation at http://escomp.github.io/cesm (but it may take a few minutes for the new version to appear).
25 | 
26 | For more details on working with sphinx-based documentation, including how to add new versions, see:
27 | - https://github.com/ESCOMP/CTSM/wiki/Directions-for-editing-CLM-documentation-on-github-and-sphinx
28 | - https://github.com/ESMCI/cime/wiki/Working-with-Sphinx-and-reStructuredText
29 | 


--------------------------------------------------------------------------------
/describe_version:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | """Describes the CESM version and any local modifications"""
 3 | 
 4 | import os
 5 | import argparse
 6 | import subprocess
 7 | 
 8 | def commandline_args():
 9 |     """Parse and return command-line arguments
10 |     """
11 | 
12 |     # We don't really need an argument parser, since there currently aren't any
13 |     # arguments. But providing this allows supporting a '--help' option with a
14 |     # description.
15 | 
16 |     description = """
17 | Script for describing the CESM version and any local modifications
18 | 
19 | Simply run:
20 | 
21 |     ./describe_version
22 | 
23 | without any arguments.
24 | 
25 | You may want to redirect the output to a file, such as:
26 | 
27 |     ./describe_version > current_version.txt
28 | """
29 | 
30 |     parser = argparse.ArgumentParser(
31 |         description=description,
32 |         formatter_class=argparse.RawTextHelpFormatter)
33 | 
34 |     args = parser.parse_args()
35 |     return args
36 | 
37 | def main():
38 |     """Main function for describe_version"""
39 |     # We currently don't actually need any arguments, but call this to allow '--help' usage
40 |     _ = commandline_args()
41 | 
42 |     # Allow this script to run correctly even if invoked from some other directory; note that
43 |     # we assume that the script resides in the top level of the CESM checkout.
44 |     cesmroot = os.path.dirname(os.path.realpath(__file__))
45 | 
46 |     separator = 72*'-' + '\n'
47 | 
48 |     # The '--long' option to git describe forces it to always show the hash, even if we're
49 |     # on a tag.
50 |     git_describe = subprocess.check_output(['git', 'describe', '--long'],
51 |                                            cwd=cesmroot,
52 |                                            universal_newlines=True)
53 |     print(separator + 'git describe:\n' + git_describe + separator)
54 | 
55 |     git_status = subprocess.check_output(['git', 'status'],
56 |                                          cwd=cesmroot,
57 |                                          universal_newlines=True)
58 |     print(separator + 'git status:\n' + git_status + separator)
59 | 
60 |     fleximod = os.path.join('bin', 'git-fleximod')
61 |     fleximod_status = subprocess.check_output([fleximod, 'status'],
62 |                                               cwd=cesmroot,
63 |                                               universal_newlines=True)
64 |     print(separator + 'git-fleximod status:\n' + fleximod_status + separator)
65 | 
66 | if __name__ == "__main__":
67 |     main()
68 | 


--------------------------------------------------------------------------------
/cime_config/testmods_dirs/allactive/defaultio/user_nl_cice:
--------------------------------------------------------------------------------
 1 | !----------------------------------------------------------------------------------
 2 | ! Have commented out the following due to known problems on restart
 3 | ! f_albice	= 'dxxxx'
 4 | ! f_albsno	= 'dxxxx'
 5 | ! f_albpnd	= 'dxxxx'
 6 | !----------------------------------------------------------------------------------
 7 | histfreq	= 'd','x','x','x','x'
 8 | histfreq_n	=  1,1,1,1,1
 9 | f_hi		= 'dxxxx'
10 | f_hs		= 'dxxxx'
11 | f_Tsfc		= 'dxxxx'
12 | f_aice		= 'dxxxx'
13 | f_uvel		= 'dxxxx'
14 | f_vvel		= 'dxxxx'
15 | f_fswdn		= 'dxxxx'
16 | f_flwdn		= 'dxxxx'
17 | f_snow		= 'dxxxx'
18 | f_snow_ai	= 'dxxxx'
19 | f_rain		= 'dxxxx'
20 | f_rain_ai	= 'dxxxx'
21 | f_faero_atm	= 'dxxxx'
22 | f_faero_ocn	= 'dxxxx'
23 | f_sst		= 'dxxxx'
24 | f_sss		= 'dxxxx'
25 | f_uocn		= 'dxxxx'
26 | f_vocn		= 'dxxxx'
27 | f_frzmlt	= 'dxxxx'
28 | f_fswfac	= 'dxxxx'
29 | f_fswabs	= 'dxxxx'
30 | f_fswabs_ai	= 'dxxxx'
31 | f_alidr		= 'dxxxx'
32 | f_alidf		= 'dxxxx'
33 | f_alvdr		= 'dxxxx'
34 | f_alvdf		= 'dxxxx'
35 | f_coszen	= 'dxxxx'
36 | f_flat		= 'dxxxx'
37 | f_flat_ai	= 'dxxxx'
38 | f_fsens		= 'dxxxx'
39 | f_fsens_ai	= 'dxxxx'
40 | f_flwup		= 'dxxxx'
41 | f_flwup_ai	= 'dxxxx'
42 | f_evap		= 'dxxxx'
43 | f_evap_ai	= 'dxxxx'
44 | f_Tair		= 'dxxxx'
45 | f_Tref		= 'dxxxx'
46 | f_Qref		= 'dxxxx'
47 | f_congel	= 'dxxxx'
48 | f_frazil	= 'dxxxx'
49 | f_snoice	= 'dxxxx'
50 | f_meltt		= 'dxxxx'
51 | f_melts		= 'dxxxx'
52 | f_meltb		= 'dxxxx'
53 | f_meltl		= 'dxxxx'
54 | f_fresh		= 'dxxxx'
55 | f_fresh_ai	= 'dxxxx'
56 | f_fsalt		= 'dxxxx'
57 | f_fsalt_ai	= 'dxxxx'
58 | f_fhocn		= 'dxxxx'
59 | f_fhocn_ai	= 'dxxxx'
60 | f_fswthru	= 'dxxxx'
61 | f_fswthru_ai	= 'dxxxx'
62 | f_strairx	= 'dxxxx'
63 | f_strairy	= 'dxxxx'
64 | f_strtltx	= 'dxxxx'
65 | f_strtlty	= 'dxxxx'
66 | f_strcorx	= 'dxxxx'
67 | f_strcory	= 'dxxxx'
68 | f_strocnx	= 'dxxxx'
69 | f_strocny	= 'dxxxx'
70 | f_strintx	= 'dxxxx'
71 | f_strinty	= 'dxxxx'
72 | f_strength	= 'dxxxx'
73 | f_opening	= 'dxxxx'
74 | f_divu		= 'dxxxx'
75 | f_shear		= 'dxxxx'
76 | f_sig1		= 'dxxxx'
77 | f_sig2		= 'dxxxx'
78 | f_dvidtt	= 'dxxxx'
79 | f_dvidtd	= 'dxxxx'
80 | f_daidtt	= 'dxxxx'
81 | f_daidtd	= 'dxxxx'
82 | f_dardg1dt	= 'dxxxx'
83 | f_dardg2dt	= 'dxxxx'
84 | f_dvirdgdt	= 'dxxxx'
85 | f_iage		= 'dxxxx'
86 | f_ardg		= 'dxxxx'
87 | f_vrdg		= 'dxxxx'
88 | f_alvl		= 'dxxxx'
89 | f_vlvl		= 'dxxxx'
90 | f_FY		= 'dxxxx'
91 | f_aisnap	= 'dxxxx'
92 | f_hisnap	= 'dxxxx'
93 | f_aicen		= 'dxxxx'
94 | f_vicen		= 'dxxxx'
95 | f_trsig		= 'dxxxx'
96 | f_icepresent	= 'dxxxx'
97 | f_fsurf_ai	= 'dxxxx'
98 | f_fcondtop_ai	= 'dxxxx'
99 | 


--------------------------------------------------------------------------------
/.github/workflows/preview_namelist.yaml:
--------------------------------------------------------------------------------
 1 | on:
 2 |   push:
 3 |     branches: [ master, cesm3.0-alphabranch ]
 4 |   pull_request:
 5 |     branches: [ master, cesm3.0-alphabranch ]
 6 | 
 7 | jobs:
 8 |   preview_namelists:
 9 |     runs-on: ubuntu-latest
10 |     strategy:
11 |       fail-fast: false
12 |       matrix:
13 |         # Lots of python versions, pare down if we don't support a version
14 |         python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
15 |         config:
16 |           - {"compset": "B1850CM", "res": "ne30pg3_t232"} # fully coupled
17 |           - {"compset": "C_JRA", "res": "TL319_t232"}     # ocean only
18 |           - {"compset": "C1850MARBL_JRA", "res": "TL319_t232"}     # ocean only, with BGC
19 |           - {"compset": "DTEST", "res": "TL319_t232"}     # ice only
20 |           - {"compset": "I1850Clm60SpCru", "res": "f10_f10_mg37"} # land only
21 |           - {"compset": "QPC6HIST", "res": "f09_f09_mg17"} # atm only
22 |           - {"compset": "T1850Gg", "res": "f09_g17_gris4"} # glc only
23 |     steps:
24 |       - uses: actions/checkout@v4
25 | 
26 |       - name: Setup python ${{ matrix.python-version }}
27 |         uses: actions/setup-python@v5
28 |         with:
29 |           python-version: ${{ matrix.python-version }}
30 | 
31 |       - name: Set up non-python environment (need xmllint and a fake ESMF make file)
32 |         run: |
33 |           git config --global user.email "testing@github.actions"
34 |           git config --global user.name "Github Actions Testing"
35 |           echo "ESMFMKFILE=$HOME/esmf.mk" >> ${GITHUB_ENV}
36 |           echo "ESMF_VERSION_MAJOR=8" > ${HOME}/esmf.mk
37 |           echo "ESMF_VERSION_MINOR=8" >> ${HOME}/esmf.mk
38 |           sudo apt-get update
39 |           sudo apt-get install libxml2-utils
40 |           sudo apt-get install subversion
41 | 
42 |       - name: Checkout CESM
43 |         run: |
44 |           $GITHUB_WORKSPACE/bin/git-fleximod update
45 | 
46 |       - name: Get WW3 grid data
47 |         # WW3 data needs to be prestaged for buildnml to work correctly
48 |         run: |
49 |           svn co https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata/wav/ww3 /home/runner/cesm/inputdata/wav/ww3          
50 |           
51 |       - name: Create new cases, run case.setup, and then preview namelists
52 |         run: |
53 |           cd $GITHUB_WORKSPACE/cime/scripts
54 |           ./create_newcase --run-unsupported --mach ubuntu-latest --compset ${{ matrix.config.compset }} --res ${{ matrix.config.res }} --case $GITHUB_WORKSPACE/cases/${{ matrix.config.compset }}_${{ matrix.python-version }}
55 |           cd $GITHUB_WORKSPACE/cases/${{ matrix.config.compset }}_${{ matrix.python-version }}
56 |           ./case.setup
57 |           ./preview_namelists
58 | 


--------------------------------------------------------------------------------
/ChangeLog_template:
--------------------------------------------------------------------------------
  1 | March 2016
  2 | 
  3 | The CESM testing and porting database records all changes to CESM tags in 
  4 | development and their associated regression tests. The ChangeLogs generated
  5 | by the testdb can be used as the SVN ChangeLog and replace this template.
  6 | 
  7 | The testdb can be viewed by CESM SVN developers at:
  8 | 
  9 | https://csegweb.cgd.ucar.edu/testdb
 10 | 
 11 | Old template format for reference:
 12 | ===============================================================
 13 |  
 14 | Tag name: 
 15 | Originator(s): 
 16 | Date: 
 17 | One-line Summary:
 18 | 
 19 | List all component tags that were used to create cesm tag:
 20 | 
 21 |   scripts  
 22 |   drv
 23 |   share     
 24 |   cam
 25 |   clm     
 26 |   cice     
 27 |   cism        
 28 |   pop     
 29 |   datm       
 30 |   dlnd    
 31 |   dice    
 32 |   docn
 33 |   dead  
 34 |   stubs
 35 |   timing  
 36 |   esmf_wrf_timemgr
 37 |   mct     
 38 |   pio     
 39 | 
 40 | 
 41 | Describe the changes for each component above that was changed relative 
 42 | to the previous cesm tag in this sequence (this includes description
 43 | of new functionality and how user can use it):
 44 | 
 45 | Bugs fixed: 
 46 | 
 47 | Describe any changes made to scripts/build system:
 48 | 
 49 | Describe any substantial timing or memory changes:
 50 | 
 51 | Code reviewed by:
 52 | 
 53 | Summary of pre-tag testing:
 54 | 
 55 |   Regression tests:
 56 |   regression test suite only tests original capability 
 57 |   regression test status of FAIL1 implies non-bfb
 58 |   regression test status of FAIL2 implies restart failure
 59 |   DO NOT create tag if FAIL2 status is obtained in any of the
 60 |   following regression tests
 61 | 
 62 |   Restart tests:
 63 |   Restart test suite tests both new and old capability 
 64 |   DO NOT create tag if FAIL status is obtained in any of the
 65 |   following restart tests
 66 | 
 67 |   Machine (bluefire):
 68 |     Restart tests:
 69 |     Regression tests:
 70 | 
 71 |   Machine (lynx_pgi):
 72 |     Restart tests:
 73 |     Regression tests:
 74 | 
 75 |   Machine (edinburgh_lahey):
 76 |     Restart tests:
 77 |     Regression tests:
 78 | 
 79 |   Machine (lynx_gnu):
 80 |     Restart tests:
 81 |     Regression tests:
 82 | 
 83 | 
 84 | Summarize any change to answers (regressions tests status is FAIL1)
 85 | 
 86 |   - what code configurations:
 87 |   - what platforms/compilers:
 88 |   - nature of change (roundoff; larger than roundoff but same climate; new
 89 |     climate): 
 90 | 
 91 | If bitwise differences were observed, how did you show they were no worse
 92 | than roundoff?
 93 | 
 94 | If this tag changes climate describe the run(s) done to evaluate the new
 95 | climate in enough detail that it(they) could be reproduced, i.e.,
 96 | 
 97 |   - platform/compilers:
 98 |   - MSS location of output:
 99 | 
100 | MSS location of control simulations used to validate new climate:
101 | 
102 | ===============================================================
103 | 


--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/config.yml:
--------------------------------------------------------------------------------
 1 | blank_issues_enabled: true
 2 | contact_links:
 3 |   - name: CESM forums
 4 |     url: https://bb.cgd.ucar.edu/cesm/
 5 |     about: For support with model use, troubleshooting, etc., please use the CESM forums
 6 |   - name: CIME issue
 7 |     url: https://github.com/ESMCI/cime/issues/new/choose
 8 |     about: Issue with CIME (scripting infrastructure, share code, etc.)
 9 |   - name: CAM issue
10 |     url: https://bb.cgd.ucar.edu/cesm/forums/cam.133/
11 |     about: Issue with CAM (Community Atmosphere Model) - please report via forums
12 |   - name: CTSM / CLM issue
13 |     url: https://github.com/ESCOMP/CTSM/issues/new/choose
14 |     about: Issue with CTSM / CLM (Community Terrestrial Systems Model / Community Land Model)
15 |   - name: POP issue
16 |     url: https://github.com/ESCOMP/POP2-CESM/issues/new/choose
17 |     about: Issue with POP in CESM (Parallel Ocean Program)
18 |   - name: MOM issue
19 |     url: https://github.com/ESCOMP/MOM_interface/issues/new/choose
20 |     about: Issue with MOM in CESM (Modular Ocean Model)
21 |   - name: CICE issue
22 |     url: https://github.com/ESCOMP/CESM_CICE/issues/new/choose
23 |     about: Issue with CICE in CESM
24 |   - name: CICE5 issue
25 |     url: https://github.com/ESCOMP/CESM_CICE5/issues/new/choose
26 |     about: Issue with (old) CICE5
27 |   - name: CISM issue
28 |     url: https://github.com/ESCOMP/cism-wrapper/issues/new/choose
29 |     about: Issue with CISM (Community Ice Sheet Model)
30 |   - name: MOSART issue
31 |     url: https://github.com/ESCOMP/MOSART/issues/new/choose
32 |     about: Issue with MOSART in CESM (Model for Scale Adaptive River Transport)
33 |   - name: RTM issue
34 |     url: https://github.com/ESCOMP/RTM/issues/new/choose
35 |     about: Issue with RTM (River Transport Model)
36 |   - name: WW3 issue
37 |     url: https://github.com/ESCOMP/WW3-CESM/issues/new/choose
38 |     about: Issue with WW3 in CESM (WaveWatch III)
39 |   - name: CMEPS issue
40 |     url: https://github.com/ESCOMP/CMEPS/issues/new/choose
41 |     about: Issue with CMEPS (NUOPC Community Mediator for Earth Prediction Systems)
42 |   - name: CDEPS issue
43 |     url: https://github.com/ESCOMP/CDEPS/issues/new/choose
44 |     about: Issue with CDEPS (Community Data Models for Earth Prediction Systems)
45 |   - name: CESM_share issue
46 |     url: https://github.com/ESCOMP/CESM_share/issues/new/choose
47 |     about: Issue with CESM share code
48 |   - name: PIO issue
49 |     url: https://github.com/NCAR/ParallelIO/issues/new/choose
50 |     about: Issue with the ParallelIO (PIO) library
51 |   - name: CCS_config issue
52 |     url: https://github.com/ESMCI/ccs_config_cesm/issues/new/choose
53 |     about: Issue with CESM configuration data (grids, machines, etc.)
54 |   - name: CPL7 issue
55 |     url: https://github.com/ESCOMP/CESM_CPL7andDataComps/issues/new/choose
56 |     about: Issue with (old) MCT-based driver/coupler (CPL7)
57 |   - name: FMS issue
58 |     url: https://github.com/ESCOMP/FMS_interface/issues/new/choose
59 |     about: Issue with FMS in CESM
60 |   - name: MCT issue
61 |     url: https://github.com/MCSclimate/MCT/issues/new/choose
62 |     about: Issue with the Model Coupling Toolkit (MCT) library
63 |   - name: CESM Experiments Database issue
64 |     url: https://github.com/NCAR/CESM_expdb2/issues/new/choose
65 |     about: Issue with the CESM Experiments Database
66 | 


--------------------------------------------------------------------------------
/cime_config/testfiles/ExpectedTestFails.xml:
--------------------------------------------------------------------------------
  1 | <?xml version= "1.0"?>
  2 | 
  3 | <expectedFails version="1.1">
  4 | 
  5 |   <!-- Notes about the format of this file:
  6 | 
  7 |        The required elements for a given failure are just:
  8 | 
  9 |          <test name="...">
 10 |            <phase name="...">
 11 |              <status>...</status>
 12 |            </phase>
 13 |          </test>
 14 | 
 15 |        There can be multiple phase blocks in a given test block.
 16 | 
 17 |        In addition, a number of optional elements are allowed, which
 18 |        currently are just for human consumption (not parsed by any
 19 |        scripts):
 20 | 
 21 |        - A phase block can contain an "issue" element, which gives the
 22 |        issue number associated with this failure. (#123 refers to issue
 23 |        #123 in the ESCOMP/ctsm repository. Issues in other repositories
 24 |        should be specified as ORG/repo#123 - e.g., ESMCI/cime#123.)
 25 | 
 26 |        - A phase block can contain a "comment" element, which gives any
 27 |        sort of comment you desire.
 28 |   -->
 29 | 
 30 | <!-- this file requires at least one entree -->
 31 | 
 32 |   <test name="SMS.res.compset.machine_compiler.clm-nofireemis">
 33 |     <phase name="RUN">
 34 |       <status>FAIL</status>
 35 |       <issue>CDEPS/#243</issue>
 36 |     </phase>
 37 |   </test>
 38 | <!-- These tests were fixed with updates to ccs_config and to the mom model.
 39 |   <test name="SMS_Ld2.ne30pg3_t232.BMT1850.derecho_nvhpc.allactive-defaultio">
 40 |     <phase name="MODEL_BUILD">
 41 |       <status>FAIL</status>
 42 |       <issue>https://github.com/NCAR/MOM6/issues/324</issue>
 43 |       <comment>atan function</comment>
 44 |     </phase>
 45 |   </test>
 46 |   <test name="SMS_Ld2.ne30pg3_t232.BLT1850.derecho_nvhpc.allactive-defaultio">
 47 |     <phase name="MODEL_BUILD">
 48 |       <status>FAIL</status>
 49 |       <issue>https://github.com/NCAR/MOM6/issues/324</issue>
 50 |       <comment></comment>
 51 |     </phase>
 52 |   </test>
 53 |   <test name="SMS_Ld2.ne30pg3_t232.BHISTC_MTso.derecho_nvhpc.allactive-defaultio">
 54 |     <phase name="MODEL_BUILD">
 55 |       <status>FAIL</status>
 56 |       <issue>https://github.com/NCAR/MOM6/issues/324</issue>
 57 |       <comment></comment>
 58 |     </phase>
 59 |   </test>
 60 |   <test name="SMS_Ld2.ne30pg3_t232.BHISTC_LTso.derecho_nvhpc.allactive-defaultio">
 61 |     <phase name="MODEL_BUILD">
 62 |       <status>FAIL</status>
 63 |       <issue>https://github.com/NCAR/MOM6/issues/324</issue>
 64 |       <comment></comment>
 65 |     </phase>
 66 |   </test>
 67 | 
 68 |   <test name="SMS_Ld2_D.ne30pg3_t232.B1850C_MTt4s.derecho_nvhpc.allactive-defaultio">
 69 |     <phase name="SHAREDLIB_BUILD">
 70 |       <status>FAIL</status>
 71 |       <issue></issue>
 72 |       <comment>FMS file mpp.F90 fails to compile in debug mode</comment>
 73 |     </phase>
 74 |   </test>
 75 | 
 76 |   <test name="SMS_Ld2_D.ne30pg3_t232.BHISTC_MTt4s.derecho_nvhpc.allactive-defaultio">
 77 |     <phase name="SHAREDLIB_BUILD">
 78 |       <status>FAIL</status>
 79 |       <issue></issue>
 80 |       <comment></comment>
 81 |     </phase>
 82 |   </test>
 83 |   <test name="SMS_Ld2.ne30pg3_t232.B1850C_LTso.derecho_nvhpc.allactive-defaultio">
 84 |     <phase name="MODEL_BUILD">
 85 |       <status>FAIL</status>
 86 |       <issue></issue>
 87 |       <comment></comment>
 88 |     </phase>
 89 |   </test>
 90 |   <test name="SMS_Ld2.ne30pg3_t232.B1850C_MTso.derecho_nvhpc.allactive-defaultio">
 91 |     <phase name="MODEL_BUILD">
 92 |       <status>FAIL</status>
 93 |       <issue></issue>
 94 |       <comment></comment>
 95 |     </phase>
 96 |   </test>
 97 | 
 98 | -->
 99 | 
100 | </expectedFails>
101 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/.github/workflows/pytest.yaml:
--------------------------------------------------------------------------------
 1 | # Run this job on pushes to `main`, and for pull requests. If you don't specify
 2 | # `branches: [main], then this actions runs _twice_ on pull requests, which is
 3 | # annoying.
 4 | 
 5 | on:
 6 |   push:
 7 |     branches: [main]
 8 |   pull_request:
 9 |     branches: [main]
10 | 
11 | jobs:
12 |   test:
13 |     runs-on: ubuntu-latest
14 |     steps:
15 |       - uses: actions/checkout@v4
16 | 
17 |       # If you wanted to use multiple Python versions, you'd have specify a matrix in the job and
18 |       # reference the matrixe python version here.
19 |       - uses: actions/setup-python@v5
20 |         with:
21 |           python-version: '3.12'
22 | 
23 |       # Cache the installation of Poetry itself, e.g. the next step. This prevents the workflow
24 |       # from installing Poetry every time, which can be slow. Note the use of the Poetry version
25 |       # number in the cache key, and the "-0" suffix: this allows you to invalidate the cache
26 |       # manually if/when you want to upgrade Poetry, or if something goes wrong. This could be
27 |       # mildly cleaner by using an environment variable, but I don't really care.
28 |       - name: cache poetry install
29 |         uses: actions/cache@v4
30 |         with:
31 |           path: ~/.local
32 |           key: poetry-1.8.2
33 | 
34 |       # Install Poetry. You could do this manually, or there are several actions that do this.
35 |       # `snok/install-poetry` seems to be minimal yet complete, and really just calls out to
36 |       # Poetry's default install script, which feels correct. I pin the Poetry version here
37 |       # because Poetry does occasionally change APIs between versions and I don't want my
38 |       # actions to break if it does.
39 |       #
40 |       # The key configuration value here is `virtualenvs-in-project: true`: this creates the
41 |       # venv as a `.venv` in your testing directory, which allows the next step to easily
42 |       # cache it.
43 |       - uses: snok/install-poetry@v1
44 |         with:
45 |           version: 1.8.2
46 |           virtualenvs-create: true
47 |           virtualenvs-in-project: true
48 | 
49 |       # Cache your dependencies (i.e. all the stuff in your `pyproject.toml`). Note the cache
50 |       # key: if you're using multiple Python versions, or multiple OSes, you'd need to include
51 |       # them in the cache key. I'm not, so it can be simple and just depend on the poetry.lock.
52 |       - name: cache deps
53 |         id: cache-deps
54 |         uses: actions/cache@v4
55 |         with:
56 |           path: .venv
57 |           key: pydeps-${{ hashFiles('**/poetry.lock') }}
58 | 
59 |       # Install dependencies. `--no-root` means "install all dependencies but not the project
60 |       # itself", which is what you want to avoid caching _your_ code. The `if` statement
61 |       # ensures this only runs on a cache miss.
62 |       - run: poetry install --no-interaction --no-root
63 |         if: steps.cache-deps.outputs.cache-hit != 'true'
64 | 
65 |       # Now install _your_ project. This isn't necessary for many types of projects -- particularly
66 |       # things like Django apps don't need this. But it's a good idea since it fully-exercises the
67 |       # pyproject.toml and makes that if you add things like console-scripts at some point that
68 |       # they'll be installed and working.
69 |       - run: poetry install --no-interaction
70 | 
71 |       # And finally run tests. I'm using pytest and all my pytest config is in my `pyproject.toml`
72 |       # so this line is super-simple. But it could be as complex as you need.
73 |       - run: |
74 |           git config --global user.name "${GITHUB_ACTOR}"
75 |           git config --global user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com"
76 |           poetry run pytest --doctest-modules
77 | #      - name: Setup tmate session
78 | #        if: ${{ failure() }}
79 | #        uses: mxschmitt/action-tmate@v3
80 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/git_fleximod/gitmodules.py:
--------------------------------------------------------------------------------
 1 | # git_fleximod/gitmodules.py
 2 | from pathlib import Path
 3 | from configparser import RawConfigParser, ConfigParser
 4 | from .lstripreader import LstripReader
 5 | 
 6 | 
 7 | class GitModules(RawConfigParser):
 8 |     def __init__(
 9 |         self,
10 |         logger,
11 |         confpath=Path.cwd(),
12 |         conffile=".gitmodules",
13 |         includelist=None,
14 |         excludelist=None,
15 |     ):
16 |         """
17 |         confpath: Path to the directory containing the .gitmodules file (defaults to the current working directory).
18 |         conffile: Name of the configuration file (defaults to .gitmodules).
19 |         includelist: Optional list of submodules to include.
20 |         excludelist: Optional list of submodules to exclude.
21 |         """
22 |         self.logger = logger
23 |         self.logger.debug(
24 |             "Creating a GitModules object {} {} {} {}".format(
25 |                 confpath, conffile, includelist, excludelist
26 |             )
27 |         )
28 |         super().__init__()
29 |         self.conf_file = Path(confpath) / Path(conffile)
30 |         if self.conf_file.exists():
31 |             self.read_file(LstripReader(str(self.conf_file)), source=conffile)
32 |         self.includelist = includelist
33 |         self.excludelist = excludelist
34 |         self.isdirty = False
35 | 
36 |     def reload(self):
37 |         self.clear()
38 |         if self.conf_file.exists():
39 |             self.read_file(LstripReader(str(self.conf_file)), source=self.conf_file)
40 | 
41 |     def set(self, name, option, value):
42 |         """
43 |         Sets a configuration value for a specific submodule:
44 |         Ensures the appropriate section exists for the submodule.
45 |         Calls the parent class's set method to store the value.
46 |         """
47 |         self.isdirty = True
48 |         self.logger.debug("set called {} {} {}".format(name, option, value))
49 |         section = f'submodule "{name}"'
50 |         if not self.has_section(section):
51 |             self.add_section(section)
52 |         super().set(section, option, str(value))
53 | 
54 |     # pylint: disable=redefined-builtin, arguments-differ
55 |     def get(self, name, option, raw=False, vars=None, fallback=None):
56 |         """
57 |         Retrieves a configuration value for a specific submodule:
58 |         Uses the parent class's get method to access the value.
59 |         Handles potential errors if the section or option doesn't exist.
60 |         """
61 |         self.logger.debug("git get called {} {}".format(name, option))
62 |         section = f'submodule "{name}"'
63 |         try:
64 |             return ConfigParser.get(
65 |                 self, section, option, raw=raw, vars=vars, fallback=fallback
66 |             )
67 |         except ConfigParser.NoOptionError:
68 |             return None
69 | 
70 |     def save(self):
71 |         if self.isdirty:
72 |             self.logger.info("Writing {}".format(self.conf_file))
73 |             with open(self.conf_file, "w") as fd:
74 |                 self.write(fd)
75 |         self.isdirty = False
76 | 
77 |     def __del__(self):
78 |         self.save()
79 | 
80 |     def sections(self):
81 |         """Strip the submodule part out of section and just use the name"""
82 |         self.logger.debug("calling GitModules sections iterator")
83 |         names = []
84 |         for section in ConfigParser.sections(self):
85 |             name = section[11:-1]
86 |             if self.includelist and name not in self.includelist:
87 |                 continue
88 |             if self.excludelist and name in self.excludelist:
89 |                 continue
90 |             names.append(name)
91 |         return names
92 | 
93 |     def items(self, name, raw=False, vars=None):
94 |         self.logger.debug("calling GitModules items for {}".format(name))
95 |         section = f'submodule "{name}"'
96 |         return ConfigParser.items(section, raw=raw, vars=vars)
97 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/tests/test_d_complex.py:
--------------------------------------------------------------------------------
  1 | # tests/test_d_complex.py
  2 | 
  3 | from tests.utils_for_tests import normalize_whitespace
  4 | 
  5 | 
  6 | def test_complex_checkout(git_fleximod, complex_repo, logger):
  7 |     status = git_fleximod(complex_repo, "status")
  8 |     logger.debug("test_complex_checkout status:\n" + status.stdout)
  9 |     assert (
 10 |         "ToplevelOptional not checked out, aligned at tag v5.3.2"
 11 |         in normalize_whitespace(status.stdout)
 12 |     )
 13 |     assert (
 14 |         "ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0"
 15 |         in normalize_whitespace(status.stdout)
 16 |     )
 17 |     assert (
 18 |         "AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0"
 19 |         in normalize_whitespace(status.stdout)
 20 |     )
 21 |     assert "Complex not checked out, aligned at tag testtag02" in normalize_whitespace(
 22 |         status.stdout
 23 |     )
 24 |     assert (
 25 |         "AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0"
 26 |         in normalize_whitespace(status.stdout)
 27 |     )
 28 | 
 29 |     # This should checkout and update test_submodule and complex_sub
 30 |     result = git_fleximod(complex_repo, "update")
 31 |     assert result.returncode == 0
 32 | 
 33 |     status = git_fleximod(complex_repo, "status")
 34 |     assert (
 35 |         "ToplevelOptional not checked out, aligned at tag v5.3.2"
 36 |         in normalize_whitespace(status.stdout)
 37 |     )
 38 |     assert "ToplevelRequired at tag MPIserial_2.5.0" in normalize_whitespace(
 39 |         status.stdout
 40 |     )
 41 |     assert "AlwaysRequired at tag MPIserial_2.4.0" in normalize_whitespace(
 42 |         status.stdout
 43 |     )
 44 |     assert "Complex at tag testtag02" in normalize_whitespace(status.stdout)
 45 | 
 46 |     # now check the complex_sub
 47 |     root = complex_repo / "modules" / "complex"
 48 |     assert not (root / "libraries" / "gptl" / ".git").exists()
 49 |     assert not (root / "libraries" / "mpi-serial" / ".git").exists()
 50 |     assert (root / "modules" / "mpi-serial" / ".git").exists()
 51 |     assert not (root / "modules" / "mpi-serial2" / ".git").exists()
 52 |     assert (root / "modules" / "mpi-sparse" / ".git").exists()
 53 |     assert (root / "modules" / "mpi-sparse" / "m4").exists()
 54 |     assert not (root / "modules" / "mpi-sparse" / "README").exists()
 55 | 
 56 |     # update a single optional submodule
 57 | 
 58 |     result = git_fleximod(complex_repo, "update ToplevelOptional")
 59 |     assert result.returncode == 0
 60 | 
 61 |     status = git_fleximod(complex_repo, "status")
 62 |     assert "ToplevelOptional at tag v5.3.2" in normalize_whitespace(status.stdout)
 63 |     assert "ToplevelRequired at tag MPIserial_2.5.0" in normalize_whitespace(
 64 |         status.stdout
 65 |     )
 66 |     assert "AlwaysRequired at tag MPIserial_2.4.0" in normalize_whitespace(
 67 |         status.stdout
 68 |     )
 69 |     assert "Complex at tag testtag02" in normalize_whitespace(status.stdout)
 70 |     assert (
 71 |         "AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0"
 72 |         in normalize_whitespace(status.stdout)
 73 |     )
 74 | 
 75 |     # Finally update optional
 76 |     result = git_fleximod(complex_repo, "update --optional")
 77 |     assert result.returncode == 0
 78 | 
 79 |     status = git_fleximod(complex_repo, "status")
 80 |     assert "ToplevelOptional at tag v5.3.2" in normalize_whitespace(status.stdout)
 81 |     assert "ToplevelRequired at tag MPIserial_2.5.0" in normalize_whitespace(
 82 |         status.stdout
 83 |     )
 84 |     assert "AlwaysRequired at tag MPIserial_2.4.0" in normalize_whitespace(
 85 |         status.stdout
 86 |     )
 87 |     assert "Complex at tag testtag02" in normalize_whitespace(status.stdout)
 88 |     assert "AlwaysOptional at tag MPIserial_2.3.0" in normalize_whitespace(
 89 |         status.stdout
 90 |     )
 91 | 
 92 |     # now check the complex_sub
 93 |     root = complex_repo / "modules" / "complex"
 94 |     assert not (root / "libraries" / "gptl" / ".git").exists()
 95 |     assert not (root / "libraries" / "mpi-serial" / ".git").exists()
 96 |     assert (root / "modules" / "mpi-serial" / ".git").exists()
 97 |     assert (root / "modules" / "mpi-serial2" / ".git").exists()
 98 |     assert (root / "modules" / "mpi-sparse" / ".git").exists()
 99 |     assert (root / "modules" / "mpi-sparse" / "m4").exists()
100 |     assert not (root / "modules" / "mpi-sparse" / "README").exists()
101 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/README.md:
--------------------------------------------------------------------------------
  1 | # git-fleximod
  2 | 
  3 | Flexible, Enhanced Submodule Management for Git
  4 | 
  5 | ## Overview
  6 | 
  7 | Git-fleximod is a Python-based tool that extends Git's submodule and sparse checkout capabilities, offering additional features for managing submodules in a more flexible and efficient way.
  8 | 
  9 | ## Installation
 10 | 
 11 |   If you choose to locate git-fleximod in your path you can access it via command: git fleximod
 12 | 
 13 | ## Usage
 14 | 
 15 |     Basic Usage:
 16 |         git fleximod <command> [options]
 17 |     Available Commands:
 18 |         status: Display the status of submodules.
 19 |         update: Update submodules to the tag indicated in .gitmodules variable fxtag.
 20 |         test: Make sure that fxtags and submodule hashes are consistant,
 21 |               make sure that official urls (as defined by fxDONOTUSEurl) are set
 22 |               make sure that fxtags are defined for all submodules
 23 |     Additional Options:
 24 |         See git fleximod --help for more details.
 25 | 
 26 | ## Supported .gitmodules Variables
 27 | 
 28 |     fxtag: Specify a specific tag or hash to checkout for a submodule.  Branches are not acceptable.
 29 |     fxrequired: Mark a submodule's checkout behavior, with allowed values:
 30 |       - ToplevelRequired: Top-level and required (checked out only when this is the Toplevel module).
 31 |       - ToplevelOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module).
 32 |       - AlwaysRequired:       Always required (always checked out).
 33 |       - AlwaysOptional:       Always optional (checked out with --optional flag).
 34 |     fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths.
 35 |     fxDONOTUSEurl: This is the url used in the test subcommand to assure that protected branches do not point to forks
 36 |            **NOTE** the fxDONOTUSEurl variable is only used to identify the official project repository and should not be
 37 |            changed by users.  Use the url variable to change to a fork if desired.
 38 | 
 39 | ## Sparse Checkouts
 40 | 
 41 |     To enable sparse checkout for a submodule, set the fxsparse variable
 42 |     in the .gitmodules file to the path of a file containing the desired
 43 |     sparse checkout paths. Git-fleximod will automatically configure
 44 |     sparse checkout based on this file when applicable commands are run.
 45 |     See [git-sparse-checkout](https://git-scm.com/docs/git-sparse-checkout#_internalsfull_pattern_set)
 46 |     for details on the format of this file.
 47 | 
 48 | ## Tests
 49 | 
 50 |    The git fleximod test action is designed to be used by, for example, github workflows
 51 |    to assure that protected branches are consistant with respect to submodule hashes and fleximod fxtags
 52 | 
 53 | ## Examples
 54 | 
 55 | Here are some common usage examples:
 56 | 
 57 | Update all submodules, including optional ones:
 58 | ```bash
 59 |   git fleximod update --optional
 60 | ```
 61 | 
 62 | Updating a specific submodule to the fxtag indicated in .gitmodules:
 63 | 
 64 | ```bash
 65 |     git fleximod update submodule-name
 66 | ```
 67 | Example .gitmodules entry:
 68 | ```ini, toml
 69 |     [submodule "cosp2"]
 70 |         path = src/physics/cosp2/src
 71 |         url = https://github.com/CFMIP/COSPv2.0
 72 |         fxsparse = ../.cosp_sparse_checkout
 73 |         fxrequired = AlwaysRequired
 74 |         fxtag = v2.1.4cesm
 75 | ```
 76 | Explanation:
 77 | 
 78 | This entry indicates that the submodule named cosp2 at tag v2.1.4cesm
 79 | should be checked out into the directory src/physics/cosp2/src
 80 | relative to the .gitmodules directory.  It should be checked out from
 81 | the URL https://github.com/CFMIP/COSPv2.0 and use sparse checkout as
 82 | described in the file ../.cosp_sparse_checkout relative to the path
 83 | directory.  It should be checked out anytime this .gitmodules entry is
 84 | read.
 85 | 
 86 | Additional example:
 87 | ```ini, toml
 88 |     [submodule "cime"]
 89 |         path = cime
 90 |         url = https://github.com/jedwards4b/cime
 91 |         fxrequired = ToplevelRequired
 92 |         fxtag = cime6.0.198_rme01
 93 | ```
 94 | 
 95 | Explanation:
 96 | 
 97 | This entry indicates that the submodule cime should be checked out
 98 | into a directory named cime at tag cime6.0.198_rme01 from the URL
 99 | https://github.com/jedwards4b/cime.  This should only be done if
100 | the .gitmodules file is at the top level of the repository clone.
101 | 
102 | ## Contributing
103 | 
104 | We welcome contributions! Please see the CONTRIBUTING.md file for guidelines.
105 | 
106 | ## License
107 | 
108 | Git-fleximod is released under the MIT License.
109 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/tests/test_e_complex_update.py:
--------------------------------------------------------------------------------
  1 | # tests/test_e_complex_update.py
  2 | from tests.utils_for_tests import normalize_whitespace
  3 | 
  4 | 
  5 | def test_complex_update(git_fleximod, complex_update, logger):
  6 |     logger.debug("Starting test_complex_update")
  7 |     status = git_fleximod(complex_update, "status")
  8 |     assert (
  9 |         "ToplevelOptional not checked out, aligned at tag v5.3.2"
 10 |         in normalize_whitespace(status.stdout)
 11 |     )
 12 |     assert (
 13 |         "ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0"
 14 |         in normalize_whitespace(status.stdout)
 15 |     )
 16 |     assert (
 17 |         "AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0"
 18 |         in normalize_whitespace(status.stdout)
 19 |     )
 20 |     assert (
 21 |         "Complex not checked out, out of sync at tag testtag02, expected tag is testtag3"
 22 |         in normalize_whitespace(status.stdout)
 23 |     )
 24 |     assert (
 25 |         "AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0"
 26 |         in normalize_whitespace(status.stdout)
 27 |     )
 28 | 
 29 |     # This should checkout and update test_submodule and complex_sub
 30 |     result = git_fleximod(complex_update, "update")
 31 |     assert result.returncode == 0
 32 | 
 33 |     status = git_fleximod(complex_update, "status")
 34 |     assert (
 35 |         "ToplevelOptional not checked out, aligned at tag v5.3.2"
 36 |         in normalize_whitespace(status.stdout)
 37 |     )
 38 |     assert "ToplevelRequired at tag MPIserial_2.5.0" in normalize_whitespace(
 39 |         status.stdout
 40 |     )
 41 |     assert "AlwaysRequired at tag MPIserial_2.4.0" in normalize_whitespace(
 42 |         status.stdout
 43 |     )
 44 |     assert "Complex at tag testtag3" in normalize_whitespace(status.stdout)
 45 | 
 46 |     # now check the complex_sub
 47 |     root = complex_update / "modules" / "complex"
 48 |     assert not (root / "libraries" / "gptl" / ".git").exists()
 49 |     assert not (root / "libraries" / "mpi-serial" / ".git").exists()
 50 |     assert (root / "modules" / "mpi-serialAR" / ".git").exists()
 51 |     assert (root / "modules" / "mpi-serialSAR" / ".git").exists()
 52 |     assert not (root / "modules" / "mpi-serial2" / ".git").exists()
 53 |     assert (root / "modules" / "mpi-sparse" / ".git").exists()
 54 |     assert (root / "modules" / "mpi-sparse" / "m4").exists()
 55 |     assert not (root / "modules" / "mpi-sparse" / "README").exists()
 56 | 
 57 |     # update a single optional submodule
 58 | 
 59 |     result = git_fleximod(complex_update, "update ToplevelOptional")
 60 |     assert result.returncode == 0
 61 | 
 62 |     status = git_fleximod(complex_update, "status")
 63 |     assert "ToplevelOptional at tag v5.3.2" in normalize_whitespace(status.stdout)
 64 |     assert "ToplevelRequired at tag MPIserial_2.5.0" in normalize_whitespace(
 65 |         status.stdout
 66 |     )
 67 |     assert "AlwaysRequired at tag MPIserial_2.4.0" in normalize_whitespace(
 68 |         status.stdout
 69 |     )
 70 |     assert "Complex at tag testtag3" in normalize_whitespace(status.stdout)
 71 |     assert (
 72 |         "AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0"
 73 |         in normalize_whitespace(status.stdout)
 74 |     )
 75 | 
 76 |     # Finally update optional
 77 |     result = git_fleximod(complex_update, "update --optional")
 78 |     assert result.returncode == 0
 79 | 
 80 |     status = git_fleximod(complex_update, "status")
 81 |     assert "ToplevelOptional at tag v5.3.2" in normalize_whitespace(status.stdout)
 82 |     assert "ToplevelRequired at tag MPIserial_2.5.0" in normalize_whitespace(
 83 |         status.stdout
 84 |     )
 85 |     assert "AlwaysRequired at tag MPIserial_2.4.0" in normalize_whitespace(
 86 |         status.stdout
 87 |     )
 88 |     assert "Complex at tag testtag3" in normalize_whitespace(status.stdout)
 89 |     assert "AlwaysOptional at tag MPIserial_2.3.0" in normalize_whitespace(
 90 |         status.stdout
 91 |     )
 92 | 
 93 |     # now check the complex_sub
 94 |     root = complex_update / "modules" / "complex"
 95 |     assert not (root / "libraries" / "gptl" / ".git").exists()
 96 |     assert not (root / "libraries" / "mpi-serial" / ".git").exists()
 97 |     assert not (root / "modules" / "mpi-serial" / ".git").exists()
 98 |     assert (root / "modules" / "mpi-serialAR" / ".git").exists()
 99 |     assert (root / "modules" / "mpi-serialSAR" / ".git").exists()
100 |     assert (root / "modules" / "mpi-sparse" / ".git").exists()
101 |     assert (root / "modules" / "mpi-serial2" / ".git").exists()
102 |     assert (root / "modules" / "mpi-sparse" / "m4").exists()
103 |     assert not (root / "modules" / "mpi-sparse" / "README").exists()
104 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/tests/test_b_update.py:
--------------------------------------------------------------------------------
  1 | # tests/test_b_update.py
  2 | import time
  3 | from pathlib import Path
  4 | 
  5 | 
  6 | def test_basic_checkout(git_fleximod, test_repo, shared_repos):
  7 |     # Prepare a simple .gitmodules
  8 |     gm = shared_repos["gitmodules_content"]
  9 |     file_path = test_repo / ".gitmodules"
 10 |     repo_name = shared_repos["submodule_name"]
 11 |     repo_path = shared_repos["subrepo_path"]
 12 | 
 13 |     file_path.write_text(gm)
 14 | 
 15 |     # Run the command
 16 |     result = git_fleximod(test_repo, f"update {repo_name}")
 17 | 
 18 |     # Assertions
 19 |     assert result.returncode == 0
 20 |     assert Path(
 21 |         test_repo / repo_path
 22 |     ).exists()  # Did the submodule directory get created?
 23 |     if "sparse" in repo_name:
 24 |         assert Path(
 25 |             test_repo / f"{repo_path}/m4"
 26 |         ).exists()  # Did the submodule sparse directory get created?
 27 |         assert not Path(
 28 |             test_repo / f"{repo_path}/README"
 29 |         ).exists()  # Did only the submodule sparse directory get created?
 30 | 
 31 | 
 32 | def test_local_modification_scenarios(git_fleximod, test_repo, shared_repos):
 33 |     """
 34 |     Test three scenarios for local modifications:
 35 |     1. Local mods, repo in sync: update leaves local mods alone.
 36 |     2. Local mods, repo out-of-sync, no conflict: update brings repo up to date, local mods retained, message shown.
 37 |     3. Local mods, repo out-of-sync, conflict: update fails with error.
 38 |     """
 39 |     repo_name = shared_repos["submodule_name"]
 40 |     repo_path = shared_repos["subrepo_path"]
 41 |     submodule_dir = test_repo / repo_path
 42 | 
 43 |     # Ensure submodule is checked out and at intended tag
 44 |     gm = shared_repos["gitmodules_content"]
 45 |     (test_repo / ".gitmodules").write_text(gm)
 46 |     result = git_fleximod(test_repo, f"update {repo_name}")
 47 |     assert result.returncode == 0
 48 |     assert submodule_dir.exists()
 49 |     test_file = submodule_dir / "README"
 50 |     if not test_file.exists():
 51 |         # README must exist in the repository for this test. If not, skip this test.
 52 |         return
 53 | 
 54 |     # --- Scenario 1: Local mods, repo in sync ---
 55 |     original_content = test_file.read_text()
 56 |     local_mod_content = f"local modification {time.time()}\n"
 57 |     test_file.write_text(original_content + local_mod_content)
 58 |     result1 = git_fleximod(test_repo, f"update {repo_name}")
 59 |     assert result1.returncode == 0
 60 |     assert (
 61 |         test_file.read_text() == original_content + local_mod_content
 62 |     ), "Local modification was overwritten when repo was in sync!"
 63 | 
 64 |     # --- Scenario 2: Local mods, repo out-of-sync, no conflict ---
 65 |     # Simulate out-of-sync by checking out previous commit/tag in submodule
 66 |     import subprocess
 67 | 
 68 |     # Try to checkout previous commit (if possible)
 69 |     log = (
 70 |         subprocess.check_output(["git", "log", "--pretty=oneline"], cwd=submodule_dir)
 71 |         .decode()
 72 |         .splitlines()
 73 |     )
 74 |     if len(log) > 1:
 75 |         prev_hash = log[1].split()[0]
 76 |         subprocess.check_call(["git", "checkout", prev_hash], cwd=submodule_dir)
 77 |         # Make a local mod that does not conflict
 78 |         test_file.write_text(original_content + local_mod_content)
 79 |         result2 = git_fleximod(test_repo, f"update {repo_name}")
 80 |         assert result2.returncode == 0
 81 |         # Should retain local mod and show message
 82 |         assert (
 83 |             test_file.read_text() == original_content + local_mod_content
 84 |         ), "Local modification was lost after update with no conflict!"
 85 |         status = git_fleximod(test_repo, f"status {repo_name}")
 86 |         assert "modified files" in status.stdout or "modified" in status.stdout.lower()
 87 | 
 88 |     # --- Scenario 3: Local mods, repo out-of-sync, conflict ---
 89 |     # Simulate conflict by modifying file and checking out previous commit that changes the same file
 90 |     if len(log) > 2:
 91 |         # Recover original README file
 92 |         subprocess.check_call(["git", "restore", "README"], cwd=submodule_dir)
 93 |         # Reset to a further previous commit\
 94 |         conflict_hash = log[2].split()[0]
 95 |         subprocess.check_call(["git", "checkout", conflict_hash], cwd=submodule_dir)
 96 |         # Overwrite file with conflicting content
 97 |         test_file.write_text("conflicting local mod\n")
 98 |         try:
 99 |             git_fleximod(test_repo, f"update {repo_name}")
100 |         except Exception as e:
101 |             assert "ERROR" in str(e) or "Failed to checkout" in str(
102 |                 e
103 |             ), "Expected error not raised for conflict scenario!"
104 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/git_fleximod/cli.py:
--------------------------------------------------------------------------------
  1 | from pathlib import Path
  2 | import argparse, os, sys
  3 | 
  4 | __version__ = "1.1.1"
  5 | 
  6 | 
  7 | class CustomArgumentParser(argparse.ArgumentParser):
  8 |     def print_help(self, file=None):
  9 |         # First print the default help message
 10 |         super().print_help(file)
 11 | 
 12 |         # Then append the contents of README.md
 13 |         candidate_paths = [
 14 |             Path(sys.prefix) / "share" / "git_fleximod" / "README.md",
 15 |             Path(__file__).resolve().parent.parent / "README.md",  # fallback for dev
 16 |         ]
 17 |         for path in candidate_paths:
 18 |             if os.path.exists(path):
 19 |                 with open(path) as f:
 20 |                     print(f.read(), file=file)
 21 |                     return
 22 |         print("README.md not found.", file=file)
 23 | 
 24 | 
 25 | def find_root_dir(filename=".gitmodules"):
 26 |     """
 27 |     Finds the highest directory in tree which contains a file called filename.
 28 | 
 29 |     >>> import tempfile, os
 30 |     >>> cwd = os.getcwd()
 31 |     >>> with tempfile.TemporaryDirectory() as tmp:
 32 |     ...     subdir = Path(tmp) / 'subdir'
 33 |     ...     subdir.mkdir()
 34 |     ...     f = Path(tmp) / '.gitmodules'
 35 |     ...     _ = f.write_text('')
 36 |     ...     os.chdir(subdir)
 37 |     ...     result = find_root_dir('.gitmodules') == str(tmp)
 38 |     ...     os.chdir(cwd)
 39 |     ...     result
 40 |     True
 41 |     """
 42 |     d = Path.cwd()
 43 |     root = Path(d.root)
 44 |     dirlist = []
 45 |     dl = d
 46 |     while dl != root:
 47 |         dirlist.append(dl)
 48 |         dl = dl.parent
 49 |     dirlist.append(root)
 50 |     dirlist.reverse()
 51 | 
 52 |     for dl in dirlist:
 53 |         attempt = dl / filename
 54 |         if attempt.is_file():
 55 |             return str(dl)
 56 |     return None
 57 | 
 58 | 
 59 | def get_parser():
 60 |     description = """
 61 |     %(prog)s manages checking out groups of gitsubmodules with additional support for Earth System Models
 62 |     """
 63 |     parser = CustomArgumentParser(
 64 |         description=description, formatter_class=argparse.RawDescriptionHelpFormatter
 65 |     )
 66 | 
 67 |     #
 68 |     # user options
 69 |     #
 70 |     choices = ["update", "status", "test"]
 71 |     parser.add_argument(
 72 |         "action",
 73 |         choices=choices,
 74 |         default="update",
 75 |         help=f"Subcommand of git-fleximod, choices are {choices[:-1]}",
 76 |     )
 77 | 
 78 |     parser.add_argument(
 79 |         "components",
 80 |         nargs="*",
 81 |         help="Specific component(s) to checkout. By default, "
 82 |         "all required submodules are checked out.",
 83 |     )
 84 | 
 85 |     parser.add_argument(
 86 |         "-C",
 87 |         "--path",
 88 |         default=find_root_dir(),
 89 |         help="Toplevel repository directory.  Defaults to top git directory relative to current.",
 90 |     )
 91 | 
 92 |     parser.add_argument(
 93 |         "-g",
 94 |         "--gitmodules",
 95 |         nargs="?",
 96 |         default=".gitmodules",
 97 |         help="The submodule description filename. " "Default: %(default)s.",
 98 |     )
 99 | 
100 |     parser.add_argument(
101 |         "-x",
102 |         "--exclude",
103 |         nargs="*",
104 |         help="Component(s) listed in the gitmodules file which should be ignored.",
105 |     )
106 |     parser.add_argument(
107 |         "-f",
108 |         "--force",
109 |         action="store_true",
110 |         default=False,
111 |         help="Override cautions and update or checkout over locally modified repository.",
112 |     )
113 | 
114 |     parser.add_argument(
115 |         "-o",
116 |         "--optional",
117 |         action="store_true",
118 |         default=False,
119 |         help="By default only the required submodules "
120 |         "are checked out. This flag will also checkout the "
121 |         "optional submodules relative to the toplevel directory.",
122 |     )
123 | 
124 |     parser.add_argument(
125 |         "-v",
126 |         "--verbose",
127 |         action="count",
128 |         default=0,
129 |         help="Output additional information to "
130 |         "the screen and log file. This flag can be "
131 |         "used up to two times, increasing the "
132 |         "verbosity level each time.",
133 |     )
134 | 
135 |     parser.add_argument(
136 |         "--no-mods-details",
137 |         action="store_true",
138 |         default=False,
139 |         help="Suppress details on local mods in status output.",
140 |     )
141 | 
142 |     parser.add_argument(
143 |         "-V",
144 |         "--version",
145 |         action="version",
146 |         version=f"%(prog)s {__version__}",
147 |         help="Print version and exit.",
148 |     )
149 | 
150 |     #
151 |     # developer options
152 |     #
153 |     parser.add_argument(
154 |         "--backtrace",
155 |         action="store_true",
156 |         help="DEVELOPER: show exception backtraces as extra " "debugging output",
157 |     )
158 | 
159 |     parser.add_argument(
160 |         "-d",
161 |         "--debug",
162 |         action="store_true",
163 |         default=False,
164 |         help="DEVELOPER: output additional debugging "
165 |         "information to the screen and log file.",
166 |     )
167 | 
168 |     return parser
169 | 


--------------------------------------------------------------------------------
/cime_config/config_compsets.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0"?>
  2 | 
  3 | <compsets version="2.0">
  4 | 
  5 |   <help>
  6 |     =========================================
  7 |     compset naming convention
  8 |     =========================================
  9 |     The compset longname below has the specified order
 10 |     atm, lnd, ice, ocn, river, glc wave cesm-options
 11 | 
 12 |     The notation for the compset longname is
 13 |     TIME_ATM[%phys]_LND[%phys]_ICE[%phys]_OCN[%phys]_ROF[%phys]_GLC[%phys]_WAV[%phys][_ESP%phys][_BGC%phys]
 14 |     Where for the specific compsets below the following is supported
 15 |     TIME = Time period (e.g. 2000, HIST, RCP8...)
 16 |     ATM  = [CAM40, CAM50, CAM60]
 17 |     LND  = [CLM45, CLM50, SLND]
 18 |     ICE  = [CICE, DICE, SICE]
 19 |     OCN  = [MOM6, DOCN, SOCN]
 20 |     ROF  = [RTM, MOSART, SROF]
 21 |     GLC  = [CISM2, DGLC, SGLC]
 22 |     WAV  = [WW3, DWAV, XWAV, SWAV]
 23 |     ESP  = [SESP]
 24 |     BGC  = optional BGC scenario
 25 | 
 26 |     The OPTIONAL %phys attributes specify submodes of the given system
 27 |     For example DOCN%DOM is the data ocean model for DOCN
 28 |     ALL the possible %phys choices for each component are listed.
 29 |     ALL data models must have a %phys option that corresponds to the data  model mode
 30 | 
 31 |     Each compset node is associated with the following elements
 32 |     - lname
 33 |     - alias
 34 |     - support  (optional description of the support level for this compset)
 35 |     Each compset node can also have the following attributes
 36 |     - grid  (optional regular expression match for grid to work with the compset)
 37 |   </help>
 38 | 
 39 |   <!-- Concentration driven compsets for CESM3 -->
 40 | 
 41 |   <compset>
 42 |     <alias>B1850C_MTt4s</alias>
 43 |     <lname>1850C_CAM70%MT%CT4S_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3</lname>
 44 |   </compset>
 45 |   <compset>
 46 |     <alias>B1850CM</alias>
 47 |     <lname>1850C_CAM70%MT%CT4S_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3</lname>
 48 |   </compset>
 49 | 
 50 |   <compset>
 51 |     <alias>B1850C_MTso</alias>
 52 |     <lname>1850C_CAM70%MT_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3</lname>
 53 |   </compset>
 54 | 
 55 |   <compset>
 56 |     <alias>BHISTC_MTso</alias>
 57 |     <lname>HISTC_CAM70%MT_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3</lname>
 58 |   </compset>
 59 | 
 60 |   <compset>
 61 |     <alias>BHISTC_MTt4s</alias>
 62 |     <lname>HISTC_CAM70%MT%CT4S_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3</lname>
 63 |   </compset>
 64 | 
 65 |   <compset>
 66 |     <alias>BHISTCM</alias>
 67 |     <lname>HISTC_CAM70%MT%CT4S_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3</lname>
 68 |   </compset>
 69 | 
 70 |   <compset>
 71 |     <alias>BHISTC_MTso</alias>
 72 |     <lname>HISTC_CAM70%MT_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3</lname>
 73 |   </compset>
 74 | <!--  Not yet available in cam component
 75 |   <compset>
 76 |     <alias>B1850C_WAt4ma</alias>
 77 |     <lname>1850C_CAM70%HT%CT4MA_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3</lname>
 78 |   </compset>
 79 | 
 80 |   <compset>
 81 |     <alias>BHISTC_WAt4ma</alias>
 82 |     <lname>1850_CAM70%HT%CT4MA_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_SWAV</lname>
 83 |   </compset>
 84 | -->
 85 |   <compset>
 86 |     <alias>B1850C_LTso</alias>
 87 |     <lname>1850C_CAM70%LT_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3</lname>
 88 |   </compset>
 89 | 
 90 |   <compset>
 91 |     <alias>BHISTC_LTso</alias>
 92 |     <lname>HISTC_CAM70%LT_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3</lname>
 93 |   </compset>
 94 | 
 95 | 
 96 |   <!-- Emissions driven compsets for CESM3 -->
 97 | 
 98 |   <compset>
 99 |     <alias>B1850E_MTt4s</alias>
100 |     <lname>1850E_CAM70%MT%CT4S_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3</lname>
101 |   </compset>
102 |   <compset>
103 |     <alias>B1850EM</alias>
104 |     <lname>1850E_CAM70%MT%CT4S_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3</lname>
105 |   </compset>
106 |   
107 |   <compset>
108 |     <alias>BHISTE_MTt4s</alias>
109 |     <lname>HISTE_CAM70%MT%CT4S_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3</lname>
110 |   </compset>
111 |   <compset>
112 |     <alias>BHISTEM</alias>
113 |     <lname>HISTE_CAM70%MT%CT4S_CLM60%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_WW3</lname>
114 |   </compset>
115 | 
116 |   <!-- SOM compsets -->
117 | 
118 |   <compset>
119 |     <alias>ETEST</alias>
120 |     <lname>2000_CAM60_CLM50%SP_CICE_DOCN%SOM_MOSART_SGLC_SWAV_TEST</lname>
121 |   </compset>
122 | 
123 |   <compset>
124 |     <alias>E1850TEST</alias>
125 |     <lname>1850_CAM60_CLM50%SP_CICE_DOCN%SOM_MOSART_SGLC_SWAV_TEST</lname>
126 |   </compset>
127 | 
128 |   <entries>
129 |     <entry id="RUN_STARTDATE">
130 |       <values>
131 | 	<value compset="1850[CE]?_"     >0001-01-01</value>
132 | 	<value compset="2000[CE]?_"     >0001-01-01</value>
133 | 	<value compset="HIST[CE]?_"     >1850-01-01</value>
134 | 	<value compset="5505[CE]?_"     >1955-01-01</value>
135 | 	<value compset="RCP[2468][CE]?_">2005-01-01</value>
136 | 	<value compset="2013[CE]?_"     >2013-01-01</value>
137 |       </values>
138 |     </entry>
139 |     <entry id="LND_SETS_DUST_EMIS_DRV_FLDS">
140 |       <values match="first">
141 |       <value compset="_CAM7.*_CLM6">FALSE</value>
142 |       </values>
143 |     </entry>
144 |   </entries>
145 | 
146 | </compsets>
147 | 


--------------------------------------------------------------------------------
/tools/statistical_ensemble_test/README:
--------------------------------------------------------------------------------
  1 | ------------------------------------------
  2 | CESM-ECT (CESM Ensemble Consistency Test:
  3 | ------------------------------------------
  4 | 
  5 | CESM-ECT is a suite of tests to determine whether a new
  6 | simulation set up (new machine, compiler, etc.) is statistically
  7 | distinguishable from an accepted ensemble.  The verification tools in
  8 | the CESM-ECT suite are:
  9 | 
 10 | CAM-ECT - detects issues in CAM and CLM (12 month runs)
 11 | UF-CAM-ECT - detects issues in CAM and CLM (9 time step runs)
 12 | POP-ECT - detects issues in POP and CICE (12 month runs)
 13 | 
 14 | The ECT process involves comparing runs generated with
 15 | the new scenario ( 3 for CAM-ECT and UF-CAM-ECT, and 1 for POP-ECT)
 16 | to an ensemble built on a trusted machine (currently
 17 | cheyenne). The python ECT tools are located in the pyCECT
 18 | subdirectory or https://github.com/NCAR/PyCECT/releases.
 19 | 
 20 | -OR-
 21 | 
 22 | We now provide a web server for CAM-ECT and UF-CAM-ECT,  where
 23 | you can upload the (3) generated runs for comparison to our ensemble.
 24 | Please see the webpage at http://www.cesm.ucar.edu/models/cesm2/verification/
 25 | for further instructions.
 26 | 
 27 | -----------------------------------
 28 | Creating or obtaining a summary file:
 29 | -----------------------------------
 30 | 
 31 | Before the test can be run, a summary file is needed of the ensemble
 32 | runs to which the comparison will be made. Ensemble summary files
 33 | (NetCDF) for existing tags for CAM-ECT, UF-CAM-ECT, and POP-ECT that
 34 | were created by CSEG are located (respectively) in the CESM input data
 35 | directories:
 36 | 
 37 | $CESMDATAROOT/inputdata/validation/ensembles
 38 | $CESMDATAROOT/inputdata/validation/uf_ensembles
 39 | $CESMDATAROOT/inputdata/validation/pop_ensembles
 40 | 
 41 | If none of our ensembles are suitable for your needs, then you may create
 42 | your own ensemble (and summary file) using the following instructions:
 43 | 
 44 | (1) To create a new ensemble, use the ensemble.py script in this directory.
 45 | This script creates and compiles a case, then creates clones of the
 46 | original case, where the initial temperature perturbation is slightly modified
 47 | for each ensemble member.  At this time, cime includes functionality
 48 | to create ensembles for CAM-ECT, UF-CAM-ECT, and POP-ECT.
 49 | 
 50 | (2) Use --ect <pop,cam> to specify whether ensemble is for CAM or POP.
 51 | (See 'python ensemble.py -h' for additional details).
 52 | 
 53 | (3) Use --ensemble <size> to specify the ensemble size.
 54 | Recommended ensemble sizes:
 55 | CAM-ECT: 151
 56 | UF-CAM-ECT: 350
 57 | POP-ECT 40
 58 | 
 59 | (4) Examples:
 60 | 
 61 | CAM-ECT:
 62 | 
 63 | python ensemble.py --case /glade/scratch/cesm_user/cesm_tag/ensemble/ensemble.cesm_tag.000 --mach cheyenne   --ensemble 151 --ect cam --project P99999999
 64 | 
 65 | 
 66 | UF-CAM-ECT:
 67 | 
 68 | python ensemble.py --case /glade/scratch/cesm_user/cesm_tag/uf_ensemble/ensemble.cesm_tag.uf.000 --mach cheyenne  --ensemble 350 --uf --ect cam --project P99999999
 69 | 
 70 | POP-ECT:
 71 | 
 72 | python ensemble.py --case /glade/scratch/cesm_user/cesm_tag/uf_ensemble/ensemble.cesm_tag.000 --mach cheyenne  --ensemble 40 --ect pop --project P99999999
 73 | 
 74 | Notes:
 75 |        (a) ensemble.py accepts (most of) the argumenets of create_newcase
 76 | 
 77 |        (b) case name must end in ".000" and include the full path
 78 | 
 79 |        (c) ensemble size must be specified, and suggested defaults are listed
 80 |        	   above. Note that for CAM-ECT and UF-CAM-ECT, the ensemble size
 81 | 	   needs to be larger than the number of variables that ECT will evaluate.
 82 | 
 83 | 
 84 | (5) Once all ensemble simulations have run successfully, copy every cam history
 85 | file (*.cam.h0.*) for CAM-ECT and UF-CAM-ECT) or monthly pop history file
 86 | (*.pop.h.*) for POP-ECT from each ensemble run directory into a separate directory.
 87 | Next create the ensemble summary using the pyCECT tool pyEnsSum.py (for CAM-ECT and
 88 | UF-CAM-ECT) or pyEnsSumPop.py (for POP-ECT).  For details see README_pyEnsSum.rst
 89 | and README_pyEnsSumPop.rst with the pyCECT tools.
 90 | 
 91 | -------------------
 92 | Creating test runs:
 93 | -------------------
 94 | 
 95 | (1) Once an ensemble summary file has been created or chosen to
 96 | use from $CESMDATAROOT/inputdata/validation, the simulation
 97 | run(s) to be verified by ECT must be created via script ensemble.py.
 98 | 
 99 | NOTE: It is important that the **same** resolution and compset be used in the
100 | individual runs as in the ensemble.  The NetCDF ensemble summary file global
101 | attributes give this information.
102 | 
103 | (2) For example, for CAM-ECT:
104 | 
105 | python ensemble.py --case /glade/scratch/cesm_user/cesm_tag/camcase.cesm_tag.000 --ect cam --mach cheyenne --project P99999999
106 | --compset   F2000climo --res f19_f19_mg17
107 | For example, for UF-CAM-ECT:
108 | 
109 | python ensemble.py --case /glade/scratch/cesm_user/cesm_tag/uf.camcase.cesm_tag.000 --ect cam --uf --mach cheyenne --project P99999999 --compset   F2000climo --res f19_f19_mg17
110 | 
111 | For example, for POP-ECT:
112 | 
113 | python ensemble.py --case /glade/scratch/cesm_user/cesm_tag/popcase.cesm_tag.000 --ect pop --mach cheyenne  --project P99999999 --compset   G --res T62_g17
114 | 
115 | (3) Next verify the new simulation(s) with the pyCECT tool pyCECT.py (see
116 | README_pyCECT.rst with the pyCECT tools).
117 | 


--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
  1 | CESM2 Copyright and Terms of Use
  2 | 
  3 | Copyright (c) 2018, University Corporation for Atmospheric Research
  4 | (UCAR) All rights reserved.
  5 | 
  6 | The Community Earth System Model (CESM) was developed primarily in
  7 | cooperation with the National Science Foundation, the Department of
  8 | Energy, the National Aeronautics and Space Administration, and the
  9 | University Corporation for Atmospheric Research National Center for
 10 | Atmospheric Research.
 11 | 
 12 | THIS SOFTWARE IS PROVIDED BY UCAR AND ANY CONTRIBUTORS "AS IS" AND ANY
 13 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 14 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 15 | PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UCAR OR ANY CONTRIBUTORS BE
 16 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 17 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 18 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 19 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 20 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 21 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 22 | THE POSSIBILITY OF SUCH DAMAGE.
 23 | 
 24 | The following components are copyrighted and may only be used, modified,
 25 | or redistributed under the terms indicated below.
 26 | 
 27 | Code		 Institution		Copyright	Terms of Use/Disclaimer
 28 | ----		 -----------	 	---------	-----------------------
 29 | AER RRTMG	Atmospheric		Copyright	AER RRTMG Copyright
 30 |     		and			2002-2010, 
 31 | 		Environmental 		Atmospheric 
 32 | 		Research, Inc.		and 
 33 | 					Environmental 
 34 | 					Research, Inc.
 35 | 
 36 | Common          University Corporation  Copyright 2017,   CIME License
 37 | Infrastructure  for Atmospheric         University
 38 | for Modeling    Research (UCAR) and DOE Corporation for
 39 | the Earth       BER E3SM project team   Atmospheric
 40 | (CIME)          members, including      Research (UCAR);
 41 |                 those at SNL and ANL    Copyright 2017,
 42 |                                         Sandia
 43 |                                         Corporation;
 44 |                                         Copyright 2017,
 45 |                                         UChicago Argonne,
 46 |                                         LLC
 47 | 
 48 | ESMF		 University of 		Copyright 	University of Illinois/NCSA Open Source License
 49 | 		 Illinois/NCSA 		2002-2009, 
 50 | 					University of 
 51 | 					Illinois/NCSA 
 52 | 					Open Source 
 53 | 					License
 54 | 
 55 | Functionally    UCAR/NCAR, UC Berkeley  Copyright 2017,  FATES License
 56 | Assembled       and DOE                 University
 57 | Terrestrial                             Corporation for
 58 | Ecosystem                               Atmospheric
 59 | Simulator                               Research (UCAR),
 60 | (FATES)                                 the U.S.
 61 |                                         Department of
 62 |                                         Energy, and the
 63 |                                         University of
 64 |                                         California
 65 | 
 66 | CISM Community  NCAR/LANL/ORNL/SNL/     Copyright       GNU Lesser General Public License v. 3
 67 | Ice Sheet       LBNL/NYU/U. Bristol/    2004-2018,      CISM is free software: you can redistribute it
 68 | Model           U. Edinburgh/           GNU Lesser      and/or modify it under the terms of the GNU
 69 |                 U. Montana/U. Swansea   General Public  Lesser General Public License as published by the
 70 |                                         License v. 3    Free Software Foundation, either version 3 of the
 71 |                                                         License, or (at your option) any later version.
 72 | 		 
 73 | Inf_NaN		Lahey			Copyright(c)	Copies of this source code, or standalone compiled 
 74 | _Detection 	Computer		2003, Lahey	files derived from this source may not be sold 
 75 | module		Systems, Inc.		Computer	without permission from Lahey Computers Systems. 
 76 | 					Systems, Inc.	All or part of this module may be freely incorporated 
 77 | 							into executable programs which are offered for sale. 
 78 | 							Otherwise, distribution of all or part of this file is 
 79 | 							permitted, provided this copyright notice and header 
 80 | 							are included.
 81 | 					 
 82 | International	 N/A			Copyright 2003, ISCCP Simulator Software
 83 | Satellite	 			2010, Steve 
 84 | Cloud					Klein and Mark 
 85 | Climatology				Webb
 86 | Project
 87 | (ISCCP)
 88 | 
 89 | MCT		Argonne			Copyright 2011,	MCT Copyright
 90 | 		National		University of
 91 | 		Laboratory		Chicago 
 92 | 		 
 93 | Parallel Ocean  Los Alamos National     Copyright 2015, Los Alamos National Security, LLC
 94 | Program (POP2)  Laboratory              Los Alamos
 95 |                                         National
 96 |                                         Security, LLC
 97 | 
 98 | The Los Alamos  Los Alamos National     Copyright 2017, Los Alamos National Security, LLC
 99 | Sea Ice Model   Laboratory              Los Alamos
100 | (CICE)                                  National
101 |                                         Security, LLC
102 | 
103 | Spherical       Los Alamos National     Copyright 2015, Los Alamos National Security, LLC
104 | Coordinate      Laboratory              Los Alamos
105 | Remapping and                           National
106 | Interpolation                           Security, LLC
107 | Package (SCRIP)
108 | 
109 | XML/Lite	Wadsack-Allen		Copyright 2001, The documentation for the Perl XML-Lite 
110 | 		Digital Group		2010 	  	module is no longer available on-line. 
111 | 					Wadsack-Allen 
112 | 					Digital Group
113 | 		
114 | For more details on the copyrights of the above components, see
115 | http://www.cesm.ucar.edu/models/cesm2/copyright.html
116 | 
117 | There may be other third party tools and libraries that are integrated
118 | into CESM, and they may have their own copyright notices and terms.
119 | 


--------------------------------------------------------------------------------
/doc/source/introduction.rst:
--------------------------------------------------------------------------------
  1 | .. _introduction:
  2 | 
  3 | ========================
  4 | Introduction (|version|)
  5 | ========================
  6 | 
  7 | How To Use This Document
  8 | ------------------------
  9 | 
 10 | This guide instructs both novice and experienced users on downloading,
 11 | building and running `CESM2
 12 | <http://www.cesm.ucar.edu/models/cesm2>`_.
 13 | 
 14 | CESM2 is built on the `CIME framework <http://github.com/ESMCI/cime>`_.
 15 | 
 16 | The majority of the CESM2 User's Guide is contained in the `CIME`_ documentation.
 17 | 
 18 | If you are a new user, we recommend reading the first few sections of
 19 | the `CIME`_ documentation which is written so that, as much as
 20 | possible, individual sections stand on their own and the `CIME`_
 21 | documentation guide can be scanned and sections read in a relatively
 22 | ad hoc order.
 23 | 
 24 | .. code-block:: console 
 25 | 
 26 |     Throughout the guide, this presentation style indicates shell
 27 |     commands and options, fragments of code, namelist variables, etc.
 28 | 
 29 | .. note:: 
 30 | 
 31 |    Variables presented as ``$VAR`` in this guide typically refer to variables in XML files
 32 |    in a CESM case. From within a case directory, you can determine the value of such a
 33 |    variable with ``./xmlquery VAR``. In some instances, ``$VAR`` refers to a shell
 34 |    variable or some other variable; we try to make these exceptions clear.
 35 |     
 36 | Please feel free to provide feedback to the `CESM forum <https://bb.cgd.ucar.edu/>`_ about how to improve the
 37 | documentation. 
 38 | 
 39 | CESM Model Version Naming Conventions
 40 | -------------------------------------
 41 | 
 42 | CESM model release versions include three numbers separated by a period (.)
 43 | - CESM X.Y.Z
 44 | 
 45 | -  X - corresponds to the major release number indicating significant
 46 |    science changes.
 47 | 
 48 | -  Y - corresponds to the addition of new infrastructure and new science
 49 |    capabilities for targeted components.
 50 | 
 51 | -  Z - corresponds to release bug fixes and machine updates.
 52 | 
 53 | When refering to CESM2, it is understood that the all versions of the
 54 | CESM2.Y.Z series of models are included. 
 55 | 
 56 | CESM Overview
 57 | =============
 58 | 
 59 | The Community Earth System Model (CESM) is a coupled climate model for
 60 | simulating Earth's climate system. Composed of separate models
 61 | simultaneously simulating the Earth's atmosphere, ocean, land, river
 62 | run-off, land-ice, and sea-ice, plus one central coupler/moderator
 63 | component, CESM allows researchers to conduct fundamental research
 64 | into the Earth's past, present, and future climate states.
 65 | 
 66 | CESM can be run on a number of different `hardware platforms
 67 | <http://www.cesm.ucar.edu/models/cesm2/cesm/machines.html>`__, and
 68 | has a relatively flexible design with respect to `processor layout
 69 | <http://esmci.github.io/cime/versions/master/html/users_guide/pes-threads.html>`__
 70 | of components.
 71 | 
 72 | The CESM project is a cooperative effort among U.S. climate
 73 | researchers.  Primarily supported by the `National Science
 74 | Foundation(NSF) <https://www.nsf.gov/>`_ and centered at the `National
 75 | Center for Atmospheric Research (NCAR) <https://ncar.ucar.edu/>`_ in
 76 | Boulder, Colorado, the CESM project enjoys close collaborations with
 77 | the `U.S. Department of Energy (DOE) <https://energy.gov/>`_ and the
 78 | `National Aeronautics and Space Administration (NASA)
 79 | <http://www.nasa.gov>`_.  Scientific development of the CESM is guided
 80 | by the CESM working groups, which meet twice a year. The main CESM
 81 | workshop is held each year in June to showcase results from the
 82 | various working groups and coordinate future CESM developments among
 83 | the working groups. The `CESM website <http://www.cesm.ucar.edu/>`__
 84 | provides more information on the CESM project, such as the management
 85 | structure, the scientific working groups, downloadable source code,
 86 | and online archives of data from previous CESM experiments.
 87 | 
 88 | CESM2 Software/Operating System Prerequisites
 89 | ---------------------------------------------
 90 | 
 91 | The following are the external system and software requirements for
 92 | installing and running CESM2.
 93 | 
 94 | -  UNIX style operating system such as CNL, AIX or Linux
 95 | 
 96 | -  python >= 2.7
 97 | 
 98 | -  perl 5 
 99 | 
100 | -  subversion client (version 1.8 or greater but less than v1.11) for downloading CAM, POP, and WW3
101 | 
102 | -  git client (1.8 or greater)
103 | 
104 | -  Fortran compiler with support for Fortran 2003
105 | 
106 | -  C compiler
107 | 
108 | -  MPI (although CESM does not absolutely require it for running on one processor)
109 | 
110 | -  `NetCDF 4.3 or newer <http://www.unidata.ucar.edu/software/netcdf/>`_.
111 | 
112 | -  `ESMF 5.2.0 or newer (optional) <http://www.earthsystemmodeling.org/>`_.
113 | 
114 | -  `pnetcdf 1.7.0 is required and 1.8.1 is optional but recommended <http://trac.mcs.anl.gov/projects/parallel-netcdf/>`_
115 | 
116 | -  `Trilinos <https://trilinos.github.io/>`_ may be required for certain configurations 
117 | 
118 | -  `LAPACK <http://www.netlib.org/lapack/>`_ and `BLAS <http://www.netlib.org/blas/>`_
119 | 
120 | -  `CMake 2.8.6 or newer <http://www.cmake.org/>`_ 
121 | 
122 | .. warning:: NetCDF must be built with the same Fortran compiler as CESM. In the netCDF build the FC environment variable specifies which Fortran compiler to use. CESM is written mostly in Fortran, netCDF is written in C. Because there is no standard way to call a C program from a Fortran program, the Fortran to C layer between CESM and netCDF will vary depending on which Fortran compiler you use for CESM. When a function in the netCDF library is called from a Fortran application, the netCDF Fortran API calls the netCDF C library. If you do not use the same compiler to build netCDF and CESM you will in most cases get errors from netCDF saying certain netCDF functions cannot be found.
123 | 
124 | Parallel-netCDF, also referred to as pnetcdf, is optional. If a user
125 | chooses to use pnetcdf, version 1.7.0 or later should be used with CESM.
126 | It is a library that is file-format compatible with netCDF, and provides
127 | higher performance by using MPI-IO. Pnetcdf is enabled by setting the
128 | ``$PNETCDF_PATH`` Makefile variable in the ``Macros.make`` file.
129 | 
130 | .. _CIME: http://esmci.github.io/cime
131 | 


--------------------------------------------------------------------------------
/doc/source/conf.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #
  3 | # cesm_quickstart documentation build configuration file, created by
  4 | # sphinx-quickstart on Wed Mar 29 12:57:57 2017.
  5 | #
  6 | # This file is execfile()d with the current directory set to its
  7 | # containing dir.
  8 | #
  9 | # Note that not all possible configuration values are present in this
 10 | # autogenerated file.
 11 | #
 12 | # All configuration values have a default; values that are commented out
 13 | # serve to show the default.
 14 | 
 15 | # If extensions (or modules to document with autodoc) are in another directory,
 16 | # add these directories to sys.path here. If the directory is relative to the
 17 | # documentation root, use os.path.abspath to make it absolute, like shown here.
 18 | #
 19 | import os
 20 | import sys
 21 | # Note that we need a specific version of sphinx_rtd_theme. This can be obtained with:
 22 | # pip install git+https://github.com/esmci/sphinx_rtd_theme.git@version-dropdown-with-fixes
 23 | import sphinx_rtd_theme
 24 | sys.path.insert(0, os.path.abspath('../../manage_externals'))
 25 | 
 26 | 
 27 | # -- General configuration ------------------------------------------------
 28 | 
 29 | # If your documentation needs a minimal Sphinx version, state it here.
 30 | #
 31 | # needs_sphinx = '1.0'
 32 | 
 33 | # Add any Sphinx extension module names here, as strings. They can be
 34 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 35 | # ones.
 36 | extensions = ['sphinx.ext.intersphinx',
 37 |     'sphinx.ext.todo',
 38 |     'sphinx.ext.coverage',
 39 |     'sphinx.ext.githubpages',
 40 |     'sphinx.ext.viewcode',
 41 |     'sphinx.ext.todo',
 42 |     'sphinxcontrib.programoutput'
 43 | ]
 44 | 
 45 | # Add any paths that contain templates here, relative to this directory.
 46 | templates_path = ['_templates']
 47 | 
 48 | # The suffix(es) of source filenames.
 49 | # You can specify multiple suffix as a list of string:
 50 | #
 51 | source_suffix = ['.rst', '.md']
 52 | #source_suffix = '.rst'
 53 | 
 54 | # The master toctree document.
 55 | master_doc = 'index'
 56 | 
 57 | # General information about the project.
 58 | project = u'CESM'
 59 | copyright = u'2020, U.S. National Science Foundation'
 60 | author = u'Staff of NSF/NCAR CESM Software Engineering Group' 
 61 | 
 62 | # The version info for the project you're documenting, acts as replacement for
 63 | # |version| and |release|, also used in various other places throughout the
 64 | # built documents.
 65 | #
 66 | # The short X.Y version.
 67 | version = u'Upcoming/Beta Version'
 68 | # The full version, including alpha/beta/rc tags.
 69 | release = u'Upcoming/Beta Version'
 70 | 
 71 | # The language for content autogenerated by Sphinx. Refer to documentation
 72 | # for a list of supported languages.
 73 | #
 74 | # This is also used if you do content translation via gettext catalogs.
 75 | # Usually you set "language" from the command line for these cases.
 76 | language = None
 77 | 
 78 | # List of patterns, relative to source directory, that match files and
 79 | # directories to ignore when looking for source files.
 80 | # This patterns also effect to html_static_path and html_extra_path
 81 | exclude_patterns = []
 82 | 
 83 | # The name of the Pygments (syntax highlighting) style to use.
 84 | pygments_style = 'sphinx'
 85 | 
 86 | # If true, `todo` and `todoList` produce output, else they produce nothing.
 87 | todo_include_todos = True
 88 | 
 89 | 
 90 | # -- Options for HTML output ----------------------------------------------
 91 | 
 92 | # The theme to use for HTML and HTML Help pages.  See the documentation for
 93 | # a list of builtin themes.
 94 | #
 95 | html_theme = 'sphinx_rtd_theme'
 96 | 
 97 | # Theme options are theme-specific and customize the look and feel of a theme
 98 | # further.  For a list of options available for each theme, see the
 99 | # documentation.
100 | #
101 | # The 'versions' option needs to have at least two versions to work, but it doesn't need
102 | # to have all versions: others will be added dynamically. Note that this maps from version
103 | # names to html links. The current version can link to the current location (i.e., do
104 | # nothing). For the other version, we just add a place-holder; its name and value are
105 | # unimportant because these versions will get replaced dynamically.
106 | html_theme_options = {}
107 | html_theme_options['versions'] = {version: ''}
108 | html_theme_options['versions']['[placeholder]'] = ''
109 | 
110 | # Add any paths that contain custom static files (such as style sheets) here,
111 | # relative to this directory. They are copied after the builtin static files,
112 | # so a file named "default.css" will overwrite the builtin "default.css".
113 | html_static_path = ['_static']
114 | 
115 | 
116 | # -- Options for HTMLHelp output ------------------------------------------
117 | 
118 | # Output file base name for HTML help builder.
119 | htmlhelp_basename = 'cesm_quickstartdoc'
120 | 
121 | 
122 | # -- Options for LaTeX output ---------------------------------------------
123 | 
124 | latex_elements = {
125 |     # The paper size ('letterpaper' or 'a4paper').
126 |     #
127 |     # 'papersize': 'letterpaper',
128 | 
129 |     # The font size ('10pt', '11pt' or '12pt').
130 |     #
131 |     # 'pointsize': '10pt',
132 | 
133 |     # Additional stuff for the LaTeX preamble.
134 |     #
135 |     # 'preamble': '',
136 | 
137 |     # Latex figure (float) alignment
138 |     #
139 |     # 'figure_align': 'htbp',
140 | }
141 | 
142 | # Grouping the document tree into LaTeX files. List of tuples
143 | # (source start file, target name, title,
144 | #  author, documentclass [howto, manual, or own class]).
145 | latex_documents = [
146 |     (master_doc, 'cesm_quickstart.tex', u'CESM Quickstart Guide',
147 |      u'Staff of NSF/NCAR CESM Software Engineering Group', 'manual'),
148 | ]
149 | 
150 | 
151 | # -- Options for manual page output ---------------------------------------
152 | 
153 | # One entry per manual page. List of tuples
154 | # (source start file, name, description, authors, manual section).
155 | man_pages = [
156 |     (master_doc, 'cesm', u'CESM',
157 |      [author], 1)
158 | ]
159 | 
160 | 
161 | # -- Options for Texinfo output -------------------------------------------
162 | 
163 | # Grouping the document tree into Texinfo files. List of tuples
164 | # (source start file, target name, title, author,
165 | #  dir menu entry, description, category)
166 | texinfo_documents = [
167 |     (master_doc, 'cesm', u'cesm',
168 |      author, 'cesm', 'One line description of project.',
169 |      'Miscellaneous'),
170 | ]
171 | 
172 | # Example configuration for intersphinx: refer to the Python standard library.
173 | intersphinx_mapping = {'https://docs.python.org/': None}
174 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/tests/conftest.py:
--------------------------------------------------------------------------------
  1 | import pytest
  2 | from git_fleximod.gitinterface import GitInterface
  3 | import subprocess
  4 | import logging
  5 | 
  6 | 
  7 | @pytest.fixture(scope="session")
  8 | def logger():
  9 |     logging.basicConfig(
 10 |         level=logging.INFO,
 11 |         format="%(name)s - %(levelname)s - %(message)s",
 12 |         handlers=[logging.StreamHandler()],
 13 |     )
 14 |     llogger = logging.getLogger(__name__)
 15 |     return llogger
 16 | 
 17 | 
 18 | all_repos = [
 19 |     {
 20 |         "subrepo_path": "modules/test",
 21 |         "submodule_name": "test_submodule",
 22 |         "status1": "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0",
 23 |         "status2": "test_submodule at tag MPIserial_2.4.0",
 24 |         "status3": "test_submodule at tag MPIserial_2.4.0",
 25 |         "status4": "test_submodule at tag MPIserial_2.4.0",
 26 |         "gitmodules_content": """
 27 |     [submodule "test_submodule"]
 28 |     path = modules/test
 29 |     url = https://github.com/ESMCI/mpi-serial.git
 30 |     fxtag = MPIserial_2.4.0
 31 |     fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git
 32 |     fxrequired = ToplevelRequired
 33 | """,
 34 |     },
 35 |     {
 36 |         "subrepo_path": "modules/test_optional",
 37 |         "submodule_name": "test_optional",
 38 |         "status1": "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0",
 39 |         "status2": "test_optional at tag MPIserial_2.4.0",
 40 |         "status3": "test_optional not checked out, out of sync at tag MPIserial_2.5.4, expected tag is MPIserial_2.4.0 (optional)",
 41 |         "status4": "test_optional at tag MPIserial_2.4.0",
 42 |         "gitmodules_content": """
 43 |      [submodule "test_optional"]
 44 |     path = modules/test_optional
 45 |     url = https://github.com/ESMCI/mpi-serial.git
 46 |     fxtag = MPIserial_2.4.0
 47 |     fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git
 48 |     fxrequired = ToplevelOptional
 49 | """,
 50 |     },
 51 |     {
 52 |         "subrepo_path": "modules/test_alwaysoptional",
 53 |         "submodule_name": "test_alwaysoptional",
 54 |         "status1": "test_alwaysoptional MPIserial_2.3.0 is out of sync with .gitmodules e5cf35c",
 55 |         "status2": "test_alwaysoptional at hash e5cf35c",
 56 |         "status3": "out of sync at tag MPIserial_2.5.4, expected tag is e5cf35c",
 57 |         "status4": "test_alwaysoptional at hash e5cf35c",
 58 |         "gitmodules_content": """
 59 |     [submodule "test_alwaysoptional"]
 60 |     path = modules/test_alwaysoptional
 61 |     url = https://github.com/ESMCI/mpi-serial.git
 62 |     fxtag = e5cf35c
 63 |     fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git
 64 |     fxrequired = AlwaysOptional
 65 | """,
 66 |     },
 67 |     {
 68 |         "subrepo_path": "modules/test_sparse",
 69 |         "submodule_name": "test_sparse",
 70 |         "status1": "test_sparse at tag MPIserial_2.5.0",
 71 |         "status2": "test_sparse at tag MPIserial_2.5.0",
 72 |         "status3": "test_sparse at tag MPIserial_2.5.0",
 73 |         "status4": "test_sparse at tag MPIserial_2.5.0",
 74 |         "gitmodules_content": """
 75 |     [submodule "test_sparse"]
 76 |     path = modules/test_sparse
 77 |     url = https://github.com/ESMCI/mpi-serial.git
 78 |     fxtag = MPIserial_2.5.0
 79 |     fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git
 80 |     fxrequired = AlwaysRequired
 81 |     fxsparse = ../.sparse_file_list
 82 | """,
 83 |     },
 84 | ]
 85 | 
 86 | 
 87 | @pytest.fixture(params=all_repos)
 88 | def shared_repos(request):
 89 |     repo_data = request.param
 90 |     return repo_data
 91 | 
 92 | 
 93 | @pytest.fixture
 94 | def get_all_repos():
 95 |     return all_repos
 96 | 
 97 | 
 98 | def write_sparse_checkout_file(fp):
 99 |     sparse_content = """m4
100 | """
101 |     fp.write_text(sparse_content)
102 | 
103 | 
104 | # pylint: disable=redefined-outer-name
105 | @pytest.fixture
106 | def test_repo(shared_repos, tmp_path, logger):
107 |     subrepo_path = shared_repos["subrepo_path"]
108 |     submodule_name = shared_repos["submodule_name"]
109 |     test_dir = tmp_path / "testrepo"
110 |     test_dir.mkdir()
111 |     str_path = str(test_dir)
112 |     gitp = GitInterface(str_path, logger)
113 |     assert test_dir.joinpath(".git").is_dir()
114 |     (test_dir / "modules").mkdir()
115 |     if "sparse" in submodule_name:
116 |         (test_dir / subrepo_path).mkdir()
117 |         # Add the sparse checkout file
118 |         write_sparse_checkout_file(test_dir / "modules" / ".sparse_file_list")
119 |         gitp.git_operation("add", "modules/.sparse_file_list")
120 |     else:
121 |         gitp = GitInterface(str(test_dir), logger)
122 |         gitp.git_operation(
123 |             "submodule",
124 |             "add",
125 |             "--depth",
126 |             "1",
127 |             "--name",
128 |             submodule_name,
129 |             "https://github.com/ESMCI/mpi-serial.git",
130 |             subrepo_path,
131 |         )
132 |         assert test_dir.joinpath(".gitmodules").is_file()
133 |         gitp.git_operation("add", subrepo_path)
134 |     gitp.git_operation("commit", "-a", "-m", '"add submod"')
135 |     test_dir2 = tmp_path / "testrepo2"
136 |     gitp.git_operation("clone", test_dir, test_dir2)
137 |     return test_dir2
138 | 
139 | 
140 | @pytest.fixture
141 | def complex_repo(tmp_path, logger):
142 |     test_dir = tmp_path / "testcomplex"
143 |     test_dir.mkdir()
144 |     str_path = str(test_dir)
145 |     gitp = GitInterface(str_path, logger)
146 |     gitp.git_operation(
147 |         "remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2"
148 |     )
149 |     gitp.git_operation("fetch", "origin")
150 |     gitp.git_operation("checkout", "v0.0.1")
151 |     return test_dir
152 | 
153 | 
154 | @pytest.fixture
155 | def complex_update(tmp_path, logger):
156 |     test_dir = tmp_path / "testcomplex"
157 |     test_dir.mkdir()
158 |     str_path = str(test_dir)
159 |     gitp = GitInterface(str_path, logger)
160 |     gitp.git_operation(
161 |         "remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2"
162 |     )
163 |     gitp.git_operation("fetch", "origin")
164 |     gitp.git_operation("checkout", "v0.0.2")
165 | 
166 |     return test_dir
167 | 
168 | 
169 | @pytest.fixture
170 | def git_fleximod():
171 |     def _run_fleximod(path, args, user_input=None):
172 |         cmd = ["git", "fleximod"] + args.split()
173 |         result = subprocess.run(
174 |             cmd,
175 |             cwd=path,
176 |             input=user_input,
177 |             stdout=subprocess.PIPE,
178 |             stderr=subprocess.PIPE,
179 |             text=True,
180 |             check=False,
181 |         )
182 |         if result.returncode:
183 |             print(result.stdout)
184 |             print(result.stderr)
185 |         return result
186 | 
187 |     return _run_fleximod
188 | 


--------------------------------------------------------------------------------
/.gitmodules:
--------------------------------------------------------------------------------
  1 | # This is a git submodule file with additional support for
  2 | # git-fleximod (https://github.com/jedwards4b/git-fleximod)
  3 | #
  4 | # The additional flags supported by git-fleximod are
  5 | # fxtag - the tag associated with the submodule, this tag can be tested for
  6 | #         consistancy with the submodule hash using git-fleximod status
  7 | #         the hash can be updated to the tag using git-fleximod update
  8 | #
  9 | # fxrequired - indicates if a given submodule should be checked out on install
 10 | #              submoudules can be toplevel or internal and required or optional
 11 | #              toplevel means that the submodule should only be checked out if the
 12 | #              module is the toplevel of the git repo (is not a submodule itself)
 13 | #              internal means that the submodule is needed by the component whether
 14 | #              the component is toplevel or the submodule of another repo
 15 | #              required means that the submodule should always be checked out
 16 | #              optional means that the submodule should only be checked out if the
 17 | #              optional flag is provided to git-fleximod or the submodule name is
 18 | #              explicitly listed on the git-fleximod command line.
 19 | #
 20 | # fxsparse -   this is a path to a git sparse checkout file indicating that the
 21 | #              submodule should be checked out in sparse mode
 22 | #
 23 | # fxDONOTUSEurl    -   this field is used by git-fleximod test to insure that the url is pointing
 24 | #              to the official url of the repo and not to an unofficial fork.
 25 | #              It is intended for use of github workflows to test commits to protected
 26 | #              repository branches.
 27 | #
 28 | 
 29 | [submodule "ccs_config"]
 30 |         path = ccs_config
 31 |         url = https://github.com/ESMCI/ccs_config_cesm.git
 32 |         fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git
 33 |         fxtag = ccs_config_cesm1.0.68
 34 |         fxrequired = ToplevelRequired
 35 | 
 36 | [submodule "cime"]
 37 |         path = cime
 38 |         url = https://github.com/ESMCI/cime
 39 |         fxDONOTUSEurl = https://github.com/ESMCI/cime
 40 | 	fxtag = cime6.1.146
 41 |         fxrequired = ToplevelRequired
 42 | 
 43 | [submodule "fms"]
 44 |         path = libraries/FMS
 45 |         url = https://github.com/ESCOMP/FMS_interface
 46 |         fxDONOTUSEurl = https://github.com/ESCOMP/FMS_interface
 47 |         fxrequired = ToplevelRequired
 48 |         fxtag = fi_250428
 49 | 
 50 | [submodule "share"]
 51 |         path = share
 52 |         url = https://github.com/ESCOMP/CESM_share
 53 |         fxDONOTUSEurl = https://github.com/ESCOMP/CESM_share
 54 |         fxtag = share1.1.16
 55 |         fxrequired = ToplevelRequired
 56 | 
 57 | [submodule "cam"]
 58 |         path = components/cam
 59 |         url = https://www.github.com/ESCOMP/CAM
 60 |         fxDONOTUSEurl = https://www.github.com/ESCOMP/CAM
 61 |         fxtag = cam6_4_133
 62 |         fxrequired = ToplevelRequired
 63 | 
 64 | [submodule "clm"]
 65 |         path = components/clm
 66 |         url = https://github.com/ESCOMP/CTSM
 67 |         fxDONOTUSEurl = https://github.com/ESCOMP/CTSM
 68 |         fxrequired = ToplevelRequired
 69 |         fxtag = ctsm5.4.003
 70 | 
 71 | [submodule "cice"]
 72 |         path = components/cice
 73 |         url = https://github.com/ESCOMP/CESM_CICE
 74 |         fxDONOTUSEurl = https://github.com/ESCOMP/CESM_CICE
 75 |         fxrequired = ToplevelRequired
 76 |         fxtag = cesm3_cice6_6_1_6
 77 | 
 78 | [submodule "mom"]
 79 |         path = components/mom
 80 |         url = https://github.com/ESCOMP/MOM_interface
 81 |         fxDONOTUSEurl = https://github.com/ESCOMP/MOM_interface
 82 |         fxrequired = ToplevelRequired
 83 |         fxtag = mi_251121
 84 | 
 85 | [submodule "cism"]
 86 |         path = components/cism
 87 |         url = https://github.com/ESCOMP/cism-wrapper.git
 88 |         fxDONOTUSEurl = https://github.com/ESCOMP/cism-wrapper.git
 89 |         fxtag = cismwrap_2_2_011
 90 |         fxrequired = ToplevelRequired
 91 | 
 92 | [submodule "cdeps"]
 93 |         path = components/cdeps
 94 |         url = https://github.com/ESCOMP/CDEPS
 95 |         fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS
 96 |         fxrequired = ToplevelRequired
 97 |         fxtag = cdeps1.0.84
 98 | 
 99 | [submodule "cmeps"]
100 |         path = components/cmeps
101 |         url = https://github.com/ESCOMP/CMEPS.git
102 |         fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git
103 |         fxrequired = ToplevelRequired
104 | 	fxtag = cmeps1.1.27
105 | 
106 | [submodule "rtm"]
107 |         path = components/rtm
108 |         url = https://github.com/ESCOMP/RTM
109 |         fxDONOTUSEurl = https://github.com/ESCOMP/RTM
110 |         fxrequired = ToplevelRequired
111 |         fxtag = rtm1_0_89
112 | 
113 | [submodule "ww3"]
114 |         path = components/ww3
115 |         url = https://github.com/ESCOMP/WW3_interface
116 |         fxDONOTUSEurl = https://github.com/ESCOMP/WW3_interface
117 |         fxtag = main_0.0.18
118 |         fxrequired = ToplevelRequired
119 | 
120 | [submodule "mizuroute"]
121 |         path = components/mizuroute
122 |         url = https://github.com/ESCOMP/mizuRoute
123 |         fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute
124 |         fxrequired = ToplevelRequired
125 |         fxtag = v3.0.1
126 | 
127 | [submodule "mosart"]
128 |         path = components/mosart
129 |         url = https://github.com/ESCOMP/MOSART
130 |         fxDONOTUSEurl = https://github.com/ESCOMP/MOSART
131 |         fxrequired = ToplevelRequired
132 |         fxtag = mosart1.1.12
133 | 
134 | [submodule "parallelio"]
135 |         path = libraries/parallelio
136 |         url = https://github.com/NCAR/ParallelIO
137 |         fxDONOTUSEurl = https://github.com/NCAR/ParallelIO
138 |         fxrequired = ToplevelRequired
139 |         fxtag = pio2_6_6
140 | 
141 | [submodule "mpi-serial"]
142 |         path = libraries/mpi-serial
143 |         url = https://github.com/ESMCI/mpi-serial
144 |         fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial
145 |         fxtag = MPIserial_2.5.4
146 |         fxrequired = ToplevelRequired
147 | 
148 | [submodule "pysect"]
149 |         path = tools/statistical_ensemble_test/pyCECT
150 |         url = https://github.com/NCAR/PyCECT
151 |         fxDONOTUSEurl = https://github.com/NCAR/PyCECT
152 |         fxrequired = ToplevelRequired
153 |         fxtag = 3.2.2
154 | 
155 | [submodule "tools/CUPiD"]
156 |         path = tools/CUPiD
157 |         url = https://github.com/NCAR/CUPiD.git
158 |         fxDONOTUSEurl = https://github.com/NCAR/CUPiD.git
159 |         fxrequired = ToplevelRequired
160 |         fxtag = v0.4.0
161 | 
162 | [submodule "FTorch"]
163 | 	path = libraries/FTorch
164 | 	url = https://github.com/ESCOMP/FTorch_interface
165 |   fxtag = v0.0.5
166 |   fxrequired = ToplevelOptional
167 |   fxDONOTUSEurl = https://github.com/ESCOMP/FTorch_interface
168 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/git_fleximod/gitinterface.py:
--------------------------------------------------------------------------------
  1 | # git_fleximod/gitinterface.py
  2 | import sys
  3 | from . import utils
  4 | from pathlib import Path
  5 | import asyncio
  6 | 
  7 | 
  8 | class GitInterface:
  9 |     def __init__(self, repo_path, logger):
 10 |         if isinstance(repo_path, str):
 11 |             self.repo_path = Path(repo_path).resolve()
 12 |         elif isinstance(repo_path, Path):
 13 |             self.repo_path = repo_path.resolve()
 14 |         else:
 15 |             raise TypeError("repo_path must be a str or Path object")
 16 |         if logger is not None:
 17 |             self.logger = logger
 18 |         else:
 19 |             import logging
 20 | 
 21 |             self.logger = logging.getLogger(__name__)
 22 |             if not self.logger.hasHandlers():
 23 |                 handler = logging.StreamHandler()
 24 |                 formatter = logging.Formatter(
 25 |                     "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
 26 |                 )
 27 |                 handler.setFormatter(formatter)
 28 |                 self.logger.addHandler(handler)
 29 |                 self.logger.setLevel(logging.DEBUG)
 30 |         self.logger.debug("Initialize GitInterface for {}".format(repo_path))
 31 |         try:
 32 |             import git
 33 | 
 34 |             self._use_module = True
 35 |             try:
 36 |                 self.repo = git.Repo(str(self.repo_path))  # Initialize GitPython repo
 37 |             except git.exc.InvalidGitRepositoryError:
 38 |                 self.git = git
 39 |                 self._init_git_repo()
 40 |             msg = "Using GitPython interface to git"
 41 |         except ImportError:
 42 |             self._use_module = False
 43 |             if not (self.repo_path / ".git").exists():
 44 |                 self._init_git_repo()
 45 |             msg = "Using shell interface to git"
 46 |         self.logger.info(msg)
 47 | 
 48 |     def _git_command(self, operation, *args):
 49 |         """
 50 |         Internal: Build git command or call GitPython.
 51 |         >>> gi = GitInterface('.', None)
 52 |         >>> isinstance(gi._git_command('status'), list) or isinstance(gi._git_command('status'), str)
 53 |         True
 54 |         """
 55 |         self.logger.info(operation)
 56 |         if self._use_module and operation != "submodule":
 57 |             try:
 58 |                 return getattr(self.repo.git, operation)(*args)
 59 |             except Exception as e:
 60 |                 sys.exit(e)
 61 |         else:
 62 |             return ["git", "-C", str(self.repo_path), operation] + list(args)
 63 | 
 64 |     def _init_git_repo(self):
 65 |         if self._use_module:
 66 |             self.repo = self.git.Repo.init(str(self.repo_path))
 67 |         else:
 68 |             command = ("git", "-C", str(self.repo_path), "init")
 69 |             utils.execute_subprocess(command)
 70 | 
 71 |     def _git_operation_command(self, operation, args):
 72 |         newargs = []
 73 |         for a in args:
 74 |             # Do not use ssh interface
 75 |             if isinstance(a, str):
 76 |                 a = a.replace("git@github.com:", "https://github.com/")
 77 |             newargs.append(a)
 78 | 
 79 |         return self._git_command(operation, *newargs)
 80 | 
 81 |     # pylint: disable=unused-argument
 82 |     def git_operation(self, operation, *args, **kwargs):
 83 |         """
 84 |         Run a git operation and return status and output.
 85 |         >>> gi = GitInterface('.', None)
 86 |         >>> status, output = gi.git_operation('status')
 87 |         >>> isinstance(status, int)
 88 |         True
 89 |         >>> isinstance(output, str)
 90 |         True
 91 |         """
 92 |         newargs = []
 93 |         for a in args:
 94 |             # Do not use ssh interface
 95 |             if isinstance(a, str):
 96 |                 a = a.replace("git@github.com:", "https://github.com/")
 97 |             newargs.append(a)
 98 | 
 99 |         command = self._git_command(operation, *newargs)
100 |         if isinstance(command, list):
101 |             try:
102 |                 status, output = utils.execute_subprocess(
103 |                     command, status_to_caller=True, output_to_caller=True
104 |                 )
105 |                 return status, output.rstrip()
106 |             except Exception as e:
107 |                 sys.exit(e)
108 |         else:
109 |             return 0, command
110 | 
111 |     # pylint: disable=unused-argument
112 |     async def git_operation_async(self, operation, *args, **kwargs):
113 |         """
114 |         Asynchronously run a git operation.
115 |         (Doc test not run: async context)
116 |         """
117 |         command = self._git_operation_command(operation, args)
118 |         if isinstance(command, list):
119 |             try:
120 |                 process = await asyncio.create_subprocess_exec(
121 |                     *command,
122 |                     stdout=asyncio.subprocess.PIPE,
123 |                     stderr=asyncio.subprocess.PIPE,
124 |                 )
125 |                 stdout, stderr = await process.communicate()
126 |                 status = process.returncode
127 |                 output = stdout.decode().strip() if stdout else stderr.decode().strip()
128 |                 return status, output
129 |             except Exception as e:
130 |                 sys.exit(e)
131 |         else:
132 |             return 0, command
133 | 
134 |     def config_get_value(self, section, name):
135 |         """
136 |         Get a git config value.
137 |         >>> gi = GitInterface('.', None)
138 |         >>> gi.config_get_value('user', 'name') is None or isinstance(gi.config_get_value('user', 'name'), str)
139 |         True
140 |         """
141 |         if self._use_module:
142 |             config = self.repo.config_reader()
143 |             try:
144 |                 val = config.get_value(section, name)
145 |             except:
146 |                 val = None
147 |             return val
148 |         else:
149 |             cmd = (
150 |                 "git",
151 |                 "-C",
152 |                 str(self.repo_path),
153 |                 "config",
154 |                 "--get",
155 |                 f"{section}.{name}",
156 |             )
157 |             output = utils.execute_subprocess(cmd, output_to_caller=True)
158 |             return output.strip()
159 | 
160 |     def config_set_value(self, section, name, value):
161 |         """
162 |         Set a git config value.
163 |         (Doc test not run: modifies config)
164 |         """
165 |         if self._use_module:
166 |             with self.repo.config_writer() as writer:
167 |                 if "." in section:
168 |                     section = section.replace(".", ' "') + '"'
169 |                 writer.set_value(section, name, value)
170 |             writer.release()  # Ensure changes are saved
171 |         else:
172 |             cmd = (
173 |                 "git",
174 |                 "-C",
175 |                 str(self.repo_path),
176 |                 "config",
177 |                 f"{section}.{name}",
178 |                 value,
179 |             )
180 |             self.logger.info(cmd)
181 |             utils.execute_subprocess(cmd, output_to_caller=True)
182 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
  1 | # Contributor Code of Conduct
  2 | _The Contributor Code of Conduct is for participants in our software projects and community._
  3 | 
  4 | ## Our Pledge
  5 | We, as contributors, creators, stewards, and maintainers (participants), of **git-fleximod** pledge to make participation in
  6 | our software, system or hardware project and community a safe, productive, welcoming and inclusive experience for everyone.
  7 | All participants are required to abide by this Code of Conduct.
  8 | This includes respectful treatment of everyone regardless of age, body size, disability, ethnicity, gender identity or expression,
  9 | level of experience, nationality, political affiliation, veteran status, pregnancy, genetic information, physical appearance, race,
 10 | religion, or sexual orientation, as well as any other characteristic protected under applicable US federal or state law.
 11 | 
 12 | ## Our Standards
 13 | Examples of behaviors that contribute to a positive environment include:
 14 | 
 15 | * All participants are treated with respect and consideration, valuing a diversity of views and opinions
 16 | * Be considerate, respectful, and collaborative
 17 | * Communicate openly with respect for others, critiquing ideas rather than individuals and gracefully accepting criticism
 18 | * Acknowledging the contributions of others
 19 | * Avoid personal attacks directed toward other participants
 20 | * Be mindful of your surroundings and of your fellow participants
 21 | * Alert UCAR staff and suppliers/vendors if you notice a dangerous situation or someone in distress
 22 | * Respect the rules and policies of the project and venue
 23 | 
 24 | Examples of unacceptable behavior include, but are not limited to:
 25 | 
 26 | * Harassment, intimidation, or discrimination in any form
 27 | * Physical, verbal, or written abuse by anyone to anyone, including repeated use of pronouns other than those requested
 28 | * Unwelcome sexual attention or advances
 29 | * Personal attacks directed at other guests, members, participants, etc.
 30 | * Publishing others' private information, such as a physical or electronic address, without explicit permission
 31 | * Alarming, intimidating, threatening, or hostile comments or conduct
 32 | * Inappropriate use of nudity and/or sexual images
 33 | * Threatening or stalking anyone, including a participant
 34 | * Other conduct which could reasonably be considered inappropriate in a professional setting
 35 | 
 36 | ## Scope
 37 | This Code of Conduct applies to all spaces managed by the Project whether they be physical, online or face-to-face.
 38 | This includes project code, code repository, associated web pages, documentation, mailing lists, project websites and wiki pages,
 39 | issue tracker, meetings, telecons, events, project social media accounts, and any other forums created by the project team which the
 40 | community uses for communication.
 41 | In addition, violations of this Code of Conduct outside these spaces may affect a person's ability to participate within them.
 42 | Representation of a project may be further defined and clarified by project maintainers.
 43 | 
 44 | ## Community Responsibilities
 45 | Everyone in the community is empowered to respond to people who are showing unacceptable behavior.
 46 | They can talk to them privately or publicly.
 47 | Anyone requested to stop unacceptable behavior is expected to comply immediately.
 48 | If the behavior continues concerns may be brought to the project administrators or to any other party listed in the
 49 | [Reporting](#reporting) section below.
 50 | 
 51 | ## Project Administrator Responsibilities
 52 | Project administrators are responsible for clarifying the standards of acceptable behavior and are encouraged to model appropriate
 53 | behavior and provide support when people in the community point out inappropriate behavior.
 54 | Project administrator(s) are normally the ones that would be tasked to carry out the actions in the [Consequences](#consequences)
 55 | section below.
 56 | 
 57 | Project administrators are also expected to keep this Code of Conduct updated with the main one housed at UCAR, as listed below in
 58 | the [Attribution](#attribution) section.
 59 | 
 60 | ## Reporting
 61 | Instances of unacceptable behavior can be brought to the attention of the project administrator(s) who may take any action as
 62 | outlined in the [Consequences](#consequences) section below.
 63 | However, making a report to a project administrator is not considered an 'official report' to UCAR.
 64 | 
 65 | Instances of unacceptable behavior may also be reported directly to UCAR pursuant to [UCAR's Harassment Reporting and Complaint
 66 | Procedure](https://www2.fin.ucar.edu/procedures/hr/harassment-reporting-and-complaint-procedure), or anonymously through [UCAR's
 67 | EthicsPoint Hotline](https://www2.fin.ucar.edu/ethics/anonymous-reporting).
 68 | 
 69 | Complaints received by UCAR will be handled pursuant to the procedures outlined in UCAR's Harassment Reporting and Complaint
 70 | Procedure.
 71 | Complaints to UCAR will be held as confidential as practicable under the circumstances, and retaliation against a person who
 72 | initiates a complaint or an inquiry about inappropriate behavior will not be tolerated.
 73 | 
 74 | Any Contributor can use these reporting methods even if they are not directly affiliated with UCAR.
 75 | The Frequently Asked Questions (FAQ) page for reporting is [here](https://www2.fin.ucar.edu/procedures/hr/reporting-faqs).
 76 | 
 77 | ## Consequences
 78 | Upon receipt of a complaint, the project administrator(s) may take any action deemed necessary and appropriate under the
 79 | circumstances.
 80 | Such action can include things such as: removing, editing, or rejecting comments, commits, code, wiki edits, email, issues, and
 81 | other contributions that are not aligned to this Code of Conduct, or banning temporarily or permanently any contributor for other
 82 | behaviors that are deemed inappropriate, threatening, offensive, or harmful.
 83 | Project administrators also have the right to report violations to UCAR HR and/or UCAR's Office of Diversity, Equity and Inclusion
 84 | (ODEI), as well as a participant's home institution and/or law enforcement.
 85 | In the event an incident is reported to UCAR, UCAR will follow its Harassment Reporting and Complaint Procedure.
 86 | 
 87 | ## Process for Changes
 88 | All UCAR managed projects are required to adopt this Contributor Code of Conduct.
 89 | Adoption is assumed even if not expressly stated in the repository.
 90 | Projects should fill in sections where prompted with project-specific information, including, project name and adoption date.
 91 | 
 92 | Projects that adopt this Code of Conduct need to stay up to date with UCAR's Contributor Code of Conduct, linked with a DOI in the
 93 | [Attribution](#attribution) section below.
 94 | Projects can make limited substantive changes to the Code of Conduct, however, the changes must be limited in scope and may not
 95 | contradict the UCAR Contributor Code of Conduct.
 96 | 
 97 | ## Attribution
 98 | This Code of Conduct was originally adapted from the [Contributor Covenant](http://contributor-covenant.org/version/1/4), version
 99 | 1.4.
100 | We then aligned it with the UCAR Participant Code of Conduct, which also borrows from the American Geophysical Union (AGU) Code of
101 | Conduct.
102 | The UCAR Participant Code of Conduct applies to both UCAR employees as well as participants in activities run by UCAR.
103 | The original version of this for all software projects that have strong management from UCAR or UCAR staff is available on the UCAR
104 | website at https://doi.org/10.5065/6w2c-a132.
105 | The date that it was adopted by this project was **Feb/13/2018**.
106 | When responding to complaints, UCAR HR and ODEI will do so based on the latest published version.
107 | Therefore, any project-specific changes should follow the [Process for Changes](#process-for-changes) section above.
108 | 


--------------------------------------------------------------------------------
/doc/source/downloading_cesm.rst:
--------------------------------------------------------------------------------
  1 | .. _downloading:
  2 | 
  3 | =============================
  4 | Downloading CESM2 (|version|)
  5 | =============================
  6 | 
  7 | Downloading the code and scripts
  8 | --------------------------------
  9 | 
 10 | Starting with CESM2, releases are available through a public GitHub
 11 | repository, `http://github.com/ESCOMP/CESM <http://github.com/ESCOMP/CESM>`_. 
 12 | 
 13 | Access to the code requires both git and Subversion client software in
 14 | place that is compatible with GitHub and our Subversion server
 15 | software.  You will need access to the command line clients, ``git``
 16 | (v1.8 or greater) and ``svn`` (v1.8 or greater but less than v1.11).  
 17 | Currently, our Subversion server
 18 | software is at version 1.8.17. For more information or to download
 19 | open source tools, visit `Subversion <http://subversion.apache.org/>`_
 20 | and `git downloads <https://git-scm.com/downloads>`_.
 21 | 
 22 | With valid git and svn clients installed on the machine where CESM2 will be
 23 | built and run, the user may download the latest version of the release
 24 | code:
 25 | 
 26 | .. code-block:: console
 27 | 
 28 |     git clone https://github.com/ESCOMP/CESM.git my_cesm_sandbox
 29 |     cd my_cesm_sandbox
 30 | 
 31 | To checkout a previous version of CESM2, first view the available versions:
 32 | 
 33 | .. code-block:: console
 34 | 
 35 |     git tag --list 'release-cesm2*'
 36 | 
 37 | To checkout a specific CESM2 release tag type, for example CESM2.0.1:
 38 | 
 39 | .. code-block:: console 
 40 | 
 41 |     git checkout release-cesm2.0.1
 42 | 
 43 | Finally, to checkout all the individual model components,
 44 | run the **checkout_externals** script from /path/to/my_cesm_sandbox.
 45 | 
 46 | .. code-block:: console
 47 | 
 48 |     ./manage_externals/checkout_externals
 49 | 
 50 | The **checkout_externals** script will read the configuration file called ``Externals.cfg`` and
 51 | will download all the external component models and CIME into /path/to/my_cesm_sandbox. 
 52 | 
 53 | Details regarding the CESM2 checkout process are available in the CESM GitHub repo
 54 | `README <http://github.com/ESCOMP/CESM/blob/master/README.rst>`_
 55 | To see more details regarding the checkout_externals script from the command line, type:
 56 | 
 57 | .. code-block:: console
 58 | 
 59 |     ./manage_externals/checkout_externals --help
 60 | 
 61 | 
 62 | .. warning:: When contacting the Subversion server for the first time, you may need to
 63 |              accept an authentication certification. If you experience problems such as
 64 |              ``checkout_externals`` hanging: Run ``svn ls
 65 |              https://svn-ccsm-models.cgd.ucar.edu/ww3/release_tags``, permanently
 66 |              accepting the certificate when prompted, then retry the CESM download
 67 |              (starting over at the top of these instructions).
 68 | 
 69 | .. warning:: If a problem was encountered during checkout_externals, which may happen with an older version of the svn client software, it may appear to have downloaded successfully, but in fact only a partial checkout has occurred. 
 70 | 
 71 | To confirm a successful download of all components, you can run ``checkout_externals``
 72 | with the status flag to show the status of the externals:
 73 | 
 74 | .. code-block:: console
 75 | 
 76 |     ./manage_externals/checkout_externals -S
 77 | 
 78 | This should show a clean status for all externals, with no characters in the first two
 79 | columns of output, as in this example:
 80 | 
 81 | .. code-block:: console
 82 | 
 83 |    Processing externals description file : Externals.cfg
 84 |    Processing externals description file : Externals_CLM.cfg
 85 |    Processing externals description file : ../Externals_cime.cfg
 86 |    Processing externals description file : Externals_POP.cfg
 87 |    Processing externals description file : Externals_CISM.cfg
 88 |    Processing externals description file : .gitmodules
 89 |    Processing submodules description file : .gitmodules
 90 |    Processing externals description file : Externals_CAM.cfg
 91 |    Checking status of externals: clm, fates, ptclm, mosart, cime, cmeps, ww3, cice, fms, pop, cvmix, marbl, cism, source_cism, rtm, cdeps, fox, mom, cam, silhs, clubb, pumas, atmos_phys, cosp2, chem_proc, atmos_cubed_sphere, carma, 
 92 |        ./cime
 93 |    e-o ./cime/src/drivers/nuopc/
 94 |        ./components/cam
 95 |        ./components/cam/chem_proc
 96 |        ./components/cam/src/atmos_phys
 97 |        ./components/cam/src/dynamics/fv3/atmos_cubed_sphere
 98 |        ./components/cam/src/physics/carma/base
 99 |        ./components/cam/src/physics/clubb
100 |        ./components/cam/src/physics/cosp2/src
101 |        ./components/cam/src/physics/pumas
102 |        ./components/cam/src/physics/silhs
103 |        ./components/cdeps
104 |        ./components/cdeps/fox
105 |        ./components/cice
106 |        ./components/cism
107 |        ./components/cism/source_cism
108 |        ./components/clm
109 |        ./components/clm/src/fates
110 |        ./components/clm/tools/PTCLM
111 |    e-o ./components/mom
112 |        ./components/mosart
113 |        ./components/pop
114 |        ./components/pop/externals/CVMix
115 |        ./components/pop/externals/MARBL
116 |        ./components/rtm
117 |        ./components/ww3
118 |    e-o ./libraries/FMS
119 | 
120 | 
121 | You should now have a default copy of the CESM2 source code in your /path/to/my_cesm_sandbox.
122 | 
123 | These components are optional and are not needed to run CESM2.
124 | 
125 | .. code-block:: console
126 | 
127 |    e-o ./cime/src/drivers/nuopc/
128 |    e-o ./components/mom
129 |    e-o ./libraries/FMS
130 | 
131 | 
132 | If there were problems obtaining an external, you might instead see something like:
133 | 
134 | .. code-block:: console
135 | 
136 |    e-  ./components/cam
137 | 
138 | This might happen if there was an unexpected interruption while downloading.  
139 | First try rerunning ``./manage_externals/checkout_externals``.
140 | If there is still a problem, try running with logging turned on using:
141 | 
142 | .. code-block:: console
143 | 
144 |    ./manage_externals/checkout_externals --logging
145 | 
146 | Check the ``manage_externals.log`` file to see what errors are reported.
147 | 
148 | Downloading input data
149 | ----------------------
150 | 
151 | Input datasets are needed to run the model. CESM input data are
152 | available through a separate Subversion input data repository.
153 | 
154 | .. warning:: The input data repository contains datasets for many configurations and resolutions and is well over 10 TByte in total size. DO NOT try to download the entire dataset.
155 | 
156 | Datasets can be downloaded on a case by case basis as needed and CESM
157 | provides tools to check and download input data automatically.
158 | 
159 | A local input data directory should exist on the local disk, and it also 
160 | needs to be set in the CESM scripts via the variable ``$DIN_LOC_ROOT.``
161 | For supported machines, this variable is preset. For generic machines,
162 | this variable is set via the ``--input-dir`` argument to **create_newcase**.
163 | It is recommended that all users of a given filesystem share the same ``$DIN_LOC_ROOT`` directory.
164 | 
165 | The files in the subdirectories of ``$DIN_LOC_ROOT`` should be
166 | write-protected. This prevents these files from being accidentally
167 | modified or deleted. The directories in ``$DIN_LOC_ROOT`` should generally
168 | be group writable, so the directory can be shared among multiple users.
169 | 
170 | As part of the process of generating the CESM executable, the utility,
171 | **check_input_data** located in each case directory
172 | is called, and it attempts to locate all required input data for the
173 | case based upon file lists generated by components. If the required
174 | data is not found on local disk in ``$DIN_LOC_ROOT``, then the data
175 | will be downloaded automatically by the scripts or it can be
176 | downloaded by the user by invoking **check_input_data** with the ``--download``
177 | command argument. If you want to download the input data manually you
178 | should do it before you build CESM.
179 | 
180 | It is possible for users to download the data using svn subcommands
181 | directly, but use of the **check_input_data** script is highly recommended
182 | to ensure that only the required datasets are downloaded. 
183 | 
184 | .. warning:: Again, users are **STRONGLY DISCOURAGED** from downloading the entire input dataset from the repository.
185 | 
186 | 


--------------------------------------------------------------------------------
/tools/statistical_ensemble_test/ensemble.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | from __future__ import print_function
  3 | import os, sys, getopt
  4 | import random
  5 | from single_run import process_args_dict, single_case
  6 | 
  7 | # ==============================================================================
  8 | # set up and submit 12-month (original) or 9-time step (uf) run.  then create
  9 | # clones for a complete ensemble or a set of (3) test cases
 10 | # ==============================================================================
 11 | 
 12 | # generate <num_pick> positive random integers in [0, end-1]
 13 | # can't have any duplicates
 14 | def random_pick(num_pick, end):
 15 |     ar = range(0, end)
 16 |     rand_list = random.sample(ar, num_pick)
 17 |     # for i in rand_list:
 18 |     #    print i
 19 |     return rand_list
 20 | 
 21 | 
 22 | # get the pertlim corressponding to the random int
 23 | # modified 10/23 to go to 2000 (previously only 1st 900 unique)
 24 | def get_pertlim_uf(rand_num):
 25 |     i = rand_num
 26 |     if i == 0:
 27 |         ptlim = 0
 28 |     elif i > 2000:
 29 |         print("don't support sizes > 2000")
 30 |     elif i <= 1800:  # [1 - 1800]
 31 |         if i <= 900:  # [1-900]
 32 |             j = 2 * int((i - 1) / 100) + 101
 33 |         elif i <= 1000:  # [901 - 1000]
 34 |             j = 2 * int((i - 1) / 100) + 100
 35 |         elif i <= 1800:  # [1001-1800]
 36 |             j = 2 * int((i - 1001) / 100) + 102
 37 |         k = (i - 1) % 100  # this is just last 2 digits of i-1
 38 |         if i % 2 != 0:  # odd
 39 |             ll = j + int(k / 2) * 18
 40 |             ippt = str(ll).zfill(3)
 41 |             ptlim = '0.' + ippt + 'd-13'
 42 |         else:  # even
 43 |             ll = j + int((k - 1) / 2) * 18
 44 |             ippt = str(ll).zfill(3)
 45 |             ptlim = '-0.' + ippt + 'd-13'
 46 |     else:  # [1801 - 2000]
 47 |         if i <= 1900:  # [1801-1900]
 48 |             j = 1
 49 |         else:  # [1901-2000]
 50 |             j = 2
 51 |         k = (i - 1) % 100
 52 |         if i % 2 != 0:  # odd
 53 |             ll = j + int(k / 2) * 2
 54 |             ippt = str(ll).zfill(3)
 55 |             ptlim = '0.' + ippt + 'd-13'
 56 |         else:  # even
 57 |             ll = j + int((k - 1) / 2) * 2
 58 |             ippt = str(ll).zfill(3)
 59 |             ptlim = '-0.' + ippt + 'd-13'
 60 | 
 61 |     return ptlim
 62 | 
 63 | 
 64 | def main(argv):
 65 | 
 66 |     caller = "ensemble.py"
 67 | 
 68 |     # directory with single_run.py and ensemble.py
 69 |     stat_dir = os.path.dirname(os.path.realpath(__file__))
 70 |     print("STATUS: stat_dir = " + stat_dir)
 71 | 
 72 |     opts_dict, case_flags = process_args_dict(caller, argv)
 73 | 
 74 |     # default is verification mode (3 runs)
 75 |     run_type = "verify"
 76 |     if opts_dict["ect"] == "pop":
 77 |         clone_count = 0
 78 |     else:
 79 |         clone_count = 2
 80 | 
 81 |     uf = opts_dict["uf"]
 82 | 
 83 |     # check for run_type change (i.e., if doing ensemble instead of verify)
 84 |     ens_size = opts_dict["ensemble"]
 85 |     if ens_size > 0:
 86 |         run_type = "ensemble"
 87 |         clone_count = ens_size - 1
 88 |         if ens_size > 2000:
 89 |             print("Error: cannot have an ensemble size greater than 2000.")
 90 |             sys.exit()
 91 |         print("STATUS: ensemble size = " + str(ens_size))
 92 | 
 93 |     #where to start ensemble
 94 |     start = opts_dict["ens_start"]
 95 |     if start < 0:
 96 |         start = 0
 97 |     if run_type == "ensemble":
 98 |         if start >= ens_size:    
 99 |             print("Error: cannot start the ensemble at a number larger than the ensemble size.")
100 |             sys.exit()
101 |         print("STATUS: ensemble start = " + str(start))
102 |     else:
103 |         #don't allow a mid start when doing verifcation runs
104 |         start = 0
105 | 
106 | 
107 |     # generate random pertlim(s) for verify
108 |     if run_type == "verify":
109 |         if opts_dict["ect"] == "pop":
110 |             rand_ints = random_pick(1, 40)
111 |         else:  # cam
112 |             if uf:
113 |                 end_range = 350
114 |             else:
115 |                 end_range = 150
116 |             rand_ints = random_pick(3, end_range)
117 | 
118 |     # now create cases
119 |     # create first case - then clone
120 |     if run_type == "verify":
121 |         opts_dict["pertlim"] = get_pertlim_uf(rand_ints[0])
122 |     else:  # full ensemble
123 |         opts_dict["pertlim"] = "0"
124 | 
125 |     # first case
126 |     if start == 0:
127 |         print("STATUS: creating first case ...")
128 |         single_case(opts_dict, case_flags, stat_dir)
129 |         begin_i = 1
130 |     else :
131 |         begin_i = start
132 | 
133 |     # clone?
134 |     if clone_count > 0:
135 | 
136 |         # now clone
137 |         print("STATUS: cloning additional cases ...")
138 | 
139 |         # scripts dir
140 |         print("STATUS: stat_dir = " + stat_dir)
141 |         ret = os.chdir(stat_dir)
142 |         ret = os.chdir("../../cime/scripts")
143 |         scripts_dir = os.getcwd()
144 |         print("STATUS: scripts dir = " + scripts_dir)
145 | 
146 |         # we know case name ends in '.0000' (already checked)
147 |         clone_case = opts_dict["case"]
148 |         case_pfx = clone_case[:-5]
149 | 
150 |         for i in range(begin_i, clone_count + 1):  # 1: clone_count
151 |             if run_type == "verify":
152 |                 this_pertlim = get_pertlim_uf(rand_ints[i])
153 |             else:  # full ensemble
154 |                 this_pertlim = get_pertlim_uf(i)
155 | 
156 | 
157 |             #allow for 4 digit numbers
158 |             iens = '{0:04d}'.format(i)
159 |             new_case = case_pfx + "." + iens
160 | 
161 |             os.chdir(scripts_dir)
162 |             print("STATUS: creating new cloned case: " + new_case)
163 | 
164 |             clone_args = " --keepexe --case " + new_case + " --clone " + clone_case
165 |             print("        with args: " + clone_args)
166 | 
167 |             command = scripts_dir + "/create_clone" + clone_args
168 |             ret = os.system(command)
169 | 
170 |             print("STATUS: running setup for new cloned case: " + new_case)
171 |             os.chdir(new_case)
172 |             command = "./case.setup"
173 |             ret = os.system(command)
174 | 
175 |             # adjust perturbation
176 |             if opts_dict["ect"] == "pop":
177 |                 if run_type == "verify":  # remove old init_ts_perturb
178 |                     f = open("user_nl_pop", "r+")
179 |                     all_lines = f.readlines()
180 |                     f.seek(0)
181 |                     for line in all_lines:
182 |                         if line.find("init_ts_perturb") == -1:
183 |                             f.write(line)
184 |                     f.truncate()
185 |                     f.close()
186 |                     text = "init_ts_perturb = " + this_pertlim
187 |                 else:
188 |                     text = "\ninit_ts_perturb = " + this_pertlim
189 | 
190 |                 # now append new pertlim
191 |                 with open("user_nl_pop", "a") as f:
192 |                     f.write(text)
193 | 
194 |             else:
195 |                 if run_type == "verify":  # remove old pertlim first
196 |                     f = open("user_nl_cam", "r+")
197 |                     all_lines = f.readlines()
198 |                     f.seek(0)
199 |                     for line in all_lines:
200 |                         if line.find("pertlim") == -1:
201 |                             f.write(line)
202 |                     f.truncate()
203 |                     f.close()
204 |                     text = "pertlim = " + this_pertlim
205 |                 else:
206 |                     text = "\npertlim = " + this_pertlim
207 | 
208 |                 # now append new pertlim
209 |                 with open("user_nl_cam", "a") as f:
210 |                     f.write(text)
211 | 
212 |             # preview namelists
213 |             command = "./preview_namelists"
214 |             ret = os.system(command)
215 | 
216 |             # submit?
217 |             if opts_dict["ns"] == False:
218 |                 command = "./case.submit"
219 |                 ret = os.system(command)
220 | 
221 |     # Final output
222 |     if run_type == "verify":
223 |         if opts_dict["ect"] == "pop":
224 |             print("STATUS: ---POP-ECT VERIFICATION CASE COMPLETE---")
225 |             print("Set up one case using the following init_ts_perturb value:")
226 |             print(get_pertlim_uf(rand_ints[0]))
227 |         else:
228 |             print("STATUS: ---CAM-ECT VERIFICATION CASES COMPLETE---")
229 |             print("Set up three cases using the following pertlim values:")
230 |             print(
231 |                 get_pertlim_uf(rand_ints[0])
232 |                 + "   "
233 |                 + get_pertlim_uf(rand_ints[1])
234 |                 + "   "
235 |                 + get_pertlim_uf(rand_ints[2])
236 |             )
237 |     else:
238 |         print("STATUS: --ENSEMBLE CASES COMPLETE---")
239 | 
240 | 
241 | if __name__ == "__main__":
242 |     main(sys.argv[1:])
243 | 


--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
  1 | ==================================
  2 |  The Community Earth System Model
  3 | ==================================
  4 | 
  5 | See the CESM web site for documentation and information:
  6 | 
  7 | http://www.cesm.ucar.edu
  8 | 
  9 | The CESM Quickstart Guide is available at:
 10 | 
 11 | http://escomp.github.io/cesm
 12 | 
 13 | This repository provides tools for managing the external components that
 14 | make up a CESM tag - alpha, beta and release. CESM tag creation should
 15 | be coordinated through CSEG at NCAR.
 16 | 
 17 | This repository is also connected to slack at http://cesm2.slack.com
 18 | 
 19 | .. sectnum::
 20 | 
 21 | .. contents::
 22 | 
 23 | Software requirements
 24 | =====================
 25 | 
 26 | Software requirements for installing, building and running CESM
 27 | ---------------------------------------------------------------
 28 | 
 29 | Installing, building and running CESM requires:
 30 | 
 31 | * a Unix-like operating system (Linux, AIX, OS X, etc.)
 32 | 
 33 | * git client version 1.8 or newer
 34 | 
 35 | * python3 version 3.8 or newer
 36 | 
 37 | * perl version 5
 38 | 
 39 | * build tools gmake and cmake
 40 | 
 41 | * Fortran and C compilers
 42 | 
 43 |   * See `Details on Fortran compiler versions`_ below for more information
 44 | 
 45 | * LAPACK and BLAS libraries
 46 | 
 47 | * a NetCDF library version 4.3 or newer built with the same compiler you
 48 |   will use for CESM
 49 | 
 50 |   * a PnetCDF library is optional, but recommended
 51 | 
 52 | * a functioning MPI environment (unless you plan to run on a single core
 53 |   with the CIME mpi-serial library)
 54 | 
 55 | Details on Fortran compiler versions
 56 | ------------------------------------
 57 | The Fortran compiler must support Fortran 2003 features. However, even
 58 | among mainstream Fortran compilers that claim to support Fortran 2003,
 59 | we have found numerous bugs. Thus, many compiler versions do *not* build
 60 | or run CESM properly (see
 61 | https://wiki.ucar.edu/display/ccsm/Fortran+Compiler+Bug+List for more
 62 | details on older Fortran compiler versions).
 63 | 
 64 | CESM2 is tested on several different systems with newer Fortran compilers:
 65 | Please see `CESM Compiler/Machine Tests <https://docs.google.com/spreadsheets/d/15QUqsXD1Z0K_rYNTlykBvjTRt8s0XcQw0cfAj9DZbj0/edit#gid=0>`_
 66 | for a spreadsheet of the current results.
 67 | 
 68 | More details on porting CESM
 69 | ----------------------------
 70 | 
 71 | For more details on porting CESM to a new machine, see
 72 | http://esmci.github.io/cime/versions/master/html/users_guide/porting-cime.html
 73 | 
 74 | Obtaining the full model code and associated scripting infrastructure
 75 | =====================================================================
 76 | 
 77 | CESM is now released via github. You will need some familiarity with git in order
 78 | to modify the code and commit these changes. However, to simply checkout and run the
 79 | code, no git knowledge is required other than what is documented in the following steps.
 80 | 
 81 | To obtain the CESM code you need to do the following:
 82 | 
 83 | #. Clone the repository. ::
 84 | 
 85 |       git clone https://github.com/escomp/cesm.git my_cesm_sandbox
 86 | 
 87 |    This will create a directory ``my_cesm_sandbox/`` in your current working directory.
 88 | 
 89 | #. Go into the newly created CESM repository and determine what version of CESM you want.
 90 |    To see what cesm tags are available, simply issue the **git tag** command. ::
 91 | 
 92 |       cd my_cesm_sandbox
 93 |       git tag
 94 | 
 95 | #. Do a git checkout of the tag you want. If you want to checkout cesm3_0_beta02, you would issue the following. ::
 96 | 
 97 |       git checkout cesm3_0_beta02
 98 | 
 99 |    (It is normal and expected to get a message about being in 'detached
100 |    HEAD' state. For now you can ignore this, but it becomes important if
101 |    you want to make changes to your Externals.cfg file and commit those
102 |    changes to a branch.)
103 | 
104 | #. Run the script **bin/git-fleximod update**. ::
105 | 
106 |       ./bin/git-fleximod update
107 | 
108 |    The **git fleximod** script is a git extension that will
109 |    populate the cesm directory with the relevant versions of each of the
110 |    components along with the CIME infrastructure code.
111 | 
112 | At this point you have a working version of CESM.
113 | 
114 | To see full details of how to set up a case, compile and run, see the CIME documentation at http://esmci.github.io/cime/ .
115 | 
116 | More details on git fleximod
117 | ----------------------------------
118 | 
119 | The file **.gitmodules** in your top-level CESM directory tells
120 | **git fleximod** which tag/branch of each component should be
121 | brought in to generate your sandbox.
122 | 
123 | NOTE: git fleximod will always attempt
124 | to make the working copy exactly match the externals description. For
125 | example, if you manually modify an external without updating .gitmodules,
126 | (e.g. switch to a different tag), then rerunning git fleximod
127 | will warn you and can restore the original version by using the --force option
128 | below documentation `Customizing your CESM sandbox`_ for more details.
129 | 
130 | **You need to rerun git-fleximod update whenever .gitmodules has
131 | changed** (unless you have already manually updated the relevant
132 | external(s) to have the correct branch/tag checked out). Common times
133 | when this is needed are:
134 | 
135 | * After checking out a new CESM branch/tag
136 | 
137 | * After merging some other CESM branch/tag into your currently
138 |   checked-out branch
139 | 
140 | To see more details of **git-fleximod**, issue ::
141 | 
142 |   ./bin/git-fleximod --help
143 | 
144 | Customizing your CESM sandbox
145 | =============================
146 | 
147 | There are several use cases to consider when you want to customize or modify your CESM sandbox.
148 | 
149 | Switching to a different CESM tag
150 | ---------------------------------
151 | 
152 | If you have already checked out a tag and **HAVE NOT MADE ANY
153 | MODIFICATIONS** it is simple to change your sandbox. Say that you
154 | checked out cesm3_0_beta01 but really wanted to have cesm3_0_beta02
155 | you would simply do the following::
156 | 
157 |   git checkout cesm3_0_beta02
158 |   ./bin/git-fleximod update
159 | 
160 | You should **not** use this method if you have any ongoing CESM cases that were created from
161 | this sandbox. In these cases, it is often easiest to do a second **git
162 | clone**.
163 | 
164 | Pointing to a different version of a component
165 | ----------------------------------------------
166 | 
167 | Each entry in **.gitmodules** has the following form (we use CAM as an
168 | example below)::
169 | 
170 |   [submodule "cam"]
171 |     path = components/cam
172 |     url = https://www.github.com/ESCOMP/CAM
173 |     fxDONOTUSEurl = https://www.github.com/ESCOMP/CAM
174 |     fxtag = cam6_4_016
175 |     fxrequired = ToplevelRequired
176 | 
177 | Each entry specifies either a tag or a hash. To point to a new tag:
178 | 
179 | #. Modify the relevant fxtag entry/entries in **.gitmodules** (e.g., changing
180 |    ``cam6_4_016`` to ``cam6_4_017`` above)
181 | 
182 | #. Checkout the new component(s)::
183 | 
184 |      ./bin/git-fleximod update cam
185 | 
186 | Keep in mind that changing individual components from a tag may result
187 | in an invalid model (won't compile, won't run, not scientifically
188 | meaningful) and is unsupported.
189 | 
190 | Committing your change to .gitmodules
191 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
192 | 
193 | After making this change, it's a good idea to commit the change in your
194 | local CESM git repository. First create a CESM branch in your local
195 | repository, then commit it.  For example::
196 | 
197 |   git checkout -b my_cesm_branch
198 |   git add .gitmodules components/cam
199 |   git commit -m "Update CAM to cam5_4_144"
200 | 
201 | Modifying a component
202 | ---------------------
203 | 
204 | If you'd like to modify a component via a branch and point to that
205 | branch in your CESM sandbox, use the following procedure (again, using
206 | CAM as an example):
207 | 
208 | #. Create a CAM branch. Let's assume you have created this branch and called it
209 |    **my_branch**.
210 | 
211 | #. Update **.gitmodules** to point to a hash on your branch. You can replace the
212 |    **tag** entry with a **hash** entry, as follows, note that we have also changed the url to
213 |    point to a personal fork::
214 | 
215 |     [submodule "cam"]
216 |       path = components/cam
217 |       url = https://www.github.com/mycamfork/CAM
218 |       fxDONOTUSEurl = https://www.github.com/ESCOMP/CAM
219 |       fxtag = 94eaf83
220 |       fxrequired = ToplevelRequired
221 | 
222 | 
223 | #. Checkout your branch::
224 | 
225 |      ./bin/git-fleximod update cam
226 | 
227 | It's a good idea to commit your **.gitmodules** file changes. See the above
228 | documentation, `Committing your change to .gitmodules`_.
229 | 
230 | Developer setup
231 | ===============
232 | 
233 | Developers who have not already done so should follow the recommended
234 | `one-time <https://github.com/esmci/cime/wiki/CIME-Git-Workflow#configure-git-one-time>`_
235 | setup directions for git. Developers may also want to set up
236 | `ssh <https://help.github.com/articles/connecting-to-github-with-ssh/>`_
237 | keys and switch to using the ``git@github.com:ESCOMP/cesm.git`` form of the github URLs.
238 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/git_fleximod/metoflexi.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | from configparser import ConfigParser
  3 | import sys
  4 | import shutil
  5 | from pathlib import Path
  6 | import argparse
  7 | import logging
  8 | from git_fleximod.gitinterface import GitInterface
  9 | from git_fleximod.gitmodules import GitModules
 10 | from git_fleximod import utils
 11 | 
 12 | logger = None
 13 | 
 14 | 
 15 | def find_root_dir(filename=".git"):
 16 |     d = Path.cwd()
 17 |     root = Path(d.root)
 18 |     while d != root:
 19 |         attempt = d / filename
 20 |         if attempt.is_dir():
 21 |             return d
 22 |         d = d.parent
 23 |     return None
 24 | 
 25 | 
 26 | def get_parser():
 27 |     description = """
 28 |     %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models
 29 |     """
 30 |     parser = argparse.ArgumentParser(
 31 |         description=description, formatter_class=argparse.RawDescriptionHelpFormatter
 32 |     )
 33 | 
 34 |     parser.add_argument(
 35 |         "-e",
 36 |         "--externals",
 37 |         nargs="?",
 38 |         default="Externals.cfg",
 39 |         help="The externals description filename. " "Default: %(default)s.",
 40 |     )
 41 | 
 42 |     parser.add_argument(
 43 |         "-C",
 44 |         "--path",
 45 |         default=find_root_dir(),
 46 |         help="Toplevel repository directory.  Defaults to top git directory relative to current.",
 47 |     )
 48 | 
 49 |     parser.add_argument(
 50 |         "-g",
 51 |         "--gitmodules",
 52 |         nargs="?",
 53 |         default=".gitmodules",
 54 |         help="The submodule description filename. " "Default: %(default)s.",
 55 |     )
 56 |     parser.add_argument(
 57 |         "-v",
 58 |         "--verbose",
 59 |         action="count",
 60 |         default=0,
 61 |         help="Output additional information to "
 62 |         "the screen and log file. This flag can be "
 63 |         "used up to two times, increasing the "
 64 |         "verbosity level each time.",
 65 |     )
 66 |     parser.add_argument(
 67 |         "-d",
 68 |         "--debug",
 69 |         action="store_true",
 70 |         default=False,
 71 |         help="DEVELOPER: output additional debugging "
 72 |         "information to the screen and log file.",
 73 |     )
 74 | 
 75 |     return parser
 76 | 
 77 | 
 78 | def commandline_arguments(args=None):
 79 |     parser = get_parser()
 80 | 
 81 |     options = parser.parse_args(args)
 82 |     handlers = [logging.StreamHandler()]
 83 | 
 84 |     if options.debug:
 85 |         try:
 86 |             open("fleximod.log", "w")
 87 |         except PermissionError:
 88 |             sys.exit("ABORT: Could not write file fleximod.log")
 89 |         level = logging.DEBUG
 90 |         handlers.append(logging.FileHandler("fleximod.log"))
 91 |     elif options.verbose:
 92 |         level = logging.INFO
 93 |     else:
 94 |         level = logging.WARNING
 95 |     # Configure the root logger
 96 |     logging.basicConfig(
 97 |         level=level, format="%(name)s - %(levelname)s - %(message)s", handlers=handlers
 98 |     )
 99 | 
100 |     return (options.path, options.gitmodules, options.externals)
101 | 
102 | 
103 | class ExternalRepoTranslator:
104 |     """
105 |     Translates external repositories configured in an INI-style externals file.
106 |     """
107 | 
108 |     def __init__(self, rootpath, gitmodules, externals):
109 |         self.rootpath = rootpath
110 |         if gitmodules:
111 |             self.gitmodules = GitModules(logger, confpath=rootpath)
112 |         self.externals = (rootpath / Path(externals)).resolve()
113 |         print(f"Translating {self.externals}")
114 |         self.git = GitInterface(rootpath, logger)
115 | 
116 |     #    def __del__(self):
117 |     #        if (self.rootpath / "save.gitignore"):
118 | 
119 |     def translate_single_repo(
120 |         self, section, tag, url, path, efile, hash_, sparse, protocol
121 |     ):
122 |         """
123 |         Translates a single repository based on configuration details.
124 | 
125 |         Args:
126 |             rootpath (str): Root path of the main repository.
127 |             gitmodules (str): Path to the .gitmodules file.
128 |             tag (str): The tag to use for the external repository.
129 |             url (str): The URL of the external repository.
130 |             path (str): The relative path within the main repository for the external repository.
131 |             efile (str): The external file or file containing submodules.
132 |             hash_ (str): The commit hash to checkout (if applicable).
133 |             sparse (str):  Boolean indicating whether to use sparse checkout (if applicable).
134 |             protocol (str): The protocol to use (e.g., 'git', 'http').
135 |         """
136 |         assert protocol != "svn", "SVN protocol is not currently supported"
137 |         print(f"Translating repository {section}")
138 |         if efile:
139 |             file_path = Path(path) / Path(efile)
140 |             newroot = (self.rootpath / file_path).parent.resolve()
141 |             if not newroot.exists():
142 |                 newroot.mkdir(parents=True)
143 |             logger.info("Newroot is {}".format(newroot))
144 |             newt = ExternalRepoTranslator(newroot, ".gitmodules", efile)
145 |             newt.translate_repo()
146 |         if protocol == "externals_only":
147 |             if tag:
148 |                 self.gitmodules.set(section, "fxtag", tag)
149 |             if hash_:
150 |                 self.gitmodules.set(section, "fxtag", hash_)
151 | 
152 |             self.gitmodules.set(section, "fxDONOTUSEurl", url)
153 |             if sparse:
154 |                 self.gitmodules.set(section, "fxsparse", sparse)
155 |             self.gitmodules.set(section, "fxrequired", "ToplevelRequired")
156 |         else:
157 |             newpath = self.rootpath / Path(path)
158 |             if newpath.exists():
159 |                 shutil.rmtree(newpath)
160 |                 logger.info("Creating directory {}".format(newpath))
161 |             newpath.mkdir(parents=True)
162 |             if tag:
163 |                 logger.info("cloning {}".format(section))
164 |                 try:
165 |                     self.git.git_operation(
166 |                         "clone", "-b", tag, "--depth", "1", url, path
167 |                     )
168 |                 except:
169 |                     self.git.git_operation("clone", url, path)
170 |                     with utils.pushd(newpath):
171 |                         ngit = GitInterface(newpath, logger)
172 |                         ngit.git_operation("checkout", tag)
173 |             if hash_:
174 |                 self.git.git_operation("clone", url, path)
175 |                 git = GitInterface(newpath, logger)
176 |                 git.git_operation("fetch", "origin")
177 |                 git.git_operation("checkout", hash_)
178 |             if sparse:
179 |                 print("setting as sparse submodule {}".format(section))
180 |                 sparsefile = newpath / Path(sparse)
181 |                 newfile = newpath / ".git" / "info" / "sparse-checkout"
182 |                 print(f"sparsefile {sparsefile} newfile {newfile}")
183 |                 shutil.copy(sparsefile, newfile)
184 | 
185 |             logger.info("adding submodule {}".format(section))
186 |             self.gitmodules.save()
187 |             self.git.git_operation(
188 |                 "submodule", "add", "-f", "--name", section, url, path
189 |             )
190 |             self.git.git_operation("submodule", "absorbgitdirs")
191 |             self.gitmodules.reload()
192 |             if tag:
193 |                 self.gitmodules.set(section, "fxtag", tag)
194 |             if hash_:
195 |                 self.gitmodules.set(section, "fxtag", hash_)
196 | 
197 |             self.gitmodules.set(section, "fxDONOTUSEurl", url)
198 |             if sparse:
199 |                 self.gitmodules.set(section, "fxsparse", sparse)
200 |             self.gitmodules.set(section, "fxrequired", "ToplevelRequired")
201 | 
202 |     def translate_repo(self):
203 |         """
204 |         Translates external repositories defined within an external file.
205 | 
206 |         Args:
207 |             rootpath (str): Root path of the main repository.
208 |             gitmodules (str): Path to the .gitmodules file.
209 |             external_file (str): The path to the external file containing repository definitions.
210 |         """
211 |         econfig = ConfigParser()
212 |         econfig.read((self.rootpath / Path(self.externals)))
213 | 
214 |         for section in econfig.sections():
215 |             if section == "externals_description":
216 |                 logger.info("skipping section {}".format(section))
217 |                 return
218 |             logger.info("Translating section {}".format(section))
219 |             tag = econfig.get(section, "tag", raw=False, fallback=None)
220 |             url = econfig.get(section, "repo_url", raw=False, fallback=None)
221 |             path = econfig.get(section, "local_path", raw=False, fallback=None)
222 |             efile = econfig.get(section, "externals", raw=False, fallback=None)
223 |             hash_ = econfig.get(section, "hash", raw=False, fallback=None)
224 |             sparse = econfig.get(section, "sparse", raw=False, fallback=None)
225 |             protocol = econfig.get(section, "protocol", raw=False, fallback=None)
226 | 
227 |             self.translate_single_repo(
228 |                 section, tag, url, path, efile, hash_, sparse, protocol
229 |             )
230 | 
231 | 
232 | def _main():
233 |     rootpath, gitmodules, externals = commandline_arguments()
234 |     global logger
235 |     logger = logging.getLogger(__name__)
236 |     with utils.pushd(rootpath):
237 |         t = ExternalRepoTranslator(Path(rootpath), gitmodules, externals)
238 |         logger.info("Translating {}".format(rootpath))
239 |         t.translate_repo()
240 | 
241 | 
242 | if __name__ == "__main__":
243 |     sys.exit(_main())
244 | 


--------------------------------------------------------------------------------
/doc/source/quickstart.rst:
--------------------------------------------------------------------------------
  1 | .. _quickstart:
  2 | 
  3 | ============================================
  4 | Quick Start: CESM Model Workflow (|version|)
  5 | ============================================
  6 | 
  7 | The following quick start guide is for versions of CESM2 that have
  8 | already been ported to the local target machine. CESM2 is built on the
  9 | CIME (Common Infrastructure for Modeling Earth) framework.
 10 | Please refer to the `CIME Porting Documentation <http://esmci.github.io/cime/versions/master/html/users_guide/porting-cime.html>`_ if CIME has not
 11 | yet been ported to the target machine. 
 12 | 
 13 | If you are new to CESM2, please consider reading the
 14 | `CIME Case Control System Part 1: Basic Usage guide <https://esmci.github.io/cime/index.html>`_ first.
 15 | 
 16 | This is the procedure for quickly setting up and running a CESM2 case.
 17 | 
 18 | Download CESM2 (see `Downloading CESM2 <downloading_cesm.html>`_).
 19 | 
 20 | Select a component set, and a resolution for your case.  Details of available
 21 | component sets and resolutions are available from the `query_config`_ tool located
 22 | in the ``my_cesm_sandbox/cime/scripts`` directory
 23 | 
 24 | .. code-block:: console
 25 | 
 26 |     cd my_cesm_sandbox/cime/scripts
 27 |     ./query_config --help
 28 | 
 29 | See the `supported component sets <http://www.cesm.ucar.edu/models/cesm2/config/compsets.html>`_,
 30 | `supported model resolutions <http://www.cesm.ucar.edu/models/cesm2/config/grids.html>`_ and `supported
 31 | machines <http://www.cesm.ucar.edu/models/cesm2/config/machines.html>`_ for a complete list of CESM2
 32 | supported component sets, grids and computational platforms.
 33 | 
 34 | .. note:: 
 35 | 
 36 |    Variables presented as ``$VAR`` in this guide typically refer to variables in XML files
 37 |    in a CESM case. From within a case directory, you can determine the value of such a
 38 |    variable with ``./xmlquery VAR``. In some instances, ``$VAR`` refers to a shell
 39 |    variable or some other variable; we try to make these exceptions clear.
 40 | 
 41 | Create a case
 42 | ==============
 43 | 
 44 | The `create_newcase`_ command creates a case directory containing the scripts and XML
 45 | files to configure a case (see below) for the requested resolution, component set, and
 46 | machine. **create_newcase** has three required arguments: ``--case``, ``--compset`` and
 47 | ``--res`` (invoke **create_newcase --help** for help).
 48 | 
 49 | On machines where a project or account code is needed (including NCAR's machines), you
 50 | must either specify the ``--project`` argument to **create_newcase** or set the
 51 | ``$PROJECT`` variable in your shell environment.
 52 | 
 53 | If running on a supported machine, that machine will
 54 | normally be recognized automatically and therefore it is *not* required
 55 | to specify the ``--machine`` argument to **create_newcase**. 
 56 | 
 57 | Invoke **create_newcase** as follows:
 58 | 
 59 | .. code-block:: console
 60 | 
 61 |     ./create_newcase --case CASENAME --compset COMPSET --res GRID
 62 | 
 63 | where:
 64 | 
 65 | - ``CASENAME`` defines the name of your case (stored in the ``$CASE`` XML variable). This
 66 |   is a very important piece of metadata that will be used in filenames, internal metadata
 67 |   and directory paths. **create_newcase** will create the *case directory* with the same
 68 |   name as the ``CASENAME``. If ``CASENAME`` is simply a name (not a path), the case
 69 |   directory is created in the directory where you executed create_newcase. If ``CASENAME``
 70 |   is a relative or absolute path, the case directory is created there, and the name of the
 71 |   case will be the last component of the path. The full path to the case directory will be
 72 |   stored in the ``$CASEROOT`` XML variable. See `CESM2 Experiment Casenames
 73 |   <http://www.cesm.ucar.edu/models/cesm2/naming_conventions.html#casenames>`_ for
 74 |   details regarding CESM experiment case naming conventions.
 75 | 
 76 | - ``COMPSET`` is the `component set <http://www.cesm.ucar.edu/models/cesm2/config/compsets.html>`_.
 77 | 
 78 | - ``GRID`` is the model `resolution <http://www.cesm.ucar.edu/models/cesm2/config/grids.html>`_.
 79 | 
 80 | Here is an example on NCAR machine cheyenne with the ``$USER`` shell environment variable
 81 | set to your cheyenne login name:
 82 | 
 83 | .. code-block:: console
 84 | 
 85 |     ./create_newcase --case /glade/scratch/$USER/cases/b.e20.B1850.f19_g17.test --compset B1850 --res f19_g17 
 86 | 
 87 | 
 88 | Setting up the case run script
 89 | ==============================
 90 | 
 91 | Issuing the `case.setup`_ command creates scripts needed to run the model
 92 | along with namelist ``user_nl_xxx`` files, where xxx denotes the set of components
 93 | for the given case configuration. Before invoking **case.setup**, modify
 94 | the ``env_mach_pes.xml`` file in the case directory using the `xmlchange`_ command
 95 | as needed for the experiment.
 96 | 
 97 | cd to the case directory. Following the example from above:
 98 | 
 99 | .. code-block:: console
100 | 
101 |     cd /glade/scratch/$USER/cases/b.e20.B1850.f19_g17.test
102 | 
103 | Modify settings in ``env_mach_pes.xml`` (optional). (Note: To edit any of
104 | the env xml files, use the `xmlchange`_ command.
105 | invoke **xmlchange --help** for help.)
106 | 
107 | Invoke the **case.setup** command.
108 | 
109 | .. code-block:: console
110 | 
111 |     ./case.setup  
112 | 
113 | 
114 | Build the executable using the case.build command
115 | =================================================
116 | 
117 | Modify build settings in ``env_build.xml`` (optional).
118 | 
119 | Run the build script.
120 | 
121 | .. code-block:: console
122 | 
123 |     ./case.build 
124 | 
125 | Users of the NCAR cheyenne system should consider using 
126 | `qcmd <https://www2.cisl.ucar.edu/resources/computational-systems/cheyenne/running-jobs/submitting-jobs-pbs>`_
127 | to compile CESM2 on a compute node as follows:
128 | 
129 | .. code-block:: console
130 | 
131 |     qcmd -- ./case.build
132 | 
133 | The CESM executable will appear in the directory given by the XML variable ``$EXEROOT``,
134 | which can be queried using:
135 | 
136 | .. code-block:: console
137 |    
138 |    ./xmlquery EXEROOT
139 | 
140 | 
141 | Run the case
142 | ============
143 | 
144 | Modify runtime settings in ``env_run.xml`` (optional). Two settings you may want to change
145 | now are:
146 | 
147 | 1. Run length: By default, the model is set to run for 5 days based on the ``$STOP_N`` and
148 |    ``$STOP_OPTION`` variables:
149 | 
150 |    .. code-block:: console
151 | 
152 |       ./xmlquery STOP_OPTION,STOP_N
153 | 
154 |    These default settings can be useful in `troubleshooting
155 |    <http://esmci.github.io/cime/versions/master/html/users_guide/troubleshooting.html>`_ runtime problems
156 |    before submitting for a longer time, but will not allow the model to run long enough to
157 |    produce monthly history climatology files. In order to produce history files, increase
158 |    the run length to a month or longer:
159 | 
160 |    .. code-block:: console
161 | 
162 |       ./xmlchange STOP_OPTION=nmonths,STOP_N=1
163 | 
164 | 2. You can set the ``$DOUT_S`` variable to FALSE to turn off short term archiving:
165 | 
166 |    .. code-block:: console
167 | 
168 |       ./xmlchange DOUT_S=FALSE
169 | 
170 | Submit the job to the batch queue using the **case.submit** command.
171 | 
172 | .. code-block:: console
173 | 
174 |     ./case.submit
175 | 
176 | When the job is complete, most output will *NOT* be written under the case directory, but
177 | instead under some other directories (on NCAR's cheyenne machine, these other directories
178 | will be in ``/glade/scratch/$USER``). Review the following directories and files, whose
179 | locations can be found with **xmlquery** (note: **xmlquery** can be run with a list of
180 | comma separated names and no spaces):
181 | 
182 | .. code-block:: console
183 | 
184 |    ./xmlquery RUNDIR,CASE,CASEROOT,DOUT_S,DOUT_S_ROOT
185 | 
186 | - ``$RUNDIR``
187 | 
188 |   This directory is set in the ``env_run.xml`` file. This is the
189 |   location where CESM2 was run. There should be log files there for every
190 |   component (i.e. of the form cpl.log.yymmdd-hhmmss) if ``$DOUT_S == FALSE``. 
191 |   Each component writes its own log file. Also see whether any restart or history files were
192 |   written. To check that a run completed successfully, check the last
193 |   several lines of the cpl.log file for the string "SUCCESSFUL
194 |   TERMINATION OF CPL7-cesm".
195 | 
196 | - ``$DOUT_S_ROOT/$CASE``
197 | 
198 |   ``$DOUT_S_ROOT`` refers to the short term archive path location on local disk.
199 |   This path is used by the case.st_archive script when ``$DOUT_S = TRUE``.
200 |   See `CESM Model Output File Locations <http://www.cesm.ucar.edu/models/cesm2/naming_conventions.html#modelOutputLocations>`_
201 |   for details regarding the component model output filenames and locations. 
202 | 
203 |   ``$DOUT_S_ROOT/$CASE`` is the short term archive directory for this case. If ``$DOUT_S`` is
204 |   FALSE, then no archive directory should exist. If ``$DOUT_S`` is TRUE, then
205 |   log, history, and restart files should have been copied into a directory
206 |   tree here.
207 | 
208 | - ``$DOUT_S_ROOT/$CASE/logs``
209 | 
210 |   The log files should have been copied into this directory if the run completed successfully
211 |   and the short-term archiver is turned on with ``$DOUT_S = TRUE``. Otherwise, the log files
212 |   are in the ``$RUNDIR``.
213 | 
214 | - ``$CASEROOT``
215 | 
216 |   There could be standard out and/or standard error files output from the batch system.
217 | 
218 | - ``$CASEROOT/CaseDocs``
219 | 
220 |   The case namelist files are copied into this directory from the ``$RUNDIR``.
221 | 
222 | - ``$CASEROOT/timing``
223 | 
224 |   There should be two timing files there that summarize the model performance.
225 | 
226 | 
227 | .. _CIME: http://esmci.github.io/cime
228 | .. _query_config: http://esmci.github.io/cime/versions/master/html/users_guide/introduction-and-overview.html#discovering-available-cases-with-query-config
229 | .. _create_newcase: http://esmci.github.io/cime/versions/master/html/users_guide/create-a-case.html
230 | .. _xmlchange: http://esmci.github.io/cime/versions/master/html/Tools_user/xmlchange.html
231 | .. _case.setup: http://esmci.github.io/cime/versions/master/html/users_guide/setting-up-a-case.html
232 | 


--------------------------------------------------------------------------------
/cime_config/testlist_allactive.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0"?>
  2 | <testlist version="2.0">
  3 |   <test name="ERS" grid="ne30pg3_t232" compset="1850_CAM60_CLM50%BGC-CROP_CICE_MOM6_MOSART_DGLC%NOEVOLVE_SWAV" testmods="allactive/defaultio">
  4 |     <machines>
  5 |       <machine name="derecho" compiler="intel" category="prealpha"/>
  6 |     </machines>
  7 |     <options>
  8 |       <option name="wallclock"> 01:00:00 </option>
  9 |     </options>
 10 |   </test>
 11 |   <test name="ERS" grid="ne30pg3_t232" compset="1850_CAM70%LT_CLM60%BGC-CROP_CICE_MOM6%MARBL-BIO_MOSART_DGLC%NOEVOLVE_SWAV" testmods="allactive/defaultio">
 12 |     <machines>
 13 |       <machine name="derecho" compiler="gnu" category="prealpha"/>
 14 |     </machines>
 15 |     <options>
 16 |       <option name="wallclock"> 01:00:00 </option>
 17 |     </options>
 18 |   </test>
 19 |   <test name="SMS_D_Ld1" grid="ne30pg3_t232" compset="1850_CAM70%LT_CLM60%BGC-CROP_CICE_MOM6%MARBL-BIO_MOSART_DGLC%NOEVOLVE_SWAV" testmods="allactive/defaultio">
 20 |     <machines>
 21 |       <machine name="derecho" compiler="intel" category="prealpha"/>
 22 |     </machines>
 23 |     <options>
 24 |       <option name="wallclock"> 01:00:00 </option>
 25 |     </options>
 26 |   </test>
 27 |   <test name="ERS_Ld5" grid="ne30pg3_t232" compset="B1850C_LTso" testmods="allactive/defaultio">
 28 |     <machines>
 29 |       <machine name="derecho" compiler="intel" category="prealpha"/>
 30 |     </machines>
 31 |     <options>
 32 |       <option name="wallclock"> 01:00:00 </option>
 33 |     </options>
 34 |   </test>
 35 |   <test name="ERS_Ld5" grid="ne30pg3_t232" compset="BHISTC_LTso" testmods="allactive/defaultio">
 36 |     <machines>
 37 |       <machine name="derecho" compiler="intel" category="prealpha"/>
 38 |     </machines>
 39 |     <options>
 40 |       <option name="wallclock"> 01:00:00 </option>
 41 |     </options>
 42 |   </test>
 43 |   <test name="ERR_Ld5" grid="ne30pg3_t232" compset="B1850C_LTso" testmods="allactive/defaultio">
 44 |     <machines>
 45 |       <machine name="derecho" compiler="gnu" category="prealpha"/>
 46 |     </machines>
 47 |     <options>
 48 |       <option name="wallclock"> 01:00:00 </option>
 49 |     </options>
 50 |   </test>
 51 |   <test name="ERS_Ld5" grid="ne30pg3_t232" compset="B1850C_LTso" testmods="allactive/decstart">
 52 |     <machines>
 53 |       <machine name="derecho" compiler="intel" category="prealpha"/>
 54 |     </machines>
 55 |     <options>
 56 |       <option name="wallclock"> 01:00:00 </option>
 57 |     </options>
 58 |   </test>
 59 |   <test name="ERS_Ld5" grid="ne30pg3_t232" compset="BHISTC_LTso" testmods="allactive/decstart">
 60 |     <machines>
 61 |       <machine name="derecho" compiler="intel" category="prealpha"/>
 62 |     </machines>
 63 |     <options>
 64 |       <option name="wallclock"> 01:00:00 </option>
 65 |     </options>
 66 |   </test>
 67 | 
 68 |   <test name="ERI" grid="ne30pg3_t232" compset="B1850C_LTso" testmods="allactive/defaultio">
 69 |     <machines>
 70 |       <machine name="derecho" compiler="intel" category="prealpha"/>
 71 |     </machines>
 72 |     <options>
 73 |       <option name="wallclock"> 02:00:00 </option>
 74 |     </options>
 75 |   </test>
 76 |   <test name="SMS_Ld2" grid="ne30pg3_t232" compset="B1850C_LTso" testmods="allactive/defaultio">
 77 |     <machines>
 78 |       <machine name="derecho" compiler="intel" category="aux_cime_baselines"/>
 79 |       <machine name="derecho" compiler="gnu" category="prealpha"/>
 80 |       <machine name="derecho" compiler="intel" category="github"/>
 81 |     </machines>
 82 |     <options>
 83 |       <option name="wallclock"> 00:30:00 </option>
 84 |     </options>
 85 |   </test>
 86 |   <test name="SMS_Ld2" grid="ne30pg3_t232" compset="B1850C_MTt4s" testmods="allactive/defaultio">
 87 |     <machines>
 88 |       <machine name="derecho" compiler="gnu" category="prealpha"/>
 89 |       <machine name="derecho" compiler="intel" category="github"/>
 90 |     </machines>
 91 |     <options>
 92 |       <option name="wallclock"> 00:30:00 </option>
 93 |     </options>
 94 |   </test>
 95 | 
 96 |   <test name="SMS_Ld2_D" grid="ne30pg3_t232" compset="B1850C_MTt4s" testmods="allactive/defaultio">
 97 |     <machines>
 98 |       <machine name="derecho" compiler="intel" category="github"/>
 99 |     </machines>
100 |     <options>
101 |       <option name="wallclock"> 00:30:00 </option>
102 |     </options>
103 |   </test>
104 | 
105 |   <test name="SMS_Ld2_D" grid="ne30pg3_t232" compset="BHISTC_MTt4s" testmods="allactive/defaultio">
106 |     <machines>
107 |       <machine name="derecho" compiler="intel" category="github"/>
108 |     </machines>
109 |     <options>
110 |       <option name="wallclock"> 00:30:00 </option>
111 |     </options>
112 |   </test>
113 | 
114 |   <test name="SMS_Ld2" grid="ne30pg3_t232" compset="B1850C_MTso" testmods="allactive/defaultio">
115 |     <machines>
116 |       <machine name="derecho" compiler="intel" category="github"/>
117 |     </machines>
118 |     <options>
119 |       <option name="wallclock"> 00:30:00 </option>
120 |     </options>
121 |   </test>
122 | 
123 |   <test name="SMS_Ld2" grid="ne30pg3_t232" compset="BHISTC_MTso" testmods="allactive/defaultio">
124 |     <machines>
125 |       <machine name="derecho" compiler="intel" category="github"/>
126 |     </machines>
127 |     <options>
128 |       <option name="wallclock"> 00:30:00 </option>
129 |     </options>
130 |   </test>
131 | <!-- These are not defined in config_compsets.xml
132 |   <test name="SMS_Ld2" grid="ne30pg3_t232" compset="B1850C_HTs1" testmods="allactive/defaultio">
133 |     <machines>
134 |       <machine name="derecho" compiler="intel" category="github"/>
135 |     </machines>
136 |     <options>
137 |       <option name="wallclock"> 00:30:00 </option>
138 |     </options>
139 |   </test>
140 |   <test name="SMS_Ld2" grid="ne30pg3_t232" compset="BHISTC_HTs1" testmods="allactive/defaultio">
141 |     <machines>
142 |       <machine name="derecho" compiler="intel" category="github"/>
143 |     </machines>
144 |     <options>
145 |       <option name="wallclock"> 00:30:00 </option>
146 |     </options>
147 |   </test>
148 | -->
149 |   <test name="SMS_Ld2" grid="ne30pg3_t232" compset="B1850C_LTso" testmods="allactive/defaultio">
150 |     <machines>
151 |       <machine name="derecho" compiler="intel" category="github"/>
152 |     </machines>
153 |     <options>
154 |       <option name="wallclock"> 00:30:00 </option>
155 |     </options>
156 |   </test>
157 | 
158 |   <test name="SMS_Ld2" grid="ne30pg3_t232" compset="BHISTC_LTso" testmods="allactive/defaultio">
159 |     <machines>
160 |       <machine name="derecho" compiler="intel" category="github"/>
161 |     </machines>
162 |     <options>
163 |       <option name="wallclock"> 00:30:00 </option>
164 |     </options>
165 |   </test>
166 | 
167 | 
168 |   
169 |   <!-- MCC tests do not work because FMS fails at initialization of MOM -->
170 |   <test name="MCC" grid="ne30pg3_t232" compset="B1850C_MTt4s" testmods="allactive/defaultio">
171 |     <machines>
172 |       <machine name="derecho" compiler="intel" category="prealpha"/>
173 |     </machines>
174 |     <options>
175 |       <option name="wallclock"> 02:00:00 </option>
176 |     </options>
177 |   </test>
178 |   <test name="ERS_Ld5" grid="f09_t232" compset="E1850TEST" testmods="cice/default">
179 |     <machines>
180 |       <machine name="derecho" compiler="intel" category="prebeta"/>
181 |     </machines>
182 |     <options>
183 |       <option name="wallclock"> 00:30:00 </option>
184 |     </options>
185 |   </test>
186 |   <test name="ERS_Ld7" grid="ne30pg3_t232" compset="B1850C_LSC" testmods="allactive/defaultio">
187 |     <machines>
188 |       <machine name="derecho"   compiler="intel" category="prealpha"/>
189 |     </machines>
190 |     <options>
191 |       <option name="wallclock"> 00:30:00 </option>
192 |     </options>
193 |   </test>
194 |   <!-- PET tests do not work because of issues with openmp in the se dycore -->
195 |   <test name="PET_PM" grid="ne30pg3_t232" compset="B1850C_LTso" testmods="allactive/defaultiomi">
196 |     <machines>
197 |       <machine name="derecho" compiler="gnu" category="prealpha"/>
198 |       <machine name="derecho" compiler="intel" category="prealpha"/>
199 |     </machines>
200 |     <options>
201 |       <option name="wallclock"> 01:30:00 </option>
202 |     </options>
203 |   </test>
204 |   <test name="PFS" grid="ne30pg3_t232" compset="B1850C_LTso" testmods="allactive/defaultio">
205 |     <machines>
206 |       <machine name="derecho" compiler="intel" category="prealpha"/>
207 |       <machine name="derecho" compiler="gnu"   category="prealpha"/>
208 |       <machine name="derecho" compiler="intel" category="prebeta"/>
209 |     </machines>
210 |     <options>
211 |       <option name="wallclock"> 00:40:00 </option>
212 |     </options>
213 |   </test>
214 |   <test name="PFS" grid="ne30pg3_t232" compset="B1850C_MTt4s" testmods="allactive/maxthroughputb">
215 |     <machines>
216 |       <machine name="derecho" compiler="intel" category="aux_perf"/>
217 |     </machines>
218 |     <options>
219 |       <option name="wallclock"> 00:40:00 </option>
220 |     </options>
221 |   </test>
222 |   <test compset="FW1850" grid="f09_f09_mt232" name="PFS" testmods="allactive/maxthroughputfw">
223 |     <machines>
224 |       <machine name="derecho" compiler="intel" category="aux_perf"/>
225 |     </machines>
226 |   </test>
227 |   <test name="ERS_D_Ld3" grid="ne30pg3_t232" compset="B1850C_LTso" testmods="allactive/defaultio">
228 |    <machines>
229 |      <machine name="derecho" compiler="intel" category="prealpha"/>
230 |    </machines>
231 |    <options>
232 |      <option name="wallclock"> 02:00:00 </option>
233 |      <option name="comment">Include a debug B compset exact restart test.</option>
234 |    </options>
235 |  </test>
236 |  <test name="SMS_D_Ld1" grid="ne30pg3_t232" compset="B1850C_LTso" testmods="allactive/aoflux_ogrid">
237 |    <machines>
238 |      <machine name="derecho" compiler="intel" category="prebeta"/>
239 |    </machines>
240 |    <options>
241 |      <option name="wallclock"> 00:30:00 </option>
242 |      <option name="comment">Include a short debug test using the ocn grid for aoflux calculations rather than the default xgrid.</option>
243 |    </options>
244 |  </test>
245 |  <test name="SMS_D_Ld1" grid="ne30pg3_t232" compset="B1850C_LTso" testmods="allactive/aoflux_agrid">
246 |    <machines>
247 |      <machine name="derecho" compiler="intel" category="prebeta"/>
248 |    </machines>
249 |    <options>
250 |      <option name="wallclock"> 00:30:00 </option>
251 |      <option name="comment">Include a short debug test using the atm grid for aoflux calculations rather than the default xgrid.</option>
252 |    </options>
253 |  </test>
254 |   <test name="FUNITSHARE_P1x1_Mmpi-serial" grid="ne30pg3_t232" compset="B1850C_LTso">
255 |     <machines>
256 |       <machine name="derecho" compiler="intel" category="prealpha"/>
257 |     </machines>
258 |     <options>
259 |       <option name="wallclock">00:30:00</option>
260 |       <option name="comment">This test runs the CESM_share Fortran unit tests. We're abusing the system test infrastructure to run these, so that a run of the test suite results in the unit tests being run as well. These unit tests are run as a GitHub action for each share PR, so this system test is less about catching code problems and more about making sure the unit testing infrastructure continues to work in each CESM tag. Grid and compset are irrelevant here. Needs to be mpi-serial in order to load the necessary modules.</option>
261 |     </options>
262 |   </test>
263 | </testlist>
264 | 


--------------------------------------------------------------------------------
/.lib/git-fleximod/git_fleximod/utils.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | """
  3 | Common public utilities for manic package
  4 | 
  5 | """
  6 | 
  7 | import logging
  8 | import os
  9 | import subprocess
 10 | import sys
 11 | from threading import Timer
 12 | from pathlib import Path
 13 | 
 14 | LOCAL_PATH_INDICATOR = "."
 15 | # ---------------------------------------------------------------------
 16 | #
 17 | # functions to massage text for output and other useful utilities
 18 | #
 19 | # ---------------------------------------------------------------------
 20 | from contextlib import contextmanager
 21 | 
 22 | 
 23 | @contextmanager
 24 | def pushd(new_dir):
 25 |     """context for chdir. usage: with pushd(new_dir)"""
 26 |     previous_dir = os.getcwd()
 27 |     os.chdir(new_dir)
 28 |     try:
 29 |         yield
 30 |     finally:
 31 |         os.chdir(previous_dir)
 32 | 
 33 | 
 34 | def log_process_output(output):
 35 |     """Log each line of process output at debug level so it can be
 36 |     filtered if necessary. By default, output is a single string, and
 37 |     logging.debug(output) will only put log info heading on the first
 38 |     line. This makes it hard to filter with grep.
 39 | 
 40 |     """
 41 |     output = output.split("\n")
 42 |     for line in output:
 43 |         logging.debug(line)
 44 | 
 45 | 
 46 | def printlog(msg, **kwargs):
 47 |     """Wrapper script around print to ensure that everything printed to
 48 |     the screen also gets logged.
 49 | 
 50 |     """
 51 |     logging.info(msg)
 52 |     if kwargs:
 53 |         print(msg, **kwargs)
 54 |     else:
 55 |         print(msg)
 56 |     sys.stdout.flush()
 57 | 
 58 | 
 59 | def find_upwards(root_dir, filename):
 60 |     """
 61 |     Find a file in root dir or any of its parents.
 62 | 
 63 |     >>> import tempfile, os
 64 |     >>> with tempfile.TemporaryDirectory() as tmp:
 65 |     ...     f = os.path.join(tmp, 'testfile.txt')
 66 |     ...     open(f, 'w').close()
 67 |     ...     find_upwards(tmp, 'testfile.txt').name == 'testfile.txt'
 68 |     True
 69 |     """
 70 |     d = Path(root_dir)
 71 |     root = Path(d.root)
 72 |     while d != root:
 73 |         attempt = d / filename
 74 |         if attempt.exists():
 75 |             return attempt
 76 |         d = d.parent
 77 |     return None
 78 | 
 79 | 
 80 | def last_n_lines(the_string, n_lines, truncation_message=None):
 81 |     """
 82 |     Returns the last n lines of the given string.
 83 |     """
 84 | 
 85 |     lines = the_string.splitlines(True)
 86 |     if len(lines) <= n_lines:
 87 |         return_val = the_string
 88 |     else:
 89 |         lines_subset = lines[-n_lines:]
 90 |         str_truncated = "".join(lines_subset)
 91 |         if truncation_message:
 92 |             str_truncated = truncation_message + "\n" + str_truncated
 93 |         return_val = str_truncated
 94 | 
 95 |     return return_val
 96 | 
 97 | 
 98 | def indent_string(the_string, indent_level):
 99 |     """
100 |     Indents the given string by a given number of spaces.
101 | 
102 |     >>> indent_string('a\\nb', 2)
103 |     '  a\\n  b'
104 |     """
105 |     lines = the_string.splitlines(True)
106 |     padding = " " * indent_level
107 |     lines_indented = [padding + line for line in lines]
108 |     return "".join(lines_indented)
109 | 
110 | 
111 | # ---------------------------------------------------------------------
112 | #
113 | # error handling
114 | #
115 | # ---------------------------------------------------------------------
116 | 
117 | 
118 | def fatal_error(message):
119 |     """
120 |     Error output function
121 |     """
122 |     logging.error(message)
123 |     raise RuntimeError("{0}ERROR: {1}".format(os.linesep, message))
124 | 
125 | 
126 | # ---------------------------------------------------------------------
127 | #
128 | # Data conversion / manipulation
129 | #
130 | # ---------------------------------------------------------------------
131 | def str_to_bool(bool_str):
132 |     """Convert a sting representation of as boolean into a true boolean.
133 | 
134 |     Conversion should be case insensitive.
135 |     """
136 |     value = None
137 |     str_lower = bool_str.lower()
138 |     if str_lower in ("true", "t"):
139 |         value = True
140 |     elif str_lower in ("false", "f"):
141 |         value = False
142 |     if value is None:
143 |         msg = (
144 |             'ERROR: invalid boolean string value "{0}". '
145 |             'Must be "true" or "false"'.format(bool_str)
146 |         )
147 |         fatal_error(msg)
148 |     return value
149 | 
150 | 
151 | REMOTE_PREFIXES = ["http://", "https://", "ssh://", "git@"]
152 | 
153 | 
154 | def is_remote_url(url):
155 |     """check if the user provided a local file path instead of a
156 |     remote. If so, it must be expanded to an absolute
157 |     path.
158 | 
159 |     """
160 |     remote_url = False
161 |     for prefix in REMOTE_PREFIXES:
162 |         if url.startswith(prefix):
163 |             remote_url = True
164 |     return remote_url
165 | 
166 | 
167 | def split_remote_url(url):
168 |     """check if the user provided a local file path or a
169 |     remote. If remote, try to strip off protocol info.
170 | 
171 |     """
172 |     remote_url = is_remote_url(url)
173 |     if not remote_url:
174 |         return url
175 | 
176 |     for prefix in REMOTE_PREFIXES:
177 |         url = url.replace(prefix, "")
178 | 
179 |     if "@" in url:
180 |         url = url.split("@")[1]
181 | 
182 |     if ":" in url:
183 |         url = url.split(":")[1]
184 | 
185 |     return url
186 | 
187 | 
188 | def expand_local_url(url, field):
189 |     """check if the user provided a local file path instead of a
190 |     remote. If so, it must be expanded to an absolute
191 |     path.
192 | 
193 |     Note: local paths of LOCAL_PATH_INDICATOR have special meaning and
194 |     represent local copy only, don't work with the remotes.
195 | 
196 |     """
197 |     remote_url = is_remote_url(url)
198 |     if not remote_url:
199 |         if url.strip() == LOCAL_PATH_INDICATOR:
200 |             pass
201 |         else:
202 |             url = os.path.expandvars(url)
203 |             url = os.path.expanduser(url)
204 |             if not os.path.isabs(url):
205 |                 msg = (
206 |                     'WARNING: Externals description for "{0}" contains a '
207 |                     "url that is not remote and does not expand to an "
208 |                     "absolute path. Version control operations may "
209 |                     "fail.\n\nurl={1}".format(field, url)
210 |                 )
211 |                 printlog(msg)
212 |             else:
213 |                 url = os.path.normpath(url)
214 |     return url
215 | 
216 | 
217 | # ---------------------------------------------------------------------
218 | #
219 | # subprocess
220 | #
221 | # ---------------------------------------------------------------------
222 | 
223 | # Give the user a helpful message if we detect that a command seems to
224 | # be hanging.
225 | _HANGING_SEC = 300
226 | 
227 | 
228 | def _hanging_msg(working_directory, command):
229 |     print(
230 |         """
231 | 
232 | Command '{command}'
233 | from directory {working_directory}
234 | has taken {hanging_sec} seconds. It may be hanging.
235 | 
236 | The command will continue to run, but you may want to abort
237 | git-fleximod with ^C and investigate. A possible cause of hangs is git
238 | requires authentication to access a private repository. On some
239 | systems, git requests for authentication information will not
240 | be displayed to the user. In this case, the program will appear to
241 | hang. Ensure you can run git manually and access all
242 | repositories without entering your authentication information.
243 | 
244 | """.format(
245 |             command=command,
246 |             working_directory=working_directory,
247 |             hanging_sec=_HANGING_SEC,
248 |         )
249 |     )
250 | 
251 | 
252 | def execute_subprocess(commands, status_to_caller=False, output_to_caller=False):
253 |     """Wrapper around subprocess.check_output to handle common
254 |     exceptions.
255 | 
256 |     check_output runs a command with arguments and waits
257 |     for it to complete.
258 | 
259 |     check_output raises an exception on a nonzero return code.  if
260 |     status_to_caller is true, execute_subprocess returns the subprocess
261 |     return code, otherwise execute_subprocess treats non-zero return
262 |     status as an error and raises an exception.
263 | 
264 |     """
265 |     cwd = os.getcwd()
266 |     msg = "In directory: {0}\nexecute_subprocess running command:".format(cwd)
267 |     logging.info(msg)
268 |     commands_str = " ".join(str(element) for element in commands)
269 |     logging.info(commands_str)
270 |     return_to_caller = status_to_caller or output_to_caller
271 |     status = -1
272 |     output = ""
273 |     hanging_timer = Timer(
274 |         _HANGING_SEC,
275 |         _hanging_msg,
276 |         kwargs={"working_directory": cwd, "command": commands_str},
277 |     )
278 |     hanging_timer.start()
279 |     try:
280 |         output = subprocess.check_output(
281 |             commands, stderr=subprocess.STDOUT, universal_newlines=True
282 |         )
283 |         log_process_output(output)
284 |         status = 0
285 |     except OSError as error:
286 |         msg = failed_command_msg(
287 |             "Command execution failed. Does the executable exist?", commands
288 |         )
289 |         logging.error(error)
290 |         fatal_error(msg)
291 |     except ValueError as error:
292 |         msg = failed_command_msg(
293 |             "DEV_ERROR: Invalid arguments trying to run subprocess", commands
294 |         )
295 |         logging.error(error)
296 |         fatal_error(msg)
297 |     except subprocess.CalledProcessError as error:
298 |         # Only report the error if we are NOT returning to the
299 |         # caller. If we are returning to the caller, then it may be a
300 |         # simple status check. If returning, it is the callers
301 |         # responsibility determine if an error occurred and handle it
302 |         # appropriately.
303 |         msg_context = "Process did not run successfully; " "returned status {0}".format(
304 |             error.returncode
305 |         )
306 |         msg = failed_command_msg(msg_context, commands, output=error.output)
307 |         if not return_to_caller:
308 |             logging.error(error)
309 |             logging.error(msg)
310 |             log_process_output(error.output)
311 |             fatal_error(msg)
312 |         status = error.returncode
313 |     finally:
314 |         hanging_timer.cancel()
315 | 
316 |     if status_to_caller and output_to_caller:
317 |         ret_value = (status, output)
318 |     elif status_to_caller:
319 |         ret_value = status
320 |     elif output_to_caller:
321 |         ret_value = output
322 |     else:
323 |         ret_value = None
324 | 
325 |     return ret_value
326 | 
327 | 
328 | def failed_command_msg(msg_context, command, output=None):
329 |     """Template for consistent error messages from subprocess calls.
330 | 
331 |     If 'output' is given, it should provide the output from the failed
332 |     command
333 |     """
334 | 
335 |     if output:
336 |         output_truncated = last_n_lines(
337 |             output, 20, truncation_message="[... Output truncated for brevity ...]"
338 |         )
339 |         errmsg = (
340 |             "Failed with output:\n" + indent_string(output_truncated, 4) + "\nERROR: "
341 |         )
342 |     else:
343 |         errmsg = ""
344 | 
345 |     command_str = " ".join(command)
346 |     errmsg += """In directory
347 |     {cwd}
348 | {context}:
349 |     {command}
350 | """.format(
351 |         cwd=os.getcwd(), context=msg_context, command=command_str
352 |     )
353 | 
354 |     if output:
355 |         errmsg += "See above for output from failed command.\n"
356 | 
357 |     return errmsg
358 | 


--------------------------------------------------------------------------------