├── requirements.txt ├── scripts ├── pull-image ├── jwst-sync ├── hst-sync ├── hst-crds-config ├── run-interactive ├── build-image ├── image-config ├── pathfinder ├── runtests └── sync-test-cache ├── templates ├── slim └── latest ├── .gitignore ├── cache_volumes ├── info.txt └── api-testing │ └── submission-api.ipynb ├── .env.template ├── Dockerfile └── README.md /requirements.txt: -------------------------------------------------------------------------------- 1 | wheel 2 | psutil~=5.7.2 3 | roman_datamodels 4 | stsynphot~=1.1.0 5 | git+https://github.com/spacetelescope/jwst 6 | jupyter 7 | boto3 -------------------------------------------------------------------------------- /scripts/pull-image: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source scripts/image-config 4 | 5 | # docker pull alphasentaurii/crds-docker-testing:slim 6 | 7 | docker pull $DOCKER_IMAGE -------------------------------------------------------------------------------- /templates/slim: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export CACHE_SRC="cache_volumes" 3 | export DOWNLOAD=0 4 | export SYNC=0 5 | export RUN_PARS="-it --privileged -u root" # start container as root -------------------------------------------------------------------------------- /templates/latest: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export CACHE_SRC="crds_cache" 3 | export DOWNLOAD=1 # clones github.com/spacetelescope/crds-test-cache 4 | export SYNC=1 # syncs all mappings and select refs into crds-default-test-cache 5 | export RUN_PARS="-it --privileged -u root" #"-it" -------------------------------------------------------------------------------- /scripts/jwst-sync: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export CRDS_SERVER_URL=https://jwst-crds.stsci.edu 3 | export CRDS_CONTEXT=latest 4 | python -m crds.sync --all --stats --log-time --check-sha1sum --repair-files --organize=flat 5 | python -m crds.sync --files jwst_niriss_flat_0000.fits jwst_miri_flat_0006.fits --stats --log-time -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Include 2 | cache_volumes/!*info.txt 3 | cache_volumes/!*submission-api.ipynb 4 | # Exclude 5 | .DS_Store 6 | cache_volumes/crds-cache-default-test 7 | cache_volumes/crds-cache-test 8 | cache_volumes/api-rkein 9 | cache_volumes/testdata 10 | cache_volumes/*.tgz 11 | cache_volumes/tmp_cache 12 | cache_volumes/crds_cache 13 | .env 14 | .env.bk 15 | *.dockercontext -------------------------------------------------------------------------------- /scripts/hst-sync: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source ${HOME}/scripts/hst-crds-config 3 | export CRDS_SERVER_URL=https://hst-crds.stsci.edu 4 | export CRDS_CONTEXT=latest 5 | python -m crds.sync --all --stats --log-time --check-sha1sum --repair-files --organize=flat 6 | python -m crds.sync --files l2d0959cj_pfl.fits n7p1032ao_apt.fits q5417413o_pct.fits xaf1429el_wcp.fits y2r1559to_apt.fits y2r16006o_pct.fits y951738kl_hv.fits yas2005el_hv.fits p7d1548qj_idc.fits 3241637sm_tmt.fits 41g16069m_tmg.fits 43h1909cm_tmc.fits 43h1240om_tmc.fits --stats --log-time 7 | python -m crds.sync --contexts hst_cos.imap --fetch-references --log-time --stats -------------------------------------------------------------------------------- /scripts/hst-crds-config: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This is useful offsite when working on a small scale with a personal computer. 4 | # Because this downloads CRDS files directly to a local cache, once downloaded 5 | # files are accessed from local disk rather than from VPN or the CRDS server, 6 | # avoiding repeat downloads of potentially gigabytes of data. 7 | export CRDS_SERVER_URL="https://hst-crds.stsci.edu" 8 | # export CRDS_PATH=${HOME}/cache_volumes 9 | export iref=${CRDS_PATH}/references/hst/wfc3/ 10 | export jref=${CRDS_PATH}/references/hst/acs/ 11 | export oref=${CRDS_PATH}/references/hst/stis/ 12 | export lref=${CRDS_PATH}/references/hst/cos/ 13 | export nref=${CRDS_PATH}/references/hst/nicmos/ 14 | export uref=${CRDS_PATH}/references/hst/wfcpc2/ 15 | export uref_linux=$uref 16 | -------------------------------------------------------------------------------- /scripts/run-interactive: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | CONTAINERNAME=${1:-"unruffled_archimedes"} 3 | source scripts/image-config 4 | 5 | if [[ "${CRDS_ROOT}" != "none" ]]; then 6 | MOUNTS="--mount type=bind,source=$CRDS_ROOT,target=/home/developer/crds" 7 | else 8 | MOUNTS = "" 9 | fi 10 | 11 | 12 | MOUNTS="${MOUNTS} --mount type=bind,source="$(pwd)"/cache_volumes,target=/home/developer/cache_volumes" 13 | 14 | 15 | existing=`docker ps -aqf "name=${CONTAINERNAME}"` 16 | 17 | if [[ -z $existing ]]; then 18 | # Run the container 19 | docker run --name $CONTAINERNAME ${RUN_PARS} -p 8000:8050 -p 8889:8889 $MOUNTS $DOCKER_IMAGE 20 | else 21 | # shell into existing container 22 | docker container start $CONTAINERNAME 23 | docker container exec ${RUN_PARS} $CONTAINERNAME /bin/bash 24 | fi -------------------------------------------------------------------------------- /cache_volumes/info.txt: -------------------------------------------------------------------------------- 1 | To mount existing crds cache test data into a running container, place the folders inside here. 2 | 3 | crds-cache-default-test 4 | crds-cache-test 5 | 6 | Set DOWNLOAD=0 to skip downloading crds-cache-test from github 7 | Set SYNC=0 to skip syncing crds-cache-default-test 8 | 9 | You can also overwrite these from within the container by running: 10 | $ CACHE_SRC=cache_volumes 11 | $ DOWNLOAD=1 12 | $ SYNC=1 13 | $ sh scripts/sync-test-cache $CACHE_SRC $DOWNLOAD $SYNC 14 | 15 | ---- 16 | 17 | The API-testing folder contains a jupyter-notebook with some starter code for doing additional custom tests/troubleshooting for things like reference file submissions. With cache_volumes mounted in the container, any files you add to it while the container is running will be accessible to the notebook. -------------------------------------------------------------------------------- /scripts/build-image: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source scripts/image-config 3 | 4 | if [[ -z $CRDS_SERVER_URL ]]; then 5 | CRDS_SERVER_URL="https://hst-crds.stsci.edu" 6 | fi 7 | 8 | if [[ -z $CRDS_CONTEXT ]]; then 9 | CRDS_CONTEXT="latest" 10 | fi 11 | 12 | docker build -f Dockerfile -t ${DOCKER_IMAGE} \ 13 | --build-arg BASE_IMAGE=$BASE_IMAGE \ 14 | --build-arg DOWNLOAD=$DOWNLOAD \ 15 | --build-arg SYNC=$SYNC \ 16 | --build-arg CRDS_REPO=$CRDS_REPO \ 17 | --build-arg CRDS_BRANCH=$CRDS_BRANCH \ 18 | --build-arg CRDS_CONFIG_OFFSITE=$CRDS_CONFIG_OFFSITE \ 19 | --build-arg CRDS_READONLY_CACHE=$CRDS_READONLY_CACHE \ 20 | --build-arg MAST_API_TOKEN=$MAST_API_TOKEN \ 21 | --build-arg CRDS_CONTEXT=$CRDS_CONTEXT \ 22 | --build-arg CRDS_SERVER_URL=$CRDS_SERVER_URL \ 23 | --build-arg CACHE_SRC=$CACHE_SRC \ 24 | --build-arg CRDS_TEST_ROOT=$CRDS_TEST_ROOT \ 25 | --build-arg CRDS_PATH=$CRDS_PATH \ 26 | . 27 | -------------------------------------------------------------------------------- /scripts/image-config: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ######################################### 3 | # USER SETTINGS - configure these in .env 4 | ######################################### 5 | if [[ ! -f .env ]]; then 6 | echo ".env file not found - using template" 7 | cp .env.template .env 8 | fi 9 | 10 | source .env 11 | export IMAGE_TAG=$IMAGE_TAG 12 | export MAST_API_TOKEN=$MAST_API_TOKEN 13 | export CRDS_CONTEXT=$CRDS_CONTEXT 14 | export CRDS_SERVER_URL=$CRDS_SERVER_URL 15 | export GH_USER=$GH_USER 16 | export CRDS_BRANCH=$CRDS_BRANCH 17 | export CRDS_SERVER_URL=$CRDS_SERVER_URL 18 | source scripts/pathfinder $CRDS_ROOT 19 | 20 | export IMAGE_REPO=alphasentaurii/crds-docker-testing 21 | export DOCKER_IMAGE=${IMAGE_REPO}:${IMAGE_TAG} 22 | export CRDS_REPO=https://github.com/${GH_USER}/crds 23 | export BASE=debian 24 | export BASE_TAG=bookworm-slim 25 | export BASE_IMAGE=${BASE}:${BASE_TAG} 26 | export DEV_HOME="." 27 | export THOME=/home/developer 28 | 29 | if [[ "${IMAGE_TAG}" == "latest" || "${IMAGE_TAG}" == "slim" ]]; then 30 | source templates/${IMAGE_TAG} 31 | else 32 | export CACHE_SRC=$CACHE_SRC #"cache_volumes" 33 | export DOWNLOAD=$DOWNLOAD 34 | export SYNC=$SYNC 35 | export RUN_PARS=$RUN_PARS 36 | fi 37 | 38 | export CRDS_TEST_ROOT=${THOME}/${CACHE_SRC} 39 | export CRDS_PATH=${CRDS_TEST_ROOT}/crds-cache-default-test 40 | export CRDS_TESTING_CACHE=${CRDS_TEST_ROOT}/crds-cache-test 41 | -------------------------------------------------------------------------------- /scripts/pathfinder: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | CRDS_ROOT=${1:-""} 3 | 4 | base=`basename $(pwd)` 5 | if [[ "${base}" == scripts ]]; then 6 | dpath=`dirname $(pwd)` 7 | echo $dpath 8 | elif [[ "${base}" == crds-docker-testing|| "${base}" == developer ]]; then 9 | dpath=`pwd` 10 | else 11 | echo "Run this script from root directory of crds-docker-testing repo and try again." 12 | exit 1 13 | fi 14 | 15 | echo "Path to this repo: ${dpath}" 16 | 17 | if [[ -z $CRDS_ROOT ]]; then 18 | parent=`dirname $dpath` 19 | gparent=`dirname $parent` 20 | checkpaths=( $parent $gparent $dpath ) 21 | for p in ${checkpaths[@]} 22 | do 23 | crds_repo_path="${p}/crds" 24 | if [[ -d $crds_repo_path ]]; then 25 | echo "CRDS repo path found: ${crds_repo_path}" 26 | export CRDS_ROOT=$crds_repo_path 27 | break 28 | fi 29 | done 30 | # still haven't found crds client - assumes not mounting 31 | if [[ -z $CRDS_ROOT ]]; then 32 | CRDS_ROOT="none" 33 | fi 34 | else 35 | # verify existence of CRDS_ROOT 36 | crds_repo_path=$CRDS_ROOT 37 | if [[ -d $crds_repo_path ]]; then 38 | echo "CRDS repo path found: ${crds_repo_path}" 39 | elif [[ "${crds_repo_path}" == "none" ]]; then 40 | echo "CRDS_ROOT=${crds_repo_path} - skipping mount" 41 | else 42 | echo "Path does not exist: ${crds_repo_path} - Please revise and try again." 43 | fi 44 | fi 45 | 46 | export SHOME=$dpath 47 | export CRDS_ROOT=$crds_repo_path 48 | -------------------------------------------------------------------------------- /scripts/runtests: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source ${HOME}/scripts/hst-crds-config 3 | # source ${HOME}/scripts/image-config 4 | 5 | # check for mounted test cache, otherwise use crds_cache 6 | cachemnt=${HOME}/cache_volumes 7 | localcache=${HOME}/crds_cache 8 | 9 | # if image was built with test cache (image_tag="latest") and mounted cache folder is empty, 10 | # untar and use crds_cache, otherwise use the mounted directory ("cache_volumes") as test root 11 | if [[ ! -d "${cachemnt}/crds-cache-default-test" && -e "${localcache}.tgz" ]]; then 12 | echo "decompressing cache archive - please wait a moment..." 13 | tar -xzf "${localcache}.tgz" 14 | export CRDS_TEST_ROOT=${localcache} 15 | else 16 | export CRDS_TEST_ROOT=${cachemnt} 17 | fi 18 | 19 | export CRDS_PATH=${CRDS_TEST_ROOT}/crds-cache-default-test 20 | export CRDS_TESTING_CACHE=${CRDS_TEST_ROOT}/crds-cache-test 21 | 22 | if [[ -z $CRDS_SERVER_URL ]]; then 23 | export CRDS_SERVER_URL=https://hst-crds.stsci.edu 24 | echo "CRDS SERVER URL set to: ${CRDS_SERVER_URL}" 25 | fi 26 | 27 | 28 | # check crds installation 29 | resp=`crds list --status` 30 | status=`echo $resp | cut -f1 -d'='` 31 | 32 | cd ${HOME}/crds 33 | 34 | if [[ -z $status ]]; then 35 | echo "Installing CRDS client code" 36 | # install mounted crds source code 37 | ./install && pip install -e .[submission,test,docs,synphot] 38 | pip install --upgrade .["submission","test"] 39 | fi 40 | 41 | # set CRDS CONTEXT 42 | if [[ -z $CRDS_CONTEXT ]]; then 43 | export CRDS_CONTEXT=latest 44 | context_num=`echo $resp | cut -f8 -d'=' | cut -f2 -d'_' | cut -f1 -d'.'` 45 | latest_context="hst_${context_num}.pmap" 46 | export CRDS_CONTEXT=$latest_context 47 | fi 48 | 49 | echo "CRDS_SERVER_URL: ${CRDS_SERVER_URL}" 50 | echo "CRDS_CONTEXT: ${CRDS_CONTEXT}" 51 | 52 | 53 | ./runtests 54 | cd $HOME 55 | -------------------------------------------------------------------------------- /.env.template: -------------------------------------------------------------------------------- 1 | # copy contents of this file into .env 2 | 3 | ##### image tag ##### 4 | # choose which image tag to use ("latest" or "slim") or enter custom tag name 5 | ## "latest" : built-in test data in /home/developer/crds_cache 6 | ## "slim" : mount your existing test data in cache_volumes 7 | # ## enter a custom tag (e.g. 'dev') if building your own image (or use "latest") 8 | 9 | # LATEST: If you don't have the crds test cache data or want to start fresh (sets DOWNLOAD and SYNC = 1) 10 | IMAGE_TAG="latest" # ~4.26 GB 11 | 12 | # For SLIM or custom image tag: container will expect test cache to be located inside the mounted "cache_volumes" directory at runtime 13 | # IMAGE_TAG="slim" # ~3.34 GB 14 | 15 | # If using custom tag, the vars below MUST be uncommented and set 16 | # IMAGE_TAG="mycustomtag" 17 | # RUN_PARS="-it --privileged -u root" 18 | # CACHE_SRC="cache_volumes" 19 | # DOWNLOAD=0 20 | # SYNC=0 21 | 22 | ##### API Testing config (optional - not used in test suite) ##### 23 | # optionally set MAST token as env var (for API testing/jupyter) 24 | MAST_API_TOKEN="" 25 | CRDS_CONTEXT="" 26 | CRDS_SERVER_URL="" 27 | 28 | ##### CRDS client code ##### 29 | # Modify these to install your branch during image build 30 | # Leave defaults if mounting into container (installed repo will be overwritten by mounted repo) 31 | GH_USER="spacetelescope" 32 | CRDS_BRANCH="master" 33 | # To mount your local crds client repo into the container (useful for live editing from host IDE): 34 | # > if your local crds repo is in same top level directory as crds-docker-testing: leave CRDS_ROOT value blank 35 | # > if your local crds repo is in some other location, set CRDS_ROOT to the abs local path 36 | # > to Skip mounting (no live editing from host IDE) set to "none" 37 | CRDS_ROOT="" # /abs/path/to/my/crds/client/code 38 | 39 | # Configure CRDS for offsite use with a dynamic reference file cache. 40 | CRDS_READONLY_CACHE=0 41 | CRDS_CONFIG_OFFSITE=1 42 | 43 | ##### test suite data ##### 44 | # If not using latest or slim image templates, uncomment and set vars below 45 | # CACHE_SRC="cache_volumes" 46 | # DOWNLOAD=0 47 | # SYNC=0 48 | # RUN_PARS="-it --privileged -u root" -------------------------------------------------------------------------------- /scripts/sync-test-cache: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | CACHE_SRC=${1:-"cache_volumes"} 3 | DOWNLOAD_CACHE=${2:-0} 4 | SYNC_CACHE=${3:-0} 5 | COMPRESS=${4:-0} 6 | TIME=/usr/bin/time 7 | logfile="${HOME}/sync.log" 8 | 9 | if [[ "${CACHE_SRC}" == "/grp/crds/cache" ]]; then 10 | echo "Readonly cache - skipping" 11 | exit 12 | fi 13 | 14 | export CRDS_TEST_ROOT="${HOME}/${CACHE_SRC}" 15 | if [[ ! -d $CRDS_TEST_ROOT ]]; then 16 | mkdir -p ${CRDS_TEST_ROOT} 17 | fi 18 | chmod 755 $CRDS_TEST_ROOT 19 | export CRDS_PATH=$CRDS_TEST_ROOT/crds-cache-default-test 20 | export CRDS_TESTING_CACHE=$CRDS_TEST_ROOT/crds-cache-test 21 | source ${HOME}/scripts/hst-crds-config 22 | 23 | if [[ $DOWNLOAD_CACHE == 1 ]]; then 24 | # setup fresh test cache directories similar to crds/setup_test_cache 25 | if [[ -d $CRDS_TESTING_CACHE ]]; then 26 | cd $CRDS_TEST_ROOT && rm -rf crds-cache-test 27 | fi 28 | mkdir -p ${CRDS_TEST_ROOT}/crds-cache-default-test && cd $CRDS_TEST_ROOT 29 | git clone https://github.com/spacetelescope/crds-cache-test.git 30 | cd $DEV_HOME 31 | fi 32 | 33 | if [[ $SYNC_CACHE == 1 ]]; then 34 | resp=`crds list --status` 35 | status=`echo $resp | cut -f1 -d'='` 36 | if [[ -z $status ]]; then 37 | cd ${HOME}/crds && ./install && pip install -e .[submission,test,docs,synphot] 38 | pip install --upgrade .["submission","test"] && cd $HOME 39 | fi 40 | cd $CRDS_TEST_ROOT 41 | echo "***** RUNNING HST SYNC *****" 42 | set -o pipefail && $TIME --verbose -o ${HOME}/hstsync.log ${HOME}/scripts/hst-sync 43 | hst_sync_exit_status=$? 44 | echo "HST Sync exit status ${hst_sync_exit_status}" 45 | 46 | echo "***** RUNNING JWST SYNC *****" 47 | set -o pipefail && $TIME --verbose -o ${HOME}/jwstsync.log ${HOME}/scripts/jwst-sync 48 | jwst_sync_exit_status=$? 49 | echo "JWST Sync exit status ${jwst_sync_exit_status}" 50 | cd $HOME 51 | fi 52 | 53 | chmod -R 755 $CRDS_TEST_ROOT 54 | 55 | if [[ -d "${HOME}/crds_cache" && "${COMPRESS}" == 1 ]]; then 56 | if [[ -e "${HOME}/crds_cache.tgz" ]]; then 57 | rm ${HOME}/crds_cache.tgz 58 | fi 59 | echo "Compressing CRDS test cache as crds_cache.tgz" 60 | tar -czf crds_cache.tgz crds_cache/ 61 | chmod 777 crds_cache.tgz 62 | rm -rf crds_cache/ 63 | fi -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG BASE_IMAGE=debian:bookworm-slim 2 | FROM ${BASE_IMAGE} 3 | 4 | RUN apt update && \ 5 | apt upgrade --assume-yes && \ 6 | ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive && \ 7 | apt install --assume-yes \ 8 | libxrender1 \ 9 | csh \ 10 | fitsverify \ 11 | git \ 12 | python3 \ 13 | python3-pip \ 14 | python3-venv \ 15 | sudo \ 16 | tcsh \ 17 | make \ 18 | gcc \ 19 | htop \ 20 | wget \ 21 | git \ 22 | tar \ 23 | curl \ 24 | time \ 25 | vim \ 26 | awscli 27 | 28 | RUN useradd -s /bin/bash -m developer && usermod -aG sudo developer 29 | ENV DEV_HOME=/home/developer 30 | ADD scripts/ ${DEV_HOME}/scripts 31 | RUN chown -R developer:developer /home/developer && chmod +x /home/developer/scripts/* 32 | 33 | USER developer 34 | WORKDIR /home/developer 35 | RUN python3 -m venv /home/developer/venv 36 | ENV PATH=/home/developer/venv/bin:${PATH} 37 | ENV LD_LIBRARY_PATH=/home/developer/venv/LD_LIBRARY_PATH 38 | COPY requirements.txt ${DEV_HOME}/. 39 | RUN pip install --upgrade pip && \ 40 | pip install -r ${DEV_HOME}/requirements.txt && \ 41 | pip uninstall --yes crds 42 | COPY .env ${DEV_HOME}/. 43 | # clone CRDS client repo 44 | ARG CRDS_REPO=https://github.com/spacetelescope/crds 45 | ARG CRDS_BRANCH=master 46 | # run test cache setup 47 | RUN git clone --branch $CRDS_BRANCH ${CRDS_REPO}.git && cd crds && ./install --dev && cd $DEV_HOME 48 | # Set ENVIRONMENT vars 49 | ARG CRDS_CONFIG_OFFSITE=1 50 | ENV CRDS_CONFIG_OFFSITE=$CRDS_CONFIG_OFFSITE 51 | ARG CRDS_READONLY_CACHE=0 52 | ENV CRDS_READONLY_CACHE=$CRDS_READONLY_CACHE 53 | ARG MAST_API_TOKEN 54 | ENV MAST_API_TOKEN=$MAST_API_TOKEN 55 | ARG CRDS_CONTEXT=latest 56 | ENV CRDS_CONTEXT=$CRDS_CONTEXT 57 | ARG CRDS_SERVER_URL=https://hst-crds.stsci.edu 58 | ENV CRDS_SERVER_URL=$CRDS_SERVER_URL 59 | ARG CACHE_SRC=cache_volumes 60 | ENV CACHE_SRC=$CACHE_SRC 61 | ARG CRDS_TEST_ROOT=/home/developer/$CACHE_SRC 62 | ENV CRDS_TEST_ROOT=$CRDS_TEST_ROOT 63 | ARG CRDS_TESTING_CACHE=$CRDS_TEST_ROOT/crds-cache-test 64 | ENV CRDS_TESTING_CACHE=$CRDS_TESTING_CACHE 65 | ARG CRDS_PATH=$CRDS_TEST_ROOT/crds-cache-default-test 66 | ENV CRDS_PATH=$CRDS_PATH 67 | 68 | # Get test cache data if requested 69 | ARG DOWNLOAD=1 70 | ARG SYNC=1 71 | RUN scripts/sync-test-cache $CACHE_SRC $DOWNLOAD $SYNC 1 72 | # reset download/sync to zero 73 | ENV DOWNLOAD=0 74 | ENV SYNC=0 75 | CMD /bin/bash 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CRDS DOCKER TESTING 2 | 3 | Running the CRDS client test suite on Docker 4 | 5 | Mount your own fork of the CRDS client to run the test suite with live code changes 6 | 7 | 8 | ## Installation and Setup 9 | 10 | Steps: 11 | 12 | 1. Clone this repo 13 | 14 | 2. Copy '.env.template' to a new file named '.env'. Configure settings as needed: 15 | 16 | a. use built-in test cache (2.85GB) --> set IMAGE_TAG="latest" 17 | 18 | b. mount your own existing test cache --> set IMAGE_TAG="slim" 19 | 20 | c. mount existing cache and create a custom image (advanced) --> set IMAGE_TAG to your custom tag 21 | 22 | 3. Pull image (`scripts/pull-image`) or Build image (`scripts/build-image`) 23 | 24 | 4. Run container: `scripts/run-interactive` 25 | 26 | 5. Run test suite 27 | 28 | 29 | ### 1. Clone Repo 30 | 31 | Clone this repo into the same parent directory as your CRDS client code so they are at the same directory level, or set the path to your CRDS client code explicitly by editing `.env` and setting `CRDS_ROOT` to the absolute path of your repo folder. (NOTE this is assuming you want to mount the crds repo into the running container, which allows you to make on the fly changes using your IDE). 32 | 33 | 34 | ### 2. Configure Settings 35 | 36 | Copy '.env.template' to a new file named '.env'. 37 | If mounting your own test cache data, move/copy these directories into the "cache_volumes" folder: 38 | 39 | * crds-cache-test 40 | * crds-cache-default-test 41 | 42 | ```bash 43 | $ mv crds-cache-default-test/ crds-docker-testing/cache_volumes/. 44 | $ mv crds-cache-test/ crds-docker-testing/cache_volumes/. 45 | ``` 46 | 47 | These (and anything else you place inside "cache_volumes") will be mounted into the container at runtime. You can always re-sync if necessary from inside the container. 48 | 49 | Edit `.env` to select an image tag and configure any additional settings: 50 | 51 | * "latest" : built-in test data 52 | * "slim" : no test data (mount your own) 53 | * custom tag (if building your own image) 54 | 55 | About Image Tag Options 56 | 57 | **"latest": built-in test cache ready to go** 58 | 59 | If you don't have any test cache data or want to start with a fresh new copy. 60 | 61 | **"slim": mount your existing test cache** 62 | 63 | If you already have an existing test cache. 64 | 65 | 66 | ### 3. Pull or Build the docker image 67 | 68 | The `cache_volumes` folder in this repo will be mounted into the running container. This means anything else you need for testing can be made accessible by placing it in this folder. E.g. for API testing, you can run a jupyter notebook inside the container and view it in a browser over port 8888. 69 | 70 | 71 | #### Option 1 (simplest): PULL image from Docker Hub 72 | 73 | ```bash 74 | # Pull image 75 | scripts/pull-image 76 | ``` 77 | 78 | NOTE - you can still mount your own test cache into this image by placing it in "cache_volumes" - it will not be overwritten. The benefit of using "slim" is a smaller download size. 79 | 80 | 81 | ### Option 2 (advanced): BUILD docker image locally 82 | 83 | The settings differ for "latest" and "slim" image tags, but keep in mind the scripts that use these default settings are for convenience. Anything can be overridden inside the container by modifying environment variables and also by using crds scripts directly. The key difference is that `latest` includes a built-in crds cache already downloaded, while `slim` does not. 84 | 85 | *2A - Download and sync turned ON* 86 | 87 | At image build time the `sync-test-cache` script will download and sync test cache data similar to `crds/setup_test_cache`. This is turned ON by default for the "latest" image tag. The Sync process can take up to ~21 mins depending on your internet connection. 88 | 89 | Default settings for "latest" image_tag: 90 | 91 | - cache folder = "/home/developer/crds_cache" 92 | - This folder is created and synced at image build time 93 | - A compressed (.tgz) archive of the cache is created and the folder deleted during image build 94 | - At container runtime, the archive is extracted and used as CRDS_PATH for tests 95 | - The built-in cache can be updated (synced again) with CRDS using the sync-test-cache script from inside a running container, e.g.: `scripts/sync-test-cache crds_cache 0 1` 96 | 97 | 98 | *2B - Download and sync turned OFF* 99 | 100 | If you already have `crds-cache-default-test` and `crds-cache-test` installed locally and want to mount these into the container instead of redownloading, resyncing, set DOWNLOAD=0, SYNC=0 then move/copy these directories into the "cache_volumes" folder. If your test cache isn't up to date, you can simply set `export SYNC=1` once you're inside the container and run the sync-test-cache script 101 | 102 | ```bash 103 | $ vi .env # assuming you've already copied it from .env.template 104 | ########## .env ########## 105 | IMAGE_TAG="slim" 106 | ######################################## 107 | ``` 108 | 109 | Once the settings are configured, build the image: 110 | 111 | ```bash 112 | $ scripts/build-image 113 | ``` 114 | 115 | Default settings for "slim" image_tag: 116 | 117 | - cache folder = "/home/developer/cache_volumes" 118 | - This folder is mounted at container run time 119 | - tests will expect contents of this folder to include crds-cache-test and crds-cache-default-test 120 | - The mounted cache directories can be synced/updated with CRDS using the sync-test-cache script from inside a running container, e.g.: `scripts/sync-test-cache cache_volumes 0 1` 121 | 122 | 123 | ### 3. Run the container 124 | 125 | ```bash 126 | $ scripts/run-interactive 127 | ``` 128 | 129 | ### 4. Run the test suite 130 | 131 | ```bash 132 | # Run the test suite (from inside running container) 133 | developer@localhost:~$ scripts/runtests 134 | ``` 135 | 136 | Note: you may want to run a subset of tests using pytest markers. To do this, just make sure you have the right environment variables set (i.e. CRDS_TEST_ROOT, CRDS_PATH, etc are set to the desired paths), cd into `crds` and run e.g. `pytest -m roman` 137 | 138 | 139 | ### 5. Troubleshooting 140 | 141 | If you get "File Not Found" test failures, you may need to set a more recent CRDS context e.g. "latest" or "jwst-edit" before running sync: 142 | 143 | ```bash 144 | developer@localhost:~$ export CRDS_CONTEXT="jwst_1015.pmap" 145 | ``` 146 | 147 | The `sync-test-cache` script can be run from inside the running container: 148 | 149 | ```bash 150 | developer@localhost:~$ CACHE_SRC=cache_volumes 151 | developer@localhost:~$ DOWNLOAD=0 152 | developer@localhost:~$ SYNC=1 153 | developer@localhost:~$ scripts/sync-test-cache $CACHE_SRC $DOWNLOAD $SYNC 154 | ``` 155 | -------------------------------------------------------------------------------- /cache_volumes/api-testing/submission-api.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Submission API Testing Starter Pack\n", 8 | "\n", 9 | "CRDS env vars need to be set on the command line before starting the notebook:\n", 10 | "\n", 11 | "```bash\n", 12 | "$ export CRDS_PATH=/home/developer/cache_volumes/crds-cache-default-test\n", 13 | "$ export CRDS_SERVER_URL=https://roman-crds-dev.stsci.edu\n", 14 | "$ export CRDS_CONTEXT=roman_0039.pmap\n", 15 | "```\n", 16 | "\n", 17 | "For submission testing, you may also want to create and set a MAST auth token:\n", 18 | "\n", 19 | "https://auth.mast.stsci.edu/token\n", 20 | "\n", 21 | "```bash\n", 22 | "export MAST_API_TOKEN=\"1234567abcde\"\n", 23 | "```" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "# IMPORT LIBRARIES\n", 33 | "\n", 34 | "import os\n", 35 | "import crds\n", 36 | "from crds import log\n", 37 | "from crds.submit import rc_submit\n", 38 | "from pprint import pprint as pp" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "\n", 48 | "log.set_verbose(0) # 80 for debugging web activity, 0 for no debug, 50 for basic flow" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": null, 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "# ENVIRONMENT SETUP\n", 58 | "\n", 59 | "HOME = %pwd\n", 60 | "CACHE_VOL = os.path.join(HOME, \"cache_volumes\")\n", 61 | "PROJECT = 'roman' # hst, jwst, roman\n", 62 | "SERVER = 'dev' # dev, test, cit\n", 63 | "CONTEXT = f\"{PROJECT}-edit\" # -edit, -operational, or _number like \"_0038\"\n", 64 | "\n", 65 | "#https://auth.mast.stsci.edu/token\n", 66 | "MAST_API_TOKEN = os.environ[\"MAST_API_TOKEN\"]\n", 67 | "\n", 68 | "# use the test suite cache or create a new folder\n", 69 | "CRDS_PATH = os.path.join(HOME, \"cache_volumes\", \"crds-cache-default-test\")\n", 70 | "APICACHE = os.path.join(CACHE_VOL, \"apicache\")\n", 71 | "TESTDATA = os.path.join(CACHE_VOL, \"testdata\")\n", 72 | "os.makedirs(APICACHE, exist_ok=True)\n", 73 | "os.makedirs(TESTDATA, exist_ok=True)\n", 74 | "\n", 75 | "\n", 76 | "os.environ[\"CRDS_SERVER_URL\"] = \"https://\" + PROJECT + \"-crds-\" + SERVER + \".stsci.edu\"\n", 77 | "os.environ[\"CRDS_PATH\"] = CRDS_PATH # or APICACHE" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": null, 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "# CRDS STATUS CHECK\n", 87 | "\n", 88 | "! crds list --status\n", 89 | "\n", 90 | "# CRDS Summary = 'The __rationale__ variable is no longer maintained and will be removed in a future release.'\n", 91 | "# CRDS Version = '11.16.15.dev3+g739ffdd3.d20221018, b11.4.0, 64d96076d89b32a5687a6b77bb910ab93b3a99b3'\n", 92 | "# CRDS_MODE = 'auto'\n", 93 | "# CRDS_PATH = '/home/developer/cache_volumes/crds-cache-default-test'\n", 94 | "# CRDS_SERVER_URL = 'https://roman-crds-dev.stsci.edu'\n", 95 | "# Cache Locking = 'enabled, multiprocessing'\n", 96 | "# Effective Context = 'roman_0039.pmap'\n", 97 | "# Last Synced = '2022-10-18 18:49:15.168687'\n", 98 | "# Python Executable = '/home/developer/venv/bin/python3'\n", 99 | "# Python Version = '3.9.2.final.0'\n", 100 | "# Readonly Cache = False" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "# TRY SYNCING CACHE WITH SERVER\n", 110 | "# CONTEXT = roman-edit\n", 111 | "! crds sync --contexts CONTEXT --repair-files --purge-mappings --purge-references\n", 112 | "\n", 113 | "# CRDS - INFO - Symbolic context 'roman-edit' resolves to 'roman_0044.pmap'\n", 114 | "# CRDS - INFO - Removing all context pickles. Use --save-pickles to recreate for specified contexts.\n", 115 | "# CRDS - INFO - Fetching /home/developer/cache_volumes/crds-cache-default-test/mappings/roman/roman_wfi_distortion_0008.rmap 3.8 K bytes (1 / 3 files) (0 / 4.9 K bytes)\n", 116 | "# CRDS - INFO - Fetching /home/developer/cache_volumes/crds-cache-default-test/mappings/roman/roman_wfi_0042.imap 741 bytes (2 / 3 files) (3.8 K / 4.9 K bytes)\n", 117 | "# CRDS - INFO - Fetching /home/developer/cache_volumes/crds-cache-default-test/mappings/roman/roman_0044.pmap 364 bytes (3 / 3 files) (4.6 K / 4.9 K bytes)\n", 118 | "# CRDS - INFO - 0 errors\n", 119 | "# CRDS - INFO - 0 warnings\n", 120 | "# CRDS - INFO - 5 infos" 121 | ] 122 | }, 123 | { 124 | "cell_type": "markdown", 125 | "metadata": {}, 126 | "source": [ 127 | "## Submission Testing\n", 128 | "\n", 129 | "1. Create submission object\n", 130 | "2. Update fields\n", 131 | "3. Add file(s)\n", 132 | "4. Submit\n", 133 | "5. (optional) Confirm submission and check filemap" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": null, 139 | "metadata": {}, 140 | "outputs": [], 141 | "source": [ 142 | "# 1. create a submission object\n", 143 | "\n", 144 | "submission = rc_submit.Submission(PROJECT, SERVER)" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": null, 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "# 2. update fields\n", 154 | "\n", 155 | "submission[\"deliverer\"] = \"rkein\"\n", 156 | "submission[\"other_email\"] = \"\"\n", 157 | "submission[\"file_type\"] = \"distortion\"\n", 158 | "submission[\"history_updated\"] = True\n", 159 | "submission[\"descrip_updated\"] = True\n", 160 | "submission[\"useafter_updated\"] = True\n", 161 | "submission[\"useafter_matches\"] = True\n", 162 | "submission[\"pedigree_updated\"] = True\n", 163 | "submission[\"keywords_checked\"] = True\n", 164 | "submission[\"compliance_verified\"] = True\n", 165 | "submission[\"ingest_files\"] = True\n", 166 | "submission[\"etc_delivery\"] = False\n", 167 | "submission[\"calpipe_version\"] = \"xxx, 1.0\"\n", 168 | "submission[\"replacement_files\"] = False\n", 169 | "submission[\"old_reference_files\"] = \"xxx\"\n", 170 | "submission[\"replacing_badfiles\"] = False\n", 171 | "submission[\"jira_issue\"] = \"\"\n", 172 | "submission[\"table_rows_changed\"] = \"xxx\"\n", 173 | "submission[\"modes_affected\"] = \"xxxx\"\n", 174 | "submission[\"correctness_testing\"] = \"lots\"\n", 175 | "submission[\"additional_considerations\"] = \"a few\"\n", 176 | "submission[\"change_level\"] = \"SEVERE\"\n", 177 | "submission[\"description\"] = \"roman submission api test\"\n", 178 | "submission[\"instrument\"] = \"wfi\"" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": null, 184 | "metadata": {}, 185 | "outputs": [], 186 | "source": [ 187 | "# 3. add file(s)\n", 188 | "f = f\"{APICACHE}/roman/roman_wfi_distortion_nine.asdf\"\n", 189 | "submission.add_file(f)" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": null, 195 | "metadata": {}, 196 | "outputs": [], 197 | "source": [ 198 | "result = submission.submit()" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": null, 204 | "metadata": {}, 205 | "outputs": [], 206 | "source": [ 207 | "# submission.update_filemap()\n", 208 | "submission.file_map\n", 209 | "\n", 210 | "# {'roman_wfi_distortion_nine.asdf': 'roman_wfi_distortion_0028.asdf',\n", 211 | "# 'roman_0048.pmap': 'roman_0049.pmap',\n", 212 | "# 'roman_wfi_0046.imap': 'roman_wfi_0047.imap',\n", 213 | "# 'roman_wfi_distortion_0012.rmap': 'roman_wfi_distortion_0013.rmap'}" 214 | ] 215 | } 216 | ], 217 | "metadata": { 218 | "kernelspec": { 219 | "display_name": "Python 3.8.11 ('base')", 220 | "language": "python", 221 | "name": "python3" 222 | }, 223 | "language_info": { 224 | "name": "python", 225 | "version": "3.8.11" 226 | }, 227 | "orig_nbformat": 4, 228 | "vscode": { 229 | "interpreter": { 230 | "hash": "ccd9a86c224ddb65511baccfbc7149f623736c0082d52ad08a98082ff67c9019" 231 | } 232 | } 233 | }, 234 | "nbformat": 4, 235 | "nbformat_minor": 2 236 | } 237 | --------------------------------------------------------------------------------