├── .github ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── LICENSE ├── README.md ├── demos └── plantcv │ ├── VIS_SV_0_z1_h1_g0_e82_1224684.png │ └── plantcv_jupyter_demo.ipynb ├── extractors ├── betydb │ ├── Dockerfile │ ├── README.md │ ├── entrypoint.sh │ ├── extractor_info.json │ └── terra_betydb.py ├── compressor │ ├── Dockerfile │ ├── README.md │ ├── entrypoint.sh │ ├── terra_compressor.py │ └── terra_compressor_queue.py ├── geostreams │ ├── Dockerfile │ ├── README.md │ ├── entrypoint.sh │ ├── extractor_info.json │ └── terra_geostreams.py └── plotclipper │ ├── Dockerfile │ ├── README.md │ ├── entrypoint.sh │ ├── extractor_info.json │ └── terra_plotclipper.py ├── meeting-notes ├── 2015-10-01_dev_team.md ├── 2015-10-08_dev_team.md ├── 2015-10-12_dev_team.md ├── 2015-10-22_dev_team.md ├── 2015-11-20_computational_pipeline.md ├── 2015_12_01_Danforth_visit.md └── 2016-01-14_Dev team.md └── scripts ├── FLIR ├── FlirRawToTemperature.m ├── GetFLIR.m └── Get_FLIR.py ├── NDVI ├── GetNDVI.m └── PlotNDVI.m ├── PS2 └── GetPS2.m ├── ShareClowderDatasetsViaSpaces.py ├── StereoVIS └── GetStereoVIS.m ├── addPlotsToGeostreams.py ├── email_xferlog ├── environmental_logger ├── environmental_logger_calculation.py ├── environmental_logger_json2netcdf.py └── environmental_logger_unittest.py ├── example-scripts ├── GeostreamDatapointPlotter.py ├── TERRAClowderUploadCurl.sh └── TERRAClowderUploadPython.py ├── extractor-logging.json ├── filecounter ├── Dockerfile ├── README.md ├── config_default.json ├── config_logging.json ├── counts.py ├── filecounter.py ├── stereoTop.csv ├── templates │ ├── dateoptions.html │ └── sensors.html ├── users.json └── utils.py ├── fullfield-preview └── test.py ├── gantrymonitor ├── Dockerfile ├── README.md ├── config_default.json ├── config_logging.json ├── gantry_scanner_service.py ├── globus_amazon.pem ├── globus_manager_service.py ├── manager.service └── scanner.service ├── geospatial ├── field_scanner_plots.R ├── range.csv ├── row.csv └── season2 │ ├── cultivar_id.csv │ ├── plots.R │ ├── plots.csv │ └── sites.csv ├── globusmonitor ├── Dockerfile_receiver ├── Dockerfile_uploader ├── config_default.json ├── config_logging.json ├── globus_amazon.pem ├── globus_monitor_service.py ├── globus_uploader_service.py ├── globusmonitor.service ├── globusmonitor_status.sh ├── globusuploader.service └── migrateJsonToPostgres.py ├── hyperspectral ├── DataProcess.py ├── README.md ├── calibration_vnir_20ms.nc ├── calibration_vnir_25ms.nc ├── calibration_vnir_30ms.nc ├── calibration_vnir_35ms.nc ├── calibration_vnir_40ms.nc ├── calibration_vnir_45ms.nc ├── calibration_vnir_50ms.nc ├── calibration_vnir_55ms.nc ├── extractor │ ├── Dockerfile │ ├── config.py │ ├── entrypoint.sh │ ├── extractor_info.json │ └── terra.hyperspectral.py ├── hyperspectral_calculation.py ├── hyperspectral_calibration.nco ├── hyperspectral_calibration_reduction.sh ├── hyperspectral_dummy.nc ├── hyperspectral_indices_make.nco ├── hyperspectral_metadata.py ├── hyperspectral_spectralon_reflectance_factory.nco ├── hyperspectral_test.py └── hyperspectral_workflow.sh ├── ip2geohash.py ├── load_file_list.py ├── plantcv ├── PlantCV_demo.ipynb ├── PlantcvClowderIndoorAnalysis.py ├── PlantcvClowderUploader.py ├── PlantcvClowderUploader_Globus.py ├── README.md └── config_custom.json ├── rebuild_scripts ├── buildClowderInstance.py ├── buildClowderInstanceLocalTest.py ├── buildClowderInstanceRogerTest.py └── loadDanforthSnapshots.py ├── reprocessing_bulk_scripts ├── get_dataset_ids_SENSOR.js ├── mongo_cleanup.js └── submit_datasets_by_list.py ├── terra_report └── upload_directories_to_clowder.py /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | 3 | Third-party contributions are highly encouraged. 4 | Our goal is to keep it as easy as possible for you contribute changes that get things working in your environment. 5 | There are a few guidelines that we need contributors to follow so that we can have a chance of keeping on top of things. 6 | 7 | There are many ways to get in touch: 8 | 9 | * report bugs 10 | * request features 11 | * support for existing databases / APIs 12 | * integrating existing algorithms into the pipeline 13 | * join discussion of open issues, features, and bugs 14 | * computing pipeline: github.com/terraref/computing-pipeline/issues 15 | * reference data: github.com/terraref/reference-data/issues 16 | * contribute algorithms to the pipeline 17 | * revise documentation 18 | 19 | New features are directed toward developing and extending the TERRA Ref computational infrastructure. 20 | 21 | If you need any help, please contact us in our [chat room](https://gitter.im/terraref/reference-data) or creating a [new issue](https://github.com/terraref/computing-pipeline/issues/new). 22 | 23 | ## Related Projects 24 | 25 | This pipeline combines a number of related projects. 26 | In many cases, new features may be discussed in this repository but eventually added directly to existing software. 27 | In these cases, please link the issues across repositories (e.g. add `will be implemented by pecanproject/bety#123` to the issue. 28 | 29 | Some of the related software we will be using and contributing to. 30 | 31 | * BETYdb: github.com/pecanproject/bety 32 | * CoGe (genomics pipeline): https://github.com/LyonsLab/coge 33 | * NCO (raster data processing): https://github.com/nco/nco 34 | * Clowder: https://opensource.ncsa.illinois.edu/bitbucket/projects/CATS/repos/clowder/browse 35 | * Breeding Management Systems 36 | * PlantCV (image processing): https://github.com/danforthcenter/plantcv 37 | 38 | ## Creating Issues 39 | 40 | - Make sure you have a GitHub account. 41 | - Search GitHub and Google to see if your issue has already been reported 42 | - Create an issue in GitHub, assuming one does not already exist. 43 | - Clearly describe the issue including steps to reproduce when it is a bug. 44 | - Make sure you fill in the earliest version that you know has the issue. 45 | - Ask @dlebauer or @robkooper to add you to the TERRA Ref project if you plan on fixing the issue. 46 | 47 | * Github Issues 48 | * [computing pipeline (infrastructure, algorithms)](github.com/terraref/computing-pipeline/issues/new) 49 | * [reference data (data products)](github.com/terraref/reference-data/issues/new) 50 | 51 | ## Contributing Text or Code 52 | 53 | ### Overview 54 | 55 | When you add a significant **new feature**, please create an issue first, to allow others to comment and give feedback. 56 | 57 | When you have created a new feature or non-trivial change to existing code, create a 'pull request'. 58 | 59 | **Branching and Pull Requests**: Although many core developers have write permissions on the TERRA Ref repositories, 60 | _please use the feature branch workflow_ (below) in order to allow pull requests, automated testing, and code review. 61 | 62 | 63 | ### Web and Desktop Interfaces 64 | 65 | If you haven't used git before, the GitHub website and GitHub desktop client allow you to do all of the following within a graphical user interface. The GitHub interface is often the easiest way to make changes even if you do know git. These make contributing easy and fun, and are well documented. 66 | 67 | Any file can be edited in the GitHub interface, and new files can be created. 68 | GitHub will create these as a new pull request. 69 | 70 | ### Using Git at the Command Line 71 | 72 | Introduce your self to GIT, make sure you use an email associated with your GitHub account. 73 | 74 | ``` 75 | git config --global user.name "John Doe" 76 | git config --global user.email johndoe@example.com 77 | ``` 78 | 79 | [Fork this repository](https://github.com/terraref/computing-pipeline/new/master#fork-destination-box) 80 | 81 | Clone your fork of this repository 82 | 83 | ``` 84 | git clone https://github.com//computing-pipeline.git 85 | ``` 86 | 87 | Setup repository to be able to fetch from the master 88 | 89 | ``` 90 | git remote add upstream https://github.com/terraref/computing-pipeline.git 91 | ``` 92 | 93 | ### Adding Features and Submitting Changes 94 | 95 | Always work in a branch rather than directly on the master branch. 96 | Branches should focus on fixing or adding a single feature or set of closely related features 97 | because this will make it easier to review and merge your contributions. 98 | If more than one person is working on the same code, make sure to keep your master branch in sync with the master of the terraref/computing-pipeline repository. 99 | 100 | Here is a simplified workflow on how add a new feature: 101 | 102 | ### Get latest version 103 | 104 | Update your master (both locally and on GitHub) 105 | 106 | ``` 107 | git fetch upstream 108 | git checkout master 109 | git merge upstream/master 110 | git push 111 | ``` 112 | 113 | ### Create a branch to do your work. 114 | 115 | A good practice is to call the branch in the form of GH- followed by the title of the issue. This makes it easier to find out the issue you are trying to solve and helps us to understand what is done in the branch. Calling a branch my-work is confusing. Names of branch can not have a space, and should be replaced with a hyphen. 116 | 117 | ``` 118 | git checkout -b GH-issuenumber-title-of-issue 119 | ``` 120 | 121 | ### Work and commit 122 | 123 | Do you work, and commit as you see fit.Make your commit messages helpful. 124 | 125 | ### Push your changes up to GitHub. 126 | 127 | If this is the first time pushing to GitHub you will need to extended command, other wise you can simply do a `git push`. 128 | 129 | ``` 130 | git push -u origin GH-issuenumber-title-of-issue 131 | ``` 132 | 133 | ### Pull Request 134 | 135 | When finished create a pull request from your branch to the main pecan repository. 136 | 137 | ## Code Of Conduct 138 | 139 | ### Summary: 140 | 141 | Harassment in code and discussion or violation of physical boundaries is completely unacceptable anywhere in TERRA-REF’s project codebases, issue trackers, chatrooms, mailing lists, meetups, and other events. Violators will be warned by the core team. Repeat violations will result in being blocked or banned by the core team at or before the 3rd violation. 142 | 143 | ### In detail 144 | 145 | Harassment includes offensive verbal comments related to gender identity, gender expression, sexual orientation, disability, physical appearance, body size, race, religion, sexual images, deliberate intimidation, stalking, sustained disruption, and unwelcome sexual attention. 146 | 147 | Individuals asked to stop any harassing behavior are expected to comply immediately. 148 | 149 | Maintainers are also subject to the anti-harassment policy. 150 | 151 | If anyone engages in harassing behavior, including maintainers, we may take appropriate action, up to and including warning the offender, deletion of comments, removal from the project’s codebase and communication systems, and escalation to GitHub support. 152 | 153 | If you are being harassed, notice that someone else is being harassed, or have any other concerns, please contact a member of the core team or email dlebauer@illinois.edu immediately. 154 | 155 | We expect everyone to follow these rules anywhere in TERRA-REF's project codebases, issue trackers, chatrooms, and mailing lists. 156 | 157 | Finally, don't forget that it is human to make mistakes! We all do. Let’s work together to help each other, resolve issues, and learn from the mistakes that we will all inevitably make from time 158 | 159 | ### Thanks 160 | 161 | Thanks to the [Fedora Code of Conduct](https://getfedora.org/code-of-conduct) and [JSConf Code of Conduct](http://jsconf.com/codeofconduct.html). 162 | 163 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 8 | 9 | 10 | 11 | 14 | 15 | 18 | 19 | ### Description 20 | 27 | 28 | 38 | ### Completion Criteria 39 | 59 | 60 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | _please edit this template before submitting a new pull request_ 2 | 3 | 6 | 7 | 10 | 11 | ## Description 12 | 18 | 19 | ## Types of changes 20 | 25 | 26 | ## Checklist: 27 | 28 | * [ ] I updated the documentation 29 | 30 | * [ ] Relevant tests (and test data) have been added or updated and they pass 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # PyCharm project files 2 | .idea 3 | 4 | # OSX files 5 | .DS_Store 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, TERRA REF 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TERRA-REF Computing Pipeline 2 | 3 | The computing pipeline is a data storage and computing system that provides researchers with access to all of the ‘raw’ data and derived plant phenotypes. 4 | 5 | ### Contact: 6 | 7 | * [Website](https://terraref.org) 8 | * email: dlebauer@illinois.edu 9 | * [GitHub Issues](https://github.com/terraref/computing-pipeline/issues) 10 | 11 | 12 | To provide input on reference data products and support for existing standards and software, please visit the [Reference Data GitHub repository](https://github.com/terraref/reference-data). 13 | 14 | * [GitHub Issues](https://github.com/terraref/reference-data/issues) 15 | 16 | 17 | ### Terms of Use 18 | 19 | Project wide - if any code or data do not have clear terms of reuse, please request that the author provide one. We use [BSD 3 Clause by default](https://opensource.org/licenses/BSD-3-Clause). 20 | 21 | Creative Commons License
25 | TERRA-REF Computing Pipeline by TERRA Reference Team is licensed under a 27 | Creative Commons Attribution 4.0 International License. 29 | -------------------------------------------------------------------------------- /demos/plantcv/VIS_SV_0_z1_h1_g0_e82_1224684.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terraref/computing-pipeline/02325cae897a19b68c6e87ca5e5d0ce05c5ef6a6/demos/plantcv/VIS_SV_0_z1_h1_g0_e82_1224684.png -------------------------------------------------------------------------------- /extractors/betydb/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM terraref/terrautils:1.2 2 | MAINTAINER Max Burnette 3 | 4 | # Install any programs needed 5 | RUN useradd -u 49044 extractor \ 6 | && mkdir -p /home/extractor/sites/ua-mac \ 7 | && chown -R extractor /home/extractor 8 | 9 | # command to run when starting docker 10 | COPY entrypoint.sh extractor_info.json *.py /home/extractor/ 11 | 12 | USER extractor 13 | ENTRYPOINT ["/home/extractor/entrypoint.sh"] 14 | CMD ["extractor"] 15 | 16 | # Setup environment variables. These are passed into the container. You can change 17 | # these to your setup. If RABBITMQ_URI is not set, it will try and use the rabbitmq 18 | # server that is linked into the container. MAIN_SCRIPT is set to the script to be 19 | # executed by entrypoint.sh 20 | ENV RABBITMQ_EXCHANGE="terra" \ 21 | RABBITMQ_VHOST="%2F" \ 22 | RABBITMQ_QUEUE="terra.betydb" \ 23 | MAIN_SCRIPT="terra_betydb.py" 24 | -------------------------------------------------------------------------------- /extractors/betydb/README.md: -------------------------------------------------------------------------------- 1 | # BETYdb extractor 2 | 3 | Generic uploader for CSV files into BETYdb traits database 4 | 5 | ## Authors 6 | 7 | * Max Burnette, National Supercomputing Applications, Urbana, Il 8 | 9 | 10 | ## Overview 11 | 12 | This extractor uploads CSV files generated by trait extractors such as canopy_cover to BETYdb. -------------------------------------------------------------------------------- /extractors/betydb/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # rabbitmq 5 | if [ "$RABBITMQ_URI" == "" ]; then 6 | # configure RABBITMQ_URI if started using docker-compose or --link flag 7 | if [ -n "$RABBITMQ_PORT_5672_TCP_ADDR" ]; then 8 | RABBITMQ_URI="amqp://guest:guest@${RABBITMQ_PORT_5672_TCP_ADDR}:${RABBITMQ_PORT_5672_TCP_PORT}/${RABBITMQ_VHOST}" 9 | fi 10 | 11 | # configure RABBITMQ_URI if rabbitmq is up for kubernetes 12 | # TODO needs implementation maybe from NDS people 13 | fi 14 | 15 | # start server if asked 16 | if [ "$1" = 'extractor' ]; then 17 | cd /home/extractor 18 | 19 | if [ "$RABBITMQ_PORT_5672_TCP_ADDR" != "" ]; then 20 | # start extractor after rabbitmq is up 21 | for i in `seq 1 10`; do 22 | if nc -z $RABBITMQ_PORT_5672_TCP_ADDR $RABBITMQ_PORT_5672_TCP_PORT ; then 23 | exec python ${MAIN_SCRIPT} 24 | fi 25 | sleep 1 26 | done 27 | fi 28 | 29 | # just launch extractor and see what happens 30 | exec python ${MAIN_SCRIPT} 31 | fi 32 | 33 | exec "$@" -------------------------------------------------------------------------------- /extractors/betydb/extractor_info.json: -------------------------------------------------------------------------------- 1 | { 2 | "@context": "http://clowder.ncsa.illinois.edu/contexts/extractors.jsonld", 3 | "name": "terra.betydb", 4 | "version": "1.1", 5 | "description": "BETYdb CSV uploader", 6 | "author": "Max Burnette", 7 | "contributors": [], 8 | "contexts": [], 9 | "repository": {"repType": "git", "repUrl": "https://github.com/terraref/computing-pipeline.git"}, 10 | "process": { 11 | "file": [ 12 | "text.csv" 13 | ] 14 | }, 15 | "external_services": [], 16 | "dependencies": [], 17 | "bibtex": [] 18 | } -------------------------------------------------------------------------------- /extractors/betydb/terra_betydb.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from pyclowder.utils import CheckMessage 4 | from pyclowder.files import download_metadata, upload_metadata 5 | from terrautils.extractors import TerrarefExtractor, is_latest_file, load_json_file, \ 6 | build_metadata, build_dataset_hierarchy 7 | from terrautils.betydb import add_arguments, get_sites, get_sites_by_latlon, submit_traits, \ 8 | get_site_boundaries 9 | from terrautils.metadata import get_extractor_metadata, get_terraref_metadata 10 | 11 | 12 | def add_local_arguments(parser): 13 | # add any additional arguments to parser 14 | add_arguments(parser) 15 | 16 | class BetyDBUploader(TerrarefExtractor): 17 | def __init__(self): 18 | super(BetyDBUploader, self).__init__() 19 | 20 | add_local_arguments(self.parser) 21 | 22 | # parse command line and load default logging configuration 23 | self.setup(sensor='stereoTop_canopyCover') 24 | 25 | # assign other argumentse 26 | self.bety_url = self.args.bety_url 27 | self.bety_key = self.args.bety_key 28 | 29 | def check_message(self, connector, host, secret_key, resource, parameters): 30 | self.start_check(resource) 31 | 32 | if not resource['name'].endswith(".csv"): 33 | self.log_skip(resource,"%s is not a CSV file" % resource['name']) 34 | return CheckMessage.ignore 35 | 36 | md = download_metadata(connector, host, secret_key, resource['id']) 37 | if get_extractor_metadata(md, self.extractor_info['name']) and not self.overwrite: 38 | self.log_skip(resource,"metadata indicates it was already processed") 39 | return CheckMessage.ignore 40 | return CheckMessage.download 41 | 42 | def process_message(self, connector, host, secret_key, resource, parameters): 43 | self.start_message(resource) 44 | 45 | with open(resource['local_paths'][0], 'r') as inputcsv: 46 | inputlines = inputcsv.readlines() 47 | 48 | if len(inputlines) <= 1: 49 | # first check if there is data besides header line 50 | self.log_info(resource, "no trait lines found in CSV; skipping upload") 51 | else: 52 | # submit CSV to BETY 53 | self.log_info(resource, "found %s trait lines; submitting CSV to bety" % str(len(inputlines)-1)) 54 | submit_traits(resource['local_paths'][0], betykey=self.bety_key) 55 | 56 | # Add metadata to original dataset indicating this was run 57 | self.log_info(resource, "updating file metadata (%s)" % resource['id']) 58 | ext_meta = build_metadata(host, self.extractor_info, resource['id'], {}, 'file') 59 | upload_metadata(connector, host, secret_key, resource['id'], ext_meta) 60 | 61 | self.end_message(resource) 62 | 63 | if __name__ == "__main__": 64 | extractor = BetyDBUploader() 65 | extractor.start() 66 | -------------------------------------------------------------------------------- /extractors/compressor/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM terraref/terrautils:1.4 2 | MAINTAINER Max Burnette 3 | 4 | # Install any programs needed 5 | RUN useradd -u 49044 extractor \ 6 | && mkdir -p /home/clowder/sites/ua-mac \ 7 | && chown -R extractor /home/clowder 8 | 9 | # command to run when starting docker 10 | COPY entrypoint.sh *.py /home/clowder/ 11 | 12 | USER extractor 13 | ENTRYPOINT ["/home/clowder/entrypoint.sh"] 14 | CMD ["extractor"] 15 | 16 | # Setup environment variables. These are passed into the container. You can change 17 | # these to your setup. If RABBITMQ_URI is not set, it will try and use the rabbitmq 18 | # server that is linked into the container. MAIN_SCRIPT is set to the script to be 19 | # executed by entrypoint.sh 20 | ENV RABBITMQ_EXCHANGE="terra" \ 21 | RABBITMQ_VHOST="%2F" \ 22 | RABBITMQ_QUEUE="terra.compressor" \ 23 | MAIN_SCRIPT="terra_compressor.py" 24 | -------------------------------------------------------------------------------- /extractors/compressor/README.md: -------------------------------------------------------------------------------- 1 | # Compressor extractor 2 | 3 | Compress geoTIFFs with LZW to reduce filesize. 4 | 5 | ## Authors 6 | 7 | * Max Burnette, National Supercomputing Applications, Urbana, Il 8 | -------------------------------------------------------------------------------- /extractors/compressor/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # rabbitmq 5 | if [ "$RABBITMQ_URI" == "" ]; then 6 | # configure RABBITMQ_URI if started using docker-compose or --link flag 7 | if [ -n "$RABBITMQ_PORT_5672_TCP_ADDR" ]; then 8 | RABBITMQ_URI="amqp://guest:guest@${RABBITMQ_PORT_5672_TCP_ADDR}:${RABBITMQ_PORT_5672_TCP_PORT}/${RABBITMQ_VHOST}" 9 | fi 10 | 11 | # configure RABBITMQ_URI if rabbitmq is up for kubernetes 12 | # TODO needs implementation maybe from NDS people 13 | fi 14 | 15 | # start server if asked 16 | if [ "$1" = 'extractor' ]; then 17 | cd /home/clowder 18 | 19 | if [ "$RABBITMQ_PORT_5672_TCP_ADDR" != "" ]; then 20 | # start extractor after rabbitmq is up 21 | for i in `seq 1 10`; do 22 | if nc -z $RABBITMQ_PORT_5672_TCP_ADDR $RABBITMQ_PORT_5672_TCP_PORT ; then 23 | exec python ${MAIN_SCRIPT} 24 | fi 25 | sleep 1 26 | done 27 | fi 28 | 29 | # just launch extractor and see what happens 30 | exec python ${MAIN_SCRIPT} 31 | fi 32 | 33 | exec "$@" -------------------------------------------------------------------------------- /extractors/compressor/terra_compressor.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | import pika 4 | 5 | 6 | RMQ_URI = os.getenv("RABBITMQ_URI", "amqp://guest:guest@127.0.0.1/%2f") 7 | RMQ_XCHNG = os.getenv("RABBITMQ_EXCHANGE", "clowder") 8 | RMQ_QUEUE = os.getenv("RABBITMQ_QUEUE", "terra.compressor") 9 | 10 | def connect(): 11 | params = pika.URLParameters(RMQ_URI+"?heartbeat_interval=300") 12 | connection = pika.BlockingConnection(params) 13 | channel = connection.channel() 14 | channel.basic_qos(prefetch_count=1) 15 | channel.queue_declare(queue=RMQ_QUEUE, durable=True) 16 | channel.exchange_declare(exchange=RMQ_XCHNG, exchange_type='topic', durable=True) 17 | channel.queue_bind(queue=RMQ_QUEUE, exchange=RMQ_XCHNG, routing_key=RMQ_QUEUE) 18 | return channel 19 | 20 | def fetch_job(chan, method, properties, body): 21 | # Get next message containing a dirpath from RabbitMQ 22 | files = os.listdir(body) 23 | for f in files: 24 | if f.endswith(".tif"): 25 | f_path = os.path.join(body, f) 26 | compress(f_path) 27 | chan.basic_ack(delivery_tag=method.delivery_tag) 28 | 29 | def compress(input_file): 30 | print("Compressing %s" % input_file) 31 | temp_out = input_file.replace(".tif", "_compress.tif") 32 | subprocess.call(["gdal_translate", "-co", "COMPRESS=LZW", input_file, temp_out]) 33 | if os.path.isfile(temp_out): 34 | os.remove(input_file) 35 | os.rename(temp_out, input_file) 36 | 37 | 38 | channel = connect() 39 | channel.basic_consume(fetch_job, queue=RMQ_QUEUE, no_ack=False) 40 | channel.start_consuming() 41 | -------------------------------------------------------------------------------- /extractors/compressor/terra_compressor_queue.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pika 3 | 4 | """ 5 | Crawl the listed directories and queue up dataset/plot folders for LZW compressor jobs. 6 | """ 7 | 8 | RMQ_URI = os.getenv("RABBITMQ_URI", "amqp://guest:guest@127.0.0.1/%2f") 9 | RMQ_XCHNG = os.getenv("RABBITMQ_EXCHANGE", "clowder") 10 | RMQ_QUEUE = os.getenv("RABBITMQ_QUEUE", "terra.compressor") 11 | 12 | tif_roots = ["/home/clowder/sites/ua-mac/Level_1_Plots/rgb_geotiff", 13 | "/home/clowder/sites/ua-mac/Level_1_Plots/ir_geotiff", 14 | "/home/clowder/sites/ua-mac/Level_1/rgb_geotiff", 15 | "/home/clowder/sites/ua-mac/Level_1/ir_geotiff"] 16 | 17 | 18 | def connect(): 19 | params = pika.URLParameters(RMQ_URI) 20 | connection = pika.BlockingConnection(params) 21 | channel = connection.channel() 22 | channel.basic_qos(prefetch_count=1) 23 | channel.queue_declare(queue=RMQ_QUEUE, durable=True) 24 | channel.exchange_declare(exchange=RMQ_XCHNG, exchange_type='topic', durable=True) 25 | channel.queue_bind(queue=RMQ_QUEUE, exchange=RMQ_XCHNG, routing_key=RMQ_QUEUE) 26 | return channel 27 | 28 | channel = connect() 29 | for root_dir in tif_roots: 30 | dates = os.listdir(root_dir) 31 | for d in dates: 32 | date_dir = os.path.join(root_dir, d) 33 | subdirs = os.listdir(date_dir) 34 | for sd in subdirs: 35 | sd_dir = os.path.join(date_dir, sd) 36 | print("Queuing %s" % sd_dir) 37 | if not channel.connection.is_open: 38 | channel = connect() 39 | channel.basic_publish(RMQ_XCHNG, RMQ_QUEUE, sd_dir) 40 | channel.connection.close() 41 | -------------------------------------------------------------------------------- /extractors/geostreams/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM terraref/terrautils:1.2 2 | MAINTAINER Max Burnette 3 | 4 | # Install any programs needed 5 | RUN useradd -u 49044 extractor \ 6 | && mkdir -p /home/extractor/sites/ua-mac \ 7 | && chown -R extractor /home/extractor 8 | 9 | # command to run when starting docker 10 | COPY entrypoint.sh extractor_info.json *.py /home/extractor/ 11 | 12 | USER extractor 13 | ENTRYPOINT ["/home/extractor/entrypoint.sh"] 14 | CMD ["extractor"] 15 | 16 | # Setup environment variables. These are passed into the container. You can change 17 | # these to your setup. If RABBITMQ_URI is not set, it will try and use the rabbitmq 18 | # server that is linked into the container. MAIN_SCRIPT is set to the script to be 19 | # executed by entrypoint.sh 20 | ENV RABBITMQ_EXCHANGE="terra" \ 21 | RABBITMQ_VHOST="%2F" \ 22 | RABBITMQ_QUEUE="terra.geostreams" \ 23 | MAIN_SCRIPT="terra_geostreams.py" 24 | -------------------------------------------------------------------------------- /extractors/geostreams/README.md: -------------------------------------------------------------------------------- 1 | # Geostreams extractor 2 | 3 | Generic uploader for CSV files into Geostreams database 4 | 5 | ## Authors 6 | 7 | * Max Burnette, National Supercomputing Applications, Urbana, Il 8 | 9 | 10 | ## Overview 11 | 12 | This extractor uploads CSV files generated by trait extractors such as canopy_cover to Geostreams. -------------------------------------------------------------------------------- /extractors/geostreams/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # rabbitmq 5 | if [ "$RABBITMQ_URI" == "" ]; then 6 | # configure RABBITMQ_URI if started using docker-compose or --link flag 7 | if [ -n "$RABBITMQ_PORT_5672_TCP_ADDR" ]; then 8 | RABBITMQ_URI="amqp://guest:guest@${RABBITMQ_PORT_5672_TCP_ADDR}:${RABBITMQ_PORT_5672_TCP_PORT}/${RABBITMQ_VHOST}" 9 | fi 10 | 11 | # configure RABBITMQ_URI if rabbitmq is up for kubernetes 12 | # TODO needs implementation maybe from NDS people 13 | fi 14 | 15 | # start server if asked 16 | if [ "$1" = 'extractor' ]; then 17 | cd /home/extractor 18 | 19 | if [ "$RABBITMQ_PORT_5672_TCP_ADDR" != "" ]; then 20 | # start extractor after rabbitmq is up 21 | for i in `seq 1 10`; do 22 | if nc -z $RABBITMQ_PORT_5672_TCP_ADDR $RABBITMQ_PORT_5672_TCP_PORT ; then 23 | exec python ${MAIN_SCRIPT} 24 | fi 25 | sleep 1 26 | done 27 | fi 28 | 29 | # just launch extractor and see what happens 30 | exec python ${MAIN_SCRIPT} 31 | fi 32 | 33 | exec "$@" -------------------------------------------------------------------------------- /extractors/geostreams/extractor_info.json: -------------------------------------------------------------------------------- 1 | { 2 | "@context": "http://clowder.ncsa.illinois.edu/contexts/extractors.jsonld", 3 | "name": "terra.geostreams", 4 | "version": "1.1", 5 | "description": "Geostreams CSV uploader", 6 | "author": "Max Burnette", 7 | "contributors": [], 8 | "contexts": [], 9 | "repository": {"repType": "git", "repUrl": "https://github.com/terraref/computing-pipeline.git"}, 10 | "process": { 11 | "file": [ 12 | "text.csv" 13 | ] 14 | }, 15 | "external_services": [], 16 | "dependencies": [], 17 | "bibtex": [] 18 | } -------------------------------------------------------------------------------- /extractors/geostreams/terra_geostreams.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | import csv 5 | 6 | from pyclowder.utils import CheckMessage 7 | from pyclowder.datasets import get_info 8 | from pyclowder.files import download_metadata, upload_metadata 9 | from terrautils.extractors import TerrarefExtractor, is_latest_file, load_json_file, \ 10 | build_metadata, build_dataset_hierarchy 11 | from terrautils.betydb import add_arguments, get_sites, get_sites_by_latlon, submit_traits, \ 12 | get_site_boundaries 13 | from terrautils.geostreams import create_datapoint_with_dependencies 14 | from terrautils.gdal import clip_raster, centroid_from_geojson 15 | from terrautils.metadata import get_extractor_metadata, get_terraref_metadata 16 | 17 | 18 | 19 | class GeostreamsUploader(TerrarefExtractor): 20 | def __init__(self): 21 | super(GeostreamsUploader, self).__init__() 22 | 23 | # parse command line and load default logging configuration 24 | self.setup(sensor='stereoTop_canopyCover') 25 | 26 | def check_message(self, connector, host, secret_key, resource, parameters): 27 | self.start_check(resource) 28 | 29 | md = download_metadata(connector, host, secret_key, resource['id']) 30 | if get_extractor_metadata(md, self.extractor_info['name']) and not self.overwrite: 31 | self.log_skip(resource,"metadata indicates it was already processed") 32 | return CheckMessage.ignore 33 | return CheckMessage.download 34 | 35 | def process_message(self, connector, host, secret_key, resource, parameters): 36 | self.start_message(resource) 37 | 38 | successful_plots = 0 39 | with open(resource['local_paths'][0], 'rb') as csvfile: 40 | reader = csv.DictReader(csvfile) 41 | for row in reader: 42 | centroid_lonlat = [row['lon'], row['lat']] 43 | time_fmt = row['dp_time'] 44 | timestamp = row['timestamp'] 45 | dpmetadata = { 46 | "source": row['source'], 47 | "value": row['value'] 48 | } 49 | trait = row['trait'] 50 | 51 | create_datapoint_with_dependencies(connector, host, secret_key, trait, 52 | (centroid_lonlat[1], centroid_lonlat[0]), time_fmt, time_fmt, 53 | dpmetadata, timestamp) 54 | successful_plots += 1 55 | 56 | # Add metadata to original dataset indicating this was run 57 | self.log_info(resource, "updating file metadata (%s)" % resource['id']) 58 | ext_meta = build_metadata(host, self.extractor_info, resource['id'], { 59 | "plots_processed": successful_plots, 60 | }, 'file') 61 | upload_metadata(connector, host, secret_key, resource['id'], ext_meta) 62 | 63 | self.end_message(resource) 64 | 65 | if __name__ == "__main__": 66 | extractor = GeostreamsUploader() 67 | extractor.start() 68 | -------------------------------------------------------------------------------- /extractors/plotclipper/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM terraref/terrautils:1.5 2 | MAINTAINER Max Burnette 3 | 4 | # Install any programs needed 5 | RUN useradd -u 49044 extractor \ 6 | && mkdir /home/extractor \ 7 | && mkdir /home/extractor/sites 8 | 9 | RUN chown -R extractor /home/extractor \ 10 | && chgrp -R extractor /home/extractor 11 | 12 | # Install PDAL 13 | RUN apt-get update \ 14 | && apt-get install -y pdal \ 15 | imagemagick \ 16 | gdal-bin \ 17 | libgdal-dev 18 | 19 | # Install laser3d science package 20 | RUN pip install --upgrade pip 21 | RUN pip install wheel laspy multiprocess plyfile \ 22 | && pip install --upgrade pyclowder \ 23 | && pip install --upgrade terraref-laser3d \ 24 | && pip install --upgrade cryptography 25 | 26 | # command to run when starting docker 27 | COPY entrypoint.sh extractor_info.json *.py /home/extractor/ 28 | 29 | USER extractor 30 | ENTRYPOINT ["/home/extractor/entrypoint.sh"] 31 | CMD ["extractor"] 32 | 33 | # Setup environment variables. These are passed into the container. You can change 34 | # these to your setup. If RABBITMQ_URI is not set, it will try and use the rabbitmq 35 | # server that is linked into the container. MAIN_SCRIPT is set to the script to be 36 | # executed by entrypoint.sh 37 | ENV RABBITMQ_EXCHANGE="terra" \ 38 | RABBITMQ_VHOST="%2F" \ 39 | RABBITMQ_QUEUE="terra.plotclipper" \ 40 | MAIN_SCRIPT="terra_plotclipper.py" 41 | -------------------------------------------------------------------------------- /extractors/plotclipper/README.md: -------------------------------------------------------------------------------- 1 | # Plot clipper extractor 2 | 3 | Clip GeoTIFF or LAS files according to plots 4 | 5 | ## Authors 6 | 7 | * Max Burnette, National Supercomputing Applications, Urbana, Il 8 | -------------------------------------------------------------------------------- /extractors/plotclipper/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # rabbitmq 5 | if [ "$RABBITMQ_URI" == "" ]; then 6 | # configure RABBITMQ_URI if started using docker-compose or --link flag 7 | if [ -n "$RABBITMQ_PORT_5672_TCP_ADDR" ]; then 8 | RABBITMQ_URI="amqp://guest:guest@${RABBITMQ_PORT_5672_TCP_ADDR}:${RABBITMQ_PORT_5672_TCP_PORT}/${RABBITMQ_VHOST}" 9 | fi 10 | 11 | # configure RABBITMQ_URI if rabbitmq is up for kubernetes 12 | # TODO needs implementation maybe from NDS people 13 | fi 14 | 15 | # start server if asked 16 | if [ "$1" = 'extractor' ]; then 17 | cd /home/extractor 18 | 19 | if [ "$RABBITMQ_PORT_5672_TCP_ADDR" != "" ]; then 20 | # start extractor after rabbitmq is up 21 | for i in `seq 1 10`; do 22 | if nc -z $RABBITMQ_PORT_5672_TCP_ADDR $RABBITMQ_PORT_5672_TCP_PORT ; then 23 | exec python ${MAIN_SCRIPT} 24 | fi 25 | sleep 1 26 | done 27 | fi 28 | 29 | # just launch extractor and see what happens 30 | exec python ${MAIN_SCRIPT} 31 | fi 32 | 33 | exec "$@" -------------------------------------------------------------------------------- /extractors/plotclipper/extractor_info.json: -------------------------------------------------------------------------------- 1 | { 2 | "@context": "http://clowder.ncsa.illinois.edu/contexts/extractors.jsonld", 3 | "name": "terra.plotclipper", 4 | "version": "1.0", 5 | "description": "TERRA-REF plot clipper", 6 | "author": "Max Burnette", 7 | "contributors": [], 8 | "contexts": [], 9 | "repository": [{"repType": "git", "repUrl": "https://github.com/terraref/computing-pipeline.git"}], 10 | "process": { 11 | "dataset": [ 12 | "file.added" 13 | ] 14 | }, 15 | "external_services": [], 16 | "dependencies": [], 17 | "bibtex": [] 18 | } -------------------------------------------------------------------------------- /meeting-notes/2015-10-01_dev_team.md: -------------------------------------------------------------------------------- 1 | # TERRAref 2 | 3 | October 1, 2015 4 | 11 am – 12 pm 5 | 6 | Meeting called by David LeBauer 7 | Attendees: Rob Kooper, Yan Liu, David Raila, Rachel Shekar, Kandace Turner 8 | 9 | 10 | ### New team members 11 | 12 | Rachel Shekar will be the primary project manager, and will work with Jay as well as other Project Managers in Steve's lab. 13 | 14 | ### Request additional budget 15 | 16 | I was asked to submit a budget that would cover increasing our data storage 10x (e.g. from 1TB/d x 6 mo / y x 4 y to 10TB/d). This data is the compressed volume. 17 | I can start by multiplying everything by 10, but mostly, I was wondering if this would change the overall architecture, and if this would require additional personnel. 18 | The request only has to be a paragraph + revised budget. But I wanted to go ahead and throw this out there 19 | 20 | Additional Information: 21 | Added 9/30/15 22 | For reference, here are notes from today's discussion about requesting additional funding for increased data volume requirement. one sensor seems to account for > 90% of total. 23 | 24 | Data volume 25 | 26 | * Issue: budgeted for only 1 PB for life of project – how much data is coming in? 27 | * David needs to know sooner rather than later, at least within an order of magnitude. If it’s 1-2 TB coming in per day, this is maybe a slight cost increase. A few PB may be primarily hardware cost, whereas tens of PB would require more technical support to install / reconfigure a system. Cost is ~$100 / TB / year, so factor this into the total dollar figure. May need to increase system admin labor. 28 | 29 | Data collection plan: Jeff White has been experimenting with different collection plans and figuring out collection times and data volumes. 30 | Notes: 31 | * The biggest data driver is the VNIR hyperspectral camera, which can generate 10-20x more data than all other sensors combined. If all sensors are on in scan mode (excluding side view scanners), it would take 48 hours to scan full field (target was 2x/day or more), and would generate 29 TB data. If VNIR is removed, this drops to 3.9 TB/day. Looking to see if one can pre-process the VNIR data, to filter it and bring the volume down. For spot mode, Jeff still sorting this out. Ben’s recommendation – in year 1, might want to collect all wavelengths, but in future years might want to remove / bin certain bands. 32 | 33 | 34 | Added 10/2/15 35 | Michelle and I had a good discussion. We agreed that we would write up a proposal at the $100/TB/y primary and $50/TB/y backup rate, plus additional funds for a data storage technician and that Michelle could provide a quote with justification on Monday. 36 | 37 | My first pass at a rough estimate follows: 38 | 39 | assume that we accumulate 10 PB at a constant over four years. This is equivalent to 10 PB for two years or 20 PB-years (area of triangle = 1/2 base x height; here base = time and height = storage volume) 40 | 41 | 20PB-years = 20,000 TB-years at $150/y. That gets us to $3,000,000, and we add Plus an FTE for four years at $70k, with indirects etc gets to +$500,000. (Whoa!!!!) So this would basically triple my current budget of $1.7m. 42 | 43 | But I will wait for Michelle for the official word. But I'd like to keep 1FTE (or more) whether we give estimates of 5PB or 10PB. 44 | 45 | 46 | Options for data storage include Blue Waters, ADS and Roger. 47 | Determine the following: 48 | • Does user need to be next to computer or if can access be remote? 49 | • What is need for storage vs. bandwidth for moving data? 50 | • Do we want tape or disk storage and access? 51 | • Will data be computed locally where collected or here? 52 | **Actions: David will have a meeting with Michelle to determine cost** 53 | 54 | ### Review of milestones and open issues on Github. 55 | https://huboard.com/terraref/reference-data#/ 56 | 57 | ### Near-term milestones 58 | David, Yan, Rob, and I are planning to demo an image processing pipeline integrated into the ISDA Clowder tool, this will be based on the PlantCV pipeline and is outlined here: https://github.com/terraref/pipeline/issues/1 59 | 60 | The pipeline was discussed and a user story was drafted. See https://docs.google.com/presentation/d/1LaOI3ESTFwn7b8x5oWZcdOWBLsCh2v4F7yqgQ836etU/edit?usp=sharing 61 | Additional discussion needed for how account management access will be handled (read only) 62 | 63 | 64 | -------------------------------------------------------------------------------- /meeting-notes/2015-10-08_dev_team.md: -------------------------------------------------------------------------------- 1 | # TERRAref Team 2 | 3 | October 8, 2015 4 | 11 am – 12 pm 5 | 6 | Meeting called by David LeBauer 7 | 8 | Present: Rob Kooper, David Lapine, Yan Liu, David Raila, Rachel Shekar, Kandace Turner 9 | 10 | ### Agenda for October Kickoff 11 | This was discussed and further decisions need to be made – primarily deciding between: Open Forum, Lightening talks, Panel discussion, and Q and A session 12 | 13 | ### Update on Virtual Standards Committee Meeting 14 | A list of decisions that need to be made by the committee are available at https://github.com/terraref/reference-data The pipeline group encouraged to give feedback on the data and metadata – is it sensible and sufficient? 15 | 16 | ### Update on Standards Committee Meeting in Arizona 17 | 18 | ### Review of milestones and open issues on Github. 19 | https://github.com/terraref/computing-pipeline 20 | See Github for notes 21 | 22 | ### Additional issues 23 | Data storage – David will present a number of options to DOE. It’s still unclear if data needs to be accessible at all times and after the project or if it can be put into storage. Look into Amazon Glacier. 24 | 25 | Action Items: 26 | 27 | **David LeBauer to develop CyberGIS statement of work** 28 | 29 | **David LeBauer to look into feasibility of Amazon Glacier** 30 | 31 | **David LeBauer to check compression, write to disk speed, and if this can be parallelized – talk to Donna Cox, Intel Expoint** 32 | 33 | **David Raila to recommend services that create unique IDs for data** 34 | 35 | **David Raila to recommend data managements system tools** 36 | -------------------------------------------------------------------------------- /meeting-notes/2015-10-12_dev_team.md: -------------------------------------------------------------------------------- 1 | # TERRAref Team 2 | 3 | October 12, 2015 4 | 11 am – 12 pm 5 | 6 | Meeting called by Kandace Turner 7 | Invited: Rob Kooper, David Raila, Rachel Shekar, Kandace Turner 8 | 9 | Reference Data Committee 10 | 11 | ### Data standards documents will be published 12 | NDS to assign DOIs 13 | NDS to automate publications 14 | ### Discussion of Danforth reporting dates 15 | are our dates the dates to report to Danforth or for Danforth to report to NSF? 16 | See https://github.com/terraref/computing-pipeline/issues 17 | 18 | ### End user how-to files 19 | All members should write user documentation, preferably during development. This can even be tagged in the code (#FAQ or similar) 20 | This should be ready by December 21 | ### Go live dates 22 | January – data can be uploaded 23 | March – ready for data capture 24 | 25 | ### new issues 26 | 27 | ** Determine who can capture and downsize data **@dlebauer 28 | 29 | ** Determine if ADS is ready ** @dlebauer and David Raila 30 | 31 | ** Check infrastructure capacity ** @dlebauer 32 | 33 | ** Create end user how-to file ** @dlebauer – due date? 34 | 35 | ** Design interface ** @robkooper 36 | 37 | ** Ensure that PlantCV runs in Clowder ** @yanliu-chn 38 | -------------------------------------------------------------------------------- /meeting-notes/2015-10-22_dev_team.md: -------------------------------------------------------------------------------- 1 | # TERRAref Team 2 | October 22, 2015 11am-12pm 3 | Reference data committee 4 | 5 | Present: Rob Kooper, David LeBauer, Jay Roloff, Yan Liu, Dan LaPine, Max Burnette, Scott Rhode 6 | # 7 | David went through is mockup workflow demo. He will post it on github 8 | 9 | Generally, the steps were 10 | - How to know which extractor to use? Need a Mask. 11 | - Create mask for suite of images 12 | - Select a few images 13 | - Button to launch vm, choose shell, Jupiter,… 14 | - Do LS of data directory 15 | - Launch Jupiter 16 | 17 | We can not see a real time build of the gantry system currently, but David showed us files on box that include pictures of plants at the field site and a diagram of data sources and flow at the site. 18 | 19 | For the Demo, we want to show default plant cv data or new data would be nice but not necessary. Need to be able to reproduce PlantCV worklflow from tutorial and embed on Clowder 20 | 21 | The group discussed renaming the project 22 | 23 | See updated github issues https://github.com/terraref/computing-pipeline 24 | -------------------------------------------------------------------------------- /meeting-notes/2015-11-20_computational_pipeline.md: -------------------------------------------------------------------------------- 1 | # TERRAref Genomics Pipeline Meeting 2 | 3 | * Date: Friday, Nov 20, 2015 4 | * Time: 11:00-12:00 pm CST 5 | 6 | ## Participants: 7 | 8 | * David LeBauer (TERRAref & TERRA cat 1, Illinois) 9 | * Vicor Jongeneel (HPCBio, Illinois) 10 | * Matt Hudson (Illinois) 11 | * Chris Fields (HPCBio, Illinois) 12 | * Noah Fahlgren (Danforth) 13 | * Rob Alba (Danforth) 14 | 15 | ## Agenda 16 | 17 | 1. Review proposal terraref/reference-data#19 18 | 2. Determine roles / interests / contributions of each participating group 19 | * TERRA REF 20 | * TERRA MEPP 21 | * HPCBio 22 | 3. Estimate data volumes (order of magnitude?) for each step 23 | 4. Define Use cases 24 | 25 | ## Teams 26 | 27 | ### TERRA REF 28 | 29 | Will provide central repository and computing resources for genomics pipeline. 30 | 31 | Core data set; sequencing to be done by Jeremy Schmutz at HudsonAlpha 32 | 33 | * 75 shared lines 34 | * 40 de novo sequences, 35 | * 15 by PacBio (? I infer this is a sort of 'gold standard'?) 36 | * 400 resequences (enough coverage with enough coverage to do assembly 37 | 38 | Estimated completion date: Mid 2016 39 | 40 | 41 | ### TERRA MEPP 42 | 43 | 44 | Amount of sequencing is to be determined, unclear from programmatic level what will be required. Pending discussions like this. 45 | 46 | 47 | ### HPC Bio 48 | 49 | Can support this project; they need clear specifications and funding 50 | 51 | ## Data volume and compute needs 52 | 53 | ### Sequence alignments 54 | 55 | * BAM can be very large (TBs); can be visualized in Jbrowse. 56 | * Sorted BAM is smaller (compressed). 57 | * Only need to keep sorted BAM. Can drop raw SAM and BAM intermediate files. 58 | * 1000 genomes, 30x ea. 4TB of BAMs + 4TB fastQ, no more than 20 TB. 59 | 60 | * CPU time: 61 | * alignment depends on aligner efficency; runs 20x faster than BWA. 10k - 100k CPU-h for re-aligning 1000 genomes 62 | * denovo assembly: depends on context 63 | 64 | ### Genome Assembly 65 | 66 | * assembling Cassava genome 5k CPU-h per genome. This will be more time than the re-alignment. 67 | * Define Use cases 68 | 69 | ## Open Questions to be addressed by larger TERRA program 70 | 71 | ### Shared germplasm 72 | 73 | * where to partition 15, 40, 400 lines in genomics diversity 74 | * What will be the coordinated germplasm used for cross-site G x E analysis? 75 | * ED: should spread 15 PacBiom through diversity of germplam + more coverage on more likely to be commercial lines 76 | 77 | 78 | ### Define data flows for the entire project 79 | 80 | * makes sense to analyze data at same place it is generated. Then deposit both raw data and derived products in common repository. 81 | * compute, browsing, visualization should be done where data is 82 | * a core standardized pipeline would be of value to individual teams 83 | 84 | 85 | ### Define Collaborations among teams (with germplasm, informatics, cyberinfrastrucutre) 86 | 87 | * **TODO** Survey of theteams to be coordinated by Rob, Rachel, and David 88 | 89 | What can teams collaborate on? 90 | * what people are required to do according to milestones, and what are the overlaps? 91 | * how can the reference team help? to what degree are teams interested in using a centralized pipeline, and in sharing data at different points in the pipeline? 92 | 93 | 94 | #### Summary 95 | 96 | The Cat5 reference platform was not designed to develop bioinformatics tools. Indeed, the focus of TERRA is to develop the technology of phenomics that has fallen behind that of genomics. 97 | 98 | HPCBio provides a wide range of bioinformatics computing services [1] and the cluster they use is biocluster [2]. The costs of using Biocluster are reasonable and both the HPCBio and IGB teams are exceptional. In addition, IGB have Galaxy and KBase available. Fees are reasonable and the quality of their work is high. The HPCBio team is enthusiastic about our project and available to assist. 99 | Access to the expertise and services of HPCBio provides a major added value to users of the reference pipeline, but has not been budgeted as an essential feature. 100 | 101 | Because the forcus of the TERRA program is to advance phenomics, the UI / NCSA team has been built around relevant expertise in ecophysiology, high performance computing, GIS, and computational workflows. 102 | The TERRA program has many experts in bioinformatics. We can thus provide a common collaborative space for the implementation of cutting edge pipelines while also supporting the use of existing and familiar tools and substantial computing power. 103 | 104 | 105 | NCSA is very generous and supportive of our efforts, and have committed to removing limitations imposed by computing power or storage space. The limiting factor will be the number of contributors to a shared infrastructure and the efficiency with which they can collaborate. 106 | 107 | #### For individual teams, protected IP 108 | 109 | We have a core set of computing allocated for use by independent researchers and teams without any requirement that the teams share code or data. Indeed, secure computing and storage is an important feature of our platform. While this resource is more limited, NCSA has staff to help researchers apply for as well as use and optimize code for HPC allocations (through xsede.org). XSEDE allocations must be renewed annually, but usually the challenge is getting people to use them. 110 | 111 | #### For collaboration and open science objectives 112 | 113 | We can provide an unprecedented level of support for open science aimed at sharing data and computing infrastructure. 114 | 115 | For more details about the computing that has been allocated, the CyberGIS group has committed 1 PB of online storage and a million compute hours on a dedicated node, plus access to many times that much computing on a shared queue [3]. 116 | Additional requests to support the 10x increase in data volume that has occurred during construction of the Lemnatec field system [4]. 117 | 118 | 119 | * [1] http://hpcbio.illinois.edu/content/services-and-fees 120 | * [2] http://help.igb.illinois.edu/Biocluster 121 | * [3] https://wiki.ncsa.illinois.edu/display/ROGER/ROGER+Technical+Summary 122 | * [4] http://terraref.ncsa.illinois.edu/articles/spectral-imaging-data-volume-compression/#on-the-upcoming-data-deluge 123 | 124 | -------------------------------------------------------------------------------- /meeting-notes/2016-01-14_Dev team.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Call in Friday 2-4 MT (3-5 CST) computational pipelines 4 | 5 | Max talked to Bob Strand 6 | Camera to be run, data will include metadata from equipment 7 | 8 | Demo – David R and Max 9 | Button will work for demo, but there will be a script to follow 10 | Secure access has not been established. 11 | Not all tools will have secure interface. 12 | David will have to work behind secure API server but it will be easier for Max to run via secure video. 13 | 14 | How can external people access data in projects folder on Roger 15 | Right now, compressed zip files 1-2 GB each, 600 GB 16 | 17 | Globus w/ Stu 18 | Our major interest is data sharing – there is a publication for the sharing system 19 | What support can Stu offer 20 | Give overview of our project and how Globus would fit in 21 | 22 | Scott, Rob and David L to simplify API to extract data – Wednesday, 1 hour 23 | 24 | Dora – 50% work on searching data get datasets that they want. Will implement functionalities of clowder to search data by multiple conditions and across databases and enhance data analysis 25 | 26 | Next milestones: 27 | 28 | * Demo at Tech Showcase 29 | * June ribbon cutting 30 | * November workshop / tri-societies meeting 31 | * Integrate Jesse's fieldbook with UIUC computing pipeline 32 | * CoGe pipeline 33 | * Get DDPSC --> UIUC pipeline fully functional 34 | 35 | 36 | Software: 37 | 38 | * TASSEL-GBS, GAPIT, MLMM, R/qtl 39 | * ASReml 40 | * photogrammetry: AgiSoft, PhotoScan, and Pix4d 41 | -------------------------------------------------------------------------------- /scripts/FLIR/FlirRawToTemperature.m: -------------------------------------------------------------------------------- 1 | [FileName,PathName,FilterIndex] = uigetfile('*.bin'); 2 | 3 | fileID = fopen([PathName FileName]); 4 | RawPixelValue = fread(fileID,[640,480],'uint16'); 5 | 6 | figure(1),title('Raw Camera Value') 7 | RawVector=reshape(RawPixelValue,1,640*480); 8 | hist(RawVector,65025); 9 | set(gca,'yscale',"log") 10 | xlabel ("raw pixel value") 11 | ylabel ("n") 12 | 13 | 14 | % Constant camera-specific parameters determined by FLIR 15 | 16 | % Plank constant - Flir 17 | 18 | % R Planck constant function of integration time and wavelength) 19 | % B Planck constant function of wavelength 20 | % F Planck constant positive value (0 - 1) 21 | % J0 global offset 22 | % J1 global gain 23 | 24 | R = 15976.1; 25 | B = 1417.3; 26 | F = 1.00; 27 | J0 = 3597; 28 | J1 = 73.549; 29 | 30 | % Constant Atmospheric transmission parameter by Flir 31 | 32 | X = 1.9; 33 | a1 = 0.006569; 34 | b1 = -0.002276; 35 | a2 = 0.01262; 36 | b2 = -0.00667; 37 | 38 | % Constant for VPD computation (sqtrH2O) 39 | H2O_K1 = 1.56E+00; 40 | H2O_K2 = 6.94E-02; 41 | H2O_K3 = -2.78E-04; 42 | H2O_K4 = 6.85E-07; 43 | 44 | % Environmental factors 45 | % According to FLIR, atmospheric absorption under 10m object distance can be 46 | % neglected, expecially under dry desert climate 47 | 48 | % H = Relative Humidity from the gantry (0 - 1) 49 | % T = air temperature in degree Celsius from the gantry 50 | % D = ObjectDistance - camera/canopy (m) 51 | % E = object emissivity, vegetation is around 0.98, bare soil around 0.93... 52 | % AmbTemp or reflective Temp (K): NEED TO BE MEASURED BEFORE/AFTER IMAGE ACQUISITION 53 | % AtmTemp or air temp (K) 54 | % ExtOpticsTemp (K) = AtmTemp 55 | % by default: AmbTemp = AtmTemp = ExtOpticsTemp 56 | 57 | H = 0.1; % gantry value 58 | T = 22.0; % gantry value 59 | D = 2.5; 60 | E = 0.98; 61 | 62 | AmbTemp = T + 273.15; % Temperature at canopy level assumed to Atmospheric temperature 63 | AtmTemp = T + 273.15; 64 | ExtOpticsTemp = 287.95; % to be specified and not used here 65 | 66 | 67 | % Theoretical object radiation = Raw pixel values (raw_pxl_val) 68 | % Flir image in 16 integer unsigned format 69 | 70 | 71 | 72 | 73 | % Step 1: Atmospheric transmission - correction factor from air temp, relative humidity and distance sensor-object; 74 | 75 | % Vapour pressure deficit call here sqrtH2O => convert relative humidity and air temperature in VPD - mmHg - 1mmHg=0.133 322 39 kPa 76 | 77 | H2OInGperM2 = H*exp(H2O_K1 + H2O_K2*T + H2O_K3*(T.^2) + H2O_K4*(T.^3)); 78 | 79 | % Atmospheric transmission correction: tao 80 | a1b1sqH2O = (a1+b1*sqrt(H2OInGperM2)); 81 | a2b2sqH2O = (a2+b2*sqrt(H2OInGperM2)); 82 | exp1 = exp(-sqrt(D/2)*a1b1sqH2O); 83 | exp2 = exp(-sqrt(D/2)*a2b2sqH2O); 84 | 85 | tao = X*exp1 + (1-X)*exp2; % Atmospheric transmission factor 86 | 87 | 88 | % Step 2: Step2: Correct raw pixel values from external factors; 89 | 90 | % General equation : Total Radiation = Object Radiation + Atmosphere Radiation + Ambient Reflection Radiation 91 | 92 | 93 | % Object Radiation: obj_rad 94 | % obj_rad = Theoretical object radiation * emissivity * atmospheric transmission 95 | % Theoretical object radiation: raw_pxl_val 96 | 97 | obj_rad = RawPixelValue.* E * tao; % FOR EACH PIXEL 98 | 99 | % Atmosphere Radiation: atm_rad 100 | % atm_rad= (1 - atmospheric transmission) * Theoretical atmospheric radiation 101 | % Theoretical atmospheric radiation: theo_atm_rad 102 | theo_atm_rad = (R*J1/(exp(B/AtmTemp)-F)) +J0; 103 | 104 | atm_rad = repmat((1 - tao).* theo_atm_rad,size(RawPixelValue)); 105 | 106 | % Ambient Reflection Radiation: amb_refl_rad 107 | % amb_refl_rad = (1 - emissivity) * atmospheric transmission * Theoretical Ambient Reflection Radiation 108 | % Theoretical Ambient Reflection Radiation: theo_amb_refl_rad 109 | theo_amb_refl_rad = (R*J1/(exp(B/AmbTemp)-F)) + J0; 110 | 111 | amb_refl_rad = repmat((1 - E) * tao * theo_amb_refl_rad,size(RawPixelValue)); 112 | 113 | % Total Radiation: corr_pxl_val 114 | corr_pxl_val= obj_rad + atm_rad + amb_refl_rad; % FOR EACH PIXEL 115 | 116 | 117 | % Step 3:RBF equation: transformation of pixel intensity in radiometric temperature from raw values or corrected values 118 | 119 | % in kelvin 120 | pxl_temp = B./log(R./(corr_pxl_val - J0).*J1+F); % FOR EACH PIXEL 121 | 122 | % in degree Celsius 123 | pxl_temp = B./log(R./(corr_pxl_val - J0).*J1+F) - 273.15; % FOR EACH PIXEL 124 | 125 | 126 | 127 | figure(2), title('Temperature in 0.01 °C') 128 | TempVector=reshape(pxl_temp,1,640*480); 129 | hist(TempVector,1000); 130 | set(gca,'yscale',"log") 131 | xlabel ("temperature") 132 | ylabel ("n") 133 | 134 | 135 | %colormap('hot') 136 | %figure(1),imagesc(At),colorbar, axis off -------------------------------------------------------------------------------- /scripts/FLIR/GetFLIR.m: -------------------------------------------------------------------------------- 1 | [FileName,PathName,FilterIndex] = uigetfile('*.bin'); 2 | 3 | fileID = fopen([PathName FileName]); 4 | A = fread(fileID,[640,480],'uint16'); 5 | 6 | 7 | %rescale to visible range 8 | Gmin=2800; 9 | Gmax=3300; 10 | 11 | At=((A-Gmin)/(Gmax-Gmin)); 12 | 13 | colormap('hot') 14 | figure(1),imagesc(At),colorbar, axis off -------------------------------------------------------------------------------- /scripts/NDVI/GetNDVI.m: -------------------------------------------------------------------------------- 1 | ## Copyright (C) 2016 LemnaTec 2 | ## 3 | ## This program is free software; you can redistribute it and/or modify it 4 | ## under the terms of the GNU General Public License as published by 5 | ## the Free Software Foundation; either version 3 of the License, or 6 | ## (at your option) any later version. 7 | ## 8 | ## This program is distributed in the hope that it will be useful, 9 | ## but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | ## GNU General Public License for more details. 12 | ## 13 | ## You should have received a copy of the GNU General Public License 14 | ## along with this program. If not, see . 15 | 16 | ## -*- texinfo -*- 17 | ## @deftypefn {Function File} {@var{retval} =} GetNDVI (@var{input1}, @var{input2}) 18 | ## 19 | ## @seealso{} 20 | ## @end deftypefn 21 | 22 | ## Author: LemnaTec 23 | ## Created: 2016-06-03 24 | 25 | function [NDVI,x,y,z,time] = GetNDVI (PathName) 26 | 27 | if nargin==0 28 | 29 | [PathName] = uigetdir(); 30 | 31 | end 32 | 33 | 34 | D=dir(PathName); 35 | 36 | % read NDVI from file 37 | Text=importdata([PathName '\' D(4).name]); 38 | fi=findstr(Text{2},'"'); 39 | NDVI=str2num(Text{2}(fi(3)+1:fi(4)-1)); 40 | 41 | % read x,y,z from file 42 | Text=importdata([PathName '\' D(3).name]); 43 | % x = row 22 y = row 23 z = row 24 44 | fi=findstr(Text{22},'"'); 45 | x=str2num(Text{22}(fi(3)+1:fi(4)-1)); 46 | 47 | fi=findstr(Text{23},'"'); 48 | y=str2num(Text{23}(fi(3)+1:fi(4)-1)); 49 | 50 | fi=findstr(Text{24},'"'); 51 | z=str2num(Text{24}(fi(3)+1:fi(4)-1)); 52 | 53 | % read time 54 | Text=importdata([PathName '\' D(3).name]); 55 | fi=findstr(Text{21},'"'); 56 | timestr=Text{21}(fi(3)+1:fi(4)-1); 57 | Year=timestr(7:10); 58 | Month=timestr(1:2); 59 | Day=timestr(4:5); 60 | Hour=timestr(12:13); 61 | Minute=timestr(15:16); 62 | 63 | time = datenum(str2num(Year), str2num(Month), str2num(Day), str2num(Hour), str2num(Minute)); 64 | 65 | endfunction 66 | -------------------------------------------------------------------------------- /scripts/NDVI/PlotNDVI.m: -------------------------------------------------------------------------------- 1 | ## Copyright (C) 2016 LemnaTec 2 | ## 3 | ## This program is free software; you can redistribute it and/or modify it 4 | ## under the terms of the GNU General Public License as published by 5 | ## the Free Software Foundation; either version 3 of the License, or 6 | ## (at your option) any later version. 7 | ## 8 | ## This program is distributed in the hope that it will be useful, 9 | ## but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | ## GNU General Public License for more details. 12 | ## 13 | ## You should have received a copy of the GNU General Public License 14 | ## along with this program. If not, see . 15 | 16 | ## -*- texinfo -*- 17 | ## @deftypefn {Function File} {@var{retval} =} PlotNDVI (@var{input1}, @var{input2}) 18 | ## 19 | ## @seealso{} 20 | ## @end deftypefn 21 | 22 | ## Author: LemnaTec 23 | ## Created: 2016-06-04 24 | 25 | function [retval] = PlotNDVI (input1, input2) 26 | 27 | [PathName] = uigetdir(); 28 | 29 | D=dir(PathName); 30 | 31 | for i=3:size(D,1) 32 | 33 | [NDVI(i-2),x(i-2),y(i-2),z(i-2),t(i-2)]=GetNDVI([PathName '\' D(i).name]); 34 | 35 | end 36 | 37 | colorMap=jet(length(unique(NDVI))); 38 | set(gcf, 'ColorMap', colorMap); 39 | h=plot(x,y); 40 | 41 | endfunction 42 | -------------------------------------------------------------------------------- /scripts/PS2/GetPS2.m: -------------------------------------------------------------------------------- 1 | 2 | [FileName,PathName,FilterIndex] = uigetfile('*.bin'); 3 | pkg image load 4 | clear M 5 | close all 6 | 7 | D=dir(PathName); 8 | 9 | 10 | 11 | for i=4:size(D,1)-1 12 | 13 | 14 | fileID = fopen([PathName D(i).name]); 15 | A = fread(fileID,[1936,1216],'uint8'); 16 | A=double(A)./255; 17 | 18 | M(i-3)=mean(mean(A)); 19 | 20 | 21 | 22 | end 23 | 24 | figure(1),subplot(3,1,1),plot(M), xlabel("frame"),ylabel("mean intensity") 25 | 26 | % get Frame 1 as Fdark 27 | fileID = fopen([PathName D(4).name]); 28 | Fdark = fread(fileID,[1936,1216],'uint8'); 29 | Fdark=double(Fdark)./255; 30 | % get Frame 2 as Fv 31 | fileID = fopen([PathName D(5).name]); 32 | F0 = fread(fileID,[1936,1216],'uint8'); 33 | % subtract Fdark 34 | F0=double(F0)./255-Fdark; 35 | 36 | fileID = fopen([PathName D(40).name]); 37 | Fm = fread(fileID,[1936,1216],'uint8'); 38 | % subtract Fdark 39 | Fm=double(Fm)./255-Fdark; 40 | FmHist=reshape(Fm,1,1936*1216); 41 | 42 | % image mask 43 | threshold=max(max(Fm))/10; 44 | 45 | mask=Fm>threshold & (Fm-F0)>=0; 46 | 47 | % clean up a bit 48 | se = strel ("square", 5); 49 | % remove background 50 | mask=imerode (mask, se); 51 | % fill holes 52 | mask=imdilate(mask, se); 53 | 54 | FvFm=(Fm-F0)./Fm.*mask; 55 | 56 | colormap('hot') 57 | figure(1),subplot(3,1,2),imagesc(FvFm),colorbar, axis off 58 | 59 | FvFm(FvFm==0)=NaN; 60 | FvFmHist=reshape(FvFm,1,1936*1216); 61 | figure(1),subplot(3,1,3),hist(FvFmHist,50),, xlim([0,1]),xlabel("Fv/Fm"),ylabel("no of pixel") 62 | 63 | -------------------------------------------------------------------------------- /scripts/ShareClowderDatasetsViaSpaces.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | import argparse 4 | import requests 5 | import json 6 | 7 | # Parse command-line arguments 8 | ########################################### 9 | def options(): 10 | """Parse command line options. 11 | 12 | Args: 13 | 14 | Returns: 15 | argparse object 16 | 17 | Raises: 18 | 19 | """ 20 | 21 | parser = argparse.ArgumentParser(description="Share Clowder Datasets by associating them with a Clowder Space.", 22 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 23 | parser.add_argument("-s", "--space", help="Clowder Space ID.", required=True) 24 | parser.add_argument("-c", "--collection", 25 | help="Clowder Collection ID. This collection should contain the Datasets to be shared.", 26 | required=True) 27 | parser.add_argument("-u", "--url", help="Clowder URL.", required=True) 28 | parser.add_argument("-U", "--username", help="Clowder username.", required=True) 29 | parser.add_argument("-p", "--password", help="Clowder password.", required=True) 30 | parser.add_argument("-v", "--verbose", help="Verbose output.", action="store_true") 31 | parser.add_argument("-n", "--dryrun", help="Dry run, do not update Datasets.", action="store_true") 32 | 33 | args = parser.parse_args() 34 | 35 | return args 36 | 37 | 38 | ########################################### 39 | 40 | # Main 41 | ########################################### 42 | def main(): 43 | """Main program. 44 | """ 45 | 46 | # Get options 47 | args = options() 48 | 49 | # Create new session 50 | session = requests.Session() 51 | session.auth = (args.username, args.password) 52 | 53 | # Does the Space exist? 54 | space = session.get(args.url + 'api/spaces/' + args.space) 55 | if space.status_code == 200: 56 | if args.verbose: 57 | print("The Space {0} returns:".format(args.space)) 58 | print(json.dumps(json.loads(space.text), indent=4, separators=(',', ': '))) 59 | else: 60 | raise StandardError("Getting Space {0} failed: Return value = {1}".format(args.space, space.status_code)) 61 | 62 | # Does the Collection exist? 63 | collection = session.get(args.url + 'api/collections/' + args.collection) 64 | if collection.status_code == 200: 65 | if args.verbose: 66 | print("The Collection {0} returns:".format(args.collection)) 67 | print(json.dumps(json.loads(collection.text), indent=4, separators=(',', ': '))) 68 | else: 69 | raise StandardError("Getting Collection {0} failed: Return value = {1}".format(args.collection, 70 | collection.status_code)) 71 | 72 | # Get all the Datasets in the Collection 73 | datasets = session.get(args.url + 'api/collections/' + args.collection + '/getDatasets') 74 | if datasets.status_code == 200: 75 | ds = json.loads(datasets.text) 76 | for dataset in ds: 77 | if args.verbose: 78 | print(json.dumps(dataset, indent=4, separators=(',', ': '))) 79 | 80 | # Add Dataset to Space 81 | if args.dryrun is False: 82 | response = session.post(args.url + 'api/spaces/' + args.space + '/addDatasetToSpace/' + dataset['id']) 83 | if response.status_code != 200: 84 | raise StandardError("Adding Dataset {0} to Space {1} failed: Response value = {2}".format( 85 | dataset['id'], args.space, response.status_code)) 86 | else: 87 | raise StandardError("Getting Datasets for Collection {0} failed: Return value = {1}".format( 88 | args.collection, datasets.status_code)) 89 | 90 | 91 | ########################################### 92 | 93 | 94 | if __name__ == '__main__': 95 | main() -------------------------------------------------------------------------------- /scripts/StereoVIS/GetStereoVIS.m: -------------------------------------------------------------------------------- 1 | close all, clear all 2 | 3 | pkg load signal 4 | 5 | [FileName,PathName,FilterIndex] = uigetfile('*.bin'); 6 | 7 | D=dir(PathName); 8 | 9 | % read left image 10 | fileID = fopen([PathName D(3).name]); 11 | 12 | Aleft = fread(fileID,[3296,2472],'uint8'); 13 | Aleft=demosaic(uint8(Aleft),'gbrg')./255; 14 | 15 | % use only half resolution due to memory constraint 16 | Aleft = Aleft(1:2:end,1:2:end,:); 17 | 18 | % read right image 19 | fileID = fopen([PathName D(5).name]); 20 | Aright = fread(fileID,[3296,2472],'uint8'); 21 | Aright=double(demosaic(uint8(Aright),'gbrg'))./255; 22 | Aright = Aright(1:2:end,1:2:end,:); 23 | % combine both images 24 | 25 | % add both images 26 | for i=1:3 27 | Atotal(:,:,i)=[Aleft(:,:,i)' Aright(:,:,i)']; 28 | end 29 | 30 | 31 | figure(1), imshow(Atotal) 32 | 33 | % align both images% add both images 34 | Aalign=[]; 35 | cutleft=250; 36 | cutright=980; 37 | for i=1:3 38 | Aalign(:,:,i)=[Aleft(1:end-cutleft,:,i)' Aright(cutright:end,:,i)']; 39 | end 40 | 41 | figure(2), imshow(Aalign); 42 | -------------------------------------------------------------------------------- /scripts/addPlotsToGeostreams.py: -------------------------------------------------------------------------------- 1 | import sys, os, json 2 | import requests 3 | from osgeo import gdal 4 | from osgeo import osr 5 | 6 | 7 | """ 8 | This script will push polygons in a shapefile into Geostreams database as sensors (plots). 9 | 10 | Plot name will be checked to avoid duplication. 11 | 12 | SENSORS 13 | gid | name | geog | created | metadata 14 | -----+-----------------+--------------------------------------------------------------------+-------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------- 15 | 13 | Range 41 Pass 2 | 01010000A0E6100000954E254166FE5BC0EE08D3E2B98940400000000000000000 | 2017-02-02 11:45:26.501508-06 | {"type":{"id":"LemnaTec","title":"LemnaTec Field Scanalyzer","sensorType":4},"name":"Range 41 Pass 2","popupContent":"Range 41 Pass 2","region":"Maricopa"} 16 | 14 | Range 14 Pass 4 | 01010000A0E6100000F030722B65FE5BC041CE78169A8940400000000000000000 | 2017-02-02 11:45:27.499092-06 | {"type":{"id":"LemnaTec","title":"LemnaTec Field Scanalyzer","sensorType":4},"name":"Range 14 Pass 4","popupContent":"Range 14 Pass 4","region":"Maricopa"} 17 | 15 | Range 20 Pass 6 | 01010000A0E610000090D7C49263FE5BC0358B04D8C68940400000000000000000 | 2017-02-02 11:45:28.511511-06 | {"type":{"id":"LemnaTec","title":"LemnaTec Field Scanalyzer","sensorType":4},"name":"Range 20 Pass 6","popupContent":"Range 20 Pass 6","region":"Maricopa"} 18 | """ 19 | 20 | 21 | # Get sensor ID from Clowder based on plot name 22 | def get_sensor_info(host, key, name): 23 | if(not host.endswith("/")): 24 | host = host+"/" 25 | 26 | url = "%sapi/geostreams/sensors?sensor_name=%s&key=%s" % (host, name, key) 27 | r = requests.get(url) 28 | if r.status_code == 200: 29 | json_data = r.json() 30 | for s in json_data: 31 | if 'name' in s and s['name'] == name: 32 | return s 33 | else: 34 | print("error searching for sensor ID") 35 | 36 | return None 37 | 38 | def create_sensor(host, key, name, geom): 39 | if(not host.endswith("/")): 40 | host = host+"/" 41 | 42 | body = { 43 | "name": name, 44 | "type": "point", 45 | "geometry": geom, 46 | "properties": { 47 | "popupContent": name, 48 | "type": { 49 | "id": "MAC Field Scanner", 50 | "title": "MAC Field Scanner", 51 | "sensorType": 4 52 | }, 53 | "name": name, 54 | "region": "Maricopa" 55 | } 56 | } 57 | 58 | url = "%sapi/geostreams/sensors?key=%s" % (host, key) 59 | r = requests.post(url, 60 | data=json.dumps(body), 61 | headers={'Content-type': 'application/json'}) 62 | if r.status_code == 200: 63 | return r.json()['id'] 64 | else: 65 | logging.error("error creating sensor") 66 | 67 | return None 68 | 69 | 70 | shpFile = sys.argv[1] 71 | outFile = "coords.csv" 72 | out = open(outFile, 'w') 73 | 74 | # OPEN SHAPEFILE 75 | ds = gdal.OpenEx( shpFile, gdal.OF_VECTOR | gdal.OF_READONLY) 76 | layerName = os.path.basename(shpFile).split('.shp')[0] 77 | lyr = ds.GetLayerByName(layerName) 78 | 79 | # load shp file 80 | lyr.ResetReading() 81 | num_records = lyr.GetFeatureCount() 82 | lyr_defn = lyr.GetLayerDefn() 83 | t_srs = lyr.GetSpatialRef() 84 | s_srs = osr.SpatialReference() 85 | s_srs.ImportFromEPSG(4326) 86 | transform = osr.CoordinateTransformation(s_srs, t_srs) 87 | transform_back = osr.CoordinateTransformation(t_srs, s_srs) 88 | 89 | fi_rangepass = lyr_defn.GetFieldIndex('RangePass') 90 | fi_range = lyr_defn.GetFieldIndex('Range') 91 | fi_pass = lyr_defn.GetFieldIndex('Pass') 92 | fi_macentry = lyr_defn.GetFieldIndex('MAC_ENTRY') 93 | 94 | # ITERATE OVER FEATURES 95 | for f in lyr: 96 | # DETERMINE SENSOR NAME 97 | plotid = f.GetFieldAsString(fi_rangepass) 98 | geom = f.GetGeometryRef() 99 | 100 | geom.Transform(transform_back) 101 | centroid = geom.Centroid() 102 | 103 | # POST IT IF IT DOESN'T EXIST 104 | host = "http://terraref.ncsa.illinois.edu/clowder" 105 | key = "" 106 | 107 | plot_name = "Range "+plotid.replace("-", " Pass ") 108 | sensor_data = get_sensor_info(host, key, plot_name) 109 | if not sensor_data: 110 | print("%s being created at: (%s, %s)" % (plot_name, centroid.GetX(), centroid.GetY()) ) 111 | sensor_id = create_sensor(host, key, plot_name, { 112 | "type": "Point", 113 | "coordinates": [centroid.GetX(), centroid.GetY(), 0] 114 | }) 115 | else: 116 | sensor_id = sensor_data['id'] 117 | c = sensor_data['geometry']['coordinates'] 118 | if c[0] == centroid.GetX() and c[1] == centroid.GetY(): 119 | print("%s [%s]: (%s, %s)" % (plot_name, sensor_id, c[0], c[1])) 120 | else: 121 | print("%s [%s]: (%s, %s) -> (%s, %s)" % (plot_name, sensor_id, c[0], c[1], centroid.GetX(), centroid.GetY())) 122 | 123 | out.write("UPDATE sensors SET geog=ST_SetSRID(ST_MakePoint(%s,%s,0), 4326) WHERE gid=%s;\n" % ( 124 | centroid.GetX(), 125 | centroid.GetY(), 126 | sensor_id)) 127 | 128 | out.close() 129 | -------------------------------------------------------------------------------- /scripts/email_xferlog: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | date=$(date --date="2 day ago" +%Y%m%d) 4 | 5 | /root/scripts/terra_report 2> /dev/null > /tmp/${date}_report 6 | 7 | echo Summary and Transfer Logs Attached for $date | mutt -s "$date Transfer Log from Gantry" -a /tmp/${date}_report -a /var/log/xferlog-$date.gz -- terrareflog@lists.illinois.edu 8 | 9 | mv /tmp/${date}_report /root/reports/ 10 | -------------------------------------------------------------------------------- /scripts/environmental_logger/environmental_logger_unittest.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This is the unit test module for environmental_logger_json2netcdf.py. 3 | It will test whether the environmental_logger_json2netcdf.py works 4 | appropriately and the validity of the imported JSON. 5 | 6 | This module will run isolated, so there's no include dependency 7 | to other files, but make sure it is in the same location as environmental_logger_json2netcdf 8 | 9 | Before running this module, make sure you have passed 10 | it a valid JSON, or all the test will be skipped 11 | 12 | To run the unit test, simply use: 13 | python environmental_logger_unittest.py 14 | ''' 15 | 16 | import unittest 17 | import sys 18 | from environmental_logger_json2netcdf import * 19 | 20 | fileLocation = sys.argv[1] 21 | 22 | 23 | class environmental_logger_json2netcdfUnitTest(unittest.TestCase): 24 | 25 | def setUp(self): 26 | self.testCase = JSONHandler(fileLocation) 27 | 28 | @unittest.skipIf(not os.path.isfile(fileLocation), 29 | "the testing JSON file does not exist") 30 | def test_canGetAWellFormattedJSON(self): 31 | ''' 32 | This test checks if the EnvironmentalLogger received a legal JSON file. Since 33 | JSONHandler just simply pass the JSON to built-in JSON module and is guaranteed 34 | to be noexcept, any error in this test case would be cause by a badly formatted 35 | JSON 36 | 37 | Skipped if the file does not exist 38 | ''' 39 | 40 | self.setUp() 41 | self.assertEqual(len(self.testCase), 4) 42 | self.assertIs(type(self.testCase[0]), list) 43 | 44 | @unittest.skipIf(not os.path.isfile(fileLocation), 45 | "the testing JSON file does not exist") 46 | def test_canGetExpectedNumberOfWavelength(self): 47 | ''' 48 | This test checks if the environmental_logger_json2netcdf can get the wavelength by 49 | testing the number of wvl collected 50 | 51 | Skipped if the file does not exist 52 | ''' 53 | 54 | self.setUp() 55 | self.assertEqual(len(self.testCase[1]), 1024) 56 | self.assertIsInstance(self.testCase[1][0], float) 57 | 58 | 59 | @unittest.skipIf(not os.path.isfile(fileLocation), 60 | "the testing JSON file does not exist") 61 | def test_canGetExpectedNumberOfSpectrum(self): 62 | ''' 63 | This test checks if the environmental_logger_json2netcdf can get the spectrum by 64 | testing whether it is a 2D-array (It is implemented as a 2D-array) 65 | 66 | Skipped if the file does not exist 67 | ''' 68 | 69 | self.setUp() 70 | self.assertEqual(len(self.testCase), 4) 71 | self.assertEqual(len(self.testCase[2]), 39) 72 | 73 | @unittest.skipIf(not os.path.isfile(fileLocation), 74 | "the testing JSON file does not exist") 75 | def test_canGetAListOfValueFromImportedJSON(self): 76 | ''' 77 | This test checks if the environmental_logger_json2netcdf can get any value from 78 | the JSON (as a list) 79 | 80 | Skipped if the file does not exist 81 | ''' 82 | 83 | self.setUp() 84 | testingJSON = [JSONMembers[u"environment_sensor_set_reading"] for JSONMembers in self.testCase[0]] 85 | self.assertIsInstance(getListOfValue(testingJSON, u"weatherStationAirPressure"), list) 86 | self.assertIsInstance(getListOfValue(testingJSON, u"weatherStationAirPressure")[0], float) 87 | self.assertEqual(len(getListOfValue(testingJSON, u"weatherStationAirPressure")), 39) 88 | 89 | @unittest.skipIf(not os.path.isfile(fileLocation), 90 | "the testing JSON file does not exist") 91 | def test_canGetAListOfRawValueFromImportedJSON(self): 92 | ''' 93 | This test checks if the environmental_logger_json2netcdf can get any raw value from 94 | the JSON (as a list) 95 | 96 | Skipped if the file does not exist 97 | ''' 98 | 99 | self.setUp() 100 | testingJSON = [JSONMembers[u"environment_sensor_set_reading"] for JSONMembers in self.testCase[0]] 101 | self.assertIsInstance(getListOfRawValue(testingJSON, u"weatherStationAirPressure"), list) 102 | self.assertIsInstance(getListOfRawValue(testingJSON, u"weatherStationAirPressure")[0], float) 103 | self.assertEqual(len(getListOfRawValue(testingJSON, u"weatherStationAirPressure")), 39) 104 | 105 | 106 | def test_canTranslateIntoLegalName(self): 107 | pass 108 | 109 | 110 | def tearDown(self): 111 | pass 112 | 113 | 114 | if __name__ == "__main__": 115 | unittest.TextTestRunner(verbosity=2).run(unittest.TestLoader().loadTestsFromTestCase(environmental_logger_json2netcdfUnitTest)) 116 | -------------------------------------------------------------------------------- /scripts/example-scripts/GeostreamDatapointPlotter.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from datetime import datetime 3 | import matplotlib.pyplot as plt 4 | import numpy as np 5 | 6 | url = "https://terraref.ncsa.illinois.edu/clowder/api/geostreams/datapoints?key=Pb3AUSqnUw&stream_id=300" 7 | 8 | response = requests.get(url) 9 | if response.status_code == 200: 10 | json = response.json() 11 | 12 | times = [] 13 | wind_speeds = [] 14 | precipitation_rate = [] 15 | surface_downwelling_shortwave_flux_in_air = [] 16 | northward_wind = [] 17 | relative_humidity = [] 18 | air_temperature = [] 19 | eastward_wind = [] 20 | surface_downwelling_photosynthetic_photon_flux_in_air = [] 21 | 22 | for datapoint in json: 23 | """Example datapoint: 24 | { 'sensor_id': '303', 25 | 'created': '2017-02-03T14:33:11Z', 26 | 'geometry': { 27 | 'type': 'Point', 28 | 'coordinates': [33.0745666667, -68.0249166667, 0] 29 | }, 30 | 'start_time': '2016-08-10T13:50:29Z', 31 | 'id': 36185, 32 | 'stream_id': '300', 33 | 'sensor_name': 'Full Field', 34 | 'end_time': '2016-08-10T13:55:00Z', 35 | 'type': 'Feature', 36 | 'properties': { 37 | 'wind_speed': 1.0890774907749077, 38 | 'precipitation_rate': 0.0, 39 | 'surface_downwelling_shortwave_flux_in_air': 43.60608856088568, 40 | 'northward_wind': -0.9997966833626167, 41 | 'relative_humidity': 60.41579335793356, 42 | 'source': u'https://terraref.ncsa.illinois.edu/clowder/datasets/5893a72c4f0c06726b1b0cda', 43 | 'source_file': u'5893a72f4f0c06726b1b0d20', 44 | 'air_temperature': 301.13597785977885, 45 | 'eastward_wind': -0.3659132309673836, 46 | 'surface_downwelling_photosynthetic_photon_flux_in_air': 152.14981549815525 47 | }}""" 48 | 49 | start_time = datapoint['start_time'] 50 | times.append(datetime.strptime(start_time, "%Y-%m-%dT%H:%M:%SZ")) 51 | 52 | p = datapoint['properties'] 53 | wind_speeds.append(p['wind_speed']) 54 | precipitation_rate.append(p['precipitation_rate']) 55 | surface_downwelling_shortwave_flux_in_air.append(p['surface_downwelling_shortwave_flux_in_air']) 56 | northward_wind.append(p['northward_wind']) 57 | relative_humidity.append(p['relative_humidity']) 58 | air_temperature.append(p['air_temperature']) 59 | eastward_wind.append(p['eastward_wind']) 60 | surface_downwelling_photosynthetic_photon_flux_in_air.append(p['surface_downwelling_photosynthetic_photon_flux_in_air']) 61 | 62 | plt.figure(1) 63 | plt.title("Wind Speed") 64 | plt.plot(times, wind_speeds, 'b-') 65 | 66 | plt.figure(2) 67 | plt.title("Precipitation Rate") 68 | plt.plot(times, precipitation_rate, 'r-') 69 | 70 | plt.figure(3) 71 | plt.title("Surface Downwelling Shortwave Flux in Air") 72 | plt.plot(times, surface_downwelling_shortwave_flux_in_air, 'g-') 73 | 74 | plt.figure(4) 75 | plt.title("Northward Wind") 76 | plt.plot(times, northward_wind, 'c-') 77 | 78 | plt.figure(5) 79 | plt.title("Relative Humidity") 80 | plt.plot(times, relative_humidity, 'm-') 81 | 82 | plt.figure(6) 83 | plt.title("Air Temperature, K") 84 | plt.plot(times, air_temperature, 'y-') 85 | 86 | plt.figure(7) 87 | plt.title("Eastward Wind") 88 | plt.plot(times, eastward_wind, 'k-') 89 | 90 | plt.figure(8) 91 | plt.title("Surface Downwelling Photosynthetic Photon Flux in Air") 92 | plt.plot(times, surface_downwelling_photosynthetic_photon_flux_in_air, 'b-') 93 | 94 | plt.show() 95 | else: 96 | print("no response") 97 | -------------------------------------------------------------------------------- /scripts/example-scripts/TERRAClowderUploadCurl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | USER="" 3 | PASS="" 4 | FILE="C:\Users\mburnet2\Documents\TERRAref\sample-data\Lemnatec-Indoor\fahlgren_et_al_2015_bellwether_jpeg\images\snapshot42549\VIS_SV_0_z3500_307595.jpg" 5 | 6 | curl -X POST -u $USER:$PASS -F "File=@$FILE;type=image/jpg" http://141.142.208.144/clowder/api/files -------------------------------------------------------------------------------- /scripts/example-scripts/TERRAClowderUploadPython.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import requests 4 | import json 5 | 6 | # Clowder path and credentials 7 | clowder_url = 'http://141.142.208.144/clowder/' 8 | clowder_username = '' 9 | clowder_password = '' 10 | 11 | sess = requests.Session() 12 | sess.auth = (clowder_username, clowder_password) 13 | 14 | # Folder containing files, and filenames to upload 15 | # All files will be uploaded to one Dataset! 16 | directory_path = r"C:\Users\mburnet2\Documents\TERRAref\sample-data\Lemnatec-Indoor\fahlgren_et_al_2015_bellwether_jpeg\images\snapshot42549" 17 | files_to_load = os.listdir(directory_path) #["VIS_SV_0_z3500_307595.jpg"] 18 | 19 | # First, create a Dataset for files to live 20 | ds_r = sess.post(clowder_url+"api/datasets/createempty", headers={"Content-Type": "application/json"}, 21 | data='{"name": "Test Dataset"}') 22 | 23 | # Did the request return success? 24 | if ds_r.status_code == 200: 25 | ds_id = ds_r.json()["id"] 26 | print("Dataset created. Adding files...") 27 | 28 | for curr_file in files_to_load: 29 | if curr_file.find(".jpg") > -1: 30 | f = open(directory_path+"\\"+curr_file, 'rb') 31 | r = sess.post(clowder_url+"api/uploadToDataset/"+ds_id, files={"File" : f}) 32 | 33 | # Did the request return success? 34 | if r.status_code == 200: 35 | print("Successfully uploaded "+curr_file+".") 36 | # TODO - Add metadata derived from filename 37 | 38 | else: 39 | print("ERROR: "+str(r)) 40 | print("Failed to upload "+curr_file+".") 41 | 42 | else: 43 | print("ERROR: "+str(ds_r)) 44 | print("Failed to create dataset.") 45 | -------------------------------------------------------------------------------- /scripts/extractor-logging.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | 4 | "formatters": { 5 | "default": { 6 | "format": "%(asctime)-15s %(levelname)-7s : %(name)s - %(message)s" 7 | } 8 | }, 9 | 10 | "handlers": { 11 | "console": { 12 | "class": "logging.StreamHandler", 13 | "formatter": "default", 14 | "level": "DEBUG", 15 | "stream": "ext://sys.stdout" 16 | }, 17 | "logstash": { 18 | "class": "logstash.TCPLogstashHandler", 19 | "level": "INFO", 20 | "host": "logger.ncsa.illinois.edu", 21 | "port": 5000, 22 | "message_type": "terra_extractor", 23 | "version": 1 24 | } 25 | }, 26 | 27 | "root": { 28 | "level": "INFO", 29 | "handlers": ["console", "logstash"] 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /scripts/filecounter/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM terraref/terrautils:1.4 2 | MAINTAINER Max Burnette 3 | 4 | RUN apt-get -y update \ 5 | && apt-get -y install curl \ 6 | && pip install flask-restful \ 7 | flask_wtf \ 8 | python-logstash \ 9 | psycopg2 \ 10 | pandas 11 | 12 | COPY *.py *.json /home/filecounter/ 13 | COPY templates /home/filecounter/templates 14 | 15 | CMD ["python", "/home/filecounter/filecounter.py"] 16 | -------------------------------------------------------------------------------- /scripts/filecounter/README.md: -------------------------------------------------------------------------------- 1 | # Pipeline File Counter 2 | 3 | ##Components 4 | - **filecounter.py** updates once every hour and will automatically update counts for the last 2 weeks. 5 | -------------------------------------------------------------------------------- /scripts/filecounter/config_default.json: -------------------------------------------------------------------------------- 1 | { 2 | "log_path": "/home/filecounter/data/log", 3 | "csv_path": "/home/extractor/sites/ua-mac/reporting", 4 | 5 | "api": { 6 | "port": "5454", 7 | "ip_address": "0.0.0.0" 8 | }, 9 | 10 | "clowder": { 11 | "host": "http://localhost:9000", 12 | "secret_key": "r1ek3rs", 13 | "user_map": {}, 14 | "primary_space": "" 15 | }, 16 | 17 | "postgres": { 18 | "database": "rulemonitor", 19 | "username": "rulemonitor", 20 | "password": "", 21 | "host": "192.168.5.169" 22 | }, 23 | "default_count_start": "2017-04-20", 24 | "default_count_end": "2019-10-01" 25 | } 26 | -------------------------------------------------------------------------------- /scripts/filecounter/config_logging.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | 4 | "formatters": { 5 | "default": { 6 | "format": "%(asctime)-15s %(levelname)-7s : %(name)s - %(message)s" 7 | } 8 | }, 9 | 10 | "handlers": { 11 | "console": { 12 | "class": "logging.StreamHandler", 13 | "formatter": "default", 14 | "level": "DEBUG", 15 | "stream": "ext://sys.stdout" 16 | }, 17 | "file": { 18 | "class": "logging.handlers.TimedRotatingFileHandler", 19 | "formatter": "default", 20 | "level": "DEBUG", 21 | "filename": "filecounter.log", 22 | "when": "D", 23 | "encoding": "utf8" 24 | }, 25 | "logstash": { 26 | "class": "logstash.TCPLogstashHandler", 27 | "level": "INFO", 28 | "host": "logger.ncsa.illinois.edu", 29 | "port": 5000, 30 | "message_type": "gantry", 31 | "version": 1 32 | } 33 | }, 34 | 35 | "loggers": { 36 | "gantry": { 37 | "level": "DEBUG", 38 | "handlers": ["file"] 39 | } 40 | }, 41 | 42 | "root": { 43 | "level": "INFO", 44 | "handlers": ["console", "logstash"] 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /scripts/filecounter/counts.py: -------------------------------------------------------------------------------- 1 | import os 2 | from collections import OrderedDict 3 | 4 | """ 5 | Dictionary of count definitions for various sensors. 6 | 7 | Types: 8 | timestamp: count timestamp directories in each date directory 9 | psql: count rows returned from specified postgres query 10 | regex: count files within each date directory that match regex 11 | Other fields: 12 | path: path containing date directories for timestamp or regex counts 13 | regex: regular expression to execute on date directory for regex counts 14 | query: postgres query to execute for psql counts 15 | parent: previous count definition for % generation (e.g. bin2tif's parent is stereoTop) 16 | """ 17 | uamac_root = "/home/clowder/sites/ua-mac/" 18 | SENSOR_COUNT_DEFINITIONS = { 19 | 20 | "stereoTop": OrderedDict([ 21 | # basic products 22 | ("stereoTop", { 23 | "path": os.path.join(uamac_root, 'raw_data/stereoTop/'), 24 | "type": 'timestamp' 25 | }), 26 | ("rgb_geotiff", { 27 | "path": os.path.join(uamac_root, 'Level_1/rgb_geotiff/'), 28 | "type": 'timestamp', 29 | "parent": "stereoTop", 30 | "extractor": "terra.stereo-rgb.bin2tif"}), 31 | ("rgb_nrmac", { 32 | "path": os.path.join(uamac_root, 'Level_2/rgb_nrmac/'), 33 | "type": 'timestamp', 34 | "parent": "rgb_geotiff", 35 | "extractor": "terra.stereo-rgb.nrmac"}), 36 | ("rgb_mask", { 37 | "path": os.path.join(uamac_root, 'Level_2/rgb_mask/'), 38 | "type": 'timestamp', 39 | "parent": "rgb_geotiff", 40 | "extractor": "terra.stereo-rgb.rgbmask"}), 41 | # plot products 42 | ("rgb_geotiff_plot", { 43 | "path": os.path.join(uamac_root, 'Level_1_Plots/rgb_geotiff/'), 44 | "type": 'plot'}), 45 | # rulechecker & fieldmosaic products 46 | ("ruledb_rgbff", { 47 | "type": "psql", 48 | "query_count": "select count(distinct file_path) from extractor_ids where output->>'rule'='Full Field' and output->>'sensor'='RGB GeoTIFFs' and output->>'date'='%s';", 49 | "query_list": "select distinct file_path from extractor_ids where output->>'rule'='Full Field' and output->>'sensor'='RGB GeoTIFFs' and output->>'date'='%s';", 50 | "parent": "rgb_geotiff", 51 | "extractor": "ncsa.rulechecker.terra"}), 52 | ("rgbff", { 53 | "path": os.path.join(uamac_root, 'Level_2/rgb_fullfield/'), 54 | "type": 'regex', 55 | "regex": ".*_rgb.tif"}), 56 | ("ruledb_nrmacff", { 57 | "type": "psql", 58 | "query_count": "select count(distinct file_path) from extractor_ids where output->>'rule'='Full Field' and output->>'sensor'='RGB GeoTIFFs NRMAC' and output->>'date'='%s';", 59 | "query_list": "select distinct file_path from extractor_ids where output->>'rule'='Full Field' and output->>'sensor'='RGB GeoTIFFs NRMAC' and output->>'date'='%s';", 60 | # Parent count is still rgb_geotiff because we are putting NRMAC in same datasets as those 61 | "parent": "rgb_geotiff", 62 | "extractor": "ncsa.rulechecker.terra"}), 63 | ("nrmacff", { 64 | "path": os.path.join(uamac_root, 'Level_2/rgb_fullfield/'), 65 | "type": 'regex', 66 | "regex": ".*_nrmac.tif"}), 67 | ("ruledb_maskff", { 68 | "type": "psql", 69 | "query_count": "select count(distinct file_path) from extractor_ids where output->>'rule'='Full Field' and output->>'sensor'='RGB GeoTIFFs Masked' and output->>'date'='%s';", 70 | "query_list": "select distinct file_path from extractor_ids where output->>'rule'='Full Field' and output->>'sensor'='RGB GeoTIFFs Masked' and output->>'date'='%s';", 71 | # Parent count is still rgb_geotiff because we are putting rgb_mask in same datasets as those 72 | "parent": "rgb_geotiff", 73 | "extractor": "ncsa.rulechecker.terra"}), 74 | ("maskff", { 75 | "path": os.path.join(uamac_root, 'Level_2/rgb_fullfield/'), 76 | "type": 'regex', 77 | "regex": ".*_mask.tif", 78 | "dispname": "Full Field", 79 | "extractor": "terra.stereo-rgb.canopycover"}), 80 | # BETYdb traits 81 | ("rgb_canopycover", { 82 | "path": os.path.join(uamac_root, 'Level_2/rgb_fullfield/'), 83 | "type": 'regex', 84 | "regex": '.*_canopycover_bety.csv', 85 | "parent": "maskff", 86 | "parent_replacer_check": ["_canopycover_bety.csv", ".tif"], 87 | "extractor": "terra.stereo-rgb.canopycover"}) 88 | ]), 89 | 90 | "flirIrCamera": OrderedDict([ 91 | # basic products 92 | ("flirIrCamera", { 93 | "path": os.path.join(uamac_root, 'raw_data/flirIrCamera/'), 94 | "type": 'timestamp'}), 95 | ("ir_geotiff", { 96 | "path": os.path.join(uamac_root, 'Level_1/ir_geotiff/'), 97 | "type": 'timestamp', 98 | "parent": "flirIrCamera", 99 | "extractor": "terra.multispectral.flir2tif"}), 100 | # plot products 101 | ("ir_geotiff_plot", { 102 | "path": os.path.join(uamac_root, 'Level_1_Plots/ir_geotiff/'), 103 | "type": 'plot'}), 104 | # rulechecker & fieldmosaic products 105 | ("ruledb_flirff", { 106 | "type": "psql", 107 | "query_count": "select count(distinct file_path) from extractor_ids where output->>'rule'='Full Field' and output->>'sensor'='Thermal IR GeoTIFFs' and output->>'date'='%s';", 108 | "query_list": "select distinct file_path from extractor_ids where output->>'rule'='Full Field' and output->>'sensor'='Thermal IR GeoTIFFs' and output->>'date'='%s';", 109 | "parent": "ir_geotiff", 110 | "extractor": "ncsa.rulechecker.terra"}), 111 | ("flirff", { 112 | "path": os.path.join(uamac_root, 'Level_2/ir_fullfield/'), 113 | "type": 'regex', 114 | "regex": ".*_thumb.tif", 115 | "dispname": "Full Field"}), 116 | # BETYdb traits 117 | ("ir_meantemp", { 118 | "path": os.path.join(uamac_root, 'Level_3/ir_meantemp/'), 119 | "type": 'regex', 120 | "regex": '.*_meantemp_bety.csv', 121 | "parent": "flirff", 122 | "parent_replacer_check": ["_meantemp_bety.csv", "_thumb.tif"], 123 | "extractor": "terra.multispectral.meantemp"}) 124 | ]), 125 | 126 | "scanner3DTop": OrderedDict([ 127 | # basic products 128 | ("scanner3DTop", { 129 | "path": os.path.join(uamac_root, 'Level_1/scanner3DTop/'), 130 | "type": 'timestamp'}), 131 | ("laser3d_las", { 132 | "path": os.path.join(uamac_root, 'Level_1/laser3d_las/'), 133 | "type": 'timestamp', 134 | "parent": "scanner3DTop", 135 | "extractor": "terra.3dscanner.ply2las"}), 136 | # plot products 137 | ("laser3d_las_plot", { 138 | "path": os.path.join(uamac_root, 'Level_1_Plots/laser3d_las/'), 139 | "type": 'plot'}), 140 | ("laser3d_canopyheight", { 141 | "path": os.path.join(uamac_root, 'Level_3/laser3d_canopyheight/'), 142 | "type": 'plot', 143 | "parent": "laser3d_las_plot", 144 | "extractor": "terra.3dscanner.las2height"}) 145 | ]), 146 | 147 | "VNIR": OrderedDict([ 148 | # basic products 149 | ("VNIR", { 150 | "path": os.path.join(uamac_root, 'raw_data/VNIR/'), 151 | "type": 'timestamp'}), 152 | ("vnir_netcdf", { 153 | "path": os.path.join(uamac_root, 'Level_1/vnir_netcdf/'), 154 | "type": 'timestamp', 155 | "parent": "VNIR", 156 | "extractor": "terra.hyperspectral"}), 157 | ("vnir_soil_masks", { 158 | "path": os.path.join(uamac_root, 'Level_1/vnir_soil_masks/'), 159 | "type": 'timestamp', 160 | "parent": "vnir_netcdf", 161 | "extractor": "terra.sunshade.soil_removal"}) 162 | ]), 163 | 164 | "SWIR": OrderedDict([ 165 | # basic products 166 | ("SWIR", { 167 | "path": os.path.join(uamac_root, 'raw_data/SWIR/'), 168 | "type": 'timestamp'}), 169 | ("swir_netcdf", { 170 | "path": os.path.join(uamac_root, 'Level_1/swir_netcdf/'), 171 | "type": 'timestamp', 172 | "parent": "SWIR", 173 | "extractor": "terra.hyperspectral"}) 174 | ]), 175 | 176 | "ps2Top": OrderedDict([ 177 | # basic products 178 | ("ps2Top", { 179 | "path": os.path.join(uamac_root, 'raw_data/ps2Top/'), 180 | "type": 'timestamp'}), 181 | ("ps2_png", { 182 | "path": os.path.join(uamac_root, 'Level_1/ps2_png/'), 183 | "type": 'timestamp', 184 | "parent": "ps2Top", 185 | "extractor": "terra.multispectral.psii2png"}) 186 | ]), 187 | 188 | "EnvironmentLogger": OrderedDict([ 189 | # basic products 190 | ("EnvironmentLogger", { 191 | "path": os.path.join(uamac_root, 'raw_data/EnvironmentLogger/'), 192 | "type": 'regex', 193 | "regex": ".*_environmentlogger.json",}), 194 | ("envlog2netcdf", { 195 | "path": os.path.join(uamac_root, 'Level_1/envlog_netcdf/'), 196 | "type": 'regex', 197 | "regex": "envlog_netcdf_.*.nc", 198 | "extractor": "terra.environmental.envlog2netcdf"}) 199 | #,("envlog2netcdf_csv", { 200 | # "path": os.path.join(uamac_root, 'Level_1/envlog_netcdf/'), 201 | # "type": 'regex', 202 | # "regex": "envlog_netcdf_.*_geo.csv", 203 | # "extractor": "terra.environmental.envlog2netcdf"}) 204 | ]) 205 | } 206 | -------------------------------------------------------------------------------- /scripts/filecounter/templates/dateoptions.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Date Options 6 | 7 | 8 |
  • 9 | Select dates and sensors 10 |
    • 11 |
    12 |
    13 | {{ form.csrf_token }} 14 | {{ form.sensors }} 15 | {{ form.start_date(class='datepicker') }} 16 | {{ form.end_date(class='datepicker') }} 17 | {{ form.submit }} 18 |
    19 | 20 | -------------------------------------------------------------------------------- /scripts/filecounter/templates/sensors.html: -------------------------------------------------------------------------------- 1 | 2 |

    List of Sensors

    3 | 4 |

    5 | {{message}} 6 |

    7 | {% for sensor in sensors %} 8 |
  • {{sensor}} 9 | (download csv) 10 | 29 |
    • 30 | {% endfor %} 31 |
    32 | Select dates for new file count 33 |
    34 | Archive existing csv files 35 | 36 | -------------------------------------------------------------------------------- /scripts/filecounter/users.json: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /scripts/filecounter/utils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import functools 3 | import flask 4 | 5 | from terrautils.extractors import load_json_file 6 | 7 | # Stores a dict of users, passwords and roles. Expected data is: 8 | # { 9 | # 'bob': { 10 | # 'password': 'secret', 11 | # 'roles': ['admin', 'viewer'] 12 | # } 13 | # } 14 | #users = load_json_file("users.json") 15 | users = { 16 | "admin": { 17 | "password": "secret", 18 | "roles": ["admin", "viewer"] 19 | }, 20 | "viewer": { 21 | "password": "secret", 22 | "roles": ["viewer"] 23 | } 24 | } 25 | 26 | 27 | def find_item(where, what): 28 | """ 29 | Tries to locate the item with either the id, or checks to see if it matches the name in the dict. 30 | 31 | :param where: dict to to be searched 32 | :param what: what to look for 33 | :return: a tuple of the key and item found, or (None, None) 34 | """ 35 | value = where.get(what, None) 36 | if value: 37 | return what, value 38 | if len(what) > 10: 39 | key = what[:10] 40 | if key in where: 41 | return key, where[key] 42 | for k, v in where.items(): 43 | if 'name' in v and v['name'] == what: 44 | return k, v 45 | return None, None 46 | 47 | def get_item(where, key, defaultvalue=None): 48 | """ 49 | Finds the key in the dict. The key can be made up of multiple pieces seperated with a period. 50 | 51 | :param where: dict to search 52 | :param key: multiple keywords combined with a period 53 | :param defaultvalue: if the key is not found return this value 54 | :return: either the defaultvalue or the value found 55 | """ 56 | x = where 57 | for k in key.split("."): 58 | if k in x: 59 | x = x[k] 60 | else: 61 | return defaultvalue 62 | return x 63 | 64 | def get_timestamp(): 65 | """ 66 | Generates a consistent timestamp. Timestamp is in ISO-8601 at UTC 67 | 68 | :return: 8601 timestamp formated in UTC. 69 | """ 70 | return datetime.datetime.utcnow().isoformat() + "Z" 71 | 72 | def check_auth(username, password): 73 | """ 74 | Checks if the given username and password match the know list of users and passwords. 75 | 76 | :param username: user to be checked 77 | :param password: password to be checked 78 | :return: true if the user exists and their password matches 79 | """ 80 | found = users.get(username, None) 81 | if found and 'password' in found and found['password'] == password: 82 | flask.g.user = username 83 | flask.g.roles = found['roles'] 84 | return True 85 | else: 86 | flask.g.user = None 87 | flask.g.roles = None 88 | return False 89 | 90 | def requires_user(*users): 91 | """ 92 | Annotation to be added to functions to see if there is given a user and matches the list of users. 93 | 94 | :param users: the list of acceptable users 95 | """ 96 | def wrapper(f): 97 | @functools.wraps(f) 98 | def wrapped(*args, **kwargs): 99 | auth = flask.request.authorization 100 | if not auth or not check_auth(auth.username, auth.password): 101 | return flask.Response(headers={'WWW-Authenticate': 'Basic realm="swarmstats"'}, status=401) 102 | elif auth.username not in users: 103 | return flask.abort(403) 104 | else: 105 | return f(*args, **kwargs) 106 | return wrapped 107 | return wrapper 108 | -------------------------------------------------------------------------------- /scripts/fullfield-preview/test.py: -------------------------------------------------------------------------------- 1 | from terrautils.betydb import get_experiments 2 | import os 3 | import json 4 | 5 | os.environ['BETYDB_KEY'] = 'JGF9SNjh94d0JhamcLauR83RLSqP6OGm2CMOdRZA' 6 | experiment_json = 'experiments.json' 7 | 8 | exp = get_experiments() 9 | print(type(exp[0])) 10 | print(len(exp)) 11 | with open(experiment_json, 'w+') as fout: 12 | json.dump(exp, fout) -------------------------------------------------------------------------------- /scripts/gantrymonitor/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu 2 | MAINTAINER Max Burnette 3 | 4 | RUN apt-get -y update \ 5 | && apt-get -y install curl \ 6 | python \ 7 | python-dev \ 8 | python-pip \ 9 | && pip install flask-restful \ 10 | requests \ 11 | python-logstash \ 12 | globusonline-transfer-api-client \ 13 | && mkdir /home/gantry 14 | 15 | COPY start_services.sh *.py *.json /home/gantry/ 16 | 17 | CMD ["/home/gantry/start_services.sh"] 18 | 19 | ENV MONITOR_API_PORT=5455 20 | 21 | # RUN w/ MOUNTED CUSTOM CONFIG, DATA, LOG FOLDERS 22 | # docker run -p 5455:5455 \ 23 | # -v /var/log/gantrymonitor:/home/gantry/data \ 24 | # -v /gantry_data:/home/gantry/sites \ 25 | # -v /gantry_data/LemnaTec/ToDelete/sites:/home/gantry/delete/sites \ 26 | # -v /var/log:/var/log \ 27 | # -d terraref/terra-gantry-monitor 28 | 29 | -------------------------------------------------------------------------------- /scripts/gantrymonitor/README.md: -------------------------------------------------------------------------------- 1 | # Monitor configuration options 2 | - _log_path_ - where to write log files 3 | - _completed_tasks_path_ - where to write completed JSON metafiles 4 | 5 | - _local_ 6 | -- _incoming_files_path_ - local disk location of sensor data accessible by monitor; can be internal Docker directory that differs from FTP log lines 7 | -- _deletion_queue_ - local disk location where symlinks of processed files will be written; can be Docker internal 8 | -- _ftp_log_path_ - where to look for xferlog files; ignored if empty string 9 | -- _file_age_monitor_paths_ - list of folders to monitor for files older than X minutes to transfer 10 | -- _min_file_age_for_transfer_mins_ - minutes old a file must be to queue for transfer in file_age_monitor_paths folders 11 | -- _file_check_frequency_secs_ - how often to check FTP logs, monitored folders, etc. for new files to queue 12 | -- _max_pending_files_ - maximum # of files that can be queued for new Globus transfers 13 | -- _globus_transfer_frequency_secs_ - how often to generate new batches of Globus transfers 14 | 15 | - _globus_ 16 | -- _source_path_ - disk location of sensor data accessible by Globus; can be different to local disk location above 17 | -- _delete_path_ - disk location to write in symlinks; can differ from internal Docker directory that is invalid outside Docker 18 | -- _destination_path_ - Globus destination root directory; files will go here in same subdirs that follow source_path 19 | -- _source_endpoint_id_ - Globus source endpoint ID 20 | -- _destination_endpoint_id_ - Globus destination endpoint ID 21 | -- _username_ - Globus username 22 | -- _password_ - Globus password 23 | -- _authentication_refresh_frequency_secs_ - how often to attempt Globus reactivation 24 | -- _max_transfer_file_count_ - maximum # of files that can be sent in one Globus transfer 25 | -- _max_active_tasks_ - maximum # of Globus transfers that can be started without completion notification from NCSA 26 | 27 | - _ncsa_api_ 28 | -- _host_ - URL of NCSA monitor on ROGER that will 'catch' the Globus transfers 29 | -- _api_check_frequency_secs_ - how often to check with NCSA monitor for Globus transfer statuses 30 | 31 | - _api_ 32 | -- _port_ - port for this local API to listen on, i.e. to submit files to queue manually 33 | -- _ip_address_ - IP for the local API; one could POST to e.g. "http://0.0.0.0:5455/files" 34 | -------------------------------------------------------------------------------- /scripts/gantrymonitor/config_default.json: -------------------------------------------------------------------------------- 1 | { 2 | "log_path": "/home/gantry/data/log", 3 | "completed_tasks_path": "/home/gantry/data/completed", 4 | 5 | "gantry": { 6 | "incoming_files_path": "/home/gantry/sites", 7 | "deletion_queue": "/home/gantry/delete/sites", 8 | "ftp_log_path": "/var/log", 9 | "file_age_monitor_paths": [], 10 | "min_file_age_for_transfer_mins": 15, 11 | "file_check_frequency_secs": 20, 12 | "max_pending_files": 40000, 13 | "globus_transfer_frequency_secs": 10, 14 | "directory_whitelist": [] 15 | }, 16 | 17 | "globus": { 18 | "source_path": "/gantry_data", 19 | "delete_path": "/gantry_data/ToDelete/sites", 20 | "source_endpoint_id": "", 21 | "destinations": { 22 | "endpoint_id": { "username": "globus_username", "password": "globus_password", "dest_path": "/~/globus"} 23 | }, 24 | "authentication_refresh_frequency_secs": 43200, 25 | "max_transfer_file_count": 1000, 26 | "max_active_tasks": 100 27 | }, 28 | 29 | "influx": { 30 | "host": "terra-logging.ncsa.illinois.edu", 31 | "port": 8086, 32 | "username": "terra", 33 | "password": "", 34 | "dbname": "transfer_db", 35 | "site_time": "+00:00" 36 | }, 37 | 38 | "ncsa_api": { 39 | "host": "http://localhost:5454", 40 | "api_check_frequency_secs": 180 41 | }, 42 | 43 | "api": { 44 | "port": "5455", 45 | "ip_address": "0.0.0.0" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /scripts/gantrymonitor/config_logging.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | 4 | "formatters": { 5 | "default": { 6 | "format": "%(asctime)-15s %(levelname)-7s : %(name)s - %(message)s" 7 | } 8 | }, 9 | 10 | "handlers": { 11 | "console": { 12 | "class": "logging.StreamHandler", 13 | "formatter": "default", 14 | "level": "DEBUG", 15 | "stream": "ext://sys.stdout" 16 | }, 17 | "file": { 18 | "class": "logging.handlers.TimedRotatingFileHandler", 19 | "formatter": "default", 20 | "level": "DEBUG", 21 | "filename": "gantry.log", 22 | "when": "D", 23 | "encoding": "utf8" 24 | }, 25 | "logstash": { 26 | "class": "logstash.TCPLogstashHandler", 27 | "level": "INFO", 28 | "host": "logger.ncsa.illinois.edu", 29 | "port": 5000, 30 | "message_type": "gantry", 31 | "version": 1 32 | } 33 | }, 34 | 35 | "loggers": { 36 | "gantry": { 37 | "level": "DEBUG", 38 | "handlers": ["file"] 39 | } 40 | }, 41 | 42 | "root": { 43 | "level": "INFO", 44 | "handlers": ["console", "logstash"] 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /scripts/gantrymonitor/globus_amazon.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFxTCCBK2gAwIBAgIQBVuNV+ohbDpXYr1UevcRNTANBgkqhkiG9w0BAQsFADBG 3 | MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRUwEwYDVQQLEwxTZXJ2ZXIg 4 | Q0EgMUIxDzANBgNVBAMTBkFtYXpvbjAeFw0xOTA1MjQwMDAwMDBaFw0yMDA2MjQx 5 | MjAwMDBaMCUxIzAhBgNVBAMTGm5leHVzLmFwaS5nbG9idXNvbmxpbmUub3JnMIIB 6 | IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAltc3aTs6zj+UxRgS4fA9tDx8 7 | DJZL+xjBAiqU8vWaiSrZl/Uw17qQVzXAhWtVSgFrCbEB+BrZvDvAqJDpB129EIVk 8 | ZHdn6S8z+vAZXVHMHmFzERLKD/qwrm7P6Cz+JcGcMVmFcoySCGBODdSHnjusKtqf 9 | eFNJRQ8BDFf40J7hC4SGDZMjz3KILC3JYPhQ3t/Uw1SiKDurzRJWlitynZyYPSJF 10 | U6y78/QsoFMtVpCs1ibBU6dK0Dt3ShoyVg6x2gcis9dsLmgu8DdwY7hH2JzJKqTb 11 | s7sWUQrx5Q8ViNPTcp604DhtkqiYiGfw7EIV4SD4FVN4LRXo9dtQh31d2v+qLwID 12 | AQABo4ICzjCCAsowHwYDVR0jBBgwFoAUWaRmBlKge5WSPKOUByeWdFv5PdAwHQYD 13 | VR0OBBYEFIgfXVUH9qhe02+GbXF7r9kJbNW9MG0GA1UdEQRmMGSCGm5leHVzLmFw 14 | aS5nbG9idXNvbmxpbmUub3JnghRuZXh1cy5hcGkuZ2xvYnVzLm9yZ4IUZ3JhcGgu 15 | YXBpLmdsb2J1cy5vcmeCGmdyYXBoLmFwaS5nbG9idXNvbmxpbmUub3JnMA4GA1Ud 16 | DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwOwYDVR0f 17 | BDQwMjAwoC6gLIYqaHR0cDovL2NybC5zY2ExYi5hbWF6b250cnVzdC5jb20vc2Nh 18 | MWIuY3JsMCAGA1UdIAQZMBcwCwYJYIZIAYb9bAECMAgGBmeBDAECATB1BggrBgEF 19 | BQcBAQRpMGcwLQYIKwYBBQUHMAGGIWh0dHA6Ly9vY3NwLnNjYTFiLmFtYXpvbnRy 20 | dXN0LmNvbTA2BggrBgEFBQcwAoYqaHR0cDovL2NydC5zY2ExYi5hbWF6b250cnVz 21 | dC5jb20vc2NhMWIuY3J0MAwGA1UdEwEB/wQCMAAwggEEBgorBgEEAdZ5AgQCBIH1 22 | BIHyAPAAdQC72d+8H4pxtZOUI5eqkntHOFeVCqtS6BqQlmQ2jh7RhQAAAWrqhqoA 23 | AAAEAwBGMEQCIEqRZarB2I5SPhtgtsKUHyIBrsl2H4AP2Q/F+/mgOULcAiA3C1fJ 24 | FLAj0u5Bv0v3oiU/yPhC2r5PSyBYvdkt2xnmFwB3AId1v+dZfPiMQ5lfvfNu/1aN 25 | R1Y2/0q1YMG06v9eoIMPAAABauqGqrYAAAQDAEgwRgIhAO2t33livh3RI+LVuceJ 26 | GfJN0GABtzDWjavvQ5RNU80sAiEA6HVyAScO42WhfyMh25HBIerklufaCg5ib6zY 27 | 1yX/Tk0wDQYJKoZIhvcNAQELBQADggEBAKJ3zHE8TrbPyZkJpYliejRG9WbSemcz 28 | je8Me5UJ3CCNh9u5NBSJ8VYc+gqViDckhdVxQKszRM0tHdYkbSzK7KqM4iC2onzW 29 | QHJpv3OgdzNg9AMDqYV3T0ngHjhTC2qq/9pDgNdWUjczsqZSq87vAPaINkykKqKm 30 | JcB2eAdIoiYWx/sVZkCG+aVst6ZLGszhn/B+ha5UF97VuNjGiePLv1MrLye5BIi6 31 | OxCmgmLxgU3sMGJynZyTyuvc3N0MCBuhpxEpbE8iKtbhm/OKx4HbPIrhHBCqsILO 32 | QFbaPBi4jeNIeKQgMnO6jtVHcGrRN0Ho8uNa10aasFCwub9UyaKEm5Y= 33 | -----END CERTIFICATE----- 34 | -----BEGIN CERTIFICATE----- 35 | MIIESTCCAzGgAwIBAgITBn+UV4WH6Kx33rJTMlu8mYtWDTANBgkqhkiG9w0BAQsF 36 | ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 37 | b24gUm9vdCBDQSAxMB4XDTE1MTAyMjAwMDAwMFoXDTI1MTAxOTAwMDAwMFowRjEL 38 | MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEVMBMGA1UECxMMU2VydmVyIENB 39 | IDFCMQ8wDQYDVQQDEwZBbWF6b24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK 40 | AoIBAQDCThZn3c68asg3Wuw6MLAd5tES6BIoSMzoKcG5blPVo+sDORrMd4f2AbnZ 41 | cMzPa43j4wNxhplty6aUKk4T1qe9BOwKFjwK6zmxxLVYo7bHViXsPlJ6qOMpFge5 42 | blDP+18x+B26A0piiQOuPkfyDyeR4xQghfj66Yo19V+emU3nazfvpFA+ROz6WoVm 43 | B5x+F2pV8xeKNR7u6azDdU5YVX1TawprmxRC1+WsAYmz6qP+z8ArDITC2FMVy2fw 44 | 0IjKOtEXc/VfmtTFch5+AfGYMGMqqvJ6LcXiAhqG5TI+Dr0RtM88k+8XUBCeQ8IG 45 | KuANaL7TiItKZYxK1MMuTJtV9IblAgMBAAGjggE7MIIBNzASBgNVHRMBAf8ECDAG 46 | AQH/AgEAMA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUWaRmBlKge5WSPKOUByeW 47 | dFv5PdAwHwYDVR0jBBgwFoAUhBjMhTTsvAyUlC4IWZzHshBOCggwewYIKwYBBQUH 48 | AQEEbzBtMC8GCCsGAQUFBzABhiNodHRwOi8vb2NzcC5yb290Y2ExLmFtYXpvbnRy 49 | dXN0LmNvbTA6BggrBgEFBQcwAoYuaHR0cDovL2NydC5yb290Y2ExLmFtYXpvbnRy 50 | dXN0LmNvbS9yb290Y2ExLmNlcjA/BgNVHR8EODA2MDSgMqAwhi5odHRwOi8vY3Js 51 | LnJvb3RjYTEuYW1hem9udHJ1c3QuY29tL3Jvb3RjYTEuY3JsMBMGA1UdIAQMMAow 52 | CAYGZ4EMAQIBMA0GCSqGSIb3DQEBCwUAA4IBAQCFkr41u3nPo4FCHOTjY3NTOVI1 53 | 59Gt/a6ZiqyJEi+752+a1U5y6iAwYfmXss2lJwJFqMp2PphKg5625kXg8kP2CN5t 54 | 6G7bMQcT8C8xDZNtYTd7WPD8UZiRKAJPBXa30/AbwuZe0GaFEQ8ugcYQgSn+IGBI 55 | 8/LwhBNTZTUVEWuCUUBVV18YtbAiPq3yXqMB48Oz+ctBWuZSkbvkNodPLamkB2g1 56 | upRyzQ7qDn1X8nn8N8V7YJ6y68AtkHcNSRAnpTitxBKjtKPISLMVCx7i4hncxHZS 57 | yLyKQXhw2W2Xs0qLeC1etA+jTGDK4UfLeC0SF7FSi8o5LL21L8IzApar2pR/ 58 | -----END CERTIFICATE----- 59 | -----BEGIN CERTIFICATE----- 60 | MIIEkjCCA3qgAwIBAgITBn+USionzfP6wq4rAfkI7rnExjANBgkqhkiG9w0BAQsF 61 | ADCBmDELMAkGA1UEBhMCVVMxEDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNj 62 | b3R0c2RhbGUxJTAjBgNVBAoTHFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4x 63 | OzA5BgNVBAMTMlN0YXJmaWVsZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1 64 | dGhvcml0eSAtIEcyMB4XDTE1MDUyNTEyMDAwMFoXDTM3MTIzMTAxMDAwMFowOTEL 65 | MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv 66 | b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj 67 | ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM 68 | 9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw 69 | IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 70 | VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L 71 | 93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm 72 | jgSubJrIqg0CAwEAAaOCATEwggEtMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ 73 | BAQDAgGGMB0GA1UdDgQWBBSEGMyFNOy8DJSULghZnMeyEE4KCDAfBgNVHSMEGDAW 74 | gBScXwDfqgHXMCs4iKK4bUqc8hGRgzB4BggrBgEFBQcBAQRsMGowLgYIKwYBBQUH 75 | MAGGImh0dHA6Ly9vY3NwLnJvb3RnMi5hbWF6b250cnVzdC5jb20wOAYIKwYBBQUH 76 | MAKGLGh0dHA6Ly9jcnQucm9vdGcyLmFtYXpvbnRydXN0LmNvbS9yb290ZzIuY2Vy 77 | MD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly9jcmwucm9vdGcyLmFtYXpvbnRydXN0 78 | LmNvbS9yb290ZzIuY3JsMBEGA1UdIAQKMAgwBgYEVR0gADANBgkqhkiG9w0BAQsF 79 | AAOCAQEAYjdCXLwQtT6LLOkMm2xF4gcAevnFWAu5CIw+7bMlPLVvUOTNNWqnkzSW 80 | MiGpSESrnO09tKpzbeR/FoCJbM8oAxiDR3mjEH4wW6w7sGDgd9QIpuEdfF7Au/ma 81 | eyKdpwAJfqxGF4PcnCZXmTA5YpaP7dreqsXMGz7KQ2hsVxa81Q4gLv7/wmpdLqBK 82 | bRRYh5TmOTFffHPLkIhqhBGWJ6bt2YFGpn6jcgAKUj6DiAdjd4lpFw85hdKrCEVN 83 | 0FE6/V1dN2RMfjCyVSRCnTawXZwXgWHxyvkQAiSr6w10kY17RSlQOYiypok1JR4U 84 | akcjMS9cmvqtmg5iUaQqqcT5NJ0hGA== 85 | -----END CERTIFICATE----- 86 | -----BEGIN CERTIFICATE----- 87 | MIID7zCCAtegAwIBAgIBADANBgkqhkiG9w0BAQsFADCBmDELMAkGA1UEBhMCVVMx 88 | EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT 89 | HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xOzA5BgNVBAMTMlN0YXJmaWVs 90 | ZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5 91 | MDkwMTAwMDAwMFoXDTM3MTIzMTIzNTk1OVowgZgxCzAJBgNVBAYTAlVTMRAwDgYD 92 | VQQIEwdBcml6b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFy 93 | ZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTswOQYDVQQDEzJTdGFyZmllbGQgU2Vy 94 | dmljZXMgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZI 95 | hvcNAQEBBQADggEPADCCAQoCggEBANUMOsQq+U7i9b4Zl1+OiFOxHz/Lz58gE20p 96 | OsgPfTz3a3Y4Y9k2YKibXlwAgLIvWX/2h/klQ4bnaRtSmpDhcePYLQ1Ob/bISdm2 97 | 8xpWriu2dBTrz/sm4xq6HZYuajtYlIlHVv8loJNwU4PahHQUw2eeBGg6345AWh1K 98 | Ts9DkTvnVtYAcMtS7nt9rjrnvDH5RfbCYM8TWQIrgMw0R9+53pBlbQLPLJGmpufe 99 | hRhJfGZOozptqbXuNC66DQO4M99H67FrjSXZm86B0UVGMpZwh94CDklDhbZsc7tk 100 | 6mFBrMnUVN+HL8cisibMn1lUaJ/8viovxFUcdUBgF4UCVTmLfwUCAwEAAaNCMEAw 101 | DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJxfAN+q 102 | AdcwKziIorhtSpzyEZGDMA0GCSqGSIb3DQEBCwUAA4IBAQBLNqaEd2ndOxmfZyMI 103 | bw5hyf2E3F/YNoHN2BtBLZ9g3ccaaNnRbobhiCPPE95Dz+I0swSdHynVv/heyNXB 104 | ve6SbzJ08pGCL72CQnqtKrcgfU28elUSwhXqvfdqlS5sdJ/PHLTyxQGjhdByPq1z 105 | qwubdQxtRbeOlKyWN7Wg0I8VRw7j6IPdj/3vQQF3zCepYoUz8jcI73HPdwbeyBkd 106 | iEDPfUYd/x7H4c7/I9vG+o1VTqkC50cRRj70/b17KSa7qWFiNyi2LSr2EIZkyXCn 107 | 0q23KXB56jzaYyWf/Wi3MOxw+3WKt21gZ7IeyLnp2KhvAotnDU0mV3HaIPzBSlCN 108 | sSi6 109 | -----END CERTIFICATE----- -------------------------------------------------------------------------------- /scripts/gantrymonitor/manager.service: -------------------------------------------------------------------------------- 1 | # Put this in /lib/systemd/system 2 | # sudo systemctl daemon-reload 3 | # sudo systemctl enable manager.service 4 | # sudo systemctl start manager.service 5 | # 6 | # sudo systemctl status manager.service 7 | 8 | [Unit] 9 | Description=Gantry Transfer Manager Service 10 | After=multi-user.target 11 | Conflicts=getty@tty1.service 12 | 13 | [Service] 14 | Type=simple 15 | WorkingDirectory=/home/mburnet2/computing-pipeline/scripts/gantrymonitor 16 | ExecStart=/usr/bin/python /home/mburnet2/computing-pipeline/scripts/gantrymonitor/globus_manager_service.py 17 | User=1006 18 | 19 | [Install] 20 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /scripts/gantrymonitor/scanner.service: -------------------------------------------------------------------------------- 1 | # Put this in /lib/systemd/system 2 | # sudo systemctl daemon-reload 3 | # sudo systemctl enable scanner.service 4 | # sudo systemctl start scanner.service 5 | # 6 | # sudo systemctl status scanner.service 7 | 8 | [Unit] 9 | Description=Gantry Scanner Service 10 | After=multi-user.target 11 | Conflicts=getty@tty1.service 12 | 13 | [Service] 14 | Type=simple 15 | WorkingDirectory=/home/mburnet2/computing-pipeline/scripts/gantrymonitor 16 | ExecStart=/usr/bin/python /home/mburnet2/computing-pipeline/scripts/gantrymonitor/gantry_scanner_service.py 17 | User=1006 18 | 19 | [Install] 20 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /scripts/geospatial/field_scanner_plots.R: -------------------------------------------------------------------------------- 1 | ##################### generate standard input#################################### 2 | ranges <- 54 3 | columns <- 16 4 | plot <- 1:(ranges * columns) 5 | grid <- data.frame(plot) 6 | grid$column <- (grid$plot - 1)%%columns + 1 7 | grid$range <- floor((grid$plot - 1)/columns) + 1 8 | 9 | for (i in (1:(ranges/2)) * 2) { 10 | grid[((i - 1) * columns + 1):(i * columns), 1] <- c((i * columns):((i - 1) * 11 | columns + 1)) 12 | } 13 | grid <- grid[order(grid$plot), ] 14 | 15 | range <- read.csv("range.csv") 16 | row <- read.csv("row.csv") 17 | column <- matrix(0, nrow = columns, ncol = 3) 18 | column <- as.data.frame(column) 19 | names(column) <- c("column", "y_west", "y_east") 20 | for (i in 1:columns) { 21 | column[i, 1] <- i 22 | column[i, 3] <- row[i * 2, 3] 23 | column[i, 2] <- row[i * 2 - 1, 2] 24 | } 25 | grid$x_south <- rep(0, ranges * columns) 26 | grid$x_north <- rep(0, ranges * columns) 27 | grid$y_west <- rep(0, ranges * columns) 28 | grid$y_east <- rep(0, ranges * columns) 29 | for (i in 1:(ranges * columns)) { 30 | grid$x_south[i] <- range[range$range == grid$range[i], ]$x_south 31 | grid$x_north[i] <- range[range$range == grid$range[i], ]$x_north 32 | grid$y_west[i] <- column[column$column == grid$column[i], ]$y_west 33 | grid$y_east[i] <- column[column$column == grid$column[i], ]$y_east 34 | } 35 | 36 | write.csv(grid, "input.csv") 37 | ################################################################################# 38 | 39 | ##################### Projection and Transformation############################### 40 | require("proj4") 41 | options(digits = 15) 42 | 43 | gantry2latlon <- function(Gx, Gy) { 44 | 45 | proj <- "+proj=utm +zone=12 +ellps=GRS80 +datum=NAD83 +units=m +no_defs" 46 | 47 | ay <- 3659974.971 48 | by <- 1.0002 49 | cy <- 0.0078 50 | ax <- 409012.2032 51 | bx <- 0.009 52 | cx <- -0.9986 53 | 54 | # gantry --> MAC 55 | Mx <- ax + bx * Gx + cx * Gy 56 | My <- ay + by * Gx + cy * Gy 57 | result <- project(cbind(Mx, My), proj, inverse = T) 58 | 59 | # MAC --> USDA 60 | result[, 1] <- result[, 1] + 2.0308287e-05 61 | result[, 2] <- result[, 2] - 1.5258894e-05 62 | return(result) 63 | } 64 | 65 | latlon2length <- function(lon1, lat1, lon2, lat2) { 66 | 67 | proj <- "+proj=utm +zone=12 +ellps=GRS80 +datum=NAD83 +units=m +no_defs" 68 | UTM_pt1 <- project(cbind(lon1, lat1), proj) 69 | UTM_pt2 <- project(cbind(lon2, lat2), proj) 70 | length <- sqrt((UTM_pt1[, 1] - UTM_pt2[, 1])^2 + (UTM_pt1[, 2] - UTM_pt2[, 71 | 2])^2) 72 | return(length) 73 | } 74 | 75 | 76 | grid$pt1X <- grid$x_south 77 | grid$pt1Y <- grid$y_west 78 | grid$pt2X <- grid$x_south 79 | grid$pt2Y <- grid$y_east 80 | grid$pt3X <- grid$x_north 81 | grid$pt3Y <- grid$y_east 82 | grid$pt4X <- grid$x_north 83 | grid$pt4Y <- grid$y_west 84 | 85 | grid$plot_W <- abs(grid$y_west - grid$y_east) 86 | grid$plot_L <- abs(grid$x_north - grid$x_south) 87 | 88 | grid$lon1 <- gantry2latlon(grid$pt1X, grid$pt1Y)[, 1] 89 | grid$lat1 <- gantry2latlon(grid$pt1X, grid$pt1Y)[, 2] 90 | grid$lon2 <- gantry2latlon(grid$pt2X, grid$pt2Y)[, 1] 91 | grid$lat2 <- gantry2latlon(grid$pt2X, grid$pt2Y)[, 2] 92 | grid$lon3 <- gantry2latlon(grid$pt3X, grid$pt3Y)[, 1] 93 | grid$lat3 <- gantry2latlon(grid$pt3X, grid$pt3Y)[, 2] 94 | grid$lon4 <- gantry2latlon(grid$pt4X, grid$pt4Y)[, 1] 95 | grid$lat4 <- gantry2latlon(grid$pt4X, grid$pt4Y)[, 2] 96 | 97 | grid$UTM_W <- latlon2length(grid$lon1, grid$lat1, grid$lon2, grid$lat2) 98 | grid$UTM_L <- latlon2length(grid$lon2, grid$lat2, grid$lon3, grid$lat3) 99 | 100 | ################################################################################## 101 | 102 | ########################### write SQL ############################################ 103 | 104 | cat("", file = "output_sql.txt", sep = "") 105 | for (i in 1:(ranges * columns)) { 106 | # tablename = 'yourtablename' sql = paste('INSERT INTO', tablename, 107 | # '(plotid,range,col,geomgantry,x,y,gantryw,gantryl,geomusda,lon,lat,utmw,utml)') 108 | # temp=paste0(grid$pt1X[i],' ',grid$pt1Y[i],', ', grid$pt2X[i],' 109 | # ',grid$pt2Y[i],', ', grid$pt3X[i],' ',grid$pt3Y[i],', ', grid$pt4X[i],' 110 | # ',grid$pt4Y[i],', ', grid$pt1X[i],' ',grid$pt1Y[i]) 111 | # geom_gantry=paste0('ST_GeomFromText('POLYGON((',temp,'))'',')') 112 | 113 | temp <- paste0(grid$lon1[i], " ", grid$lat1[i], " 353, ", grid$lon2[i], " ", 114 | grid$lat2[i], " 353, ", grid$lon3[i], " ", grid$lat3[i], " 353, ", grid$lon4[i], 115 | " ", grid$lat4[i], " 353, ", grid$lon1[i], " ", grid$lat1[i], " 353") 116 | geom_latlon <- paste0("ST_GeomFromText('POLYGON((", temp, "))'", ", 4326)") 117 | 118 | # sql = paste(sql,'VALUES (',paste(grid$plot[i],grid$range[i], 119 | # grid$column[i],geom_gantry,grid$pt1X[i],grid$pt1Y[i], 120 | # grid$plot_W[i],grid$plot_L[i],geom_latlon,grid$lon1[i],grid$lat1[i], 121 | # grid$UTM_W[i],grid$UTM_L[i],sep=', '),');') 122 | 123 | sql <- paste0("INSERT INTO sites (sitename, geometry) VALUES ( ", paste0("'MAC Field Scanner Field Plot ", 124 | grid$plot[i], " Season 2', "), geom_latlon, ");") 125 | 126 | cat(sql, file = "output_sql.txt", sep = "\n", append = TRUE) 127 | } 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | -------------------------------------------------------------------------------- /scripts/geospatial/range.csv: -------------------------------------------------------------------------------- 1 | range,x_south,x_north 2 | 54,209.857,213.821 3 | 53,205.874,209.838 4 | 52,201.89,205.854 5 | 51,197.906,201.87 6 | 50,193.923,197.887 7 | 49,189.939,193.903 8 | 48,185.955,189.919 9 | 47,181.972,185.936 10 | 46,177.988,181.952 11 | 45,174.005,177.969 12 | 44,170.021,173.985 13 | 43,166.037,170.001 14 | 42,162.054,166.018 15 | 41,158.07,162.034 16 | 40,154.086,158.05 17 | 39,150.103,154.067 18 | 38,146.119,150.083 19 | 37,142.136,146.1 20 | 36,138.152,142.116 21 | 35,134.168,138.132 22 | 34,130.185,134.149 23 | 33,126.201,130.165 24 | 32,122.217,126.181 25 | 31,118.234,122.198 26 | 30,114.25,118.214 27 | 29,110.266,114.23 28 | 28,106.283,110.247 29 | 27,102.299,106.263 30 | 26,98.316,102.28 31 | 25,94.332,98.296 32 | 24,90.348,94.312 33 | 23,86.365,90.329 34 | 22,82.381,86.345 35 | 21,78.397,82.361 36 | 20,74.414,78.378 37 | 19,70.43,74.394 38 | 18,66.446,70.41 39 | 17,62.463,66.427 40 | 16,58.479,62.443 41 | 15,54.496,58.46 42 | 14,50.512,54.476 43 | 13,46.528,50.492 44 | 12,42.545,46.509 45 | 11,38.561,42.525 46 | 10,34.577,38.541 47 | 9,30.594,34.558 48 | 8,26.61,30.574 49 | 7,22.627,26.591 50 | 6,18.643,22.607 51 | 5,14.659,18.623 52 | 4,10.676,14.64 53 | 3,6.692,10.656 54 | 2,2.708,6.672 55 | 1,-1.275,2.689 56 | -------------------------------------------------------------------------------- /scripts/geospatial/row.csv: -------------------------------------------------------------------------------- 1 | row,y_west,y_east 2 | 1,24.334,23.569 3 | 2,23.569,22.805 4 | 3,22.805,22.04 5 | 4,22.04,21.275 6 | 5,21.275,20.511 7 | 6,20.511,19.746 8 | 7,19.746,18.982 9 | 8,18.982,18.217 10 | 9,18.217,17.453 11 | 10,17.453,16.688 12 | 11,16.688,15.923 13 | 12,15.923,15.159 14 | 13,15.159,14.394 15 | 14,14.394,13.63 16 | 15,13.63,12.865 17 | 16,12.865,12.101 18 | 17,12.101,11.336 19 | 18,11.336,10.571 20 | 19,10.571,9.807 21 | 20,9.807,9.042 22 | 21,9.042,8.278 23 | 22,8.278,7.513 24 | 23,7.513,6.749 25 | 24,6.749,5.984 26 | 25,5.984,5.219 27 | 26,5.219,4.455 28 | 27,4.455,3.69 29 | 28,3.69,2.926 30 | 29,2.926,2.161 31 | 30,2.161,1.397 32 | 31,1.397,0.632 33 | 32,0.632,0 34 | -------------------------------------------------------------------------------- /scripts/geospatial/season2/cultivar_id.csv: -------------------------------------------------------------------------------- 1 | id,name 2 | 6000000241,RIL_9(SC56*TX7000)-F10 3 | 6000000242,RIL_10(SC56*TX7000)-F10 4 | 6000000243,RIL_11(SC56*TX7000)-F10 5 | 6000000244,RIL_12(SC56*TX7000)-F10 6 | 6000000245,RIL_13(SC56*TX7000)-F10 7 | 6000000246,RIL_14(SC56*TX7000)-F10 8 | 6000000247,RIL_16(SC56*TX7000)-F10 9 | 6000000248,RIL_17(SC56*TX7000)-F10 10 | 6000000249,RIL_18(SC56*TX7000)-F10 11 | 6000000250,RIL_19(SC56*TX7000)-F10 12 | 6000000251,RIL_20(SC56*TX7000)-F10 13 | 6000000252,RIL_21(SC56*TX7000)-F10 14 | 6000000253,RIL_22(SC56*TX7000)-F10 15 | 6000000254,RIL_23(SC56*TX7000)-F10 16 | 6000000255,RIL_25(SC56*TX7000)-F10 17 | 6000000256,RIL_26(SC56*TX7000)-F10 18 | 6000000257,RIL_27(SC56*TX7000)-F10 19 | 6000000258,RIL_28(SC56*TX7000)-F10 20 | 6000000259,RIL_29(SC56*TX7000)-F10 21 | 6000000260,RIL_30(SC56*TX7000)-F10 22 | 6000000261,RIL_32(SC56*TX7000)-F10 23 | 6000000262,RIL_33(SC56*TX7000)-F10 24 | 6000000263,RIL_35(SC56*TX7000)-F10 25 | 6000000264,RIL_36(SC56*TX7000)-F10 26 | 6000000265,RIL_40(SC56*TX7000)-F10 27 | 6000000266,RIL_41(SC56*TX7000)-F10 28 | 6000000267,RIL_43(SC56*TX7000)-F10 29 | 6000000268,RIL_44(SC56*TX7000)-F10 30 | 6000000269,RIL_45(SC56*TX7000)-F10 31 | 6000000270,RIL_46(SC56*TX7000)-F10 32 | 6000000271,RIL_47(SC56*TX7000)-F10 33 | 6000000272,RIL_48(SC56*TX7000)-F10 34 | 6000000273,RIL_49(SC56*TX7000)-F10 35 | 6000000274,RIL_50(SC56*TX7000)-F10 36 | 6000000275,RIL_51(SC56*TX7000)-F10 37 | 6000000276,RIL_52(SC56*TX7000)-F10 38 | 6000000277,RIL_54(SC56*TX7000)-F10 39 | 6000000278,RIL_55(SC56*TX7000)-F10 40 | 6000000279,RIL_56(SC56*TX7000)-F10 41 | 6000000280,RIL_57(SC56*TX7000)-F10 42 | 6000000281,RIL_59(SC56*TX7000)-F10 43 | 6000000282,RIL_60(SC56*TX7000)-F10 44 | 6000000283,RIL_61(SC56*TX7000)-F10 45 | 6000000284,RIL_62(SC56*TX7000)-F10 46 | 6000000285,RIL_63(SC56*TX7000)-F10 47 | 6000000286,RIL_64(SC56*TX7000)-F10 48 | 6000000287,RIL_65(SC56*TX7000)-F10 49 | 6000000288,RIL_66(SC56*TX7000)-F10 50 | 6000000289,RIL_67(SC56*TX7000)-F10 51 | 6000000290,RIL_69(SC56*TX7000)-F10 52 | 6000000291,RIL_70(SC56*TX7000)-F10 53 | 6000000292,RIL_71(SC56*TX7000)-F10 54 | 6000000293,RIL_72(SC56*TX7000)-F10 55 | 6000000294,RIL_73(SC56*TX7000)-F10 56 | 6000000295,RIL_74(SC56*TX7000)-F10 57 | 6000000296,RIL_75(SC56*TX7000)-F10 58 | 6000000297,RIL_76(SC56*TX7000)-F10 59 | 6000000298,RIL_77(SC56*TX7000)-F10 60 | 6000000299,RIL_78(SC56*TX7000)-F10 61 | 6000000300,RIL_79(SC56*TX7000)-F10 62 | 6000000301,RIL_80(SC56*TX7000)-F10 63 | 6000000302,RIL_81(SC56*TX7000)-F10 64 | 6000000303,RIL_82(SC56*TX7000)-F10 65 | 6000000304,RIL_83(SC56*TX7000)-F10 66 | 6000000305,RIL_84(SC56*TX7000)-F10 67 | 6000000306,RIL_85(SC56*TX7000)-F10 68 | 6000000307,RIL_86(SC56*TX7000)-F10 69 | 6000000308,RIL_87(SC56*TX7000)-F10 70 | 6000000309,RIL_88(SC56*TX7000)-F10 71 | 6000000310,RIL_89(SC56*TX7000)-F10 72 | 6000000311,RIL_90(SC56*TX7000)-F10 73 | 6000000312,RIL_91(SC56*TX7000)-F10 74 | 6000000313,RIL_92(SC56*TX7000)-F10 75 | 6000000314,RIL_93(SC56*TX7000)-F10 76 | 6000000315,RIL_94(SC56*TX7000)-F10 77 | 6000000316,RIL_95(SC56*TX7000)-F10 78 | 6000000317,RIL_96(SC56*TX7000)-F10 79 | 6000000318,RIL_97(SC56*TX7000)-F10 80 | 6000000319,RIL_98(SC56*TX7000)-F10 81 | 6000000320,RIL_99(SC56*TX7000)-F10 82 | 6000000321,RIL_100(SC56*TX7000)-F10 83 | 6000000322,RIL_101(SC56*TX7000)-F10 84 | 6000000323,RIL_102(SC56*TX7000)-F10 85 | 6000000324,RIL_103(SC56*TX7000)-F10 86 | 6000000325,RIL_104(SC56*TX7000)-F10 87 | 6000000326,RIL_105(SC56*TX7000)-F10 88 | 6000000327,RIL_106(SC56*TX7000)-F10 89 | 6000000328,RIL_108(SC56*TX7000)-F10 90 | 6000000329,RIL_109(SC56*TX7000)-F10 91 | 6000000330,RIL_110(SC56*TX7000)-F10 92 | 6000000331,RIL_111(SC56*TX7000)-F10 93 | 6000000332,RIL_112(SC56*TX7000)-F10 94 | 6000000333,RIL_113(SC56*TX7000)-F10 95 | 6000000334,RIL_114(SC56*TX7000)-F10 96 | 6000000335,RIL_115(SC56*TX7000)-F10 97 | 6000000336,RIL_116(SC56*TX7000)-F10 98 | 6000000337,RIL_117(SC56*TX7000)-F10 99 | 6000000338,RIL_118(SC56*TX7000)-F10 100 | 6000000339,RIL_119(SC56*TX7000)-F10 101 | 6000000340,RIL_120(SC56*TX7000)-F10 102 | 6000000341,RIL_121(SC56*TX7000)-F10 103 | 6000000342,RIL_122(SC56*TX7000)-F10 104 | 6000000343,RIL_123(SC56*TX7000)-F10 105 | 6000000344,RIL_124(SC56*TX7000)-F10 106 | 6000000345,RIL_125(SC56*TX7000)-F10 107 | 6000000346,RIL_126(SC56*TX7000)-F10 108 | 6000000347,RIL_127(SC56*TX7000)-F10 109 | 6000000348,RIL_128(SC56*TX7000)-F10 110 | 6000000349,RIL_129(SC56*TX7000)-F10 111 | 6000000350,RIL_130(SC56*TX7000)-F10 112 | 6000000351,RIL_132(SC56*TX7000)-F10 113 | 6000000352,RIL_133(SC56*TX7000)-F10 114 | 6000000353,RIL_134(SC56*TX7000)-F10 115 | 6000000354,RIL_135(SC56*TX7000)-F10 116 | 6000000355,RIL_136(SC56*TX7000)-F10 117 | 6000000356,RIL_138(SC56*TX7000)-F10 118 | 6000000357,RIL_139(SC56*TX7000)-F10 119 | 6000000358,RIL_140(SC56*TX7000)-F10 120 | 6000000359,RIL_141(SC56*TX7000)-F10 121 | 6000000360,RIL_143(SC56*TX7000)-F10 122 | 6000000361,RIL_144(SC56*TX7000)-F10 123 | 6000000362,RIL_145(SC56*TX7000)-F10 124 | 6000000363,RIL_146(SC56*TX7000)-F10 125 | 6000000364,RIL_147(SC56*TX7000)-F10 126 | 6000000365,RIL_148(SC56*TX7000)-F10 127 | 6000000366,RIL_149(SC56*TX7000)-F10 128 | 6000000367,RIL_150(SC56*TX7000)-F10 129 | 6000000368,RIL_151(SC56*TX7000)-F10 130 | 6000000369,RIL_152(SC56*TX7000)-F10 131 | 6000000370,RIL_153(SC56*TX7000)-F10 132 | 6000000371,RIL_154(SC56*TX7000)-F10 133 | 6000000372,RIL_156(SC56*TX7000)-F10 134 | 6000000373,RIL_157(SC56*TX7000)-F10 135 | 6000000374,RIL_158(SC56*TX7000)-F10 136 | 6000000375,RIL_159(SC56*TX7000)-F10 137 | 6000000376,RIL_160(SC56*TX7000)-F10 138 | 6000000377,RIL_161(SC56*TX7000)-F10 139 | 6000000378,RIL_162(SC56*TX7000)-F10 140 | 6000000379,RIL_163(SC56*TX7000)-F10 141 | 6000000380,RIL_164(SC56*TX7000)-F10 142 | 6000000381,RIL_165(SC56*TX7000)-F10 143 | 6000000382,RIL_166(SC56*TX7000)-F10 144 | 6000000383,RIL_168(SC56*TX7000)-F10 145 | 6000000384,RIL_169(SC56*TX7000)-F10 146 | 6000000385,RIL_170(SC56*TX7000)-F10 147 | 6000000386,RIL_171(SC56*TX7000)-F10 148 | 6000000387,RIL_172(SC56*TX7000)-F10 149 | 6000000388,RIL_173(SC56*TX7000)-F10 150 | 6000000389,RIL_175(SC56*TX7000)-F10 151 | 6000000390,RIL_176(SC56*TX7000)-F10 152 | 6000000391,RIL_177(SC56*TX7000)-F10 153 | 6000000392,RIL_178(SC56*TX7000)-F10 154 | 6000000393,RIL_179(SC56*TX7000)-F10 155 | 6000000394,RIL_180(SC56*TX7000)-F10 156 | 6000000395,RIL_181(SC56*TX7000)-F10 157 | 6000000396,RIL_182(SC56*TX7000)-F10 158 | 6000000397,RIL_183(SC56*TX7000)-F10 159 | 6000000398,RIL_185(SC56*TX7000)-F10 160 | 6000000399,RIL_186(SC56*TX7000)-F10 161 | 6000000400,RIL_187(SC56*TX7000)-F10 162 | 6000000401,RIL_188(SC56*TX7000)-F10 163 | 6000000402,RIL_189(SC56*TX7000)-F10 164 | 6000000403,RIL_190(SC56*TX7000)-F10 165 | 6000000404,RIL_1(SC56*TX7000)-F10 166 | 6000000405,RIL_3(SC56*TX7000)-F10 167 | 6000000406,RIL_4(SC56*TX7000)-F10 168 | 6000000407,RIL_31(SC56*TX7000)-F10 169 | 6000000408,RIL_37(SC56*TX7000)-F10 170 | 6000000409,RIL_38(SC56*TX7000)-F10 171 | 6000000410,RIL_39(SC56*TX7000)-F10 172 | 6000000411,RIL_107(SC56*TX7000)-F10 173 | 6000000412,RIL_194(SC56*TX7000)-F10 174 | 6000000413,SC56-14SC56-14 175 | 6000000414,Ton-a-Milk 176 | 6000000415,Great Scott 177 | 6000000416,TX7000TX7000 178 | -------------------------------------------------------------------------------- /scripts/geospatial/season2/plots.R: -------------------------------------------------------------------------------- 1 | require(bit64) 2 | require(data.table) 3 | plots <- fread("update_traits_cultivars.csv") 4 | cultivar_id <- fread("cultivar_id.csv") 5 | 6 | setnames(cultivar_id, 'id', 'cultivar_id') 7 | plots2 <- merge(plots, cultivar_id, by = 'name') 8 | 9 | sites <- fread("sites.csv") 10 | setnames(sites, 'id', 'site_id') 11 | 12 | sitenames <- plots2[!is.na(D_row),list(cultivar_id, sitename = paste0("MAC Field Scanner Season 2 Range ",Range,' Pass ', D_row))] 13 | 14 | sites2 <- merge(sites, sitenames, by = 'sitename') 15 | 16 | updates <- sites2[,list(update = paste0('update traits set cultivar_id = ', cultivar_id, ' where site_id = ', site_id, ';'))] 17 | 18 | writeLines(updates$update, 'updates.sql') 19 | -------------------------------------------------------------------------------- /scripts/globusmonitor/Dockerfile_receiver: -------------------------------------------------------------------------------- 1 | FROM terraref/terrautils:1.2 2 | MAINTAINER Max Burnette 3 | 4 | RUN apt-get -y update \ 5 | && apt-get -y install curl \ 6 | && pip install flask-restful \ 7 | python-logstash \ 8 | globusonline-transfer-api-client \ 9 | psycopg2 10 | 11 | COPY *.py *.json *.pem /home/globusmonitor/ 12 | 13 | ENV MONITOR_API_PORT 5454 14 | CMD ["python", "/home/globusmonitor/globus_monitor_service.py"] 15 | -------------------------------------------------------------------------------- /scripts/globusmonitor/Dockerfile_uploader: -------------------------------------------------------------------------------- 1 | FROM terraref/terrautils:1.2 2 | MAINTAINER Max Burnette 3 | 4 | RUN apt-get -y update \ 5 | && apt-get -y install curl \ 6 | && pip install flask-restful \ 7 | python-logstash \ 8 | globusonline-transfer-api-client \ 9 | psycopg2 10 | 11 | COPY *.py *.json *.pem /home/globusmonitor/ 12 | 13 | CMD ["python", "/home/globusmonitor/globus_uploader_service.py"] 14 | -------------------------------------------------------------------------------- /scripts/globusmonitor/config_default.json: -------------------------------------------------------------------------------- 1 | { 2 | "status_log_path": "/home/globusmonitor/data/log/monitor_status.json", 3 | 4 | "globus": { 5 | "incoming_files_path": "/home/globus", 6 | "influx_refresh_frequency_secs": 900, 7 | "authentication_refresh_frequency_secs": 43200, 8 | "transfer_update_frequency_secs": 120, 9 | "valid_users": {} 10 | }, 11 | 12 | "api": { 13 | "port": "5454", 14 | "ip_address": "0.0.0.0" 15 | }, 16 | 17 | "clowder": { 18 | "host": "http://localhost:9000", 19 | "secret_key": "r1ek3rs", 20 | "user_map": {}, 21 | "primary_space": "", 22 | "globus_processing_frequency": 30 23 | }, 24 | 25 | "postgres": { 26 | "database": "globusmonitor", 27 | "username": "globusmonitor", 28 | "password": "", 29 | "host": "terra-postgres" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /scripts/globusmonitor/config_logging.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | 4 | "formatters": { 5 | "default": { 6 | "format": "%(asctime)-15s %(levelname)-7s : %(name)s - %(message)s" 7 | } 8 | }, 9 | 10 | "handlers": { 11 | "console": { 12 | "class": "logging.StreamHandler", 13 | "formatter": "default", 14 | "level": "DEBUG", 15 | "stream": "ext://sys.stdout" 16 | }, 17 | "logstash": { 18 | "class": "logstash.TCPLogstashHandler", 19 | "level": "INFO", 20 | "host": "logger.ncsa.illinois.edu", 21 | "port": 5000, 22 | "message_type": "gantry", 23 | "version": 1 24 | } 25 | }, 26 | 27 | "root": { 28 | "level": "INFO", 29 | "handlers": ["console", "logstash"] 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /scripts/globusmonitor/globus_amazon.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFxTCCBK2gAwIBAgIQBVuNV+ohbDpXYr1UevcRNTANBgkqhkiG9w0BAQsFADBG 3 | MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRUwEwYDVQQLEwxTZXJ2ZXIg 4 | Q0EgMUIxDzANBgNVBAMTBkFtYXpvbjAeFw0xOTA1MjQwMDAwMDBaFw0yMDA2MjQx 5 | MjAwMDBaMCUxIzAhBgNVBAMTGm5leHVzLmFwaS5nbG9idXNvbmxpbmUub3JnMIIB 6 | IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAltc3aTs6zj+UxRgS4fA9tDx8 7 | DJZL+xjBAiqU8vWaiSrZl/Uw17qQVzXAhWtVSgFrCbEB+BrZvDvAqJDpB129EIVk 8 | ZHdn6S8z+vAZXVHMHmFzERLKD/qwrm7P6Cz+JcGcMVmFcoySCGBODdSHnjusKtqf 9 | eFNJRQ8BDFf40J7hC4SGDZMjz3KILC3JYPhQ3t/Uw1SiKDurzRJWlitynZyYPSJF 10 | U6y78/QsoFMtVpCs1ibBU6dK0Dt3ShoyVg6x2gcis9dsLmgu8DdwY7hH2JzJKqTb 11 | s7sWUQrx5Q8ViNPTcp604DhtkqiYiGfw7EIV4SD4FVN4LRXo9dtQh31d2v+qLwID 12 | AQABo4ICzjCCAsowHwYDVR0jBBgwFoAUWaRmBlKge5WSPKOUByeWdFv5PdAwHQYD 13 | VR0OBBYEFIgfXVUH9qhe02+GbXF7r9kJbNW9MG0GA1UdEQRmMGSCGm5leHVzLmFw 14 | aS5nbG9idXNvbmxpbmUub3JnghRuZXh1cy5hcGkuZ2xvYnVzLm9yZ4IUZ3JhcGgu 15 | YXBpLmdsb2J1cy5vcmeCGmdyYXBoLmFwaS5nbG9idXNvbmxpbmUub3JnMA4GA1Ud 16 | DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwOwYDVR0f 17 | BDQwMjAwoC6gLIYqaHR0cDovL2NybC5zY2ExYi5hbWF6b250cnVzdC5jb20vc2Nh 18 | MWIuY3JsMCAGA1UdIAQZMBcwCwYJYIZIAYb9bAECMAgGBmeBDAECATB1BggrBgEF 19 | BQcBAQRpMGcwLQYIKwYBBQUHMAGGIWh0dHA6Ly9vY3NwLnNjYTFiLmFtYXpvbnRy 20 | dXN0LmNvbTA2BggrBgEFBQcwAoYqaHR0cDovL2NydC5zY2ExYi5hbWF6b250cnVz 21 | dC5jb20vc2NhMWIuY3J0MAwGA1UdEwEB/wQCMAAwggEEBgorBgEEAdZ5AgQCBIH1 22 | BIHyAPAAdQC72d+8H4pxtZOUI5eqkntHOFeVCqtS6BqQlmQ2jh7RhQAAAWrqhqoA 23 | AAAEAwBGMEQCIEqRZarB2I5SPhtgtsKUHyIBrsl2H4AP2Q/F+/mgOULcAiA3C1fJ 24 | FLAj0u5Bv0v3oiU/yPhC2r5PSyBYvdkt2xnmFwB3AId1v+dZfPiMQ5lfvfNu/1aN 25 | R1Y2/0q1YMG06v9eoIMPAAABauqGqrYAAAQDAEgwRgIhAO2t33livh3RI+LVuceJ 26 | GfJN0GABtzDWjavvQ5RNU80sAiEA6HVyAScO42WhfyMh25HBIerklufaCg5ib6zY 27 | 1yX/Tk0wDQYJKoZIhvcNAQELBQADggEBAKJ3zHE8TrbPyZkJpYliejRG9WbSemcz 28 | je8Me5UJ3CCNh9u5NBSJ8VYc+gqViDckhdVxQKszRM0tHdYkbSzK7KqM4iC2onzW 29 | QHJpv3OgdzNg9AMDqYV3T0ngHjhTC2qq/9pDgNdWUjczsqZSq87vAPaINkykKqKm 30 | JcB2eAdIoiYWx/sVZkCG+aVst6ZLGszhn/B+ha5UF97VuNjGiePLv1MrLye5BIi6 31 | OxCmgmLxgU3sMGJynZyTyuvc3N0MCBuhpxEpbE8iKtbhm/OKx4HbPIrhHBCqsILO 32 | QFbaPBi4jeNIeKQgMnO6jtVHcGrRN0Ho8uNa10aasFCwub9UyaKEm5Y= 33 | -----END CERTIFICATE----- 34 | -----BEGIN CERTIFICATE----- 35 | MIIESTCCAzGgAwIBAgITBn+UV4WH6Kx33rJTMlu8mYtWDTANBgkqhkiG9w0BAQsF 36 | ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 37 | b24gUm9vdCBDQSAxMB4XDTE1MTAyMjAwMDAwMFoXDTI1MTAxOTAwMDAwMFowRjEL 38 | MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEVMBMGA1UECxMMU2VydmVyIENB 39 | IDFCMQ8wDQYDVQQDEwZBbWF6b24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK 40 | AoIBAQDCThZn3c68asg3Wuw6MLAd5tES6BIoSMzoKcG5blPVo+sDORrMd4f2AbnZ 41 | cMzPa43j4wNxhplty6aUKk4T1qe9BOwKFjwK6zmxxLVYo7bHViXsPlJ6qOMpFge5 42 | blDP+18x+B26A0piiQOuPkfyDyeR4xQghfj66Yo19V+emU3nazfvpFA+ROz6WoVm 43 | B5x+F2pV8xeKNR7u6azDdU5YVX1TawprmxRC1+WsAYmz6qP+z8ArDITC2FMVy2fw 44 | 0IjKOtEXc/VfmtTFch5+AfGYMGMqqvJ6LcXiAhqG5TI+Dr0RtM88k+8XUBCeQ8IG 45 | KuANaL7TiItKZYxK1MMuTJtV9IblAgMBAAGjggE7MIIBNzASBgNVHRMBAf8ECDAG 46 | AQH/AgEAMA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUWaRmBlKge5WSPKOUByeW 47 | dFv5PdAwHwYDVR0jBBgwFoAUhBjMhTTsvAyUlC4IWZzHshBOCggwewYIKwYBBQUH 48 | AQEEbzBtMC8GCCsGAQUFBzABhiNodHRwOi8vb2NzcC5yb290Y2ExLmFtYXpvbnRy 49 | dXN0LmNvbTA6BggrBgEFBQcwAoYuaHR0cDovL2NydC5yb290Y2ExLmFtYXpvbnRy 50 | dXN0LmNvbS9yb290Y2ExLmNlcjA/BgNVHR8EODA2MDSgMqAwhi5odHRwOi8vY3Js 51 | LnJvb3RjYTEuYW1hem9udHJ1c3QuY29tL3Jvb3RjYTEuY3JsMBMGA1UdIAQMMAow 52 | CAYGZ4EMAQIBMA0GCSqGSIb3DQEBCwUAA4IBAQCFkr41u3nPo4FCHOTjY3NTOVI1 53 | 59Gt/a6ZiqyJEi+752+a1U5y6iAwYfmXss2lJwJFqMp2PphKg5625kXg8kP2CN5t 54 | 6G7bMQcT8C8xDZNtYTd7WPD8UZiRKAJPBXa30/AbwuZe0GaFEQ8ugcYQgSn+IGBI 55 | 8/LwhBNTZTUVEWuCUUBVV18YtbAiPq3yXqMB48Oz+ctBWuZSkbvkNodPLamkB2g1 56 | upRyzQ7qDn1X8nn8N8V7YJ6y68AtkHcNSRAnpTitxBKjtKPISLMVCx7i4hncxHZS 57 | yLyKQXhw2W2Xs0qLeC1etA+jTGDK4UfLeC0SF7FSi8o5LL21L8IzApar2pR/ 58 | -----END CERTIFICATE----- 59 | -----BEGIN CERTIFICATE----- 60 | MIIEkjCCA3qgAwIBAgITBn+USionzfP6wq4rAfkI7rnExjANBgkqhkiG9w0BAQsF 61 | ADCBmDELMAkGA1UEBhMCVVMxEDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNj 62 | b3R0c2RhbGUxJTAjBgNVBAoTHFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4x 63 | OzA5BgNVBAMTMlN0YXJmaWVsZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1 64 | dGhvcml0eSAtIEcyMB4XDTE1MDUyNTEyMDAwMFoXDTM3MTIzMTAxMDAwMFowOTEL 65 | MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv 66 | b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj 67 | ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM 68 | 9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw 69 | IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 70 | VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L 71 | 93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm 72 | jgSubJrIqg0CAwEAAaOCATEwggEtMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ 73 | BAQDAgGGMB0GA1UdDgQWBBSEGMyFNOy8DJSULghZnMeyEE4KCDAfBgNVHSMEGDAW 74 | gBScXwDfqgHXMCs4iKK4bUqc8hGRgzB4BggrBgEFBQcBAQRsMGowLgYIKwYBBQUH 75 | MAGGImh0dHA6Ly9vY3NwLnJvb3RnMi5hbWF6b250cnVzdC5jb20wOAYIKwYBBQUH 76 | MAKGLGh0dHA6Ly9jcnQucm9vdGcyLmFtYXpvbnRydXN0LmNvbS9yb290ZzIuY2Vy 77 | MD0GA1UdHwQ2MDQwMqAwoC6GLGh0dHA6Ly9jcmwucm9vdGcyLmFtYXpvbnRydXN0 78 | LmNvbS9yb290ZzIuY3JsMBEGA1UdIAQKMAgwBgYEVR0gADANBgkqhkiG9w0BAQsF 79 | AAOCAQEAYjdCXLwQtT6LLOkMm2xF4gcAevnFWAu5CIw+7bMlPLVvUOTNNWqnkzSW 80 | MiGpSESrnO09tKpzbeR/FoCJbM8oAxiDR3mjEH4wW6w7sGDgd9QIpuEdfF7Au/ma 81 | eyKdpwAJfqxGF4PcnCZXmTA5YpaP7dreqsXMGz7KQ2hsVxa81Q4gLv7/wmpdLqBK 82 | bRRYh5TmOTFffHPLkIhqhBGWJ6bt2YFGpn6jcgAKUj6DiAdjd4lpFw85hdKrCEVN 83 | 0FE6/V1dN2RMfjCyVSRCnTawXZwXgWHxyvkQAiSr6w10kY17RSlQOYiypok1JR4U 84 | akcjMS9cmvqtmg5iUaQqqcT5NJ0hGA== 85 | -----END CERTIFICATE----- 86 | -----BEGIN CERTIFICATE----- 87 | MIID7zCCAtegAwIBAgIBADANBgkqhkiG9w0BAQsFADCBmDELMAkGA1UEBhMCVVMx 88 | EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT 89 | HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xOzA5BgNVBAMTMlN0YXJmaWVs 90 | ZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5 91 | MDkwMTAwMDAwMFoXDTM3MTIzMTIzNTk1OVowgZgxCzAJBgNVBAYTAlVTMRAwDgYD 92 | VQQIEwdBcml6b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFy 93 | ZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTswOQYDVQQDEzJTdGFyZmllbGQgU2Vy 94 | dmljZXMgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZI 95 | hvcNAQEBBQADggEPADCCAQoCggEBANUMOsQq+U7i9b4Zl1+OiFOxHz/Lz58gE20p 96 | OsgPfTz3a3Y4Y9k2YKibXlwAgLIvWX/2h/klQ4bnaRtSmpDhcePYLQ1Ob/bISdm2 97 | 8xpWriu2dBTrz/sm4xq6HZYuajtYlIlHVv8loJNwU4PahHQUw2eeBGg6345AWh1K 98 | Ts9DkTvnVtYAcMtS7nt9rjrnvDH5RfbCYM8TWQIrgMw0R9+53pBlbQLPLJGmpufe 99 | hRhJfGZOozptqbXuNC66DQO4M99H67FrjSXZm86B0UVGMpZwh94CDklDhbZsc7tk 100 | 6mFBrMnUVN+HL8cisibMn1lUaJ/8viovxFUcdUBgF4UCVTmLfwUCAwEAAaNCMEAw 101 | DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJxfAN+q 102 | AdcwKziIorhtSpzyEZGDMA0GCSqGSIb3DQEBCwUAA4IBAQBLNqaEd2ndOxmfZyMI 103 | bw5hyf2E3F/YNoHN2BtBLZ9g3ccaaNnRbobhiCPPE95Dz+I0swSdHynVv/heyNXB 104 | ve6SbzJ08pGCL72CQnqtKrcgfU28elUSwhXqvfdqlS5sdJ/PHLTyxQGjhdByPq1z 105 | qwubdQxtRbeOlKyWN7Wg0I8VRw7j6IPdj/3vQQF3zCepYoUz8jcI73HPdwbeyBkd 106 | iEDPfUYd/x7H4c7/I9vG+o1VTqkC50cRRj70/b17KSa7qWFiNyi2LSr2EIZkyXCn 107 | 0q23KXB56jzaYyWf/Wi3MOxw+3WKt21gZ7IeyLnp2KhvAotnDU0mV3HaIPzBSlCN 108 | sSi6 109 | -----END CERTIFICATE----- -------------------------------------------------------------------------------- /scripts/globusmonitor/globusmonitor.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=TERRA Globus Pipeline Monitor 3 | After=clowder.service 4 | 5 | [Service] 6 | User=ubuntu 7 | Group=users 8 | Restart=on-failure 9 | WorkingDirectory=/home/globusmonitor/computing-pipeline/scripts/globusmonitor 10 | ExecStart=/usr/bin/python /home/globusmonitor/computing-pipeline/scripts/globusmonitor/globus_monitor_service.py 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /scripts/globusmonitor/globusmonitor_status.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Status 3 | # The check result in Nagios convention: 4 | # 0 for OK, 1 for WARNING, 2 for CRITICAL and 3 for UNKNOWN 5 | # 6 | # Item name 7 | # A name for the check item that is unique within the host. It will be 8 | # used as Nagios' service description. It must not contain spaces, 9 | # because spaces separate the four columns. 10 | # 11 | # Performance data 12 | # You may output zero, one or more performance variables here with the 13 | # standard Nagios convention. That is varname=value;warn;crit;min;max. 14 | # If you have no performance data simply write a dash (-) instead. From 15 | # version 1.1.5 on, it is possible to output more then one variable. 16 | # Separate your variables with a pipe symbol (|). Do not used spaces. 17 | # 18 | # Check output 19 | # The last column is the output of the check. It will be displayed in 20 | # Nagios. Spaces are allowed here and only here. 21 | threshold=4000 22 | status=$( curl -f -s -X GET http://141.142.170.137:5454/status | tr '\n' ' ' ) 23 | exitcode=$? 24 | if [ $exitcode -eq 0 ]; then 25 | unprocessed=$( echo $status | sed 's/.*"unprocessed_task_count": \([0-9]*\).*/\1/' ) 26 | active=$( echo $status | sed 's/.*"active_task_count": \([0-9]*\).*/\1/' ) 27 | echo "0 globusmonitor unprocessed=$unprocessed;5000;10000|active=$active;5000;10000 All is OK." 28 | else 29 | echo "2 globusmonitor - API is not responding (${exitcode}) : $status" 30 | fi 31 | -------------------------------------------------------------------------------- /scripts/globusmonitor/globusuploader.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=TERRA Globus Pipeline Clowder Uploader 3 | After=clowder.service 4 | 5 | [Service] 6 | User=ubuntu 7 | Group=users 8 | Restart=on-failure 9 | WorkingDirectory=/home/globusmonitor/computing-pipeline/scripts/globusmonitor 10 | ExecStart=/usr/bin/python /home/globusmonitor/computing-pipeline/scripts/globusmonitor/globus_uploader_service.py 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /scripts/globusmonitor/migrateJsonToPostgres.py: -------------------------------------------------------------------------------- 1 | import os, json 2 | import psycopg2 3 | from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT 4 | 5 | """Load contents of .json file into a JSON object""" 6 | def loadJsonFile(filename): 7 | try: 8 | f = open(filename) 9 | jsonObj = json.load(f) 10 | f.close() 11 | return jsonObj 12 | except IOError: 13 | print("- unable to open %s" % filename) 14 | return {} 15 | 16 | """Return a connection to the PostgreSQL database""" 17 | def connectToPostgres(): 18 | """ 19 | If globusmonitor database does not exist yet: 20 | $ initdb /home/globusmonitor/postgres/data 21 | $ pg_ctl -D /home/globusmonitor/postgres/data -l /home/globusmonitor/postgres/log 22 | $ createdb globusmonitor 23 | """ 24 | try: 25 | conn = psycopg2.connect(dbname='globusmonitor') 26 | except: 27 | # Attempt to create database if not found 28 | conn = psycopg2.connect(dbname='postgres') 29 | conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) 30 | curs = conn.cursor() 31 | curs.execute('CREATE DATABASE globusmonitor;') 32 | curs.close() 33 | conn.commit() 34 | conn.close() 35 | 36 | conn = psycopg2.connect(dbname='globusmonitor') 37 | initializeDatabase(conn) 38 | 39 | return conn 40 | 41 | """Create PostgreSQL database tables""" 42 | def initializeDatabase(db_connection): 43 | # Table creation queries 44 | ct_tasks = "CREATE TABLE globus_tasks (globus_id TEXT PRIMARY KEY NOT NULL, status TEXT NOT NULL, received TEXT NOT NULL, completed TEXT, globus_user TEXT, contents JSON);" 45 | ct_dsets = "CREATE TABLE datasets (name TEXT PRIMARY KEY NOT NULL, clowder_id TEXT NOT NULL);" 46 | ct_colls = "CREATE TABLE collections (name TEXT PRIMARY KEY NOT NULL, clowder_id TEXT NOT NULL);" 47 | 48 | # Index creation queries 49 | ix_tasks = "CREATE UNIQUE INDEX globus_idx ON globus_tasks (globus_id);" 50 | ix_dsets = "CREATE UNIQUE INDEX dset_idx ON datasets (name);" 51 | ix_colls = "CREATE UNIQUE INDEX coll_idx ON collections (name);" 52 | 53 | # Execute each query 54 | curs = db_connection.cursor() 55 | print("Creating PostgreSQL tables...") 56 | curs.execute(ct_tasks) 57 | curs.execute(ct_dsets) 58 | curs.execute(ct_colls) 59 | print("Creating PostgreSQL indexes...") 60 | curs.execute(ix_tasks) 61 | curs.execute(ix_dsets) 62 | curs.execute(ix_colls) 63 | curs.close() 64 | db_connection.commit() 65 | 66 | print("PostgreSQL initialization complete.") 67 | 68 | """Write a Globus task into PostgreSQL, insert/update as needed""" 69 | def writeTaskToDatabase(task): 70 | gid = task['globus_id'] 71 | stat = task['status'] 72 | recv = task['received'] 73 | comp = task['completed'] 74 | guser = task['user'] 75 | jbody = json.dumps(task['contents']) 76 | 77 | # Attempt to insert, update if globus ID already exists 78 | q_insert = "INSERT INTO globus_tasks (globus_id, status, received, completed, globus_user, contents) " \ 79 | "VALUES ('%s', '%s', '%s', '%s', '%s', '%s') " \ 80 | "ON CONFLICT (globus_id) DO UPDATE " \ 81 | "SET status='%s', received='%s', completed='%s', globus_user='%s', contents='%s';" % ( 82 | gid, stat, recv, comp, guser, jbody, stat, recv, comp, guser, jbody) 83 | 84 | curs = psql_conn.cursor() 85 | #print("Writing task %s to PostgreSQL..." % gid) 86 | curs.execute(q_insert) 87 | curs.close() 88 | 89 | """Write dataset (name -> clowder_id) mapping to PostgreSQL database""" 90 | def writeDatasetRecordToDatabase(dataset_name, dataset_id): 91 | 92 | q_insert = "INSERT INTO datasets (name, clowder_id) VALUES ('%s', '%s') " \ 93 | "ON CONFLICT (name) DO UPDATE SET clowder_id='%s';" % ( 94 | dataset_name, dataset_id, dataset_id) 95 | 96 | curs = psql_conn.cursor() 97 | #print("Writing dataset %s to PostgreSQL..." % dataset_name) 98 | curs.execute(q_insert) 99 | curs.close() 100 | 101 | """Write collection (name -> clowder_id) mapping to PostgreSQL database""" 102 | def writeCollectionRecordToDatabase(collection_name, collection_id): 103 | 104 | q_insert = "INSERT INTO collections (name, clowder_id) VALUES ('%s', '%s') " \ 105 | "ON CONFLICT (name) DO UPDATE SET clowder_id='%s';" % ( 106 | collection_name, collection_id, collection_id) 107 | 108 | curs = psql_conn.cursor() 109 | #print("Writing collection %s to PostgreSQL..." % collection_name) 110 | curs.execute(q_insert) 111 | curs.close() 112 | 113 | # ERROR FILES 114 | # completed/79/7e/d0/40/797ed040-745e-11e6-8427-22000b97daec.json 115 | # completed/d4/72/8c/ae/d4728cae-7458-11e6-8427-22000b97daec.json 116 | # completed/dc/7b/c7/e4/dc7bc7e4-899d-11e6-b030-22000b92c261.json 117 | # completed/f2/b8/9d/30/f2b89d30-1872-11e6-a7cf-22000bf2d559.json 118 | #-------------------------------------------------------------------------- 119 | root = "/home/globusmonitor/computing-pipeline/scripts/globusmonitor/data" 120 | #-------------------------------------------------------------------------- 121 | 122 | PROCESS_DATASETS = True 123 | PROCESS_COLLECTIONS = True 124 | PROCESS_COMPLETED = True 125 | PROCESS_ACTIVE = True 126 | 127 | psql_conn = connectToPostgres() 128 | 129 | # Handle ID maps first 130 | commLoop = 0 131 | if PROCESS_DATASETS: 132 | print("Loading dataset map into Postgres...") 133 | ds_json = loadJsonFile(os.path.join(root, 'log/datasets.json')) 134 | for key in ds_json: 135 | if key != "": 136 | writeDatasetRecordToDatabase(key, ds_json[key]) 137 | commLoop += 1 138 | if commLoop % 100000 == 0: 139 | print('...committing after %s total writes' % commLoop) 140 | psql_conn.commit() 141 | del ds_json 142 | 143 | if PROCESS_COLLECTIONS: 144 | print("Loading collection map into Postgres...") 145 | coll_json = loadJsonFile(os.path.join(root, 'log/collections.json')) 146 | for key in coll_json: 147 | if key != "": 148 | writeCollectionRecordToDatabase(key, coll_json[key]) 149 | commLoop += 1 150 | if commLoop % 100000 == 0: 151 | print('...committing after %s total writes' % commLoop) 152 | psql_conn.commit() 153 | del coll_json 154 | 155 | # Now handle tasks 156 | active_handled = [] 157 | active_list = loadJsonFile(os.path.join(root, 'log/active_tasks.json')) 158 | if PROCESS_COMPLETED: 159 | print("Loading completed.json files into Postgres...") 160 | completed = os.path.join(root, 'completed') 161 | unproc_list = loadJsonFile(os.path.join(root, 'log/unprocessed_tasks.txt')) 162 | 163 | count = 0 164 | for root, dirs, files in os.walk(completed): 165 | for f in files: 166 | if f.endswith(".json"): 167 | fpath = os.path.join(root, f) 168 | 169 | try: 170 | taskdata = loadJsonFile(fpath) 171 | gid = taskdata['globus_id'] 172 | if gid in active_list.keys(): 173 | taskdata['status'] = 'IN PROGRESS' 174 | active_handled += gid 175 | elif gid not in unproc_list: 176 | taskdata['status'] = 'PROCESSED' 177 | writeTaskToDatabase(taskdata) 178 | count += 1 179 | except ValueError: 180 | print("...no JSON object decoded in %s" % fpath) 181 | if count % 1000 == 0: 182 | print("...loaded %s files" % count) 183 | psql_conn.commit() 184 | 185 | if PROCESS_ACTIVE: 186 | print("Loading remaining active tasks...") 187 | for gid in active_list.keys(): 188 | if gid not in active_handled: 189 | writeTaskToDatabase(active_list[gid]) 190 | psql_conn.commit() 191 | 192 | print("Data load complete.") 193 | -------------------------------------------------------------------------------- /scripts/hyperspectral/DataProcess.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Created on Mar 19, 2016 5 | 6 | @author: jeromemao 7 | 8 | This module will process the data file and export a netCDF with variables 9 | from it and dimesions (band, x, y) from its hdr file 10 | ---------------------------------------------------------------------------------------- 11 | Usage: 12 | python DataProcessPath DataPath 13 | 14 | where 15 | DataProcessPath is where this script locates 16 | DataPath is where the data file locates 17 | ***make sure the header file is in the same path since the script will automatically find the 18 | header file*** 19 | 20 | Example: 21 | python ${HOME}/terraref/computing-pipeline/scripts/hyperspectral/DataProcess.py data 22 | ---------------------------------------------------------------------------------------- 23 | Process: 24 | Professor Zender noticed that the image is "interleaved," which "ruined" the output image 25 | ''' 26 | 27 | import sys 28 | from netCDF4 import Dataset 29 | import struct 30 | import os 31 | import time 32 | from hyperspectral_metadata import isDigit 33 | 34 | #Note: datatype so far has not included type 6 and 9. They are complex and double-precison complex 35 | #key: ENVI type 36 | #value: (C type, standard size) 37 | DATATYPE = {'1':('H',2),'2':('i',4),'3':('l',4),'4':('f',4),'5':('d',8),'12':('H',4),'13':('L',4),'14':('q',8),'15':('Q',8)} 38 | 39 | class TimeMeasurement(object): 40 | ''' 41 | Supportive class; 42 | Measuring the time used by unpacking the data 43 | ''' 44 | def __init__(self,lineName): 45 | self.lineName = lineName 46 | 47 | def __enter__(self): 48 | self.startTime = time.time() 49 | 50 | def __exit__(self, *args): 51 | self.endTime = time.time() 52 | self.runningTime = (self.endTime - self.startTime) * 1000 53 | 54 | reportHandler = open("PerformanceReport.txt","w") 55 | 56 | prompt = "%s elapsed time: %.3fms, %.5fs"%(self.lineName, 57 | self.runningTime, 58 | self.runningTime/1000) 59 | reportHandler.write(prompt) 60 | print(prompt) 61 | 62 | def getDimension(fileName): 63 | ''' 64 | Acquire dimensions from related HDR file 65 | ''' 66 | fileHandler = open(fileName+'.hdr') 67 | 68 | for members in fileHandler.readlines(): 69 | if "samples" in members: 70 | x = members[members.find("=")+1:len(members)] 71 | elif "lines" in members: 72 | y = members[members.find("=")+1:len(members)] 73 | elif "bands" in members: 74 | band = members[members.find("=")+1:len(members)] 75 | 76 | return int(band.strip('\n').strip('\r')), int(x.strip('\n').strip('\r')), int(y.strip('\n').strip('\r')) 77 | 78 | def getWavelength(fileName): 79 | ''' 80 | Acquire wavelength(s) from related HDR file 81 | ''' 82 | fileHandler = open(fileName+'.hdr') 83 | wavelengthGroup = [float(x.strip('\r').strip('\n').strip(',')) for x in fileHandler.readlines() 84 | if isDigit(x.strip('\r').strip('\n').strip(','))] 85 | 86 | return wavelengthGroup 87 | 88 | def getHeaderInfo(fileName): 89 | ''' 90 | Acquire Other Information from related HDR file 91 | ''' 92 | fileHandler, infoDictionary = open(fileName+'.hdr'), dict() 93 | for members in fileHandler.readlines(): 94 | if '=' in members and 'wavelength' not in members: 95 | infoDictionary[members[0:members.find("=")-1]] = members[members.find("=")+2:].strip('\n').strip('\r') 96 | 97 | return infoDictionary 98 | 99 | def main(fileName): 100 | ''' 101 | The main function, reading the data and exporting netCDF file 102 | ''' 103 | newData = Dataset("WavelengthExp.nc","w",format="NETCDF4") 104 | dimensionBand, dimensionX, dimensionY = getDimension(fileName) 105 | wavelength, hdrInfo = getWavelength(fileName), getHeaderInfo(fileName) 106 | 107 | newData.createDimension('band', dimensionBand) 108 | newData.createDimension('x', dimensionX) 109 | newData.createDimension('y', dimensionY) 110 | newData.createDimension('wavelength', len(wavelength)) 111 | 112 | mainDataHandler, tempVariable = open('/Users/jeromemao/Desktop/terraref/data'),\ 113 | newData.createVariable('exposure_2','f8',('band', 'x', 'y'))#('band', 'x', 'y') 114 | fileSize = os.path.getsize(fileName) 115 | dataNumber, dataType, dataSize = fileSize/DATATYPE[hdrInfo['data type']][-1], DATATYPE[hdrInfo['data type']][0],\ 116 | DATATYPE[hdrInfo['data type']][-1] 117 | 118 | with TimeMeasurement("unpacking") as lineTiming: #measuring the time 119 | value = struct.unpack(dataType*dataNumber,mainDataHandler.read(dataSize*dataNumber))#reading the data from the file 120 | 121 | with TimeMeasurement("assigning value") as lineTiming: 122 | tempVariable[:,:,:] = value #TODO need a better method to assign value to avoid "de-interleaving" 123 | 124 | nestedWavelength = newData.createVariable('wavelength', 'f8',('wavelength',)) 125 | nestedWavelength[:] = wavelength 126 | headerInfo = newData.createGroup("HeaderInfo") 127 | 128 | for members in hdrInfo: 129 | setattr(headerInfo,members,hdrInfo[members]) 130 | if isDigit(hdrInfo[members]): 131 | tempVariable = headerInfo.createVariable(members,'i4') 132 | tempVariable.assignValue(int(hdrInfo[members])) 133 | 134 | newData.close() 135 | 136 | if __name__ == '__main__': 137 | main(sys.argv[1]) 138 | 139 | -------------------------------------------------------------------------------- /scripts/hyperspectral/README.md: -------------------------------------------------------------------------------- 1 | HyperSpectral Workflow 2 | ======================= 3 | 4 | # Modules 5 | 6 | * CalculationWorks.py 7 | 8 | A supporting module for environmental_logger_json2netcdf.py and hyperspectral_metadata.py. 9 | This module is in charge of all the calculation works needed in the 10 | environmental_logger_json2netcdf.py (converting the data made by environmental logger) 11 | and hyperspectral_metadata.py (group up the supporting files for data_raw). 12 | 13 | * EnvironmentalLoggerAnalyzer.py 14 | 15 | This module will read data generated by Environmental Sensor and convert to netCDF file. 16 | 17 | Please notice that there are two versions of this module (recognized by their names). The 18 | older one will be in charge of the older JSONs; it will be truncated after re-formatting all 19 | the JSONs. 20 | 21 | * hyperspectral_metadata.py 22 | 23 | This module parses JSON formatted metadata and data and header provided by LemnaTec and outputs a formatted netCDF4 file 24 | 25 | * DataProcess.py 26 | 27 | We had stopped updating this module; it is now a part of EnvironmentalLoggerAnalyzer.py 28 | 29 | * hyperspectral_workflow.sh 30 | 31 | This is the master script for Hyperscpetral workflow. All the scripts above will be called in this script. 32 | NCO/ncap2 script to process and calibrate Terraref exposure data 33 | 34 | 35 | # Prerequisites Before Deployed 36 | 37 | * VERY IMPORTANT: Please make sure that hyperspectral_workflow.sh, hyperspectral_calibration.nco, hyperspectral_metadata.py, environmental_logger_json2netcdf.py and CalculationWorks.py are in 38 | the same directory. 39 | 40 | * All the Python scripts syntactically support Python 2.7 and above. Please make sure that the Python in the running environement is 41 | in appropriate version. However, since they also need numpy and scipy and neither of them supports 3.X, hyperspectral workflow normally runs in Python 42 | 2.7 43 | 44 | * All the Pythons scripts also rely on the third-party library including: numpy, scipy, netCDF4 and HDF5; they can be installed with either Homebrew 45 | or Macports. 46 | 47 | * Hyperspectral workflow also widely dependes on NCO, a toolkit in dealing netCDF-accessible files, you can get the source code of NCO from Github by 48 | `git clone https://github.com/nco/nco.git` and then come to the directroy and `make install` to install it. It is also available in Homebrew and 49 | Macports by using `brew install nco` or `ports install nco` (macports sometimes requires sudo permission or the installation will fail) 50 | 51 | * Before running the hyperspectral_workflow.sh (the master script), please remember to update the PATH variable by using export command: 52 | `export PATH=$PATH:` or you need to have `bash` in front of each hyperspectral_workflow.sh in commands 53 | 54 | # Usage 55 | 56 | * CalculationWorks.py 57 | 58 | This module will work as a supporting module for EnvironmentalLoggerAnalyzer.py and hyperspectral_metadata.py; it will not be executed 59 | independently except for testing use. 60 | 61 | * EnvironmentalLoggerAnalyzer.py 62 | 63 | In terminal, use the following command line: 64 | 65 | `python ${HOME}/terraref/computing-pipeline/scripts/hyperspectral/environmental_logger_json2netcdf.py /projects/arpae/terraref/raw_data/ua-mac/EnvironmentLogger/2016-04-07/2016-04-07_12-00-07_enviromentlogger.json ~/rgr` 66 | 67 | It will also be called in hyperspectral_workflow.sh 68 | 69 | * hyperspectral_metadata.py 70 | 71 | In terminal, use the following command line: 72 | 73 | `python ${HOME}/terraref/computing-pipeline/scripts/hyperspectral/hyperspectral_metadata.py ${DATA}/terraref/test_metadata.json ${DATA}/terraref/data` 74 | 75 | It will also be called in hyperspectral_workflow.sh 76 | 77 | * hyperspectral_workflow.sh 78 | 79 | In terminal, use the following command line (assume the hyperspectral files are save in projects/arpae/terraref/raw_data/ua-mac/MovingSensor/VNIR/): 80 | `ls -R /projects/arpae/terraref/raw_data/ua-mac/MovingSensor/VNIR/2016-04-07/*/*_raw | hyperspectral_workflow.sh -d 1 -O /gpfs_scratch/arpae/imaging_spectrometer > ~/terraref.out 2>&1 &` 81 | 82 | 83 | # Hyperspectral Indices Extractor 84 | 85 | * hyperspectral_indices_make.nco 86 | 87 | The above script calculates the hyperspectral indices as defined in : 88 | https://github.com/terraref/documentation/blob/master/spectral_indices.md 89 | 90 | In terminal use a command line of the form: 91 | 92 | ncap2 -v -O -S hyperspectral_indices_make.nco 93 | 94 | The bands used are those nearest the required wavelength. for example: 95 | 96 | R445 band used is 4.45121e-07 m 97 | R700 band used is 6.99953e-07 m 98 | R970 band,used is 9.70152e-07 m 99 | 100 | Each index is given attributes long_name and units for example: 101 | 102 | TCARI=3.0f*( (R700-R670)-0.2f * (R700-R550) * (R700/R670) ); 103 | TCARI@long_name="Transformed chlorophyll absorption in reflectance index"; 104 | TCARI@units="1"; 105 | 106 | In the netcdf output file are the intermediate variables iR445,R445, iR450, R450 etc. These may turn out to be superfluous. 107 | 108 | 109 | 110 | 111 | 112 | -------------------------------------------------------------------------------- /scripts/hyperspectral/calibration_vnir_20ms.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terraref/computing-pipeline/02325cae897a19b68c6e87ca5e5d0ce05c5ef6a6/scripts/hyperspectral/calibration_vnir_20ms.nc -------------------------------------------------------------------------------- /scripts/hyperspectral/calibration_vnir_25ms.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terraref/computing-pipeline/02325cae897a19b68c6e87ca5e5d0ce05c5ef6a6/scripts/hyperspectral/calibration_vnir_25ms.nc -------------------------------------------------------------------------------- /scripts/hyperspectral/calibration_vnir_30ms.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terraref/computing-pipeline/02325cae897a19b68c6e87ca5e5d0ce05c5ef6a6/scripts/hyperspectral/calibration_vnir_30ms.nc -------------------------------------------------------------------------------- /scripts/hyperspectral/calibration_vnir_35ms.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terraref/computing-pipeline/02325cae897a19b68c6e87ca5e5d0ce05c5ef6a6/scripts/hyperspectral/calibration_vnir_35ms.nc -------------------------------------------------------------------------------- /scripts/hyperspectral/calibration_vnir_40ms.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terraref/computing-pipeline/02325cae897a19b68c6e87ca5e5d0ce05c5ef6a6/scripts/hyperspectral/calibration_vnir_40ms.nc -------------------------------------------------------------------------------- /scripts/hyperspectral/calibration_vnir_45ms.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terraref/computing-pipeline/02325cae897a19b68c6e87ca5e5d0ce05c5ef6a6/scripts/hyperspectral/calibration_vnir_45ms.nc -------------------------------------------------------------------------------- /scripts/hyperspectral/calibration_vnir_50ms.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terraref/computing-pipeline/02325cae897a19b68c6e87ca5e5d0ce05c5ef6a6/scripts/hyperspectral/calibration_vnir_50ms.nc -------------------------------------------------------------------------------- /scripts/hyperspectral/calibration_vnir_55ms.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terraref/computing-pipeline/02325cae897a19b68c6e87ca5e5d0ce05c5ef6a6/scripts/hyperspectral/calibration_vnir_55ms.nc -------------------------------------------------------------------------------- /scripts/hyperspectral/extractor/Dockerfile: -------------------------------------------------------------------------------- 1 | # Dockerfile for the TerraRef hyperspectral image conversion extractor 2 | # August 17, 2016 3 | FROM ubuntu:14.04 4 | MAINTAINER Yan Y. Liu 5 | 6 | # install common libraries and python modules 7 | USER root 8 | RUN apt-get update 9 | RUN apt-get upgrade -y -q 10 | RUN apt-get install -y -q build-essential m4 swig antlr libantlr-dev udunits-bin libudunits2-dev unzip cmake wget git libjpeg-dev libpng-dev libtiff-dev 11 | RUN apt-get install -y -q python-dev python-numpy python-pip python-virtualenv 12 | # set up dirs for user installed software 13 | RUN useradd -m -s /bin/bash ubuntu 14 | RUN mkdir /srv/downloads && chown -R ubuntu: /srv/downloads && \ 15 | mkdir /srv/sw && chown -R ubuntu: /srv/sw 16 | 17 | USER ubuntu 18 | # set env vars for common libraries and python paths 19 | ENV PYTHONPATH="/usr/lib/python2.7/dist-packages:${PYTHONPATH}" 20 | 21 | ## install from source 22 | 23 | # hdf5 24 | RUN cd /srv/downloads && \ 25 | wget -q https://www.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.17/src/hdf5-1.8.17.tar.gz && \ 26 | tar xfz hdf5-1.8.17.tar.gz && \ 27 | cd hdf5-1.8.17 && \ 28 | ./configure --prefix=/srv/sw/hdf5-1.8.17 && \ 29 | make && make install 30 | ENV PATH="/srv/sw/hdf5-1.8.17/bin:${PATH}" \ 31 | LD_LIBRARY_PATH="/srv/sw/hdf5-1.8.17/lib:${LD_LIBRARY_PATH}" 32 | 33 | # netcdf4 34 | RUN cd /srv/downloads && \ 35 | wget -q ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.4.1.tar.gz && \ 36 | tar xfz netcdf-4.4.1.tar.gz && \ 37 | cd netcdf-4.4.1 && \ 38 | CFLAGS="-I/srv/sw/hdf5-1.8.17/include " LDFLAGS=" -L/srv/sw/hdf5-1.8.17/lib " LIBS=" -lhdf5 -lhdf5_hl " ./configure --prefix=/srv/sw/netcdf-4.4.1 --enable-netcdf4 && \ 39 | make && make install 40 | ENV PATH="/srv/sw/netcdf-4.4.1/bin:${PATH}" \ 41 | LD_LIBRARY_PATH="/srv/sw/netcdf-4.4.1/lib:${LD_LIBRARY_PATH}" 42 | 43 | # geos 44 | RUN cd /srv/downloads && \ 45 | wget -q http://download.osgeo.org/geos/geos-3.5.0.tar.bz2 && \ 46 | tar xfj geos-3.5.0.tar.bz2 && \ 47 | cd geos-3.5.0 && \ 48 | ./configure --prefix=/srv/sw/geos --enable-python && \ 49 | make && make install 50 | ENV PATH="/srv/sw/geos/bin:${PATH}" \ 51 | PYTHONPATH="/srv/sw/geos/lib/python2.7/site-packages:${PYTHONPATH}" \ 52 | LD_LIBRARY_PATH="/srv/sw/geos/lib:${LD_LIBRARY_PATH}" 53 | 54 | # proj4 55 | RUN cd /srv/downloads && \ 56 | wget -q https://github.com/OSGeo/proj.4/archive/4.9.2.tar.gz -O proj.4-4.9.2.tar.gz && \ 57 | tar xfz proj.4-4.9.2.tar.gz && \ 58 | cd proj.4-4.9.2 && \ 59 | ./configure --prefix=/srv/sw/proj4 && \ 60 | make && make install 61 | ENV PATH="/srv/sw/proj4/bin:${PATH}" \ 62 | LD_LIBRARY_PATH="/srv/sw/proj4/lib:${LD_LIBRARY_PATH}" 63 | 64 | # gdal 65 | RUN cd /srv/downloads && \ 66 | wget -q http://download.osgeo.org/gdal/2.1.1/gdal-2.1.1.tar.gz && \ 67 | tar xfz gdal-2.1.1.tar.gz && \ 68 | cd gdal-2.1.1 && \ 69 | ./configure --with-libtiff=internal --with-geotiff=internal --with-png=internal --with-jpeg=internal --with-gif=internal --without-curl --with-python --with-hdf5=/srv/sw/hdf5-1.8.17 --with-netcdf=/srv/sw/netcdf-4.4.1 --with-geos=/srv/sw/geos/bin/geos-config --with-threads --prefix=/srv/sw/gdal && \ 70 | make && make install 71 | ENV PATH="/srv/sw/gdal/bin:${PATH}" \ 72 | PYTHONPATH="/srv/sw/gdal/lib/python2.7/site-packages:${PYTHONPATH}" \ 73 | LD_LIBRARY_PATH="/srv/sw/gdal/lib:${LD_LIBRARY_PATH}" 74 | 75 | # nco 76 | RUN cd /srv/downloads && \ 77 | wget -q https://github.com/nco/nco/archive/4.6.1.tar.gz -O nco-4.6.1.tar.gz && \ 78 | tar xfz nco-4.6.1.tar.gz && \ 79 | cd nco-4.6.1 && \ 80 | ./configure NETCDF_ROOT=/srv/sw/netcdf-4.4.1 --prefix=/srv/sw/nco-4.6.1 --enable-ncap2 --enable-udunits2 && \ 81 | make && make install 82 | ENV PATH="/srv/sw/nco-4.6.1/bin:${PATH}" \ 83 | LD_LIBRARY_PATH="/srv/sw/nco-4.6.1/lib:${LD_LIBRARY_PATH}" 84 | 85 | ENV USERHOME="/home/ubuntu" 86 | WORKDIR "${USERHOME}" 87 | 88 | ## install pyclowder 89 | # install python modules 90 | RUN cd ${USERHOME} && \ 91 | virtualenv pyenv && \ 92 | . pyenv/bin/activate && \ 93 | pip install pika && \ 94 | CC=gcc CXX=g++ USE_SETUPCFG=0 HDF5_INCDIR=/srv/sw/hdf5-1.8.17/include HDF5_LIBDIR=/srv/sw/hdf5-1.8.17/lib NETCDF4_INCDIR=/srv/sw/netcdf-4.4.1/include NETCDF4_LIBDIR=/srv/sw/netcdf-4.4.1/lib pip install netCDF4 && \ 95 | pip install git+https://opensource.ncsa.illinois.edu/stash/scm/cats/pyclowder.git@bugfix/CATS-554-add-pyclowder-support-for-dataset && \ 96 | deactivate 97 | 98 | ## install hyperspectral image converter script 99 | ENV PIPELINEDIR="${USERHOME}/computing-pipeline" 100 | RUN git clone https://github.com/terraref/computing-pipeline.git "${PIPELINEDIR}" 101 | 102 | ## create workspace directories 103 | ENV INPUTDIR="${USERHOME}/input" \ 104 | OUTPUTDIR="${USERHOME}/output" 105 | RUN mkdir -p "${INPUTDIR}" && \ 106 | mkdir -p "${OUTPUTDIR}" && \ 107 | mkdir -p "${USERHOME}/logs" \ 108 | mkdir -p "${USERHOME}/test-data" 109 | 110 | ## download test input data 111 | RUN wget -q http://141.142.168.44/nfiedata/yanliu/terraref-hyperspectral-input-sample.tgz && \ 112 | tar -xf terraref-hyperspectral-input-sample.tgz -C "./test-data" --strip-components 1 113 | 114 | ## install extractor 115 | ENV RABBITMQ_URI="" \ 116 | RABBITMQ_EXCHANGE="clowder" \ 117 | RABBITMQ_VHOST="%2F" \ 118 | RABBITMQ_QUEUE="terra.hyperspectral" \ 119 | WORKER_SCRIPT="${PIPELINEDIR}/scripts/hyperspectral/terraref.sh" 120 | COPY entrypoint.sh extractor_info.json config.py terra.hyperspectral.py ./ 121 | ENTRYPOINT ["./entrypoint.sh"] 122 | CMD ["python", "./terra.hyperspectral.py"] 123 | -------------------------------------------------------------------------------- /scripts/hyperspectral/extractor/config.py: -------------------------------------------------------------------------------- 1 | # ============================================================================= 2 | # 3 | # In order for this extractor to run according to your preferences, 4 | # the following parameters need to be set. 5 | # 6 | # Some parameters can be left with the default values provided here - in that 7 | # case it is important to verify that the default value is appropriate to 8 | # your system. It is especially important to verify that paths to files and 9 | # software applications are valid in your system. 10 | # 11 | # ============================================================================= 12 | 13 | import os 14 | 15 | # name to show in rabbitmq queue list 16 | extractorName = os.getenv('RABBITMQ_QUEUE', "terra.hyperspectral") 17 | 18 | # URL to be used for connecting to rabbitmq 19 | rabbitmqURL = os.getenv('RABBITMQ_URI', "amqp://guest:guest@localhost/%2f") 20 | 21 | # name of rabbitmq exchange 22 | rabbitmqExchange = os.getenv('RABBITMQ_EXCHANGE', "clowder") 23 | 24 | # type of files to process 25 | messageType = "*.dataset.file.added" 26 | 27 | # trust certificates, set this to false for self signed certificates 28 | sslVerify = os.getenv('RABBITMQ_SSLVERIFY', False) 29 | 30 | # Location of terraref.sh 31 | workerScript = os.getenv('WORKER_SCRIPT', "terraref.sh") 32 | 33 | # Workspace for input/output files. 34 | inputDirectory = os.getenv('INPUTDIR', "./input") 35 | outputDirectory = os.getenv('OUTPUTDIR', "./output") 36 | 37 | # The extractor will only run when all these files are present. 38 | # These are just filename postfixes for file matching. 39 | # A few other things depend on the `_raw` file. 40 | requiredInputFiles = [ 41 | '_raw', 42 | '_raw.hdr', 43 | '_image.jpg', 44 | '_metadata.json', 45 | '_frameIndex.txt', 46 | '_settings.txt' 47 | ] 48 | -------------------------------------------------------------------------------- /scripts/hyperspectral/extractor/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # If RabbitMQ URI is not set, use the default credentials; while doing so, 6 | # handle the linking scenario, where RABBITMQ_PORT_5672 is set. 7 | if [ "$RABBITMQ_URI" == "" ]; then 8 | if [ -n $RABBITMQ_PORT_5672 ]; then 9 | RABBITMQ_URI="amqp://guest:guest@${RABBITMQ_PORT_5672_TCP_ADDR}:${RABBITMQ_PORT_5672_TCP_PORT}/%2F" 10 | else 11 | RABBITMQ_URI="amqp://guest:guest@localhost:5672/%2F" 12 | fi 13 | fi 14 | 15 | . pyenv/bin/activate 16 | 17 | printf "exec %s \n\n" "$@" 18 | exec "$@" 19 | -------------------------------------------------------------------------------- /scripts/hyperspectral/extractor/extractor_info.json: -------------------------------------------------------------------------------- 1 | { 2 | "@context": "http://clowder.ncsa.illinois.edu/contexts/extractors.jsonld", 3 | "name": "terra.hyperspectral", 4 | "version": "1.0", 5 | "description": "Hyperspectral to netCDF", 6 | "author": "Rob Kooper ", 7 | "contributors": [], 8 | "contexts": [], 9 | "repository": {"repType": "git", "repUrl": "https://opensource.ncsa.illinois.edu/stash/scm/cats/pyclowder.git"}, 10 | "external_services": [], 11 | "dependencies": [], 12 | "bibtex": [] 13 | } -------------------------------------------------------------------------------- /scripts/hyperspectral/extractor/terra.hyperspectral.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | terra.hyperspectral.py 5 | 6 | This extractor will trigger when a file is added to a dataset in Clowder. 7 | It checks if all the required input files are present in the dataset while the 8 | output file is not present. The output filename is always determined from the 9 | filename of the `_raw` file. 10 | If the check is OK, it calls the `workerScript` defined in the config file to 11 | create a netCDF output file and adds that to the same dataset. 12 | """ 13 | 14 | import os 15 | import subprocess 16 | import logging 17 | from config import * 18 | import pyclowder.extractors as extractors 19 | 20 | 21 | def main(): 22 | global extractorName, messageType, rabbitmqExchange, rabbitmqURL 23 | 24 | # Set logging 25 | logging.basicConfig(format='%(levelname)-7s : %(name)s - %(message)s', level=logging.WARN) 26 | logging.getLogger('pyclowder.extractors').setLevel(logging.INFO) 27 | 28 | # Connect to rabbitmq 29 | extractors.connect_message_bus( 30 | extractorName = extractorName, 31 | messageType = messageType, 32 | rabbitmqExchange = rabbitmqExchange, 33 | rabbitmqURL = rabbitmqURL, 34 | processFileFunction = process_dataset, 35 | checkMessageFunction = check_message 36 | ) 37 | 38 | def check_message(parameters): 39 | # Check for expected input files before beginning processing 40 | if has_all_files(parameters): 41 | if has_output_file(parameters): 42 | print 'skipping, output file already exists' 43 | return False 44 | else: 45 | # Handle the message but do not download any files automatically. 46 | return "bypass" 47 | else: 48 | print 'skipping, not all input files are ready' 49 | return False 50 | 51 | # ---------------------------------------------------------------------- 52 | # Process the dataset message and upload the results 53 | def process_dataset(parameters): 54 | global extractorName, workerScript, inputDirectory, outputDirectory 55 | 56 | # Find input files in dataset 57 | files = get_all_files(parameters) 58 | 59 | # Download files to input directory 60 | for fileExt in files: 61 | files[fileExt]['path'] = extractors.download_file( 62 | channel = parameters['channel'], 63 | header = parameters['header'], 64 | host = parameters['host'], 65 | key = parameters['secretKey'], 66 | fileid = files[fileExt]['id'], 67 | # What's this argument for? 68 | intermediatefileid = files[fileExt]['id'], 69 | ext = fileExt 70 | ) 71 | # Restore temp filenames to original - script requires specific name formatting so tmp names aren't suitable 72 | files[fileExt]['old_path'] = files[fileExt]['path'] 73 | files[fileExt]['path'] = os.path.join(inputDirectory, files[fileExt]['filename']) 74 | os.rename(files[fileExt]['old_path'], files[fileExt]['path']) 75 | print 'found %s file: %s' % (fileExt, files[fileExt]['path']) 76 | 77 | # Invoke terraref.sh 78 | outFilePath = os.path.join(outputDirectory, get_output_filename(files['_raw']['filename'])) 79 | print 'invoking terraref.sh to create: %s' % outFilePath 80 | returncode = subprocess.call(["bash", workerScript, "-d", "1", "-I", inputDirectory, "-O", outputDirectory]) 81 | print 'done creating output file (%s)' % (returncode) 82 | 83 | if returncode != 0: 84 | print 'terraref.sh encountered an error' 85 | 86 | # Verify outfile exists and upload to clowder 87 | if os.path.exists(outFilePath): 88 | print 'output file detected' 89 | if returncode == 0: 90 | print 'uploading output file...' 91 | extractors.upload_file_to_dataset(filepath=outFilePath, parameters=parameters) 92 | print 'done uploading' 93 | # Clean up the output file. 94 | os.remove(outFilePath) 95 | else: 96 | print 'no output file was produced' 97 | 98 | print 'cleaning up...' 99 | # Clean up the input files. 100 | for fileExt in files: 101 | os.remove(files[fileExt]['path']) 102 | print 'done cleaning' 103 | 104 | # ---------------------------------------------------------------------- 105 | # Find as many expected files as possible and return the set. 106 | def get_all_files(parameters): 107 | global requiredInputFiles 108 | files = dict() 109 | for fileExt in requiredInputFiles: 110 | files[fileExt] = None 111 | 112 | if 'filelist' in parameters: 113 | for fileItem in parameters['filelist']: 114 | fileId = fileItem['id'] 115 | fileName = fileItem['filename'] 116 | for fileExt in files: 117 | if fileName[-len(fileExt):] == fileExt: 118 | files[fileExt] = { 119 | 'id': fileId, 120 | 'filename': fileName 121 | } 122 | return files 123 | 124 | # ---------------------------------------------------------------------- 125 | # Returns the output filename. 126 | def get_output_filename(raw_filename): 127 | return '%s.nc' % raw_filename[:-len('_raw')] 128 | 129 | # ---------------------------------------------------------------------- 130 | # Returns true if all expected files are found. 131 | def has_all_files(parameters): 132 | files = get_all_files(parameters) 133 | allFilesFound = True 134 | for fileExt in files: 135 | if files[fileExt] == None: 136 | allFilesFound = False 137 | return allFilesFound 138 | 139 | # ---------------------------------------------------------------------- 140 | # Returns true if the output file is present. 141 | def has_output_file(parameters): 142 | if 'filelist' not in parameters: 143 | return False 144 | if not has_all_files(parameters): 145 | return False 146 | files = get_all_files(parameters) 147 | outFilename = get_output_filename(files['_raw']['filename']) 148 | outFileFound = False 149 | for fileItem in parameters['filelist']: 150 | if outFilename == fileItem['filename']: 151 | outFileFound = True 152 | break 153 | return outFileFound 154 | 155 | if __name__ == "__main__": 156 | main() 157 | -------------------------------------------------------------------------------- /scripts/hyperspectral/hyperspectral_calculation.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import numpy as np 4 | import sys 5 | import json 6 | 7 | # from Dr. LeBauer, Github thread: terraref/referece-data #32 8 | CAMERA_POSITION = np.array([1.9, 0.855, 0.635]) 9 | 10 | # from Dr. LeBauer, Github thread: terraref/referece-data #32 11 | CAMERA_FOCAL_LENGTH = 24e-3 # the focal length for SWIR camera. unit:[m] 12 | 13 | # from Dr. LeBauer, Github thread: terraref/referece-data #32 14 | PIXEL_PITCH = 25e-6 #[m] 15 | 16 | # from Dr. LeBauer, Github thread: terraref/referece-data #32 17 | # Originally in 33, 04.470' N / -111, 58.485' W 18 | #print REFERENCE_POINT_LATLONG 19 | 20 | # from Dr. LeBauer, Github thread: terraref/referece-data #32 21 | GAMMA = 0 #TODO: waiting for the correct value 22 | 23 | 24 | REFERENCE_POINT = 33 + 4.47 / 60, -111 - 58.485 / 60 # from https://github.com/terraref/reference-data/issues/32 25 | 26 | LONGITUDE_TO_METER = 1 / (30.87 * 3600) 27 | LATITUDE_TO_METER = 1/ (25.906 * 3600) #varies, but has been corrected based on the actural location of the field 28 | 29 | GOOGLE_MAP_TEMPLATE = "https://maps.googleapis.com/maps/api/staticmap?size=1280x720&zoom=17&path=color:0x0000005|weight:5|fillcolor:0xFFFF0033|{pointA}|{pointB}|{pointC}|{pointD}" 30 | 31 | # from Dr. LeBauer, Github thread: terraref/referece-data #32 32 | # This matrix looks like this: 33 | # 34 | # | alphaX, gamma, u0 | 35 | # | | 36 | # A = | 0 , alphaY, v0 | 37 | # | | 38 | # | 0 , 0, 1 | 39 | # 40 | # where alphaX = alphaY = CAMERA_FOCAL_LENGTH / PIXEL_PITCH, 41 | # GAMMA is calibration constant 42 | # u0 and v0 are the center coordinate of the image (waiting to be found) 43 | # 44 | # will be used in calculating the lat long of the image 45 | 46 | ORIENTATION_MATRIX = np.array([[CAMERA_FOCAL_LENGTH / PIXEL_PITCH, GAMMA, 0], [0, CAMERA_FOCAL_LENGTH / PIXEL_PITCH, 0 ], [0, 0, 1]]) 47 | 48 | def pixel2Geographic(jsonFileLocation, headerFileLocation, cameraOption): 49 | 50 | ######################### Load necessary data ######################### 51 | with open(jsonFileLocation) as fileHandler: 52 | master = json.loads(fileHandler.read())["lemnatec_measurement_metadata"] 53 | 54 | x_gantry_pos = float(master["gantry_system_variable_metadata"]["position x [m]"]) 55 | y_gantry_pos = float(master["gantry_system_variable_metadata"]["position y [m]"]) 56 | 57 | x_camera_pos = 1.9 # From https://github.com/terraref/reference-data/issues/32 58 | y_camera_pos = 0.855 59 | 60 | if cameraOption == "SWIR": 61 | x_pixel_size = 1.930615052e-3 62 | else: 63 | x_pixel_size = 1.025e-3 64 | 65 | y_pixel_size = 0.98526434004512529576754637665e-3 66 | 67 | with open(headerFileLocation) as fileHandler: 68 | overall = fileHandler.readlines() 69 | 70 | for members in overall: 71 | if "samples" in members: 72 | x_pixel_num = int(members.split("=")[-1].strip("\n")) 73 | elif "lines" in members: 74 | y_pixel_num = int(members.split("=")[-1].strip("\n")) 75 | 76 | 77 | ######################### Do calculation ######################### 78 | 79 | x_absolute_pos = x_gantry_pos + x_camera_pos 80 | y_absolute_pos = y_gantry_pos + y_camera_pos 81 | 82 | x_final_result = np.array([x * x_pixel_size for x in range(x_pixel_num)]) + x_absolute_pos 83 | y_final_result = np.array([y * y_pixel_size for y in range(y_pixel_num)]) + y_absolute_pos 84 | 85 | ########### Sample result: x -> 0.377 [m], y -> 0.267 [m] ########### 86 | 87 | SE = x_final_result[-1] * LONGITUDE_TO_METER + REFERENCE_POINT[0], y_final_result[-1] * LATITUDE_TO_METER + REFERENCE_POINT[1] 88 | SW = x_final_result[0] * LONGITUDE_TO_METER + REFERENCE_POINT[0], y_final_result[-1] * LATITUDE_TO_METER + REFERENCE_POINT[1] 89 | NE = x_final_result[-1] * LONGITUDE_TO_METER + REFERENCE_POINT[0], y_final_result[0] * LATITUDE_TO_METER + REFERENCE_POINT[1] 90 | NW = x_final_result[0] * LONGITUDE_TO_METER + REFERENCE_POINT[0] , y_final_result[0] * LATITUDE_TO_METER + REFERENCE_POINT[1] 91 | 92 | bounding_box = [str(SE).strip("()"), str(SW).strip("()"), str(NE).strip("()"), str(NW).strip("()")] 93 | bounding_box_mapview = GOOGLE_MAP_TEMPLATE.format(pointA=bounding_box[0], 94 | pointB=bounding_box[1], 95 | pointC=bounding_box[2], 96 | pointD=bounding_box[3]) 97 | 98 | return x_final_result, y_final_result, bounding_box, bounding_box_mapview -------------------------------------------------------------------------------- /scripts/hyperspectral/hyperspectral_calibration.nco: -------------------------------------------------------------------------------- 1 | // -*-C++-*- 2 | 3 | /* Purpose: NCO/ncap2 script to process and calibrate Terraref exposure data 4 | 5 | Requirements: NCO version 4.6.2 (dated 20161116) or later 6 | 7 | Documentation: 8 | https://docs.google.com/document/d/1w_zHHlrPVKsy1mnW9wrVzAU2edVqZH8i1IZa5BZxVpo/edit#heading=h.jjfbhbos05cc # Calibration employed since 20160908 9 | https://github.com/terraref/computing-pipeline/issues/88 # Calibration employed until 20160908 10 | 11 | Usage: 12 | ncap2 -v -O -S ~/terraref/computing-pipeline/scripts/hyperspectral/hyperspectral_calibration.nco ${DATA}/terraref/whiteReference_raw.nc ~/foo.nc */ 13 | 14 | // Declare flags as RAM variables or they will clutter output file 15 | *flg_tst=1s; // [flg] Test mode 16 | *flg_prd=0s; // [flg] Production mode 17 | *flg_vnir=0s; // [flg] VNIR image 18 | *flg_swir=0s; // [flg] SWIR image 19 | 20 | // Run in test or production mode? 21 | *flg_typ=flg_prd; // [enm] Run type 22 | 23 | // Defaults for values not provided on command-line 24 | if(!exists(clb_nbr)) *clb_nbr=73; // [nbr] Calibration number 25 | if(!exists(drc_spt)) *drc_spt="."; // [sng] Script directory 26 | 27 | // Change hyperspectral camera wavelengths from nm to m (SI) 28 | wavelength*=1.0e-9; // [m] 29 | wavelength@standard_name="radiation_wavelength"; 30 | wavelength@units="meter"; 31 | 32 | /* 20161005 Set flags to separate VNIR from SWIR code when necessary */ 33 | if(min(wavelength) < 0.5e-6){ 34 | flg_vnir=1s; 35 | *x_pxl_spc=0.001025; // [m] 36 | @camera="VNIR Camera"; 37 | }else{ 38 | flg_swir=1s; 39 | *x_pxl_spc=0.001930615052; // [m] 40 | @camera="SWIR Camera"; 41 | } // !wavelength 42 | 43 | // Add georeferenced coordinates 44 | // Store x and y in double-precision in case we use lat/lon 45 | // If stored as meters from image or gantry corner, then single-precision might be adequate 46 | x@long_name="North distance from southeast corner of field"; 47 | x@units="meter"; 48 | x@orientation="The x-dimension (which runs north-south) spans what are called samples or pixels. The y-dimension (which runs east-west) spans what are called lines or scanlines or frames. Normal, non-calibration images have the same number of pixels (384 or 1600 for SWIR and VNIR, respectively) and a variable number of lines/frames (images) that the camera takes as the camera box moves east-west."; 49 | x@algorithm="Based on https://github.com/terraref/computing-pipeline/issues/144, x is derived from camera geometry including the Aperature Field-of-View (AFOV), the Horizontal Field-of-View (HFOV), and the height of the camera above the canopy (aka the Working Distance, or WD). We take WD = 2 m. Focal length (about 25 mm) is ignored in this estimate because it is much smaller than the WD. The camera geometry implies that AFOV[degrees]=2*atan(HFOV/WD). We use AFOV=21 and 44.6 degress for SWIR and VNIR, respectively. Then we solve for HFOV, and that distance is equally apportioned to 384 or 1600 pixels for SWIR or VNIR, respectively. For SWIR x = 1.93mm, for VNIR x = 1.025mm."; 50 | 51 | y@long_name="West distance from southeast corner of field"; 52 | y@units="meter"; 53 | y@orientation="The x-dimension (which runs north-south) spans what are called samples or pixels. The y-dimension (which runs east-west) spans what are called lines or scanlines or frames. Normal, non-calibration images have the same number of pixels (384 or 1600 for SWIR and VNIR, respectively) and a variable number of lines/frames (images) the camera takes as the camera box moves east-west."; 54 | y@algorithm="Based on https://github.com/terraref/computing-pipeline/issues/144. The y-dimension spans 0.9853 mm per scanline/frame (exact number is 0.98526434004512529576754637665 mm). The datestamp for each frame is stored in the *_frameindex.txt file which is archived in the time variable"; 55 | 56 | // Compute quality control diagnostics (also used in "guesstimate" calibration) 57 | xps_img_max=xps_img.max($y,$x); 58 | xps_img_max@long_name="Maximum image exposure at each wavelength"; 59 | xps_img_max@units="Counts on scale from 0 to 2^16-1 = 65535"; 60 | 61 | xps_img_min=xps_img.min($y,$x); 62 | xps_img_min@long_name="Minimum image exposure at each wavelength"; 63 | xps_img_min@units="Counts on scale from 0 to 2^16-1 = 65535"; 64 | 65 | // [W m-2 m-1] Downwelling spectral irradiance 66 | // 20160908 This is a placeholder for irradiance on the hyperspectral grid 67 | // Currently no instrument measures this 68 | // The only irradiance measured in by the Environmental Logger, which has a different (and non-overlapping) spectral grid 69 | //flx_dwn[wavelength]=2.0e9f; 70 | //flx_dwn@long_name="Downwelling spectral irradiance"; 71 | //flx_dwn@standard_name="surface_downwelling_radiative_flux_per_unit_wavelength_in_air"; 72 | //flx_dwn@units="watt meter-2 meter-1"; 73 | 74 | // Input factory calibrated Spectralon reflectance from NCO file in script directory 75 | //@fl_clb=push(@drc_spt,"/hyperspectral_spectralon_reflectance_factory.nco"); 76 | //print(@fl_clb," = %s\n"); 77 | // Directory containing this file must be CWD or in NCO_PATH environment variable 78 | #include "hyperspectral_spectralon_reflectance_factory.nco" 79 | 80 | /* Interpolate factory calibration of Spectralon target to wavelength grid of current instrument (VNIR, SWIR) */ 81 | gsl_interp_cspline(&ram_ntp_spl,wvl_clb,factory_calibrated_reflectance); 82 | factory_calibrated_reflectance_interpolated=float(gsl_spline_eval(ram_ntp_spl,wavelength)); 83 | ram_delete(ram_ntp_spl); 84 | 85 | if(flg_swir){ 86 | // As of 20161114, SWIR calibration data are insufficient, so implement "guesstimate" method 87 | 88 | /* Calibrate hyperspectral image from raw counts to reflectance 89 | Optimal calibration would have this equal 2^16-1 in each channel for a perfectly white reflector 90 | This parameter depends solely on the bit-resolution of the camera (currently 16 bits) */ 91 | *exposure_theoretical_maximum=ushort(2^16-1); // [cnt] 92 | 93 | /* Assume spectral reference target employed is "white" reference sheet that reflects all wavelengths with 95% efficiency 94 | (Some) Documentation for Spectralon reference targets at 95 | https://github.com/terraref/reference-data/issues/36 96 | https://github.com/terraref/reference-data/issues/53 97 | 98 | 20160927: fxm replace guesstimate with actual, wavelength-dependent calibrated reflectance below */ 99 | *factory_calibrated_reflectance_guesstimate=0.95f; // [frc] 100 | 101 | /* exposure_reference is exposure measured when pointing at Spectralon reference target 102 | Camera exposed to natural light reflecting from reference target for standard exposure time accumulates exposure_reference counts in each channel 103 | Expected to be large fraction (assumed here to be factory_calibrated_reflectance_guesstimate) of exposure_theoretical_maximum for white reflector 104 | However, grey reference reflector might be more optimal than white 105 | Plants have maximum reflectance of ~0.4 so calibrating at white (~0.95) not as accurate as calibrating near 0.4 106 | We hope Lemnatec tunes exposure time to achieve the greatest dynamic range, and thus precision, under a wide range of circumstances 107 | One tuning consideration is that maximum exposure for white reflector at noon is close to but does not exceed exposure_theoretical_maximum 108 | Other tuning considerations are possible... */ 109 | // 20160908: actual values are not yet available so use this guesstimate 110 | *exposure_reference_guesstimate=ushort(factory_calibrated_reflectance_guesstimate*exposure_theoretical_maximum); // [cnt] 111 | *maximum_plant_reflectance_guesstimate=0.37f; // [frc] 112 | 113 | // A perfect detector would measure no counts when aperture shut 114 | *exposure_theoretical_minimum=ushort(0); // [cnt] 115 | 116 | // A real detector exposed to darkness for standard exposure time accumulates exposure_dark counts in each channel 117 | // 20160908: actual values are not yet available for SWIR so we use this guesstimate 118 | *exposure_dark_guesstimate=ushort(10); // [cnt] 119 | 120 | } // !flg_swir 121 | 122 | // [cnt] Exposure from Spectralon reference target 123 | //if(flg_swir) xps_img_wht[wavelength]=exposure_reference_guesstimate; // Produces reflectances too small by ~10^5 124 | if(flg_swir) xps_img_wht[wavelength]=ushort(xps_img_max*(factory_calibrated_reflectance_guesstimate/maximum_plant_reflectance_guesstimate)); 125 | xps_img_wht@long_name="Exposure from Spectralon reference target"; 126 | xps_img_wht@units="Counts on scale from 0 to 2^16-1 = 65535"; 127 | 128 | // [cnt] Exposure under dark (nighttime) conditions 129 | if(flg_swir) xps_img_drk[wavelength]=xps_img_min*0.01f; 130 | xps_img_drk@long_name="Exposure under dark (nighttime) conditions"; 131 | xps_img_drk@units="Counts on scale from 0 to 2^16-1 = 65535"; 132 | 133 | // [frc] Reflectance of Spectralon reference target (from factory calibration) 134 | // 20160908: Add angular dependence? 135 | if(flg_swir) rfl_rfr_fct[wavelength]=factory_calibrated_reflectance_guesstimate; 136 | if(flg_vnir) rfl_rfr_fct[wavelength]=factory_calibrated_reflectance_interpolated; 137 | rfl_rfr_fct@long_name="Reflectance of Spectralon reference target (from factory calibration)"; 138 | rfl_rfr_fct@units="1"; 139 | 140 | // [frc] = Reflectance of image (plant reflectance) 141 | // 20160826: Pre-assigning to zero would create additional copy and increase required memory 142 | // Instead use implicit casting, and overwrite attributes propagated from rfl_rfr_fct 143 | //rfl_img[wavelength,y,x]=0.0f; 144 | rfl_img=rfl_rfr_fct*(xps_img-xps_img_drk)/(xps_img_wht-xps_img_drk); 145 | rfl_img@long_name="Reflectance of image"; 146 | rfl_img@standard_name="surface_albedo"; 147 | rfl_img@units="1"; 148 | 149 | // Compute quality control diagnostics 150 | rfl_img_max=rfl_img.max($y,$x); 151 | rfl_img_max@long_name="Maximum reflectance at each wavelength"; 152 | rfl_img_max@units="1"; 153 | 154 | rfl_img_min=rfl_img.min($y,$x); 155 | rfl_img_min@long_name="Minimum reflectance at each wavelength"; 156 | rfl_img_min@units="1"; 157 | -------------------------------------------------------------------------------- /scripts/hyperspectral/hyperspectral_calibration_reduction.sh: -------------------------------------------------------------------------------- 1 | # Purpose: Reduce raw calibration data (~10 GB) to exposures (~10 kB) used by hyperspectral_calibration.nco 2 | # Script is run off-line, usually on CSZ's machines, although that could be generalized 3 | # Input names and formats provided by Lemnatec are non-standard (e.g., 7Z) 4 | # Hence this script will need to change for every generation of white and dark references 5 | # Output files contain xps_img_[drk/wht] variables in one file per exposure time 6 | 7 | # Source: https://github.com/terraref/computing-pipeline/tree/master/scripts/hyperspectral/hyperspectral_calibration_reduction.sh 8 | 9 | # Directory for hyperspectral calibration output files 10 | drc_clb=${DATA}/terraref/clb 11 | 12 | # White reference, single pixel (px1), created by Solmaz (NB: pre-compensated for dark counts) 13 | # "Single pixel" (px1) values, can be compared to the difference (white-dark) of the following area-averaged calibrations 14 | for drc in 2016_10_21_13_14_32_20ms 2016_10_21_13_16_20_30ms 2016_10_21_13_17_21_40ms 2016_10_21_13_18_22_50ms 2016_10_21_13_15_21_25ms 2016_10_21_13_16_47_35ms 2016_10_21_13_17_51_45ms 2016_10_21_13_18_52_55ms ; do 15 | /bin/rm -f ${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/vnir*px1* 16 | xps_tm=$(echo ${drc} | cut -d '_' -f 7) 17 | ncks -O --trr var_nm=xps_img_wht --trr_wxy=955,1,1 --trr typ_in=NC_USHORT --trr typ_out=NC_USHORT --trr ntl_in=bil --trr ntl_out=bsq --trr ttl="Spectralon target with nominal visible reflectance = 0.95, as exposed to VNIR single pixel, single scanline on 20161021 ~13:15 local time in ${drc}" --trr_in=${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/CorrectedWhite_raw ~/terraref/computing-pipeline/scripts/hyperspectral/hyperspectral_dummy.nc ${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/vnir_wht_px1_${xps_tm}.nc 18 | /bin/cp ${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/vnir_wht_px1_${xps_tm}.nc ${drc_clb} 19 | done 20 | 21 | # White reference, image: 22 | for drc in 2016_10_21_13_14_32_20ms 2016_10_21_13_16_20_30ms 2016_10_21_13_17_21_40ms 2016_10_21_13_18_22_50ms 2016_10_21_13_15_21_25ms 2016_10_21_13_16_47_35ms 2016_10_21_13_17_51_45ms 2016_10_21_13_18_52_55ms ; do 23 | /bin/rm -f ${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/vnir*img* ${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/vnir*cut* ${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/vnir*avg* 24 | xps_tm=$(echo ${drc} | cut -d '_' -f 7) 25 | hdr_fl=${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/raw.hdr 26 | ydm_nbr=$(grep '^lines' ${hdr_fl} | cut -d ' ' -f 3 | tr -d '\015') 27 | echo "Calibration file ${drc}/raw has ${ydm_nbr} lines" 28 | ncks -O --trr var_nm=xps_img_wht --trr_wxy=955,1600,${ydm_nbr} --trr typ_in=NC_USHORT --trr typ_out=NC_USHORT --trr ntl_in=bil --trr ntl_out=bsq --trr ttl="Spectralon target with nominal visible reflectance = 0.95, as exposed to VNIR full image 1600 pixel and 268-298 lines on 20161021 ~13:15 local time in ${drc}. Spectralon is located in lines ~35-90 and samples (pixels) 600-1000." --trr_in=${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/raw ~/terraref/computing-pipeline/scripts/hyperspectral/hyperspectral_dummy.nc ${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/vnir_wht_img_${xps_tm}.nc 29 | # Visual inspection shows following hyperslab matches Spectralon location and shape 30 | ncks -O -F -d x,600,1000 -d y,35,90 ${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/vnir_wht_img_${xps_tm}.nc ${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/vnir_wht_cut_${xps_tm}.nc 31 | ncwa -O -a x,y ${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/vnir_wht_cut_${xps_tm}.nc ${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/vnir_wht_avg_${xps_tm}.nc 32 | /bin/cp ${HOME}/Downloads/VNIR_SpectralonRef_SinglePixel/${drc}/vnir_wht_avg_${xps_tm}.nc ${drc_clb} 33 | done 34 | 35 | # Dark reference: 36 | # 7z l VNIR-DarkRef.7z # List files 37 | # 7z e VNIR-DarkRef.7z # Extract all files to ${CWD} 38 | # 7z x VNIR-DarkRef.7z # Extract files with full paths 39 | for drc in 2016_10_19_02_58_32-20ms 2016_10_19_03_00_27-30ms 2016_10_19_03_01_51-40ms 2016_10_19_04_16_44-50ms 2016_10_19_02_59_35-25ms 2016_10_19_03_01_07-35ms 2016_10_19_04_16_16-45ms 2016_10_19_04_17_07-55ms ; do 40 | /bin/rm -f ${HOME}/Downloads/VNIR-DarkRef/${drc}/vnir*img* ${HOME}/Downloads/VNIR-DarkRef/${drc}/vnir*avg* 41 | xps_tm=$(echo ${drc} | cut -d '_' -f 6) 42 | xps_tm=$(echo ${xps_tm} | cut -d '-' -f 2) 43 | hdr_fl=${HOME}/Downloads/VNIR-DarkRef/${drc}/raw.hdr 44 | ydm_nbr=$(grep '^lines' ${hdr_fl} | cut -d ' ' -f 3 | tr -d '\015') 45 | echo "Calibration file ${drc}/raw has ${ydm_nbr} lines" 46 | ncks -O --trr var_nm=xps_img_drk --trr_wxy=955,1600,${ydm_nbr} --trr typ_in=NC_USHORT --trr typ_out=NC_USHORT --trr ntl_in=bil --trr ntl_out=bsq --trr ttl="Dark counts as exposed to VNIR full image 1600 pixel and 182-218 lines on 20161019 ~3-4 AM local time in ${drc}." --trr_in=${HOME}/Downloads/VNIR-DarkRef/${drc}/raw ~/terraref/computing-pipeline/scripts/hyperspectral/hyperspectral_dummy.nc ${HOME}/Downloads/VNIR-DarkRef/${drc}/vnir_drk_img_${xps_tm}.nc 47 | # 20161031 Dark image data fits well. No hyperslabbing necessary. Some wavelength-dependent (though unpredictable) structure in X. Little-to-no Y structure. 48 | ncwa -O -a x,y ${HOME}/Downloads/VNIR-DarkRef/${drc}/vnir_drk_img_${xps_tm}.nc ${HOME}/Downloads/VNIR-DarkRef/${drc}/vnir_drk_avg_${xps_tm}.nc 49 | /bin/cp ${HOME}/Downloads/VNIR-DarkRef/${drc}/vnir_drk_avg_${xps_tm}.nc ${drc_clb} 50 | done 51 | 52 | # Combine dark and white reference exposures in single file for each exposure duration 53 | for xps_tm in 20ms 25ms 30ms 35ms 40ms 45ms 50ms 55ms ; do 54 | ncks -O -v xps_img_wht ${drc_clb}/vnir_wht_avg_${xps_tm}.nc ${drc_clb}/calibration_vnir_${xps_tm}.nc 55 | ncks -A -v xps_img_drk ${drc_clb}/vnir_drk_avg_${xps_tm}.nc ${drc_clb}/calibration_vnir_${xps_tm}.nc 56 | done 57 | -------------------------------------------------------------------------------- /scripts/hyperspectral/hyperspectral_dummy.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terraref/computing-pipeline/02325cae897a19b68c6e87ca5e5d0ce05c5ef6a6/scripts/hyperspectral/hyperspectral_dummy.nc -------------------------------------------------------------------------------- /scripts/hyperspectral/hyperspectral_indices_make.nco: -------------------------------------------------------------------------------- 1 | iR445=min_coords(wavelength, 4.45e-7d); 2 | iR450=min_coords(wavelength, 4.5e-7d); 3 | iR470=min_coords(wavelength, 4.7e-7d); 4 | 5 | iR500=min_coords(wavelength, 5.0e-7d); 6 | iR512=min_coords(wavelength, 5.120e-7d); 7 | iR531=min_coords(wavelength, 5.310e-7d); 8 | iR540=min_coords(wavelength, 5.40e-7d); 9 | iR550=min_coords(wavelength, 5.50e-7d); 10 | iR570=min_coords(wavelength, 5.7e-7d); 11 | iR586=min_coords(wavelength, 5.86e-7d); 12 | iR590=min_coords(wavelength, 5.9e-7d); 13 | 14 | iR600=min_coords(wavelength, 6e-7d); 15 | iR650=min_coords(wavelength, 6.5e-7d); 16 | iR670=min_coords(wavelength, 6.7e-7d); 17 | iR680=min_coords(wavelength, 6.8e-7d); 18 | iR690=min_coords(wavelength, 6.9e-7d); 19 | 20 | iR700=min_coords(wavelength, 7.00e-7d); 21 | iR705=min_coords(wavelength, 7.05e-7d); 22 | iR710=min_coords(wavelength, 7.1e-7d); 23 | iR720=min_coords(wavelength, 7.2e-7d); 24 | iR740=min_coords(wavelength, 7.4e-7d); 25 | iR750=min_coords(wavelength, 7.5e-7d); 26 | iR760=min_coords(wavelength, 7.6e-7d); 27 | iR780=min_coords(wavelength, 7.8e-7d); 28 | iR790=min_coords(wavelength, 7.9e-7d); 29 | 30 | 31 | iR800=min_coords(wavelength, 8e-7d); 32 | 33 | iR900=min_coords(wavelength, 9e-7d); 34 | iR970=min_coords(wavelength, 9.7e-7d); 35 | 36 | 37 | 38 | R445=rfl_img(iR445,:,:); 39 | R450=rfl_img(iR450,:,:); 40 | R470=rfl_img(iR470,:,:); 41 | 42 | R500=rfl_img(iR500,:,:); 43 | R512=rfl_img(iR512,:,:); 44 | R531=rfl_img(iR531,:,:); 45 | R540=rfl_img(iR540,:,:); 46 | R550=rfl_img(iR550,:,:); 47 | R570=rfl_img(iR570,:,:); 48 | R586=rfl_img(iR586,:,:); 49 | R590=rfl_img(iR590,:,:); 50 | 51 | R600=rfl_img(iR600,:,:); 52 | R650=rfl_img(iR650,:,:); 53 | R670=rfl_img(iR670,:,:); 54 | R680=rfl_img(iR680,:,:); 55 | R690=rfl_img(iR690,:,:); 56 | 57 | R700=rfl_img(iR700,:,:); 58 | R705=rfl_img(iR705,:,:); 59 | R710=rfl_img(iR710,:,:); 60 | R720=rfl_img(iR720,:,:); 61 | R740=rfl_img(iR740,:,:); 62 | R750=rfl_img(iR750,:,:); 63 | R760=rfl_img(iR760,:,:); 64 | R780=rfl_img(iR780,:,:); 65 | R790=rfl_img(iR790,:,:); 66 | 67 | R800=rfl_img(iR800,:,:); 68 | 69 | R900=rfl_img(iR900,:,:); 70 | R970=rfl_img(iR970,:,:); 71 | 72 | 73 | 74 | 75 | 76 | // sanity check for wavelength coord check min ,max, mode 77 | if( fabs(wavelength(iR445)-4.45e-7) > 0.01e-7d ) 78 | print("wavelength R445 not in wavelength coord\n"); 79 | 80 | if( fabs(wavelength(iR970)-9.70e-7) > 0.01e-7d ) 81 | print("wavelength R970 not in wavelength coord\n"); 82 | 83 | if( fabs(wavelength(iR700)-7e-7) > 0.01e-7d ) 84 | print("wavelength R700 not in wavelength coord\"); 85 | 86 | 87 | NDVI= ( R900 -R680) / (R900+R680 ); 88 | NDVI@long_name="Normalized Difference Vegetation Index"; 89 | 90 | SR=R900 / R680; 91 | SR@long_name="Simple ratio"; 92 | 93 | OSAVI=1.16f*( (R800-R670) ) / ( R800+R670+0.16f); 94 | OSAVI@long_name="Optimized Soil-Adjusted Vegetation index"; 95 | 96 | WI=R900-R970; 97 | WI@long_name="water index"; 98 | 99 | CHL=R750 / R550; 100 | CHL@long_name="Chlorophyll index"; 101 | 102 | msR705= (R750-R445) / (R705-R445); 103 | msR705@long_name="Modified simple ratio 705"; 104 | 105 | 106 | TCARI=3.0f*( (R700-R670)-0.2f * (R700-R550) * (R700/R670) ); 107 | TCARI@long_name="Transformed chlorophyll absorption in reflectance index"; 108 | 109 | CarChap=R760/R500; 110 | CarChap@long_name="Carotenoid index (Chappelle)"; 111 | 112 | Car1Black=R800/R470; 113 | Car1Black@long_name="Carotenoid index (BlackBurn)"; 114 | 115 | 116 | Car2Black= ( R800 - R470 ) / (R800 + R470); 117 | Car2Black@long_name="Carotenoid index 2 (BlackBurn)"; 118 | 119 | PRI570 = (R531 - R570) / (R531+R570); 120 | PRI570@long_name="Photochemical reflectance index (570)"; 121 | 122 | 123 | SIPI =( R800 - R450) / (R800 + R650); 124 | SIPI@long_name="Structure intensive pigment index"; 125 | 126 | 127 | antGamon=R650/R550; 128 | antGamon@long_name="Anthocyanin (Gamon)"; 129 | 130 | antGitelson=( 1.0f/R550 - 1.0f/R700)*R780; 131 | antGitelson@long_name="Anthocyanin (Gitelson)"; 132 | 133 | CHLDela=( R540-R590 ) / ( R540 + R590); 134 | CHLDela@long_name="Chlorophyll content"; 135 | 136 | CI=( R750-R705 ) / ( R750 + R705); 137 | CI@long_name="Chlorophyll index"; 138 | 139 | PRI586=( R531-R586) / ( R531 + R586); 140 | PRI586@long_name="Photochemical reflectance index (586)"; 141 | 142 | PRI512=( R531-R512) / ( R531 + R512); 143 | PRI512@long_name="Photochemical reflectance index (512)"; 144 | 145 | FRI1=R690 / R600; 146 | FRI1@long_name="Fluorescence ratio index1"; 147 | 148 | FRI2=R740 / R800; 149 | FRI2@long_name="Fluorescence ratio iindices 2"; 150 | 151 | NDVI1=(R800-R670)/ (R800+R670); 152 | NDVI1@long_name="Normalized Difference Vegetation Index1"; 153 | 154 | RDVI=NDVI1*0.5f; 155 | RDVI@long_name="Renormalized Difference Vegetation Index"; 156 | 157 | RERI=R700/R670; 158 | RERI@long_name="Red edge ratio index"; 159 | 160 | 161 | ZM= R750 / R710; 162 | ZM@long_name="Red edge"; 163 | 164 | REP=700.0f +40.0f*(((R670+R780)/2-R700) /(R740-R700)); 165 | REP@long_name="Red edge position"; 166 | 167 | NDRE=(R790-R720)/(R790+R720); 168 | NDRE@long_name="Normalized difference vegetation index"; 169 | 170 | TVI=0.5f* (120.0f*(R750-R550)-200.0f*(R670-R550)); 171 | TVI@long_name="Triangular Vegetation Index"; -------------------------------------------------------------------------------- /scripts/ip2geohash.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | """ IP to GEOHASH CONVERTER 4 | 5 | Given an IP address as an argument, this script will query GeoIP2 free database to estimate 6 | the IP location latitutde/longitude and from there return a geohash. 7 | 8 | Usage: 9 | python ip2geohash.py 192.168.99.100 10 | """ 11 | 12 | import Geohash 13 | import geoip2.database 14 | import sys 15 | 16 | 17 | ip_address = sys.argv[1] 18 | 19 | try: 20 | reader = geoip2.database.Reader("GeoLite2-City.mmdb") 21 | except: 22 | print("GeoLite2-City.mmdb must be in same directory as script.") 23 | exit(1) 24 | 25 | response = reader.city(ip_address) 26 | 27 | latlon = (response.location.latitude, 28 | response.location.longitude) 29 | 30 | print(Geohash.encode(response.location.latitude, 31 | response.location.longitude)) 32 | -------------------------------------------------------------------------------- /scripts/load_file_list.py: -------------------------------------------------------------------------------- 1 | ## Load contents of Condo file dump into Postgres 2 | 3 | 4 | import os 5 | import logging 6 | import psycopg2 7 | 8 | 9 | def connectToPostgres(): 10 | """ 11 | If rulemonitor database does not exist yet: 12 | $ initdb /home/rulemonitor/postgres/data 13 | $ pg_ctl -D /home/rulemonitor/postgres/data -l /home/rulemonitor/postgres/log 14 | $ createdb rulemonitor 15 | """ 16 | 17 | pghost = os.getenv("POSTGRES_HOST", "192.168.5.83") 18 | pguser = os.getenv("POSTGRES_USER", "rulemonitor") 19 | pgpass = os.getenv("POSTGRES_PASSWORD", "BSJYngTW4k") 20 | 21 | conn = psycopg2.connect(dbname='rulemonitor', user=pguser, host=pghost, password=pgpass) 22 | logging.info("Connected to Postgres") 23 | return conn 24 | 25 | def parse_line(linestr): 26 | # 1114061452 2051065010 0 32 4258 54 54 160 54 202 47852 -- /terraref/sites/ua-mac/raw_data/stereoTop/2017-08-24/2017-08-24__12-40-01-916/ef0b016f-3069-4f00-90fb-95475cb581f6_metadata.json 27 | # 1114061470 593171087 0 7968 8147712 54 54 254 54 202 47852 -- /terraref/sites/ua-mac/raw_data/stereoTop/2017-05-22/2017-05-22__14-47-17-826/d83f0811-de9d-4406-8707-26fabfd2e845_right.bin 28 | # 1114061482 84336251 0 7968 8147712 54 54 160 54 202 47852 -- /terraref/sites/ua-mac/raw_data/stereoTop/2017-08-24/2017-08-24__12-40-01-916/ef0b016f-3069-4f00-90fb-95475cb581f6_right.bin 29 | 30 | (data, filepath) = linestr.split("--") 31 | filepath = filepath.strip().replace("/terraref/sites/ua-mac/", "/sites/ua-mac") 32 | sub = data.split(" ") 33 | 34 | return { 35 | "filepath": filepath, 36 | "filename": os.path.basename(filepath), 37 | "filesize": sub[5], 38 | "create_time": sub[6], 39 | "change_time": sub[7], 40 | "mod_time": sub[8], 41 | "access_time": sub[9], 42 | "GID": sub[10], 43 | "UID": sub[11] 44 | } 45 | 46 | def post_line(curs, linedata): 47 | q_insert = "INSERT INTO filesystem VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s);" 48 | q_data = (linedata["filepath"], linedata["filename"], linedata["filesize"], linedata["create_time"], linedata["change_time"], 49 | linedata["mod_time"], linedata["access_time"], linedata["GID"], linedata["UID"]) 50 | curs.execute(q_insert, q_data) 51 | 52 | 53 | input_file = "/home/mburnet2/filedb/all-files.txt" 54 | 55 | """ 56 | CREATE TABLE filesystem (filepath TEXT, filename TEXT, filesize BIGINT, create_time INT, change_time INT, mod_time INT, access_time INT, gid TEXT, uid TEXT); 57 | """ 58 | 59 | conn = connectToPostgres() 60 | curs = conn.cursor() 61 | lines = 0 62 | with open(input_file) as f: 63 | curr_line = f.readline() 64 | while curr_line: 65 | curr_line = f.readline() 66 | line_data = parse_line(curr_line) 67 | post_line(curs, line_data) 68 | lines += 1 69 | 70 | if lines % 10000 == 0: 71 | logging.info("inserted %s lines" % lines) 72 | conn.commit() 73 | 74 | conn.commit() 75 | curs.close() -------------------------------------------------------------------------------- /scripts/plantcv/README.md: -------------------------------------------------------------------------------- 1 | # TERRAClowderUploadPython.py 2 | 3 | This is an example script showing how one could iterate through a folder of images, create a Clowder dataset to contain them, and post all JPG images in the folder to that dataset. 4 | 5 | In this example: 6 | * **clowder_url** contains the URL of the Clowder target instance. 7 | * **directory_path** contains the folder to iterate through. 8 | * **line 21** ("name": "Test Dataset") defines the name of the dataset the images will be loaded into. 9 | 10 | This script could be modified to write filenames matching specific patterns to particular datasets. 11 | 12 | Coming soon, an example of loading associated .json files with images into their EXIF metadata to bundle image+metadata into one file. 13 | -------------------------------------------------------------------------------- /scripts/plantcv/config_custom.json: -------------------------------------------------------------------------------- 1 | { 2 | "log_path": "/home/gantry/data/log", 3 | "completed_tasks_path": "/home/gantry/data/completed", 4 | 5 | "gantry": { 6 | "incoming_files_path": "/home/gantry/snapshots", 7 | "deletion_queue": "", 8 | "ftp_log_path": "", 9 | "directory_whitelist": ["/Users/mburnette/globus/"] 10 | }, 11 | 12 | "globus": { 13 | "source_path": "/Users/mburnette/globus", 14 | "delete_path": "", 15 | "destination_path": "danforth/raw_data", 16 | }, 17 | 18 | "ncsa_api": { 19 | "host": "http://141.142.170.137:5454", 20 | "api_check_frequency_secs": 30 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /scripts/rebuild_scripts/buildClowderInstanceLocalTest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datetime 3 | import json 4 | 5 | from pyclowder.connectors import Connector 6 | from pyclowder.datasets import upload_metadata 7 | from pyclowder.files import upload_to_dataset 8 | from terrautils.extractors import build_dataset_hierarchy, build_metadata 9 | from terrautils.metadata import clean_metadata 10 | 11 | 12 | # CONNECTION SETTINGS 13 | CLOWDER_HOST = "http://141.142.22.37:9000/" 14 | CLOWDER_KEY = "r1ek3rs" 15 | CLOWDER_USER = "mburnet2@illinois.edu" 16 | CONN = Connector(None, mounted_paths={"/Users/mburnette/globus":"/Users/mburnette/globus"}) 17 | 18 | SPACE_ID = "5997333de98a9d4e498532ce" 19 | SENSOR_FOLDER = "/Users/mburnette/globus/Level_1" 20 | LOGFILE = open(os.path.join(SENSOR_FOLDER, "build_log.txt"), "w+") 21 | SENSOR_LIST = ["scanner3DTop"] 22 | TIMESTAMP_FOLDER = True 23 | DRY_RUN = False 24 | 25 | 26 | def log(string): 27 | print("%s: %s" % (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), string)) 28 | LOGFILE.write("%s: %s" % (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), string)) 29 | 30 | def loadJsonFile(jsonfile): 31 | try: 32 | f = open(jsonfile) 33 | jsonobj = json.load(f) 34 | f.close() 35 | return jsonobj 36 | except IOError: 37 | print("- unable to open %s" % jsonfile) 38 | return {} 39 | 40 | def upload_ds(conn, host, key, sensor, date, timestamp, ds_files, ds_meta): 41 | if len(ds_files) > 0: 42 | year, month, dd = date.split("-") 43 | if DRY_RUN: 44 | log("[%s] %s files" % (sensor+' - '+timestamp, len(ds_files))) 45 | return 46 | 47 | if TIMESTAMP_FOLDER: 48 | dataset_id = build_dataset_hierarchy(CLOWDER_HOST, CLOWDER_KEY, CLOWDER_USER, CLOWDER_PASS, SPACE_ID, 49 | sensor, year, month, dd, sensor+' - '+timestamp) 50 | else: 51 | dataset_id = build_dataset_hierarchy(CLOWDER_HOST, CLOWDER_KEY, CLOWDER_USER, CLOWDER_PASS, SPACE_ID, 52 | sensor, year, month, leaf_ds_name=sensor+' - '+date) 53 | 54 | log("adding files to Clowder dataset %s" % dataset_id) 55 | 56 | for FILEPATH in ds_files: 57 | upload_to_dataset(CONN, CLOWDER_HOST, CLOWDER_KEY, dataset_id, FILEPATH) 58 | if len(ds_meta.keys()) > 0: 59 | log("adding metadata to Clowder dataset %s" % dataset_id) 60 | format_md = { 61 | "@context": ["https://clowder.ncsa.illinois.edu/contexts/metadata.jsonld", 62 | {"@vocab": "https://terraref.ncsa.illinois.edu/metadata/uamac#"}], 63 | "content": ds_meta, 64 | "agent": { 65 | "@type": "cat:user", 66 | "user_id": "https://terraref.ncsa.illinois.edu/clowder/api/users/58e2a7b9fe3ae3efc1632ae8" 67 | } 68 | } 69 | upload_metadata(CONN, CLOWDER_HOST, CLOWDER_KEY, dataset_id, format_md) 70 | 71 | for sensor in SENSOR_LIST: 72 | SENSOR_DIR = os.path.join(SENSOR_FOLDER, sensor) 73 | 74 | for date in os.listdir(SENSOR_DIR): 75 | if date.startswith('.'): 76 | continue 77 | DATE_DIR = os.path.join(SENSOR_DIR, date) 78 | 79 | # Need one additional loop if there is a timestamp-level directory 80 | if TIMESTAMP_FOLDER and os.path.isdir(DATE_DIR): 81 | log("Scanning datasets in %s" % DATE_DIR) 82 | for timestamp in os.listdir(DATE_DIR): 83 | if timestamp.startswith('.'): 84 | continue 85 | TIMESTAMP_DIR = os.path.join(DATE_DIR, timestamp) 86 | DS_FILES = [] 87 | DS_META = {} 88 | 89 | # Find files and metadata in the directory 90 | for filename in os.listdir(TIMESTAMP_DIR): 91 | if filename[0] != ".": 92 | FILEPATH = os.path.join(TIMESTAMP_DIR, filename) 93 | if filename.find("metadata.json") > -1: 94 | DS_META = clean_metadata(loadJsonFile(FILEPATH), sensor) 95 | else: 96 | DS_FILES.append(FILEPATH) 97 | 98 | upload_ds(CONN, CLOWDER_HOST, CLOWDER_KEY, sensor, date, timestamp, DS_FILES, DS_META) 99 | #failz() 100 | 101 | # Otherwise the date is the dataset level 102 | elif os.path.isdir(DATE_DIR): 103 | log("Scanning datasets in %s" % SENSOR_DIR) 104 | DS_FILES = [] 105 | DS_META = {} 106 | for filename in os.listdir(DATE_DIR): 107 | if filename[0] != ".": 108 | FILEPATH = os.path.join(DATE_DIR, filename) 109 | if filename.find("metadata.json") > -1: 110 | DS_META = clean_metadata(loadJsonFile(FILEPATH), sensor) 111 | else: 112 | DS_FILES.append(FILEPATH) 113 | 114 | upload_ds(CONN, CLOWDER_HOST, CLOWDER_KEY, sensor, date, '', DS_FILES, DS_META) 115 | #failz() 116 | 117 | # Don't create a dataset for metadata only 118 | 119 | log("Completed.") 120 | LOGFILE.close() 121 | -------------------------------------------------------------------------------- /scripts/reprocessing_bulk_scripts/get_dataset_ids_SENSOR.js: -------------------------------------------------------------------------------- 1 | /* 2 | Template for script to generate a CSV containing list of dataset IDs from MongoDB. 3 | 4 | Usage: 5 | mongo clowder --quiet get_dataset_ids_SENSOR.js > list_SENSOR_YEAR.csv 6 | Output: 7 | dataset_id,dataset_name 8 | 9 | The regex is used to filter by dataset name. Using the ^ character for start of name 10 | speeds things up considerably. 11 | 12 | Sensor reference: 13 | stereoTop - 2018 14 | RGB GeoTIFFs - 2018 15 | flirIrCamera - 2018 16 | Thermal IR GeoTIFFs - 2018 17 | scanner3DTop - 2018 18 | Laser Scanner 3D LAS - 2018 19 | VNIR - 2018 20 | EnvironmentLogger netCDFs - 2018 21 | 22 | After generating the CSV, submit it to a Clowder instance for extraction using: 23 | submit_datasets_by_list.py 24 | */ 25 | 26 | db.datasets.find({"name": {$regex: /^SENSOR - YEAR.*/}}).forEach(function(ds){ 27 | print(ds._id.valueOf()+","+ds.name); 28 | }); 29 | -------------------------------------------------------------------------------- /scripts/reprocessing_bulk_scripts/submit_datasets_by_list.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import requests 4 | 5 | from pyclowder.connectors import Connector 6 | from pyclowder.datasets import submit_extraction 7 | 8 | """ 9 | This script will consume a CSV generated by get_dataset_ids_SENSOR.js and submit them to a desired extractor. 10 | The -d flag is commonly used when submitting GeoTIFFs to rulechecker that have previously been recorded in order 11 | to trigger a fieldmosaic process. 12 | 13 | Usage: 14 | python submit_datasets_by_list.py -k CLOWDERKEY -f list_SENSOR_YEAR.csv -e terra.stereo-rgb.bin2tif 15 | 16 | Sensor -> extractor reference: 17 | stereoTop -> terra.stereo-rgb.bin2tif 18 | RGB GeoTIFFs -> ncsa.rulechecker.terra 19 | flirIrCamera -> terra.multispectral.flir2tif 20 | Thermal IR GeoTIFFs -> ncsa.rulechecker.terra 21 | scanner3DTop -> terra.3dscanner.ply2las 22 | Laser Scanner 3D LAS -> terra.plotclipper 23 | VNIR -> terra.hyperspectral 24 | EnvironmentLogger netCDFs -> terra.environmental.envlog2netcdf 25 | """ 26 | 27 | parser = argparse.ArgumentParser() 28 | parser.add_argument('-k', '--key', help="Clowder key", default="") 29 | parser.add_argument('-f', '--input', help="input CSV file") 30 | parser.add_argument('-e', '--extractor', help="extractor to use", default="") 31 | parser.add_argument('-s', '--sites', help="where /sites is mounted", default="/home/clowder/sites") 32 | parser.add_argument('-h', '--host', help="Clowder host URL", default="https://terraref.ncsa.illinois.edu/clowder/") 33 | parser.add_argument('-d', '--daily', help="only submit one dataset per day", default=False, action='store_true') 34 | parser.add_argument('-t', '--test', help="only submit one dataset then exit", default=False, action='store_true') 35 | args = parser.parse_args() 36 | 37 | logging.basicConfig(filename="submit_%s.log" % args.input, level=logging.DEBUG) 38 | 39 | CONN = Connector(None, mounted_paths={"/home/clowder/sites":args.sites}) 40 | 41 | logging.info("attempting to parse %s" % args.input) 42 | sess = requests.Session() 43 | 44 | if args.daily: 45 | seen_days = [] 46 | with open(args.input, 'r') as csv: 47 | i = 0 48 | for line in csv: 49 | ds_id, ds_name = line.replace("\n", "").split(",") 50 | if len(ds_id) > 0: 51 | if args.daily: 52 | day = ds_name.split(" - ")[1].split("__")[0] 53 | if day in seen_days: 54 | continue 55 | else: 56 | seen_days.append(day) 57 | try: 58 | submit_extraction(CONN, args.host, args.key, ds_id, args.extractor) 59 | except Exception as e: 60 | logging.info("failed to submit %s [%s]" % (ds_id, e)) 61 | i+=1 62 | if (i % 1000 == 0): 63 | logging.info("submitted %s files" % i) 64 | if args.test: 65 | logging.info("submitted %s" % ds_id) 66 | break 67 | logging.info("processing completed") -------------------------------------------------------------------------------- /scripts/terra_report: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sensors=($(ls /gantry_data/LemnaTec/MovingSensor/)) 4 | date=$(date --date="2 days ago" +"%Y-%m-%d") 5 | 6 | printf '%-22s %8s %8s\n' Sensor FileCount Size 7 | for i in ${sensors[@]} 8 | do 9 | file_count=$(find /gantry_data/LemnaTec/MovingSensor/$i/$date -type f | wc -l) 10 | size=$(du -h --max-depth=0 /gantry_data/LemnaTec/MovingSensor/$i/$date | awk '{print $1}') 11 | printf '%-22s %8s %8s\n' $i $file_count $size 12 | done 13 | 14 | file_count=$(find /gantry_data/LemnaTec/EnvironmentLogger/$date -type f | wc -l) 15 | size=$(du -h --max-depth=0 /gantry_data/LemnaTec/EnvironmentLogger/$date | awk '{print $1}') 16 | printf '%-22s %8s %8s\n' EnvironmentLogger $file_count $size 17 | 18 | --------------------------------------------------------------------------------