├── README.md ├── Use Python Download the MODIS Data.ipynb └── python laads-data-download.py /README.md: -------------------------------------------------------------------------------- 1 | # Modis_Data_Download_By_Python 2 | 3 | CN as This Link 4 | -------------------------------------------------------------------------------- /Use Python Download the MODIS Data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "attachments": {}, 5 | "cell_type": "markdown", 6 | "metadata": {}, 7 | "source": [ 8 | "# Step 1\n", 9 | "Log in to your account, click Profile, click App Keys. Enter the key that Modis gets in Description.\n", 10 | "\n", 11 | " \n", 12 | "At this point, the preparations are done!\n", 13 | "\n", 14 | "The key sptography obtained is as follows F*******-****-****-****-D****D*****C\n", 15 | "\n", 16 | "# Step 2 \n", 17 | "And then log in the Website \n", 18 | "Click in order of steps: \n", 19 | "1.PRODUCTS \n", 20 | "2.TIME \n", 21 | "3.LOCATION \n", 22 | "4.FILES \n", 23 | "5.REVIEW and ORDER\n", 24 | "\n", 25 | " \n", 26 | "\n", 27 | "1.Products \n", 28 | "\n", 29 | "Here I choose `mod04-3km` data from `Terra` satellite aerosol products. \n", 30 | "\n", 31 | " Click on the right i symbol to see information about the satellite \n", 32 | "\n", 33 | "\n", 34 | " \n", 35 | "\n", 36 | "2.TIME\n", 37 | "\n", 38 | "Choose a time, a total of two options (time period and single day), please explore yourself (lazy hahaha). I chose 20190601 - 20190625\n", 39 | "\n", 40 | "\n", 41 | "\n", 42 | "3.LOCATION\n", 43 | "\n", 44 | "\n", 45 | "\n", 46 | "4.FILES\n", 47 | "\n", 48 | "Click `Select All`\n", 49 | "\n", 50 | "\n", 51 | "\n", 52 | "\n", 53 | "5.REVIEW and ORDER\n", 54 | "\n", 55 | "After the final Submit Order, the Modis takeaway you ordered was delivered to your mailbox~~~~~~~\n", 56 | "\n", 57 | "\n", 58 | "\n", 59 | "\n", 60 | "After the order is submitted successfully, the registration mailbox receives an email. \n", 61 | "\n", 62 | "\n", 63 | "\n", 64 | "\n", 65 | "\n", 66 | "Go to the arrow page and download the corresponding python version of the tool. \n", 67 | "To avoid trouble, I have prepared `python laads-data-download.py` in this project.\n", 68 | "\n", 69 | "\n", 70 | "\n", 71 | "Personally, it is recommended that the code be present inside Jupyter.\n", 72 | "\n", 73 | "But after following the official tip to remove the .txt extension, we got the script we needed!!!!\n", 74 | "\n", 75 | "Next up is the stage of downloading the data. Copy the order number, view the download link\n", 76 | "\n", 77 | "\n", 78 | "\n", 79 | "\n", 80 | "You need to switch to the directory where the script is located before opening the cmd runner.\n", 81 | "\n", 82 | "\n", 83 | "\n", 84 | "The input format as follow:\n", 85 | "\n", 86 | "```python\n", 87 | "python laads-data-download.py -s https://ladsweb.modaps.eosdis.nasa.gov/archive/orders/501348842/ -d D:\\balabala -t F*******-****-****-****-D****D*****C\n", 88 | "```\n", 89 | "\n", 90 | "Finally, explain the parameters used for the operation:\n", 91 | "\n", 92 | "```python\n", 93 | "This script will recursively download all files if they don't exist from a\n", 94 | "LAADS URL and stores them to the specified path\n", 95 | "\n", 96 | "optional arguments:\n", 97 | " -h, show this help message and exit\n", 98 | " -s URL, Recursively download files at URL\n", 99 | " -d DIR, Store directory structure in DIR\n", 100 | " -t TOK, Use app token TOK to authenticate\n", 101 | "```" 102 | ] 103 | } 104 | ], 105 | "metadata": { 106 | "kernelspec": { 107 | "display_name": "Python 3", 108 | "language": "python", 109 | "name": "python3" 110 | }, 111 | "language_info": { 112 | "codemirror_mode": { 113 | "name": "ipython", 114 | "version": 3 115 | }, 116 | "file_extension": ".py", 117 | "mimetype": "text/x-python", 118 | "name": "python", 119 | "nbconvert_exporter": "python", 120 | "pygments_lexer": "ipython3", 121 | "version": "3.7.0" 122 | } 123 | }, 124 | "nbformat": 4, 125 | "nbformat_minor": 2 126 | } 127 | -------------------------------------------------------------------------------- /python laads-data-download.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Thu Sep 19 21:19:05 2019 4 | 5 | @author: Zhi_Zheng 6 | """ 7 | 8 | # % python laads-data-download.py 9 | #!/usr/bin/env python 10 | 11 | # script supports either python2 or python3 12 | # 13 | # Attempts to do HTTP Gets with urllib2(py2) urllib.requets(py3) or subprocess 14 | # if tlsv1.1+ isn't supported by the python ssl module 15 | # 16 | # Will download csv or json depending on which python module is available 17 | # 18 | 19 | from __future__ import (division, print_function, absolute_import, unicode_literals) 20 | 21 | import argparse 22 | import os 23 | import os.path 24 | import shutil 25 | import sys 26 | 27 | try: 28 | from StringIO import StringIO # python2 29 | except ImportError: 30 | from io import StringIO # python3 31 | 32 | 33 | ################################################################################ 34 | 35 | 36 | USERAGENT = 'tis/download.py_1.0--' + sys.version.replace('\n','').replace('\r','') 37 | 38 | 39 | def geturl(url, token=None, out=None): 40 | headers = { 'user-agent' : USERAGENT } 41 | if not token is None: 42 | headers['Authorization'] = 'Bearer ' + token 43 | try: 44 | import ssl 45 | CTX = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) 46 | if sys.version_info.major == 2: 47 | import urllib2 48 | try: 49 | fh = urllib2.urlopen(urllib2.Request(url, headers=headers), context=CTX) 50 | if out is None: 51 | return fh.read() 52 | else: 53 | shutil.copyfileobj(fh, out) 54 | except urllib2.HTTPError as e: 55 | print('HTTP GET error code: %d' % e.code(), file=sys.stderr) 56 | print('HTTP GET error message: %s' % e.message, file=sys.stderr) 57 | except urllib2.URLError as e: 58 | print('Failed to make request: %s' % e.reason, file=sys.stderr) 59 | return None 60 | 61 | else: 62 | from urllib.request import urlopen, Request, URLError, HTTPError 63 | try: 64 | fh = urlopen(Request(url, headers=headers), context=CTX) 65 | if out is None: 66 | return fh.read().decode('utf-8') 67 | else: 68 | shutil.copyfileobj(fh, out) 69 | except HTTPError as e: 70 | print('HTTP GET error code: %d' % e.code(), file=sys.stderr) 71 | print('HTTP GET error message: %s' % e.message, file=sys.stderr) 72 | except URLError as e: 73 | print('Failed to make request: %s' % e.reason, file=sys.stderr) 74 | return None 75 | 76 | except AttributeError: 77 | # OS X Python 2 and 3 don't support tlsv1.1+ therefore... curl 78 | import subprocess 79 | try: 80 | args = ['curl', '--fail', '-sS', '-L', '--get', url] 81 | for (k,v) in headers.items(): 82 | args.extend(['-H', ': '.join([k, v])]) 83 | if out is None: 84 | # python3's subprocess.check_output returns stdout as a byte string 85 | result = subprocess.check_output(args) 86 | return result.decode('utf-8') if isinstance(result, bytes) else result 87 | else: 88 | subprocess.call(args, stdout=out) 89 | except subprocess.CalledProcessError as e: 90 | print('curl GET error message: %' + (e.message if hasattr(e, 'message') else e.output), file=sys.stderr) 91 | return None 92 | 93 | 94 | 95 | ################################################################################ 96 | 97 | 98 | DESC = "This script will recursively download all files if they don't exist from a LAADS URL and stores them to the specified path" 99 | 100 | 101 | def sync(src, dest, tok): 102 | '''synchronize src url with dest directory''' 103 | try: 104 | import csv 105 | files = [ f for f in csv.DictReader(StringIO(geturl('%s.csv' % src, tok)), skipinitialspace=True) ] 106 | except ImportError: 107 | import json 108 | files = json.loads(geturl(src + '.json', tok)) 109 | 110 | # use os.path since python 2/3 both support it while pathlib is 3.4+ 111 | for f in files: 112 | # currently we use filesize of 0 to indicate directory 113 | filesize = int(f['size']) 114 | path = os.path.join(dest, f['name']) 115 | url = src + '/' + f['name'] 116 | if filesize == 0: 117 | try: 118 | print('creating dir:', path) 119 | os.mkdir(path) 120 | sync(src + '/' + f['name'], path, tok) 121 | except IOError as e: 122 | print("mkdir `%s': %s" % (e.filename, e.strerror), file=sys.stderr) 123 | sys.exit(-1) 124 | else: 125 | try: 126 | if not os.path.exists(path): 127 | print('downloading: ' , path) 128 | with open(path, 'w+b') as fh: 129 | geturl(url, tok, fh) 130 | else: 131 | print('skipping: ', path) 132 | except IOError as e: 133 | print("open `%s': %s" % (e.filename, e.strerror), file=sys.stderr) 134 | sys.exit(-1) 135 | return 0 136 | 137 | 138 | def _main(argv): 139 | parser = argparse.ArgumentParser(prog=argv[0], description=DESC) 140 | parser.add_argument('-s', '--source', dest='source', metavar='URL', help='Recursively download files at URL', required=True) 141 | parser.add_argument('-d', '--destination', dest='destination', metavar='DIR', help='Store directory structure in DIR', required=True) 142 | parser.add_argument('-t', '--token', dest='token', metavar='TOK', help='Use app token TOK to authenticate', required=True) 143 | args = parser.parse_args(argv[1:]) 144 | if not os.path.exists(args.destination): 145 | os.makedirs(args.destination) 146 | return sync(args.source, args.destination, args.token) 147 | 148 | 149 | if __name__ == '__main__': 150 | try: 151 | sys.exit(_main(sys.argv)) 152 | except KeyboardInterrupt: 153 | sys.exit(-1) --------------------------------------------------------------------------------