├── .gitignore ├── clean_rgb.sh ├── clean_projected.sh ├── clean_corrected.sh ├── kangbashi.yml ├── shanghai.yml ├── las_vegas.yaml ├── tyler.yml ├── tyler.geojson ├── process.sh ├── README.md ├── main.py ├── earthexplorer.py ├── satellite.py ├── landsat-orig.rb ├── landsat.rb └── scene.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | data 3 | test 4 | *.tif 5 | .DS_Store 6 | -------------------------------------------------------------------------------- /clean_rgb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm data/**/**/**/**/merged.tif 4 | ./clean_corrected.sh 5 | -------------------------------------------------------------------------------- /clean_projected.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm data/**/**/**/**/*-projected.tif 4 | ./clean_rgb.sh 5 | -------------------------------------------------------------------------------- /clean_corrected.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm data/**/**/**/**/temp.tif 4 | rm data/**/**/**/**/color_corrected.tif 5 | -------------------------------------------------------------------------------- /kangbashi.yml: -------------------------------------------------------------------------------- 1 | north: 39.6 2 | west: 109.6 3 | south: 39.5 4 | east: 110 5 | start_year: 2000 6 | end_year: 2010 7 | max_cloud_cover: 0 8 | -------------------------------------------------------------------------------- /shanghai.yml: -------------------------------------------------------------------------------- 1 | north: 31.6 2 | west: 120.7 3 | south: 30.7 4 | east: 122.4 5 | start_year: 2000 6 | end_year: 2010 7 | max_cloud_cover: 0 8 | levels: '-channel R -level 8%,46% -channel G -level 11%,37% -channel B -level 28%,61%' 9 | -------------------------------------------------------------------------------- /las_vegas.yaml: -------------------------------------------------------------------------------- 1 | north: 36.4 2 | west: -115.6 3 | south: 35.9 4 | east: -114.7 5 | start_year: 2000 6 | end_year: 2010 7 | max_cloud_cover: 50 8 | levels: '-channel R -level 8%,46% -channel G -level 11%,37% -channel B -level 28%,61%' 9 | -------------------------------------------------------------------------------- /tyler.yml: -------------------------------------------------------------------------------- 1 | path_start: 26 2 | path_end: 26 3 | row_start: 37 4 | row_end: 38 5 | start_year: 2000 6 | end_year: 2010 7 | max_cloud_cover: 0 8 | levels: -channel R -level 12%,41% -channel G -level 16%,34% -channel B -level 24%,37% 9 | cutline: tyler.geojson 10 | -------------------------------------------------------------------------------- /tyler.geojson: -------------------------------------------------------------------------------- 1 | {"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[-95.53848266601562,32.17445009023132],[-95.53848266601562,32.5178680435577],[-95.08529663085938,32.5178680435577],[-95.08529663085938,32.17445009023132],[-95.53848266601562,32.17445009023132]]]}}]} -------------------------------------------------------------------------------- /process.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # for BAND in {4,3,2}; do 4 | # gdalwarp -t_srs EPSG:3857 LC80260382015338LGN00/LC80260382015338LGN00_B$BAND.TIF $BAND-projected.tif; 5 | # done 6 | 7 | # convert -combine {4,3,2}-projected.tif RGB.tif 8 | 9 | # convert -sigmoidal-contrast 50x12% RGB.tif RGB-corrected.tif 10 | 11 | # convert -channel B -gamma 0.925 -channel R -gamma 1.03 -channel RGB -sigmoidal-contrast 50x16% RGB.tif RGB-corrected.tif 12 | 13 | 14 | 15 | 16 | convert -sigmoidal-contrast 50x12% /Users/cgroskopf/landsat/processed/LC80260382015338LGN00/LC80260382015338LGN00_bands_432_pan.TIF RGB-corrected.tif 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | growing-cities 2 | ============== 3 | 4 | Based on Ruby code written by ProPublica for their [Las Vegas Growth Map](https://projects.propublica.org/las-vegas-growth-map/). 5 | 6 | Setup 7 | ----- 8 | 9 | Requires Python 2.7 and GDAL. 10 | 11 | ``` 12 | mkvirtualenv growing-cities 13 | pip install requests lxml cssselect invoke pyyaml rasterio scikit-image 14 | 15 | gem install net 16 | gem install nokogiri 17 | gem install fileutils 18 | 19 | # Install GDAL with Python support 20 | brew install gdal --with-python --with-python3 --with-postgresql 21 | 22 | # Hotlink GDAL python modules into virtualenv 23 | echo 'import site; site.addsitedir("/usr/local/lib/python2.7/site-packages")' >> /Users/cgroskopf/.virtualenvs/growing-cities/lib/python2.7/site-packages/homebrew.pth 24 | 25 | # Install Google Cloud utils for downloading files 26 | pip install gsutil 27 | 28 | # Install linux tar utility 29 | brew install gnu-tar --with-default-names 30 | 31 | mkdir data 32 | 33 | python main.py tyler.yml 34 | ``` 35 | 36 | TODO 37 | ---- 38 | 39 | * Purge cloud cover 40 | * Balance colors 41 | * Pansharpen Landsat 8 images 42 | * Skip nighttime images 43 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Landsat file guide: http://landsat.usgs.gov//files_will_be_provided_with_a_Landsat_scene.php 5 | Spectral band definitions: http://landsat.usgs.gov//band_designations_landsat_satellites.php 6 | Band usages: http://landsat.usgs.gov//best_spectral_bands_to_use.php 7 | """ 8 | 9 | from glob import glob 10 | import os 11 | import sys 12 | import yaml 13 | 14 | import rasterio 15 | from rasterio.tools.merge import merge 16 | 17 | from earthexplorer import EarthExplorer 18 | from scene import Scene 19 | 20 | 21 | def merge_adjacent(path_dir): 22 | print('Merging adjacent images') 23 | 24 | output_path = '%s/merged.tif' % path_dir 25 | 26 | with rasterio.drivers(): 27 | files = glob('%s/**/**/color_corrected.tif' % path_dir) 28 | 29 | sources = [rasterio.open(f) for f in files] 30 | dest, output_transform = merge(sources) 31 | 32 | profile = sources[0].profile 33 | profile.pop('affine') 34 | profile['transform'] = output_transform 35 | profile['height'] = dest.shape[1] 36 | profile['width'] = dest.shape[2] 37 | profile['driver'] = 'GTiff' 38 | 39 | with rasterio.open(output_path, 'w', **profile) as dst: 40 | dst.write(dest) 41 | 42 | 43 | def main(): 44 | output_dir = 'data' 45 | 46 | with open(sys.argv[1]) as f: 47 | config = yaml.load(f) 48 | 49 | for year in range(config['start_year'], config['end_year'] + 1): 50 | print(year) 51 | year_dir = os.path.join(output_dir, str(year)) 52 | 53 | for path in range(config['path_start'], config['path_end'] + 1): 54 | path_dir = os.path.join(year_dir, str(path)) 55 | print(path) 56 | 57 | for row in range(config['row_start'], config['row_end'] + 1): 58 | row_dir = os.path.join(path_dir, str(row)) 59 | print(row) 60 | 61 | explorer = EarthExplorer(year, path, row, max_cloud_cover=config['max_cloud_cover']) 62 | scene_ids = explorer.get_scenes() 63 | 64 | for scene_id in scene_ids[:1]: 65 | print('Processing %s' % scene_id) 66 | 67 | scene_dir = os.path.join(row_dir, scene_id) 68 | scene = Scene(scene_id, scene_dir, config['levels'], config['cutline']) 69 | scene.process() 70 | 71 | files = glob('%s/**/**/color_corrected.tif' % path_dir) 72 | 73 | if len(files) > 0: 74 | merge_adjacent(path_dir) 75 | 76 | 77 | if __name__ == '__main__': 78 | main() 79 | -------------------------------------------------------------------------------- /earthexplorer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from lxml import etree 4 | from lxml.cssselect import CSSSelector 5 | import requests 6 | 7 | 8 | class EarthExplorer(object): 9 | """ 10 | Search USGS EarthExplorer for scene identifiers. 11 | """ 12 | def __init__(self, year, path, row, max_cloud_cover=0): 13 | self.year = year 14 | self.path = path 15 | self.row = row 16 | self.max_cloud_cover = max_cloud_cover 17 | 18 | def _get_dataset_name(self): 19 | """ 20 | Get the correct EarthExplorer dataset name for the given year. 21 | """ 22 | # Landsat 8 23 | if self.year >= 2013: 24 | return 'LANDSAT_8' 25 | # Landsat 5 for broken years of Landsat 7 26 | elif self.year >= 2003: 27 | return 'LANDSAT_COMBINED' 28 | # Landsat 7 29 | elif self.year >= 1999: 30 | return 'LANDSAT_ETM' 31 | # Landsat 5 32 | elif self.year >= 1984: 33 | return 'LANDSAT_COMBINED' 34 | # Landsat 1-4 35 | elif self.year >= 1972: 36 | return 'LANDSAT_MSS1' 37 | else: 38 | raise ValueError('Landsat 1 did not come online until 1972.') 39 | 40 | def get_scenes(self): 41 | """ 42 | Fetch a scene list from the EarthExplorer API. Individual scenes are 43 | parsed and returned as :class:`.SceneID` instances. 44 | """ 45 | url = 'http://earthexplorer.usgs.gov/EE/InventoryStream/pathrow?start_path={path}&end_path={path}&start_row={row}&end_row={row}&sensor={sensor}&start_date={year}-01-01&end_date={year}-12-31'.format( 46 | path=self.path, 47 | row=self.row, 48 | sensor=self._get_dataset_name(), 49 | year=self.year 50 | ) 51 | print(url) 52 | 53 | r = requests.get(url) 54 | content = r.content 55 | xml = etree.fromstring(content) 56 | 57 | def cssselect(el, css): 58 | selector = CSSSelector('ns|%s' % css, namespaces={ 'ns': 'http://upe.ldcm.usgs.gov/schema/metadata' }) 59 | 60 | return selector(el) 61 | 62 | def cloud_filter(metadata): 63 | scene_id = cssselect(metadata, 'sceneID')[0].text 64 | cloud_cover_el = cssselect(metadata, 'cloudCoverFull')[0] 65 | 66 | value = float(cloud_cover_el.text) 67 | 68 | if value > self.max_cloud_cover: 69 | print('Skipping %s, %.0f%% cloud cover' % (scene_id, value)) 70 | return False 71 | 72 | return True 73 | 74 | def get_scene_id(metadata): 75 | scene_id_el = cssselect(metadata, 'sceneID')[0] 76 | 77 | return scene_id_el.text 78 | 79 | metadata = cssselect(xml, 'metaData') 80 | cloudless = filter(cloud_filter, metadata) 81 | scene_ids = map(get_scene_id, cloudless) 82 | 83 | return scene_ids 84 | -------------------------------------------------------------------------------- /satellite.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | 4 | class Satellite(object): 5 | """ 6 | Configuration data for a given Landsat satellite. 7 | """ 8 | def __init__(self, version): 9 | if version == 6: 10 | raise ValueError('There is no Landsat 6 data due to a rocket failure.') 11 | elif version > 8: 12 | raise ValueError('Landsat 8 is the highest active version.') 13 | 14 | self.version = version 15 | 16 | @property 17 | def sensor(self): 18 | if self.version == 8: 19 | return 'C' 20 | elif self.version == 7: 21 | return 'E' 22 | elif self.version >= 4: 23 | return 'T' 24 | else: 25 | return 'M' 26 | 27 | @property 28 | def natural_color_bands(self): 29 | """ 30 | 4-3-2 natural color. 31 | """ 32 | if self.version < 4: 33 | raise ValueError('True color is not possible for Landsats 1-3 as they did not have a blue band.') 34 | elif self.version < 8: 35 | return [3, 2, 1] 36 | else: 37 | return [4, 3, 2] 38 | 39 | @property 40 | def urban_false_color_bands(self): 41 | """ 42 | 7-6-4 false color for analyzing urban areas. 43 | 44 | http://www.exelisvis.com/Home/NewsUpdates/TabId/170/ArtMID/735/ArticleID/14305/The-Many-Band-Combinations-of-Landsat-8.aspx 45 | """ 46 | if self.version < 4: 47 | raise ValueError('Urban false color is not possible for Landsats 1-3 as they did not have short-wave infrared bands.') 48 | elif self.version < 8: 49 | return [7, 5, 3] 50 | else: 51 | return [7, 6, 4] 52 | 53 | @property 54 | def vegetation_false_color_bands(self): 55 | """ 56 | 5-4-3 color infrared for analyzing vegetation. 57 | 58 | http://www.exelisvis.com/Home/NewsUpdates/TabId/170/ArtMID/735/ArticleID/14305/The-Many-Band-Combinations-of-Landsat-8.aspx 59 | """ 60 | if self.version < 4: 61 | return [6, 5, 4] 62 | elif self.version < 8: 63 | return [4, 3, 2] 64 | else: 65 | return [5, 4, 3] 66 | 67 | @classmethod 68 | def for_year(cls, year): 69 | """ 70 | Get the preferred satellite configuration for a given year. 71 | """ 72 | if year >= 2013: 73 | return cls(8) 74 | elif year >= 1999 and year < 2003: 75 | return cls(7) 76 | elif year >= 1984: 77 | return cls(5) 78 | elif year >= 1982: 79 | return cls(4) 80 | # elif year >= 1978: 81 | # return cls(3) 82 | # elif year >= 1975: 83 | # return cls(2) 84 | # else: 85 | # return cls(1) 86 | else: 87 | raise ValueError('Only years starting in 1982 are allowed (due to change in WVS coordinates).') 88 | -------------------------------------------------------------------------------- /landsat-orig.rb: -------------------------------------------------------------------------------- 1 | require 'net/http' 2 | require 'nokogiri' 3 | require 'fileutils' 4 | 5 | class Satellite 6 | def initialize(opts = {}) 7 | @mission = "L" 8 | @sensor = opts[:sensor] 9 | @version = opts[:version].to_i 10 | end 11 | 12 | def vegetation 13 | return %w[B6 B5 B4] if @version < 4 14 | return %w[B40 B30 B20] if @version <= 5 && @sensor == "T" # 4-7 TM 15 | return %w[B4 B2 B1] if @version <= 5 # 4-7 MSS 16 | return %w[B40 B30 B20] if @version == 7 17 | return %w[B5 B4 B3] 18 | end 19 | 20 | # http://landsat.usgs.gov/consumer.php 21 | def metadata_sensor_id 22 | return "LANDSAT_8" if @version == 8 23 | return "LANDSAT_MSS1" if @version < 5 24 | return "LANDSAT_COMBINED" if @version < 7 25 | return "LANDSAT_ETM" if @version < 8 # 1999-2003 before breakage 26 | end 27 | 28 | def self.find_by_year(year) 29 | year = year.to_i 30 | return self.new({:sensor => "C", :version => "8"}) if year >= 2013 31 | return self.new({:sensor => "E", :version => "7"}) if year >= 1999 && year < 2003 # landsat 7 broke on may 31, 2003 32 | return self.new({:sensor => "T", :version => "5"}) if year >= 1984 33 | return self.new({:sensor => "T", :version => "4"}) if year >= 1982 34 | return self.new({:sensor => "M", :version => "3"}) if year >= 1978 35 | return self.new({:sensor => "M", :version => "2"}) if year >= 1975 36 | return self.new({:sensor => "M", :version => "1"}) 37 | end 38 | end 39 | 40 | class Scene 41 | attr_reader :id, :satellite, :outdir 42 | 43 | def initialize(id, outdir) 44 | @id = id 45 | @outdir = outdir 46 | end 47 | 48 | def mission 49 | "L" 50 | end 51 | 52 | def product 53 | @id[0,3] 54 | end 55 | 56 | # gs://earthengine-public/landsat/scene_list.zip 57 | # gs://earthengine-public/landsat/L5/ 58 | # gs://earthengine-public/landsat/L7/ 59 | # gs://earthengine-public/landsat/L8/ 60 | # gs://earthengine-public/landsat/LM1/ 61 | # gs://earthengine-public/landsat/LM2/ 62 | # gs://earthengine-public/landsat/LM3/ 63 | # gs://earthengine-public/landsat/LM4/ 64 | # gs://earthengine-public/landsat/LM5/ 65 | # gs://earthengine-public/landsat/LT4/ 66 | # gs://earthengine-public/landsat/PE1/ 67 | def gsproduct 68 | version.to_i > 4 ? "L#{version}" : "L#{sensor}#{version}" 69 | end 70 | 71 | def band_file_pattern 72 | id 73 | # return id if version.to_i < 7 || version.to_i > 7 74 | # return "#{mission}#{version}*_#{row}#{year}#{day}" # some odd id between L7 and path/row/day/band in L7 band files 75 | end 76 | 77 | def sensor 78 | @id[1] 79 | end 80 | 81 | def version 82 | @id[2] 83 | end 84 | 85 | def path 86 | @id[3,3] 87 | end 88 | 89 | def row 90 | @id[6,3] 91 | end 92 | 93 | def year 94 | @id[9,4] 95 | end 96 | 97 | def day 98 | @id[13,3] 99 | end 100 | 101 | def gsi 102 | @id[16,3] 103 | end 104 | 105 | def archive_version 106 | @id[19,2] 107 | end 108 | 109 | def satellite 110 | @satellite ||= Satellite.new({:sensor => sensor, :version => version}) 111 | end 112 | 113 | def zip_exists? 114 | File.exists?(File.join("#{@outdir}", "#{id}.tar.bz")) 115 | end 116 | 117 | def band_files_exist? 118 | Dir["#{@outdir}/#{id}_B*"].length > 0 119 | end 120 | 121 | def processed_files_exist? 122 | Dir["#{@outdir}/#{id}*projected*"].length > 0 123 | end 124 | 125 | def download 126 | return if zip_exists? 127 | return if band_files_exist? 128 | 129 | puts "== downloading #{gsproduct}/#{path}/#{row}/#{id} to #{@outdir}/#{id}.tar.bz" 130 | `gsutil cp gs://earthengine-public/landsat/#{gsproduct}/#{path}/#{row}/#{id}.tar.bz #{@outdir}` unless File.exists?("#{@outdir}/#{id}.tar.bz") 131 | end 132 | 133 | def unzip 134 | `cd #{@outdir} && gtar --transform 's/^.*_/#{id}_/g' -xzvf #{id}.tar.bz` 135 | end 136 | 137 | def warp(polygon_shp) 138 | return unless zip_exists? 139 | 140 | if !band_files_exist? 141 | unzip 142 | end 143 | 144 | if !processed_files_exist? 145 | satellite.vegetation.each do |band| 146 | # if it's landsat 8, convert to 8 bit 147 | if version.to_i > 7 148 | puts "== Landsat 8, converting to 8bit" 149 | `gdal_translate -of "GTiff" -co "COMPRESS=LZW" -scale 0 65535 0 255 -ot Byte #{@outdir}/#{band_file_pattern}_#{band}.TIF #{@outdir}/#{band_file_pattern}_#{band}_tmp.TIF && \ 150 | rm #{@outdir}/#{band_file_pattern}_#{band}.TIF && mv #{@outdir}/#{band_file_pattern}_#{band}_tmp.TIF #{@outdir}/#{band_file_pattern}_#{band}.TIF` 151 | end 152 | `gdalwarp -t_srs "EPSG:3857" -cutline #{polygon_shp} -crop_to_cutline #{@outdir}/#{band_file_pattern}_#{band}.TIF #{@outdir}/#{band_file_pattern}_#{band}-projected.tif` 153 | end 154 | puts "gdal_merge.py -seperate #{@outdir}/#{band_file_pattern}_{#{satellite.vegetation.join(",")}}-projected.tif -o #{@outdir}/#{band_file_pattern}_RGB-projected.tif" 155 | `gdal_merge.py -seperate #{@outdir}/#{band_file_pattern}_{#{satellite.vegetation.join(",")}}-projected.tif -o #{@outdir}/#{band_file_pattern}_RGB-projected.tif` #&& \ 156 | # convert -channel B -gamma 0.925 -channel R -gamma 1.03 -channel RGB -sigmoidal-contrast 50x16% #{@outdir}/#{id}_RGB-projected.tif #{@outdir}/#{id}_RGB-projected-corrected.tif && \ 157 | # convert -depth 8 #{@outdir}/#{id}_RGB-projected-corrected.tif #{@outdir}/#{id}_RGB-projected-corrected-8bit.tif && \ 158 | # listgeo -tfw #{@outdir}/#{id}_#{satellite.vegetation[0]}-projected.tif && \ 159 | # mv #{@outdir}/#{id}_#{satellite.vegetation[0]}-projected.tfw #{@outdir}/#{id}_RGB-projected-corrected-8bit.tfw && \ 160 | # gdal_edit.py -a_srs EPSG:3857 #{@outdir}/#{id}_RGB-projected-corrected-8bit.tif && \ 161 | # gdal_translate -a_nodata 0 #{@outdir}/#{id}_RGB-projected-corrected-8bit.tif #{@outdir}/#{id}_RGB-projected-corrected-8bit-nodata.tif` 162 | end 163 | end 164 | 165 | def tfw 166 | `listgeo -tfw #{@outdir}/#{id}_#{satellite.vegetation[0]}-projected.tif && \ 167 | mv #{@outdir}/#{id}_#{satellite.vegetation[0]}-projected.tfw #{@outdir}/#{id}_RGB-projected.tfw` 168 | end 169 | end 170 | 171 | class Processor 172 | BB = [36.42311, -115.63218, 35.90424, -114.66538] 173 | 174 | attr_reader :year, :outdir, :satellite, :scenes 175 | 176 | def initialize(year, outdir) 177 | @year = year 178 | @outdir = File.expand_path outdir 179 | @satellite = Satellite.find_by_year(year) 180 | FileUtils.mkdir_p(outdir) unless File.directory?(outdir) 181 | end 182 | 183 | def scenes 184 | @scenes ||= get_scenes 185 | end 186 | 187 | def same_size 188 | tifs = Dir["#{@outdir}/*_RGB-projected.tif"] 189 | puts tifs[0] 190 | size = `gdalinfo #{tifs[0]}`.split("\n")[3] 191 | 192 | wh = size.scan(/[\d]+/) 193 | puts wh 194 | tifs.each do |tif| 195 | `rm #{tif}.resized.tif` 196 | `gdalwarp -ts #{wh[0]} #{wh[1]} #{tif} #{tif}.resized.tif` 197 | end 198 | end 199 | 200 | def median 201 | `convert #{@outdir}/*_RGB-projected.tif -evaluate-sequence median #{@outdir}/median-out.tif` 202 | end 203 | 204 | private 205 | 206 | def get_scenes 207 | url = "http://earthexplorer.usgs.gov/EE/InventoryStream/latlong?north=#{BB[0]}&south=#{BB[2]}4&east=#{BB[3]}8&west=#{BB[1]}&sensor=#{@satellite.metadata_sensor_id}&start_date=#{@year}-01-01&end_date=#{@year}-12-31" 208 | puts "trying #{url}" 209 | scenes = Nokogiri::XML(Net::HTTP.get(URI.parse(url))).css('metaData').select {|n| n.children.css('cloudCoverFull').text.to_i < 2 }.map {|n| n.children.css('sceneID').text } 210 | scenes.map!{|q| Scene.new(q, @outdir) } 211 | puts "== got #{scenes.length} scenes" 212 | scenes 213 | end 214 | end 215 | 216 | if __FILE__ == $0 217 | start_year = ARGV[0].to_i 218 | end_year = ARGV[1].to_i 219 | outdir = ARGV[2] 220 | 221 | if ARGV.length < 3 || ARGV.length > 3 222 | puts <<-EOD 223 | 224 | usage: ruby median-blend.rb start_year end_year outdir 225 | 226 | example: ruby median-blend.rb 1972 2014 /Volumes/Gilese581c/vegas/ 227 | 228 | oh it wants you to have a POLYGON.shp in yr ../data/POLYGON.shp too thx 229 | EOD 230 | 231 | exit 1 232 | end 233 | 234 | 235 | puts "== doin #{start_year} up to #{end_year} in #{outdir}" 236 | (start_year).upto(end_year).each do |year| 237 | puts "== tryin #{year}" 238 | landsat_year = Processor.new(year, File.join(File.expand_path(outdir), year.to_s)) 239 | scenes = landsat_year.scenes 240 | 241 | q = [] 242 | scenes.map {|s| q << s } 243 | m = Mutex.new 244 | threads = (1..8).map do 245 | Thread.new do 246 | loop do 247 | scene = nil 248 | exit = false 249 | m.synchronize do 250 | if q.empty? 251 | exit = true 252 | else 253 | scene = q.shift 254 | end 255 | end 256 | Thread.exit if exit 257 | scene.download 258 | scene.warp("#{File.expand_path(File.dirname(__dir__), "..")}/data/POLYGON.shp") 259 | end 260 | end 261 | end 262 | 263 | threads.map(&:join) 264 | end 265 | end 266 | -------------------------------------------------------------------------------- /landsat.rb: -------------------------------------------------------------------------------- 1 | require 'net/http' 2 | require 'nokogiri' 3 | require 'fileutils' 4 | 5 | class Satellite 6 | def initialize(opts = {}) 7 | @mission = "L" 8 | @sensor = opts[:sensor] 9 | @version = opts[:version].to_i 10 | end 11 | 12 | def vegetation 13 | return %w[B6 B5 B4] if @version < 4 14 | return %w[B40 B30 B20] if @version <= 5 && @sensor == "T" # 4-7 TM 15 | return %w[B4 B2 B1] if @version <= 5 # 4-7 MSS 16 | return %w[B40 B30 B20] if @version == 7 17 | return %w[B5 B4 B3] 18 | end 19 | 20 | # http://landsat.usgs.gov/consumer.php 21 | def metadata_sensor_id 22 | return "LANDSAT_8" if @version == 8 23 | return "LANDSAT_MSS1" if @version < 5 24 | return "LANDSAT_COMBINED" if @version < 7 25 | return "LANDSAT_ETM" if @version < 8 # 1999-2003 before breakage 26 | end 27 | 28 | def self.find_by_year(year) 29 | year = year.to_i 30 | return self.new({:sensor => "C", :version => "8"}) if year >= 2013 31 | return self.new({:sensor => "E", :version => "7"}) if year >= 1999 && year < 2003 # landsat 7 broke on may 31, 2003 32 | return self.new({:sensor => "T", :version => "5"}) if year >= 1984 33 | return self.new({:sensor => "T", :version => "4"}) if year >= 1982 34 | return self.new({:sensor => "M", :version => "3"}) if year >= 1978 35 | return self.new({:sensor => "M", :version => "2"}) if year >= 1975 36 | return self.new({:sensor => "M", :version => "1"}) 37 | end 38 | end 39 | 40 | class Scene 41 | attr_reader :id, :satellite, :outdir 42 | 43 | def initialize(id, outdir) 44 | @id = id 45 | @outdir = outdir 46 | end 47 | 48 | def mission 49 | "L" 50 | end 51 | 52 | def product 53 | @id[0,3] 54 | end 55 | 56 | # gs://earthengine-public/landsat/scene_list.zip 57 | # gs://earthengine-public/landsat/L5/ 58 | # gs://earthengine-public/landsat/L7/ 59 | # gs://earthengine-public/landsat/L8/ 60 | # gs://earthengine-public/landsat/LM1/ 61 | # gs://earthengine-public/landsat/LM2/ 62 | # gs://earthengine-public/landsat/LM3/ 63 | # gs://earthengine-public/landsat/LM4/ 64 | # gs://earthengine-public/landsat/LM5/ 65 | # gs://earthengine-public/landsat/LT4/ 66 | # gs://earthengine-public/landsat/PE1/ 67 | def gsproduct 68 | version.to_i > 4 ? "L#{version}" : "L#{sensor}#{version}" 69 | end 70 | 71 | def band_file_pattern 72 | id 73 | # return id if version.to_i < 7 || version.to_i > 7 74 | # return "#{mission}#{version}*_#{row}#{year}#{day}" # some odd id between L7 and path/row/day/band in L7 band files 75 | end 76 | 77 | def sensor 78 | @id[1] 79 | end 80 | 81 | def version 82 | @id[2] 83 | end 84 | 85 | def path 86 | @id[3,3] 87 | end 88 | 89 | def row 90 | @id[6,3] 91 | end 92 | 93 | def year 94 | @id[9,4] 95 | end 96 | 97 | def day 98 | @id[13,3] 99 | end 100 | 101 | def gsi 102 | @id[16,3] 103 | end 104 | 105 | def archive_version 106 | @id[19,2] 107 | end 108 | 109 | def satellite 110 | @satellite ||= Satellite.new({:sensor => sensor, :version => version}) 111 | end 112 | 113 | def zip_exists? 114 | File.exists?(File.join("#{@outdir}", "#{id}.tar.bz")) 115 | end 116 | 117 | def band_files_exist? 118 | Dir["#{@outdir}/#{id}_B*"].length > 0 119 | end 120 | 121 | def processed_files_exist? 122 | Dir["#{@outdir}/#{id}*projected*"].length > 0 123 | end 124 | 125 | def download 126 | return if zip_exists? 127 | return if band_files_exist? 128 | 129 | puts "== downloading #{gsproduct}/#{path}/#{row}/#{id} to #{@outdir}/#{id}.tar.bz" 130 | `gsutil cp gs://earthengine-public/landsat/#{gsproduct}/#{path}/#{row}/#{id}.tar.bz #{@outdir}` unless File.exists?("#{@outdir}/#{id}.tar.bz") 131 | end 132 | 133 | def unzip 134 | `cd #{@outdir} && tar --transform 's/^.*_/#{id}_/g' -xzvf #{id}.tar.bz` 135 | end 136 | 137 | def warp(polygon_shp) 138 | return unless zip_exists? 139 | 140 | if !band_files_exist? 141 | unzip 142 | end 143 | 144 | if !processed_files_exist? 145 | satellite.vegetation.each do |band| 146 | # if it's landsat 8, convert to 8 bit 147 | if version.to_i > 7 148 | puts "== Landsat 8, converting to 8bit" 149 | `gdal_translate -of "GTiff" -co "COMPRESS=LZW" -scale 0 65535 0 255 -ot Byte #{@outdir}/#{band_file_pattern}_#{band}.TIF #{@outdir}/#{band_file_pattern}_#{band}_tmp.TIF && \ 150 | rm #{@outdir}/#{band_file_pattern}_#{band}.TIF && mv #{@outdir}/#{band_file_pattern}_#{band}_tmp.TIF #{@outdir}/#{band_file_pattern}_#{band}.TIF` 151 | end 152 | puts "gdalwarp -t_srs \"EPSG:3857\" #{@outdir}/#{band_file_pattern}_#{band}.TIF #{@outdir}/#{band_file_pattern}_#{band}-projected.tif" 153 | `gdalwarp -t_srs "EPSG:3857" -cutline #{polygon_shp} -crop_to_cutline #{@outdir}/#{band_file_pattern}_#{band}.TIF #{@outdir}/#{band_file_pattern}_#{band}-projected.tif` 154 | end 155 | puts "gdal_merge.py -separate #{@outdir}/#{band_file_pattern}_{#{satellite.vegetation.join(",")}}-projected.tif -o #{@outdir}/#{band_file_pattern}_RGB-projected.tif" 156 | `gdal_merge.py -separate #{@outdir}/#{band_file_pattern}_{#{satellite.vegetation.join(",")}}-projected.tif -o #{@outdir}/#{band_file_pattern}_RGB-projected.tif` #&& \ 157 | # convert -channel B -gamma 0.925 -channel R -gamma 1.03 -channel RGB -sigmoidal-contrast 50x16% #{@outdir}/#{id}_RGB-projected.tif #{@outdir}/#{id}_RGB-projected-corrected.tif && \ 158 | # convert -depth 8 #{@outdir}/#{id}_RGB-projected-corrected.tif #{@outdir}/#{id}_RGB-projected-corrected-8bit.tif && \ 159 | # listgeo -tfw #{@outdir}/#{id}_#{satellite.vegetation[0]}-projected.tif && \ 160 | # mv #{@outdir}/#{id}_#{satellite.vegetation[0]}-projected.tfw #{@outdir}/#{id}_RGB-projected-corrected-8bit.tfw && \ 161 | # gdal_edit.py -a_srs EPSG:3857 #{@outdir}/#{id}_RGB-projected-corrected-8bit.tif && \ 162 | # gdal_translate -a_nodata 0 #{@outdir}/#{id}_RGB-projected-corrected-8bit.tif #{@outdir}/#{id}_RGB-projected-corrected-8bit-nodata.tif` 163 | end 164 | end 165 | 166 | def tfw 167 | `listgeo -tfw #{@outdir}/#{id}_#{satellite.vegetation[0]}-projected.tif && \ 168 | mv #{@outdir}/#{id}_#{satellite.vegetation[0]}-projected.tfw #{@outdir}/#{id}_RGB-projected.tfw` 169 | end 170 | end 171 | 172 | class Processor 173 | # 6.4531 174 | # 3.3958 175 | # NWSE 176 | # BB = [36.42311, -115.63218, 35.90424, -114.66538] 177 | BB = [6.7, 3, 6.4, 3.7] 178 | 179 | attr_reader :year, :outdir, :satellite, :scenes 180 | 181 | def initialize(year, outdir) 182 | @year = year 183 | @outdir = File.expand_path outdir 184 | @satellite = Satellite.find_by_year(year) 185 | FileUtils.mkdir_p(outdir) unless File.directory?(outdir) 186 | end 187 | 188 | def scenes 189 | @scenes ||= get_scenes 190 | end 191 | 192 | def same_size 193 | tifs = Dir["#{@outdir}/*_RGB-projected.tif"] 194 | puts tifs[0] 195 | size = `gdalinfo #{tifs[0]}`.split("\n")[3] 196 | 197 | wh = size.scan(/[\d]+/) 198 | puts wh 199 | tifs.each do |tif| 200 | `rm #{tif}.resized.tif` 201 | `gdalwarp -ts #{wh[0]} #{wh[1]} #{tif} #{tif}.resized.tif` 202 | end 203 | end 204 | 205 | def median 206 | `convert #{@outdir}/*_RGB-projected.tif -evaluate-sequence median #{@outdir}/median-out.tif` 207 | end 208 | 209 | private 210 | 211 | def get_scenes 212 | url = "http://earthexplorer.usgs.gov/EE/InventoryStream/latlong?north=#{BB[0]}&south=#{BB[2]}4&east=#{BB[3]}8&west=#{BB[1]}&sensor=#{@satellite.metadata_sensor_id}&start_date=#{@year}-01-01&end_date=#{@year}-12-31" 213 | puts "trying #{url}" 214 | scenes = Nokogiri::XML(Net::HTTP.get(URI.parse(url))).css('metaData').select {|n| n.children.css('cloudCoverFull').text.to_i < 10 }.map {|n| n.children.css('sceneID').text } 215 | scenes.map!{|q| Scene.new(q, @outdir) } 216 | puts "== got #{scenes.length} scenes" 217 | scenes 218 | end 219 | end 220 | 221 | if __FILE__ == $0 222 | start_year = ARGV[0].to_i 223 | end_year = ARGV[1].to_i 224 | outdir = ARGV[2] 225 | 226 | if ARGV.length < 3 || ARGV.length > 3 227 | puts <<-EOD 228 | 229 | usage: ruby landsat.rb start_year end_year outdir 230 | 231 | example: ruby landsat.rb 1972 2014 /Volumes/Gilese581c/vegas/ 232 | 233 | oh it wants you to have a POLYGON.shp in yr ../data/POLYGON.shp too thx 234 | EOD 235 | 236 | exit 1 237 | end 238 | 239 | 240 | puts "== doin #{start_year} up to #{end_year} in #{outdir}" 241 | (start_year).upto(end_year).each do |year| 242 | puts "== tryin #{year}" 243 | landsat_year = Processor.new(year, File.join(File.expand_path(outdir), year.to_s)) 244 | scenes = landsat_year.scenes 245 | 246 | q = [] 247 | scenes.map {|s| q << s } 248 | m = Mutex.new 249 | threads = (1..8).map do 250 | Thread.new do 251 | loop do 252 | scene = nil 253 | exit = false 254 | m.synchronize do 255 | if q.empty? 256 | exit = true 257 | else 258 | scene = q.shift 259 | end 260 | end 261 | Thread.exit if exit 262 | scene.download 263 | scene.warp("data/NIR-24_outline.shp") 264 | end 265 | end 266 | end 267 | 268 | threads.map(&:join) 269 | end 270 | end 271 | -------------------------------------------------------------------------------- /scene.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from glob import glob 4 | import os 5 | 6 | from invoke import run 7 | from invoke.exceptions import Failure 8 | import numpy as np 9 | import rasterio 10 | from skimage import exposure, img_as_ubyte 11 | 12 | from satellite import Satellite 13 | 14 | 15 | class SceneID(str): 16 | """ 17 | A Landsat scene identifier like: LT41910561988052AAA03. 18 | 19 | Naming convention is defined at: 20 | http://landsat.usgs.gov/naming_conventions_scene_identifiers.php 21 | """ 22 | @property 23 | def sensor(self): 24 | return self[1] 25 | 26 | @property 27 | def version(self): 28 | return int(self[2]) 29 | 30 | @property 31 | def path(self): 32 | return self[3:6] 33 | 34 | @property 35 | def row(self): 36 | return self[6:9] 37 | 38 | @property 39 | def year(self): 40 | return self[9:13] 41 | 42 | @property 43 | def day(self): 44 | return self[13:16] 45 | 46 | @property 47 | def ground_station_id(self): 48 | return self[16:19] 49 | 50 | @property 51 | def archive_version(self): 52 | return self[19:21] 53 | 54 | @property 55 | def google_id(self): 56 | if self.version > 4: 57 | return 'L%s' % self.version 58 | else: 59 | return 'L%s%i' % (self.sensor, self.version) 60 | 61 | 62 | class Scene(object): 63 | """ 64 | Encasulates the processing for a single scene. 65 | """ 66 | def __init__(self, scene_id, output_dir, levels, cutline): 67 | self.scene_id = SceneID(scene_id) 68 | self.output_dir = output_dir 69 | self.levels = levels 70 | self.cutline = cutline 71 | 72 | self.satellite = Satellite(self.scene_id.version) 73 | 74 | @property 75 | def bands(self): 76 | long_name = '{base_path}_B10.TIF'.format(base_path=self.base_path) 77 | 78 | if os.path.exists(long_name): 79 | return ['B%i0' % b for b in self.satellite.natural_color_bands] 80 | else: 81 | return ['B%i' % b for b in self.satellite.natural_color_bands] 82 | 83 | @property 84 | def base_path(self): 85 | return '{output_dir}/{scene_id}'.format( 86 | output_dir=self.output_dir, 87 | scene_id=self.scene_id 88 | ) 89 | 90 | @property 91 | def zip_exists(self): 92 | tar_path = os.path.join(self.output_dir, '%s.tar.bz' % self.scene_id) 93 | 94 | return os.path.exists(tar_path) 95 | 96 | @property 97 | def band_files_exist(self): 98 | band_files = glob('%s/*_B*' % self.output_dir) 99 | 100 | return len(band_files) >= len(self.bands) 101 | 102 | @property 103 | def projected_files_exist(self): 104 | projected_files = glob('%s/*-projected.tif' % self.output_dir) 105 | 106 | return len(projected_files) >= len(self.bands) 107 | 108 | @property 109 | def merged_file_exists(self): 110 | merged_file = '%s/merged.tif' % self.output_dir 111 | 112 | return os.path.exists(merged_file) 113 | 114 | @property 115 | def color_corrected_file_exists(self): 116 | color_corrected_file = '%s/color_corrected.tif' % self.output_dir 117 | 118 | return os.path.exists(color_corrected_file) 119 | 120 | @property 121 | def crop_file_exists(self): 122 | crop_file = '%s/crop.tif' % self.output_dir 123 | 124 | return os.path.exists(crop_file) 125 | 126 | def download(self): 127 | """ 128 | Download this landsat scene. 129 | """ 130 | if self.zip_exists: 131 | return 132 | 133 | if not os.path.exists(self.output_dir): 134 | os.makedirs(self.output_dir) 135 | 136 | url = 'gs://earthengine-public/landsat/{id.google_id}/{id.path}/{id.row}/{id}.tar.bz'.format(id=self.scene_id) 137 | 138 | cmd = 'gsutil cp %s %s' % (url, self.output_dir) 139 | 140 | print(cmd) 141 | run(cmd) 142 | 143 | def unzip(self): 144 | """ 145 | Unzip this landsat scene after download. 146 | """ 147 | if not self.zip_exists: 148 | raise IOError('Archive does not exist!') 149 | 150 | if self.band_files_exist: 151 | return 152 | 153 | cmd = 'cd %s && tar --transform \'s/^.*_/%s_/g\' -xzvf %s.tar.bz' % ( 154 | self.output_dir, self.scene_id, self.scene_id 155 | ) 156 | 157 | print(cmd) 158 | run(cmd) 159 | 160 | def _convert_to_8bit(self, band): 161 | cmd = 'gdal_translate -of "GTiff" -co "COMPRESS=LZW" -scale 0 65535 0 255 -ot Byte {base_path}_{band}.TIF {base_path}_{band}_tmp.TIF'.format( 162 | base_path=self.base_path, 163 | band=band 164 | ) 165 | 166 | print(cmd) 167 | run(cmd) 168 | 169 | cmd = 'rm {base_path}_{band}.TIF && mv {base_path}_{band}_tmp.TIF {base_path}_{band}.TIF'.format( 170 | base_path=self.base_path, 171 | band=band 172 | ) 173 | 174 | print(cmd) 175 | run(cmd) 176 | 177 | def project_bands(self): 178 | if not self.band_files_exist: 179 | raise IOError('Band files do not exist!') 180 | 181 | if self.projected_files_exist: 182 | return 183 | 184 | for band in self.bands: 185 | if self.satellite.version > 7: 186 | self._convert_to_8bit(band) 187 | 188 | base_file_path = '{base_path}_{band}'.format( 189 | base_path=self.base_path, 190 | band=band, 191 | ) 192 | cmd = 'gdalwarp -t_srs "EPSG:3857" {base_file_path}.TIF {base_file_path}-projected.tif'.format( 193 | base_file_path=base_file_path, 194 | cutline=self.cutline 195 | ) 196 | 197 | print(cmd) 198 | run(cmd) 199 | 200 | def merge_bands(self): 201 | if not self.projected_files_exist: 202 | raise IOError('Projected band files do not exist!') 203 | 204 | if self.merged_file_exists: 205 | return 206 | 207 | bands = ','.join(self.bands) 208 | 209 | cmd = 'rio stack {base_path}_{{{bands}}}-projected.tif -o {output_dir}/merged.tif'.format( 210 | base_path=self.base_path, 211 | bands=bands, 212 | output_dir=self.output_dir 213 | ) 214 | 215 | print(cmd) 216 | run(cmd) 217 | 218 | def crop(self): 219 | if not self.merged_file_exists: 220 | raise IOError('Merged file does not exist') 221 | 222 | if self.crop_file_exists: 223 | return 224 | 225 | cmd = 'gdalwarp -cutline {cutline} -crop_to_cutline {output_dir}/merged.tif {output_dir}/crop.tif'.format( 226 | cutline=self.cutline, 227 | output_dir=self.output_dir 228 | ) 229 | 230 | print(cmd) 231 | run(cmd) 232 | 233 | def hist_match(self, source, template): 234 | """ 235 | Adjust the pixel values of a grayscale image such that its histogram 236 | matches that of a target image 237 | 238 | Arguments: 239 | ----------- 240 | source: np.ndarray 241 | Image to transform; the histogram is computed over the flattened 242 | array 243 | template: np.ndarray 244 | Template image; can have different dimensions to source 245 | Returns: 246 | ----------- 247 | matched: np.ndarray 248 | The transformed output image 249 | """ 250 | 251 | oldshape = source.shape 252 | source = source.ravel() 253 | template = template.ravel() 254 | 255 | # get the set of unique pixel values and their corresponding indices and 256 | # counts 257 | s_values, bin_idx, s_counts = np.unique(source, return_inverse=True, 258 | return_counts=True) 259 | t_values, t_counts = np.unique(template, return_counts=True) 260 | 261 | s_counts[0] = 0 262 | print(t_counts) 263 | 264 | # take the cumsum of the counts and normalize by the number of pixels to 265 | # get the empirical cumulative distribution functions for the source and 266 | # template images (maps pixel value --> quantile) 267 | s_quantiles = np.cumsum(s_counts) 268 | s_quantiles = (255 * s_quantiles / s_quantiles[-1]).astype(np.ubyte) #normalize 269 | t_quantiles = np.cumsum(t_counts).astype(np.float64) 270 | t_quantiles = (255 * t_quantiles / t_quantiles[-1]).astype(np.ubyte) #normalize 271 | 272 | # source_cdf, source_bin_centers = exposure.cumulative_distribution(source) 273 | # template_cdf, template_bin_centers = exposure.cumulative_distribution(template) 274 | # out = np.interp(image.flat, bin_centers, cdf) 275 | # out = np.interp(source.flat, template_bin_centers, template_cdf) 276 | 277 | # # interpolate linearly to find the pixel values in the template image 278 | # # that correspond most closely to the quantiles in the source image 279 | interp_t_values = np.interp(s_quantiles, t_quantiles, t_values).astype(np.ubyte) 280 | 281 | print interp_t_values 282 | 283 | return interp_t_values[bin_idx].reshape(oldshape) 284 | 285 | # return out.reshape(oldshape) 286 | 287 | def color_correct(self): 288 | if not self.crop_file_exists: 289 | raise IOError('Crop file does not exist') 290 | 291 | if self.color_corrected_file_exists: 292 | return 293 | 294 | print('Equalizing histograms') 295 | 296 | with rasterio.drivers(): 297 | with rasterio.open('correct.tif') as f: 298 | template = np.rollaxis(np.rollaxis(f.read(), 1), 2, 1) 299 | 300 | with rasterio.open('%s/crop.tif' % self.output_dir) as f: 301 | data = f.read() 302 | profile = f.profile 303 | 304 | rolled = np.rollaxis(np.rollaxis(data, 1), 2, 1) 305 | 306 | # new_bands = [] 307 | # 308 | # for b, band in enumerate(rolled.T): 309 | # # R 310 | # if b == 0: 311 | # in_range = (28, 130) 312 | # # G 313 | # elif b == 1: 314 | # in_range = (41, 105) 315 | # # B 316 | # elif b == 2: 317 | # in_range = (58, 120) 318 | # 319 | # new_bands.append( 320 | # exposure.rescale_intensity(band, in_range=in_range) 321 | # ) 322 | # 323 | # rescaled = np.array(new_bands).T 324 | 325 | # rescaled = img_as_ubyte(self.hist_match(rolled, template)) 326 | 327 | # rolled = exposure.adjust_gamma(rolled, 1.25) 328 | rescaled = exposure.rescale_intensity(rolled, in_range=(1, 255)) 329 | rescaled = img_as_ubyte(exposure.equalize_adapthist(rolled)) 330 | # rescaled = correct(rolled) 331 | # rescaled = img_as_ubyte(exposure.adjust_sigmoid(rolled, 0.25, 10)) 332 | 333 | unrolled = np.rollaxis(rescaled, 2) 334 | 335 | with rasterio.open('%s/color_corrected.tif' % self.output_dir, 'w', **profile) as f: 336 | f.write(unrolled) 337 | 338 | def process(self): 339 | print('{s.year} {s.day} {s.path} {s.row}'.format(s=self.scene_id)) 340 | 341 | try: 342 | self.download() 343 | self.unzip() 344 | self.project_bands() 345 | self.merge_bands() 346 | self.crop() 347 | self.color_correct() 348 | except Failure as e: 349 | print(str(e)) 350 | --------------------------------------------------------------------------------