├── requirements.txt ├── dropindexes.sh ├── .gitignore ├── sql ├── addlabelmapfk.sql ├── createlabelmap.sql └── createfieldsview.sql ├── sites ├── bavaria.geojson ├── garching.geojson ├── munich.geojson ├── munich.site ├── bavariasmall.geojson ├── bavariasmall.site ├── garching.site ├── bavaria.site └── geojson2aoi.py ├── cfg ├── bands.json.ol ├── bandsvrtsequence.json └── L2A_GIPP.xml ├── download.sh ├── pipeline.sh ├── sen2cordeletefails.sh ├── setup.sh ├── addEvalField.sh ├── query.sh ├── sen2cor.sh ├── delGranules.py ├── crop.sh ├── geojson2aoi.py ├── crop_via_vrt.sh ├── querySCL.py ├── uploadraster.sh ├── readme.md ├── uploadmeta.py ├── grid.py ├── addvrttoproduct.py ├── status.py ├── downloadtiles.py └── dbviz.ipynb /requirements.txt: -------------------------------------------------------------------------------- 1 | sentinelhub 2 | pyproj 3 | configobj 4 | -------------------------------------------------------------------------------- /dropindexes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for i in {1..136}; do 4 | echo $(date +"%d.%m.%Y %H:%M:%S") idx $i 5 | psql -d fields -c "drop index if exists bavaria_st_convexhull_idx$i"; 6 | done 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ProductDownload 2 | Sen2Cor-2.4.0-Linux64 3 | 4 | munichdemo 5 | 6 | conda 7 | 8 | data 9 | 10 | *.log 11 | 12 | *.swp 13 | 14 | gdal-2.2.1 15 | 16 | .idea 17 | 18 | *.pyc 19 | 20 | .ipy* 21 | -------------------------------------------------------------------------------- /sql/addlabelmapfk.sql: -------------------------------------------------------------------------------- 1 | /*select ROW_NUMBER() OVER () as labelid, label from labelmap where not label='' order by labelid 2 | */ 3 | update labelmap c 4 | set labelid = c2.labelid 5 | from (select ROW_NUMBER() OVER () as labelid, label from labelmap where not label='' group by label order by labelid 6 | ) c2 7 | where c2.label = c.label; -------------------------------------------------------------------------------- /sql/createlabelmap.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE public.labelmap 2 | ( 3 | name text, 4 | label text, 5 | id integer NOT NULL DEFAULT nextval('labelmap_id_seq'::regclass), 6 | doystart integer, 7 | doyend integer, 8 | labelid integer, 9 | CONSTRAINT id PRIMARY KEY (id) 10 | ) 11 | WITH ( 12 | OIDS=FALSE 13 | ); 14 | ALTER TABLE public.labelmap 15 | OWNER TO russwurm; 16 | -------------------------------------------------------------------------------- /sites/bavaria.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "name": "bavaria", 4 | "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::32632" } }, 5 | "features": [ 6 | { "type": "Feature", "properties": { "id": 2 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 737000.0, 5387000.0 ], [ 737000.0, 5345000.0 ], [ 635000.0, 5345000.0 ], [ 635000.0, 5387000.0 ], [ 737000.0, 5387000.0 ] ] ] } } 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /sql/createfieldsview.sql: -------------------------------------------------------------------------------- 1 | SELECT m.labelid, 2 | m.label, 3 | m.doystart, 4 | m.doyend, 5 | o.geom 6 | FROM labelmap m, 7 | osm o 8 | WHERE o.fclass::text = m.name AND NOT m.label = ''::text 9 | UNION ALL 10 | SELECT m.labelid, 11 | m.label, 12 | m.doystart, 13 | m.doyend, 14 | f.geom 15 | FROM labelmap m, 16 | fieldsstmelf f 17 | WHERE f.nutzung::text = m.name AND NOT m.label = ''::text; 18 | -------------------------------------------------------------------------------- /cfg/bands.json.ol: -------------------------------------------------------------------------------- 1 | { 2 | "L1C": { 3 | "10m": ["B4","B3","B2","B8"], 4 | "20m": ["B5","B6","B7","B8A","B11","B12"], 5 | "60m": ["B1","B9","B10"] 6 | }, 7 | "L2A": { 8 | "10m": ["B2","B3","B4","B8","AOT","WVP"], 9 | "20m": ["B4","B3","B2","B5","B6","B7","B11","B12","B8A","AOT","CLD","SCL","SNW","WVP"], 10 | "60m": ["B1","B2","B3","B4","B5","B6","B7","B9","B11","B12","B8A","AOT","CLD","SCL","SNW","WVP"] 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | # read configs 5 | . $1 6 | 7 | numdownload=$(cat $path/download.todo | wc -l) 8 | 9 | echo "downloading "$numdownload" products" 10 | 11 | cat $path/download.todo | while read product 12 | do 13 | 14 | if [ ! -d $path/$product.SAFE ]; then 15 | echo "downloading product: "$product 16 | sentinelhub.aws --product $product -f $path -e -t 17 | else 18 | echo "skipping product: "$product 19 | fi 20 | done 21 | -------------------------------------------------------------------------------- /cfg/bandsvrtsequence.json: -------------------------------------------------------------------------------- 1 | { 2 | "L1C": { 3 | "10m": ["B04","B03","B02","B08"], 4 | "20m": ["B05","B06","B07","B8A","B11","B12"], 5 | "60m": ["B01","B09","B10"] 6 | }, 7 | "L2A": { 8 | "10m": ["B04","B03","B02","B08","AOT","WVP"], 9 | "20m": ["B04","B03","B02","B05","B06","B07","B11","B12","B8A","AOT","CLD","SCL","SNW","WVP"], 10 | "60m": ["B01","B02","B03","B04","B05","B06","B07","B09","B11","B12","B8A","AOT","CLD","SCL","SNW","WVP"] 11 | } 12 | } 13 | 14 | -------------------------------------------------------------------------------- /pipeline.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cfg_path=$1 4 | 5 | sh query.sh $cfg_path 6 | sh download.sh $cfg_path 7 | 8 | sh warp.sh $cfg_path L1C 9 | 10 | 11 | # create a 3.84 km tile grid 60m * 2 ** 6 12 | python grid.py sites/bavaria.site sites/bavaria.geojson gridtraintest --tilesize 3840 --margin 0 13 | 14 | # add eval t/f field to tile grid 15 | bash addEvalField.sh sites/bavaria.site gridtraintest eval 0.8 "" 16 | 17 | # add train/valid field for fold 0 if eval is not true 18 | for fold in 0 1 2 3 4 5 6 7 8 9 19 | do 20 | bash addEvalField.sh sites/bavaria.site gridtraintest train$fold 0.75 "where eval = 'f'" 21 | done 22 | 23 | # create grids for sample tiles: 24 | for tilesize in 60 120 240 480 25 | do 26 | python grid.py sites/bavaria.site sites/bavaria.geojson tiles$tilesize --tilesize $tilesize 27 | done 28 | -------------------------------------------------------------------------------- /sites/garching.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": {}, 7 | "geometry": { 8 | "type": "Polygon", 9 | "coordinates": [ 10 | [ 11 | [ 12 | 11.648340225219727, 13 | 48.25874140815746 14 | ], 15 | [ 16 | 11.681900024414062, 17 | 48.25874140815746 18 | ], 19 | [ 20 | 11.681900024414062, 21 | 48.27611002352571 22 | ], 23 | [ 24 | 11.648340225219727, 25 | 48.27611002352571 26 | ], 27 | [ 28 | 11.648340225219727, 29 | 48.25874140815746 30 | ] 31 | ] 32 | ] 33 | } 34 | } 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /sites/munich.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": {}, 7 | "geometry": { 8 | "type": "Polygon", 9 | "coordinates": [ 10 | [ 11 | [ 12 | 11.425437927246094, 13 | 48.06041366222354 14 | ], 15 | [ 16 | 11.700782775878904, 17 | 48.06041366222354 18 | ], 19 | [ 20 | 11.700782775878904, 21 | 48.22124165162929 22 | ], 23 | [ 24 | 11.425437927246094, 25 | 48.22124165162929 26 | ], 27 | [ 28 | 11.425437927246094, 29 | 48.06041366222354 30 | ] 31 | ] 32 | ] 33 | } 34 | } 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /sites/munich.site: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # obtained via 4 | # python preprocessing/download/geojson2aoi.py data/bavaria/shp/aoi.geojson 5 | #area="11.5314854866,48.289743953 12.1866587382,48.2897751466 12.1866300812,48.558659954 11.5314562298,48.5586286318 11.5314854866,48.289743953" 6 | 7 | project="$HOME/projects/preprocessing" 8 | 9 | configdir=$(dirname "$0") # this directory 10 | #area=$(python $configdir/geojson2aoi.py $project/sites/munich.geojson) 11 | area="11.524285797,48.1845387572 11.5249725608,48.1845387902 11.5249724871,48.1852256095 11.5242857233,48.1852255765 11.524285797,48.1845387572" 12 | 13 | queryfile=munich.query 14 | # relevant granules (others will be deleted to save processing time) 15 | granules="32UPU 32UQU 33UTP 33UUP" 16 | 17 | maxcloud=80 18 | startdate="2017-04-01" 19 | enddate="2017-06-01" 20 | 21 | # path of data 22 | path=$project/munichdemo 23 | 24 | # warp 25 | cutline=$project/sites/munich.geojson 26 | srs=EPSG_32632 27 | -------------------------------------------------------------------------------- /sites/bavariasmall.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "name": "bavariasmall", 4 | "crs": { 5 | "type": "name", 6 | "properties": { 7 | "name": "urn:ogc:def:crs:EPSG::32632" 8 | } 9 | }, 10 | "features": [ 11 | { 12 | "type": "Feature", 13 | "properties": { 14 | "id": 2 15 | }, 16 | "geometry": { 17 | "type": "Polygon", 18 | "coordinates": [ 19 | [ 20 | [ 21 | 737000, 22 | 5387000 23 | ], 24 | [ 25 | 737000, 26 | 5382000 27 | ], 28 | [ 29 | 730000, 30 | 5382000 31 | ], 32 | [ 33 | 730000, 34 | 5387000 35 | ], 36 | [ 37 | 737000, 38 | 5387000 39 | ] 40 | ] 41 | ] 42 | } 43 | } 44 | ] 45 | } 46 | -------------------------------------------------------------------------------- /sites/bavariasmall.site: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # obtained via 4 | # python preprocessing/download/geojson2aoi.py data/bavaria/shp/aoi.geojson 5 | area="12.2142368059,48.5911876629 12.2113888618,48.5462769878 12.1166861433,48.5488821657 12.1194504397,48.593796933 12.1194504397,48.593796933" 6 | 7 | project="$HOME/projects/preprocessing" 8 | 9 | granules="33UTP 33UUP" 10 | 11 | maxcloud=50 12 | startdate="2016-01-01" 13 | enddate="2017-09-11" 14 | queryfile=bavariasmall.query 15 | #L1C=$project/data/bavaria/data/L1Czip 16 | #L1C=$project/data/bavaria/data/L1C 17 | path=$project/data/bavaria/data/L1C 18 | 19 | # warp/home/russwurm/projects/preprocessing/uploadraster.sh 20 | tifpath=$project/data/bavaria/small 21 | cutline=$project/sites/bavariasmall.geojson 22 | srs=EPSG_32632 23 | 24 | # upload 25 | dbhost="dbserver" 26 | dbase="fields" 27 | dbtable="bavariasmall" 28 | dbuser=$POSTGIS_USER 29 | dbpassword=$POSTGIS_PASSWORD 30 | 31 | # tilegrid 32 | trainvalidevaltable="tiles" 33 | footpringtable="footprints" 34 | -------------------------------------------------------------------------------- /sites/garching.site: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # obtained via 4 | # python preprocessing/download/geojson2aoi.py data/bavaria/shp/aoi.geojson 5 | #area="11.5314854866,48.289743953 12.1866587382,48.2897751466 12.1866300812,48.558659954 11.5314562298,48.5586286318 11.5314854866,48.289743953" 6 | 7 | project="$HOME/projects/preprocessing" 8 | 9 | configdir=$(dirname "$0") # this directory 10 | #area=$(python $configdir/geojson2aoi.py $project/sites/munich.geojson) 11 | area="11.5314854866,48.289743953 12.1866587382,48.2897751466 12.1866300812,48.558659954 11.5314562298,48.5586286318 11.5314854866,48.289743953" 12 | 13 | queryfile=garching.query 14 | # relevant granules (others will be deleted to save processing time) 15 | granules="32UPU 32UQU 33UTP 33UUP" 16 | 17 | maxcloud=80 18 | startdate="2017-07-01" 19 | enddate="2017-09-13" 20 | 21 | # path of data 22 | path=$project/data/bavaria/data/L1C 23 | 24 | # warp 25 | tifpath=$project/data/bavaria/data/garching 26 | cutline=$project/sites/garching.geojson 27 | srs=EPSG_32632 28 | -------------------------------------------------------------------------------- /sen2cordeletefails.sh: -------------------------------------------------------------------------------- 1 | #!bin/bash 2 | 3 | # read configs 4 | . $1 5 | 6 | sucess="Progress[%]: 100.00 : Application terminated successfully." 7 | 8 | # delete all folders, which have not 'Progress[%]: 100.00 : Application terminated successfully.' at last log line 9 | # for every line in the results file 10 | cat $path/$queryfile | while read product 11 | do 12 | # skip if already exists 13 | L2Aproductname=$(echo $product | sed 's/MSIL1C/MSIL2A/g' | sed 's/OPER/USER/g' ) 14 | echo "checking: $L2Aproductname" 15 | 16 | # skip if no logfile exists 17 | if [ ! -f "$path/$L2Aproductname.sen2cor" ] 18 | then 19 | continue 20 | fi 21 | 22 | lastlog=$(tail -n 1 $path/$L2Aproductname.sen2cor) 23 | #echo $lastlog 24 | # if last logging entry does not indicate success 25 | if [ "$lastlog" != "$sucess" ]; then 26 | # delete folder if still exists 27 | echo "detected failed $L2Aproductname.SAFE" 28 | echo $lastlog 29 | rm -ifr $path/$L2Aproductname.SAFE 30 | fi 31 | done 32 | -------------------------------------------------------------------------------- /sites/bavaria.site: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # obtained via 4 | # python preprocessing/download/geojson2aoi.py data/bavaria/shp/aoi.geojson 5 | area="12.2142368059,48.5911876629 12.1905302768,48.2139231883 10.81837963,48.2438316204 10.8319097901,48.6214929716 12.2142368059,48.5911876629" 6 | 7 | project="$HOME/projects/preprocessing" 8 | 9 | granules="32UPU 32UQU 33UTP 33UUP" 10 | 11 | maxcloud=80 12 | startdate="2016-01-01" 13 | enddate="2017-10-09" 14 | queryfile=bavaria.query 15 | #L1C=$project/data/bavaria/data/L1Czip 16 | #L1C=$project/data/bavaria/data/L1C 17 | path=$project/data/bavaria/SAFE 18 | 19 | # sen2cor target directory 20 | export L2ATARGET=$path 21 | 22 | # warp 23 | tifpath=$project/data/bavaria/tif 24 | cutline=$project/sites/bavaria.geojson 25 | srs=EPSG_32632 26 | 27 | # upload 28 | dbhost="dbserver" 29 | dbase="fields" 30 | dbtable="bavaria" 31 | dbuser=$POSTGIS_USER 32 | dbpassword=$POSTGIS_PASSWORD 33 | 34 | # tilegrid 35 | trainvalidevaltable="tiles" 36 | footpringtable="footprints" 37 | 38 | tilesize=120 39 | 40 | tiletable=tiles480 41 | tilefolder=$project/data/bavaria/tiles 42 | -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- 1 | mkdir ProductDownload 2 | cd ProductDownload 3 | 4 | wget https://github.com/kraftek/awsdownload/releases/download/1.7/ProductDownload-1.7.zip 5 | unzip ProductDownload-1.7.zip 6 | rm ProductDownload-1.7.zip 7 | cd .. 8 | 9 | #wget https://github.com/kraftek/awsdownload/releases/download/1.5.1/ProductDownload-1.5.1.zip 10 | #unzip ProductDownload-1.5.1.zip 11 | #rm ProductDownload-1.5.1.zip 12 | 13 | # sen2cor 14 | wget http://step.esa.int/thirdparties/sen2cor/2.4.0/Sen2Cor-2.4.0-Linux64.run 15 | bash Sen2Cor-2.4.0-Linux64.run 16 | rm Sen2Cor-2.4.0-Linux64.run 17 | 18 | # create anaconda environment with latest GDAL 2.2.1 19 | conda create -y -c conda-forge bkreider --prefix ./conda python=2.7 pip gdal postgis 20 | 21 | # pip packages 22 | source activate conda 23 | 24 | pip install psycopg2 25 | pip install pandas 26 | pip install shapely 27 | pip install numpy 28 | 29 | source deactivate 30 | 31 | if [[ $EUID -ne 0 ]]; then 32 | echo "Skipping installation of postgis (required for upload.sh), no root permissions" 33 | exit 1 34 | fi 35 | # database tools (raster2pgsql) 36 | sudo apt-get install postgis -y 37 | -------------------------------------------------------------------------------- /addEvalField.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] || [ -z "$4" ]; 4 | then 5 | echo Adds a boolean column to a postgresql table with a given split ratio 6 | echo 7 | echo "Usage: sh $0 site table columnname splitratio sqlwhere" 8 | echo "" 9 | echo "e.g. sh $0 sites/bavaria.site tiles train0 0.8 where eval=0 " 10 | echo "adds column 'train0' to 'tiles' table. Fills it with true/false" 11 | echo "in a t/f ratio of 0.8 where the eval field is false" 12 | exit 1 13 | fi 14 | 15 | 16 | # site config files (db connection attributes used) 17 | site=$1 18 | # table of tile geometry file 19 | table=$2 20 | columnname=$3 # eg eval, valid0, valid1 21 | split=$4 # ratio between true and false 22 | sqlwhere=$5 # e.g. 'where eval=0' 23 | 24 | if [ -z "$site" ] 25 | then 26 | echo "No config file provided! Aborting" 27 | exit 1 28 | fi 29 | 30 | # read configs 31 | . $site 32 | 33 | # database connection 34 | psql="psql -d $dbase --username=russwurm --host=$dbhost" 35 | 36 | # add column 37 | $psql -c "alter table $2 add column if not exists $columnname boolean;" 38 | 39 | # insert '0' or '1' in split ratio 40 | $psql -c "update $table set $columnname=(random() > $split) $sqlwhere;" 41 | -------------------------------------------------------------------------------- /query.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # insert config file as first argument 4 | # 5 | # $ sh downloader.sh bavaria.download 6 | 7 | # download product downloader if not exists 8 | if [ ! -f ProductDownload/ProductDownload.jar ]; then 9 | sh setup.sh 10 | fi 11 | 12 | # Assertions 13 | if [ -z "$1" ]; then 14 | echo "No config file provided! Aborting" 15 | exit 1 16 | fi 17 | if [ ! -f $1 ]; then 18 | echo "Config file $1 not found! Aborting" 19 | exit 1 20 | fi 21 | 22 | # read configs 23 | . $1 24 | 25 | echo $startdate 26 | 27 | # seems not to work for 2017 so query-> $queryfile 28 | java -jar ProductDownload/ProductDownload.jar $2 \ 29 | --mode RESUME \ 30 | --sensor S2 \ 31 | --aws \ 32 | -a $area \ 33 | --startdate $startdate \ 34 | --enddate $enddate \ 35 | --out "$path" \ 36 | --cloudpercentage $maxcloud \ 37 | --verbose \ 38 | --store AWS \ 39 | --q 40 | 41 | # rename $queryfile to queryfile name 42 | mv $path/results.txt $path/$queryfile 43 | 44 | #--unpacked 45 | # 46 | # --aws \ 47 | # --input $L1Cfolder \ 48 | 49 | 50 | #cat $L1C/$queryfile | while read product 51 | #do 52 | # echo $product # do something with $line here 53 | # #sentinelhub.aws --product $product -f $L1C -e -t 54 | #done 55 | 56 | #sentinelhub.aws --product S2A_OPER_PRD_MSIL1C_PDMC_20160908T083647_R022_V20160906T101022_20160906T101558 -f . -e 57 | -------------------------------------------------------------------------------- /sen2cor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Runs sen2cor on all products, which are located in the L1C folder (defined by site.cfg) 4 | # 5 | # usage sh sen2cor.sh site.cfg 6 | # 7 | 8 | # read configs 9 | . $1 10 | 11 | # flag -delete for deletion of previous step or -single 12 | flag=$2 13 | 14 | L2AProcess="Sen2Cor-2.4.0-Linux64/bin/L2A_Process" 15 | 16 | # default to project root config file if not set 17 | if [ -z "$L2A_GIPP_path" ]; then 18 | L2A_GIPP_path=$project/cfg/L2A_GIPP.xml 19 | fi 20 | 21 | # for every line in the results file shuf shuffles sequence such that multiple processes will process a different sequence 22 | cat $path/sen2cor.todo | shuf | while read product 23 | do 24 | 25 | L2Aproductname=$(echo $product | sed 's/MSIL1C/MSIL2A/g' | sed 's/OPER/USER/g' ) 26 | 27 | if [ "$flag" = "-delete" ]; then 28 | echo "deleting previous $L2Aproductname.SAFE" 29 | rm -r $path/$L2Aproductname.SAFE 30 | fi 31 | 32 | echo $(echo date "+%Y-%m-%d %H:%M:%S")" sen2cor: "$product" -> "$L2Aproductname 33 | #echo "$L2AProcess $path/$product.SAFE --GIP_L2A $L2A_GIPP_path > $path/$L2Aproductname.sen2cor" 34 | $L2AProcess $path/$product.SAFE --GIP_L2A $L2A_GIPP_path > $path/$L2Aproductname.sen2cor 35 | 36 | # stop loop after one image 37 | if [ "$flag" = "-single" ]; then 38 | break 39 | fi 40 | 41 | done 42 | 43 | # delete folders of unsuccessfull processes (but leave log files) 44 | #sh sen2cordeletefails.sh $1 45 | -------------------------------------------------------------------------------- /delGranules.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | from configobj import ConfigObj 3 | import re 4 | import shutil 5 | 6 | cfg_path=sys.argv[1] 7 | 8 | cfg = ConfigObj(cfg_path) 9 | 10 | # set dryrun flat 11 | dryrun=False 12 | if len(sys.argv) > 2: 13 | dryrun=sys.argv[2]=='-d' 14 | 15 | if dryrun: 16 | print "dryrun flag ('-d') active, no deletions" 17 | 18 | if "granules" not in cfg.keys(): 19 | print "no variable 'granules' in {} specified, nothing to delete...".format(cfg_path) 20 | sys.exit() 21 | 22 | relevantgranules=cfg["granules"].replace(" ","|") 23 | project=cfg["project"].replace('$HOME',os.environ["HOME"]) 24 | path=cfg["path"].replace('$project',project) 25 | 26 | regex=".*("+relevantgranules+").*" 27 | p = re.compile(regex) 28 | 29 | # some statistics 30 | nproducts=0 31 | ngranules=0 32 | ngrkept=0 33 | ngrdeleted=0 34 | 35 | for product in os.listdir(path): 36 | 37 | # skip files 38 | if os.path.isfile(os.path.join(path,product)): 39 | continue 40 | nproducts+=1 41 | 42 | if os.path.exists(os.path.join(path,product,"GRANULE")): 43 | granules = os.listdir(os.path.join(path,product,"GRANULE")) 44 | 45 | # for every granule in product 46 | for granule in granules: 47 | 48 | #skip if not folder 49 | if os.path.isfile(os.path.join(path,product,"GRANULE",granule)): 50 | continue 51 | 52 | ngranules+=1 53 | 54 | # if not matches required Granules 55 | if p.match(granule) is None: 56 | print "granule {} does not match regex {} -> deleting...".format(granule,regex) 57 | ngrdeleted+=1 58 | if not dryrun: shutil.rmtree(os.path.join(path,product,"GRANULE",granule)) 59 | else: 60 | ngrkept+=1 61 | 62 | print 63 | print "SUMMARY: {} products inspected, {} granules checked, {} deleted, {} kept".format(nproducts,ngranules,ngrdeleted,ngrkept) 64 | -------------------------------------------------------------------------------- /crop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # read configs 4 | . $1 5 | 6 | # either L1C or L2A 7 | level=$2 8 | 9 | if [ "$level" = "L1C" ]; then 10 | gdalS2prefix="SENTINEL2_L1C" 11 | todofile=cropL1C.todo 12 | elif [ "$level" = "L2A" ]; then 13 | gdalS2prefix="SENTINEL2_L2A" 14 | todofile=cropL2A.todo 15 | else 16 | echo "please specify level ('L1C' or 'L2A') as second argument" 17 | echo "e.g. \$sh warp.sh demo.cfg L1C" 18 | exit 0 19 | fi 20 | #statements 21 | 22 | # default output directory to path if not defined 23 | if [ -z "$tifpath" ]; then 24 | tifpath=$path 25 | fi 26 | 27 | # should be installed there by setup.sh 28 | gdalwarp="conda/bin/gdalwarp" 29 | 30 | # for every line in the results file $path/$queryfile 31 | while read p; 32 | do 33 | 34 | # select product name 35 | if [ "$level" = "L1C" ]; then 36 | product=$p 37 | elif [ "$level" = "L2A" ]; then 38 | # predict L2A product name 39 | product=$(echo $p | sed 's/MSIL1C/MSIL2A/g' | sed 's/OPER/USER/g' ) 40 | fi 41 | 42 | echo $product 43 | 44 | # find product's main xml file: 45 | # -> list directory filter by regex, return first match 46 | file=$(ls $path/$product.SAFE | grep -E -m 1 "(S2.*xml|MTD.*xml)") 47 | $gdalwarp -of GTiff -multi -wo NUM_THREADS=val/ALL_CPUS -crop_to_cutline -cutline $cutline "$gdalS2prefix:$path/$product.SAFE/$file:10m:$srs" "$tifpath/$product"_10m.tif 48 | $gdalwarp -of GTiff -multi -wo NUM_THREADS=val/ALL_CPUS -crop_to_cutline -cutline $cutline "$gdalS2prefix:$path/$product.SAFE/$file:20m:$srs" "$tifpath/$product"_20m.tif 49 | $gdalwarp -of GTiff -multi -wo NUM_THREADS=val/ALL_CPUS -crop_to_cutline -cutline $cutline "$gdalS2prefix:$path/$product.SAFE/$file:60m:$srs" "$tifpath/$product"_60m.tif 50 | echo $gdalwarp -of GTiff -crop_to_cutline -cutline $cutline "$gdalS2prefix:$path/$product.SAFE/$file:60m:$srs" "$tifpath/$product"_60m.tif 51 | 52 | done <$path/$todofile 53 | -------------------------------------------------------------------------------- /geojson2aoi.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | import pyproj 5 | import re 6 | import argparse 7 | 8 | """ 9 | projects polygon nodes of input geojson file to WGS84 (epsg:4326) and prints 10 | list of longitude and latitude. 11 | Only the first feature is considered. 12 | 13 | converts projected geojson to list format of https://github.com/kraftek/awsdownload 14 | """ 15 | 16 | def geojson2aoi(geojson_file,srs="epsg:4326"): 17 | with open(geojson_file,"r") as f: 18 | js = json.load(f) 19 | 20 | target_proj = pyproj.Proj(init=srs) # wgs84 lon lat 21 | source_proj = pyproj.Proj(init=getsrs(js)) # extract from geojson 22 | 23 | # Point list of the first feature 24 | pts = js["features"][0]["geometry"]["coordinates"][0] 25 | 26 | lines = [] 27 | for pt in pts: 28 | lon, lat = pyproj.transform(source_proj,target_proj,pt[0],pt[1]) 29 | lines.append("{},{}".format(lon,lat)) 30 | 31 | aoi = " ".join(lines) 32 | return aoi 33 | 34 | def getsrs(geojson): 35 | # regex to find crs 36 | regex = "EPSG:{1,2}[0-9]{4,5}" 37 | 38 | m = re.search(regex, str(geojson)) 39 | 40 | if m is None: # default to wgs84 41 | return "epsg:4236" 42 | epsg = m.group(0) 43 | epsg = epsg.replace("::",":").lower() 44 | return epsg 45 | 46 | if __name__ == "__main__": 47 | 48 | parser = argparse.ArgumentParser(description='converts geojson containing one polygon to a list of wgs84 (epsg:4236) coordinates. If multiple features are in the geojson only the first is considered') 49 | parser.add_argument('geojson', help='path to the input geojson file') 50 | parser.add_argument('--srs', default='epsg:4326', help="target srs coordinate system. Defaults to 'epsg:4326'") 51 | args = parser.parse_args() 52 | 53 | ## debug 54 | #args.geojson="data/shp/bavaria/aoi.geojson" 55 | #args.geojson="data/shp/ecuador/Ecuador_AOI.geojson" 56 | #args.aoi="download/bavaria.aoi" 57 | 58 | print(geojson2aoi(args.geojson,srs=args.srs)) 59 | -------------------------------------------------------------------------------- /sites/geojson2aoi.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | import pyproj 5 | import re 6 | import argparse 7 | 8 | """ 9 | projects polygon nodes of input geojson file to WGS84 (epsg:4326) and prints 10 | list of longitude and latitude. 11 | Only the first feature is considered. 12 | 13 | converts projected geojson to list format of https://github.com/kraftek/awsdownload 14 | """ 15 | 16 | def geojson2aoi(geojson_file,srs="epsg:4326"): 17 | with open(geojson_file,"r") as f: 18 | js = json.load(f) 19 | 20 | target_proj = pyproj.Proj(init=srs) # wgs84 lon lat 21 | source_proj = pyproj.Proj(init=getsrs(js)) # extract from geojson 22 | 23 | # Point list of the first feature 24 | pts = js["features"][0]["geometry"]["coordinates"][0] 25 | 26 | lines = [] 27 | for pt in pts: 28 | lon, lat = pyproj.transform(source_proj,target_proj,pt[0],pt[1]) 29 | lines.append("{},{}".format(lon,lat)) 30 | 31 | aoi = " ".join(lines) 32 | return aoi 33 | 34 | def getsrs(geojson): 35 | # regex to find crs 36 | regex = "EPSG:{1,2}[0-9]{4,5}" 37 | 38 | m = re.search(regex, str(geojson)) 39 | 40 | if m is None: # default to wgs84 41 | return "epsg:4236" 42 | epsg = m.group(0) 43 | epsg = epsg.replace("::",":").lower() 44 | return epsg 45 | 46 | if __name__ == "__main__": 47 | 48 | parser = argparse.ArgumentParser(description='converts geojson containing one polygon to a list of wgs84 (epsg:4236) coordinates. If multiple features are in the geojson only the first is considered') 49 | parser.add_argument('geojson', help='path to the input geojson file') 50 | parser.add_argument('--srs', default='epsg:4326', help="target srs coordinate system. Defaults to 'epsg:4326'") 51 | args = parser.parse_args() 52 | 53 | ## debug 54 | #args.geojson="data/shp/bavaria/aoi.geojson" 55 | #args.geojson="data/shp/ecuador/Ecuador_AOI.geojson" 56 | #args.aoi="download/bavaria.aoi" 57 | 58 | print(geojson2aoi(args.geojson,srs=args.srs)) 59 | -------------------------------------------------------------------------------- /crop_via_vrt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # wrapper around warp.sh and vrap via vrt 4 | # 5 | # if wrap.sh fails with L2A Sentinel 2 images. Crop images via VRTs 6 | 7 | # read configs 8 | . $1 9 | 10 | # either L1C or L2A 11 | level=$2 12 | 13 | gdalwarp=conda/bin/gdalwarp 14 | 15 | if [ "$level" = "L1C" ]; then 16 | gdalS2prefix="SENTINEL2_L1C" 17 | todofile=cropL1C.todo 18 | elif [ "$level" = "L2A" ]; then 19 | gdalS2prefix="SENTINEL2_L2A" 20 | todofile=cropL2A.todo 21 | else 22 | echo "please specify level ('L1C' or 'L2A') as second argument" 23 | echo "e.g. \$sh warp.sh demo.cfg L1C" 24 | exit 0 25 | fi 26 | #statements 27 | 28 | # default output directory to path if not defined 29 | if [ -z "$tifpath" ]; then 30 | tifpath=$path 31 | fi 32 | 33 | #bash warp.sh $1 $2 &> warp.log 34 | 35 | # catch products which have caused ERROR 6 36 | #products=$(grep -B1 "ERROR 6" warp.log | grep S2) 37 | 38 | while read p; 39 | do 40 | 41 | # select product name 42 | if [ "$level" = "L1C" ]; then 43 | product=$p 44 | elif [ "$level" = "L2A" ]; then 45 | # predict L2A product name 46 | product=$(echo $p | sed 's/MSIL1C/MSIL2A/g' | sed 's/OPER/USER/g' ) 47 | fi 48 | 49 | # create VRTs 50 | python addvrttoproduct.py $path/$product.SAFE --bandconfigfile cfg/bandsvrtsequence.json --gdalbuildvrt conda/bin/gdalbuildvrt 51 | 52 | # replace '_' with ':' e.g. EPSG_32632 -> EPSG:32632 53 | t_srs=$(echo $srs | sed 's/_/:/g') 54 | 55 | # create clips 56 | echo $path/$product 57 | $gdalwarp -multi -wo NUM_THREADS=val/ALL_CPUS -t_srs $t_srs -of GTiff -crop_to_cutline -cutline $cutline $path/$product.SAFE/10m.vrt "$tifpath/$product"_10m.tif 58 | $gdalwarp -multi -wo NUM_THREADS=val/ALL_CPUS -t_srs $t_srs -of GTiff -crop_to_cutline -cutline $cutline $path/$product.SAFE/20m.vrt "$tifpath/$product"_20m.tif 59 | $gdalwarp -multi -wo NUM_THREADS=val/ALL_CPUS -t_srs $t_srs -of GTiff -crop_to_cutline -cutline $cutline $path/$product.SAFE/60m.vrt "$tifpath/$product"_60m.tif 60 | 61 | done <$path/$todofile 62 | 63 | -------------------------------------------------------------------------------- /querySCL.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import psycopg2 3 | import argparse 4 | import os 5 | from datetime import datetime 6 | 7 | def main(): 8 | parser = argparse.ArgumentParser(description='Query database for rasters. Requires POSTGIS_USER and POSTGIS_PASSWORD environment variables to be set') 9 | parser.add_argument('filename', type=str, help='csv filename') 10 | parser.add_argument('--db', type=str, default="fields", help='database (default fields)') 11 | parser.add_argument('--host', type=str, default="dbserver", help='database (default dbserver)') 12 | 13 | args = parser.parse_args() 14 | 15 | filename=args.filename 16 | 17 | conn = psycopg2.connect('postgres://{}:{}@{}/{}'.format(os.environ["POSTGIS_USER"],os.environ["POSTGIS_PASSWORD"],args.host,args.db)) 18 | 19 | dates = pd.read_sql('Select distinct date from bavaria order by date',conn)["date"].sort_values().values 20 | 21 | frames=[] 22 | for date in dates: 23 | print "{}: query product from {}".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), date.strftime("%Y-%m-%d")) 24 | frames.append(querySCLhist(date,conn)) 25 | 26 | out = pd.concat(frames) 27 | out.to_csv(filename) 28 | 29 | def querySCLhist(date, conn): 30 | """ Query database for raster at defined date. Look at SCL band (=12) and count values 31 | returns one lined Dataframe index=date and columns sclnames""" 32 | 33 | sclnames=['nodata', 34 | 'defect', 35 | 'darkfeat', 36 | 'cloud shadow', 37 | 'vegetation', 38 | 'bare soil', 39 | 'water', 40 | 'cloud low prob', 41 | 'cloud med prob', 42 | 'cloud high prob', 43 | 'cloud cirrus', 44 | 'snow'] 45 | 46 | scldict={} 47 | for i in range(len(sclnames)): 48 | scldict[i+1] = sclnames[i] 49 | 50 | # band12 -> SCL 51 | sql=""" 52 | SELECT (stats).value as SCL,(stats).count 53 | FROM ( 54 | select ST_ValueCount(st_union(rast),12) as stats 55 | from bavaria where date='{date}'::date and level='L2A' and type='20m' 56 | ) as foo 57 | """.format(date=date.strftime("%Y-%m-%d")) 58 | 59 | sclhist = pd.read_sql(sql,conn) 60 | sclhist["scl"]=sclhist.scl.map(scldict) # replace pixel value with name 61 | sclhist = sclhist.set_index("scl").T 62 | sclhist.index=pd.to_datetime([date]) 63 | return sclhist 64 | 65 | 66 | 67 | if __name__ == "__main__": 68 | main() 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /uploadraster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ! "$1"="--help" ] || [ -z "$1" ] || [ -z "$2" ]; 4 | then 5 | echo "uploads all raster specified in the site file to the database table" 6 | echo "" 7 | echo "bash uploadraster.sh sites/bavaria.site L1C" 8 | exit 1 9 | fi 10 | 11 | # read configs 12 | . $1 13 | 14 | # either L1C or L2A 15 | level=$2 16 | 17 | 18 | psql -d $dbase -c "Create table if not exists $dbtable(rid serial primary key, rast raster, filename text);" 19 | 20 | # database connection 21 | psql="psql -d $dbase --username=russwurm --host=$dbhost" 22 | 23 | # check if table already exists 24 | if [ $($psql -tAc "SELECT 1 FROM pg_tables where tablename='$dbtable'") = '1' ]; then 25 | echo "appending to DB" 26 | appendflag="-a" 27 | else 28 | appendflag="-I" # first insert should also create an index 29 | fi 30 | 31 | # database exists 32 | # $? is 0 33 | 34 | #files="data/bavaria/test/S2A_OPER_PRD_MSIL1C_PDMC_20160522T182438_R065_V20160522T102029_20160522T102029_10m.tif" 35 | 36 | # query already present products 37 | productsindb=$($psql -c "Select distinct filename from $dbtable") 38 | 39 | 40 | echo $psql 41 | while read p; 42 | do 43 | 44 | # select product name 45 | if [ "$level" = "L1C" ]; then 46 | product=$p 47 | elif [ "$level" = "L2A" ]; then 48 | # predict L2A product name 49 | product=$(echo $p | sed 's/MSIL1C/MSIL2A/g' | sed 's/OPER/USER/g' ) 50 | fi 51 | 52 | # query if filename does not already exists 53 | #echo $filename 54 | if [[ "$productsindb" = *"$product"* ]] ; then 55 | echo "product $p already in database table $dbtable. skipping..." 56 | continue 57 | fi 58 | echo "loading $p to database" 59 | raster2pgsql -s $srs $appendflag -P $tifpath/$product*.tif -F -t $tilesize"x"$tilesize $dbtable | $psql 60 | #raster2pgsql -s $srs $appendflag -P -C -M $tifpath/$product*.tif -F -t $tilesize"x"$tilesize $dbtable > uploadsql/$product.sql 61 | done <$path/$queryfile 62 | # type := 10m, 20m, 60m or SCL 63 | 64 | # add columns 65 | $psql <```) 97 | ``` 98 | SET postgis.gdal_enabled_drivers = 'ENABLE_ALL'; 99 | ALTER SYSTEM SET postgis.gdal_enabled_drivers TO 'GTiff PNG JPEG'; 100 | SELECT pg_reload_conf(); 101 | ``` 102 | 103 | test if drivers are enabled via 104 | ``` 105 | SELECT short_name, long_name FROM ST_GDALDrivers(); 106 | ``` 107 | -------------------------------------------------------------------------------- /uploadmeta.py: -------------------------------------------------------------------------------- 1 | import psycopg2 2 | import os 3 | import sys 4 | from configobj import ConfigObj 5 | import pandas as pd 6 | import re 7 | import datetime 8 | 9 | if len(sys.argv) < 2: 10 | print "No config file provided! Aborting" 11 | sys.exit() 12 | 13 | cfg_path=sys.argv[1] 14 | 15 | cfg = ConfigObj(cfg_path) 16 | 17 | def setdefault(key,value): 18 | if key not in cfg.keys(): 19 | print "WARNING: {} not specified in {}, defaulting to '{}'".format(key,cfg_path, value) 20 | cfg[key]=value 21 | # set defaults if not defined 22 | 23 | setdefault('dbhost','dbserver') 24 | setdefault('dbase','fields') 25 | setdefault('dbtable','test') 26 | 27 | conn = psycopg2.connect('postgres://{}:{}@{}/{}'.format(os.environ["POSTGIS_USER"],os.environ["POSTGIS_PASSWORD"],cfg['dbhost'],cfg['dbase'])) 28 | cur = conn.cursor() 29 | 30 | # needs to be executed while no other process accesses table... 31 | if True: 32 | # create attributes if not exists 33 | sql=""" 34 | ALTER TABLE {0} ADD COLUMN IF NOT EXISTS year integer; 35 | ALTER TABLE {0} ADD COLUMN IF NOT EXISTS doy integer; 36 | ALTER TABLE {0} ADD COLUMN IF NOT EXISTS date date; 37 | ALTER TABLE {0} ADD COLUMN IF NOT EXISTS level varchar(5); 38 | ALTER TABLE {0} ADD COLUMN IF NOT EXISTS sat varchar(5); 39 | ALTER TABLE {0} ADD COLUMN IF NOT EXISTS type varchar(5); 40 | """.format(cfg['dbtable']) 41 | cur.execute(sql) 42 | conn.commit() 43 | 44 | # select filenames, which lack meta values 45 | sql=""" 46 | select distinct filename from {} 47 | where 48 | year is NULL or 49 | doy is NULL or 50 | date is NULL or 51 | level is NULL or 52 | sat is NULL or 53 | type is NULL 54 | """.format(cfg['dbtable']) 55 | 56 | # old format: e.g. 57 | #S2A_USER_PRD_MSIL2A_PDMC_20160831T224638_R065_V20160830T102022_20160830T102052 58 | #S2oldformat=".{3}_.{4}_.{3}_.{6}_.{4}_.{15}_.{4}_.{16}_.{16}.*" 59 | 60 | # new format: e.g. 61 | # S2A_MSIL1C_20170616T102021_N0205_R065_T32UQV_20170616T102331_60m.tif 62 | #S2newformat=".{3}_.{6}_.{15}_.{5}_.{4}_.{6}_.{15}.*" 63 | 64 | rs = pd.read_sql(sql, conn) 65 | 66 | for filename in rs["filename"]: 67 | print filename 68 | sat=re.search(r"S2A|S2B",filename).group(0) 69 | level=re.search(r"L1C|L2A",filename).group(0) 70 | datestr=re.search(r"_[0-9]{8}T",filename).group(0)[1:-1] # remove "_" and "T" 71 | date=datetime.datetime.strptime(datestr,"%Y%m%d") 72 | doy=date.timetuple().tm_yday 73 | year=date.year 74 | stype=re.search(r"(.{3})\.tif",filename).group(1) 75 | 76 | print "parsed: sat {}, level {}, date {}, doy {}, year {}, type {} from {}".format(sat,level,date.date(),doy,year,stype,filename) 77 | 78 | updatesql=""" 79 | update {} 80 | set 81 | sat=%s, 82 | level=%s, 83 | date=%s, 84 | doy=%s, 85 | year=%s, 86 | type=%s 87 | where 88 | filename='{}'; 89 | """.format(cfg['dbtable'],filename) 90 | 91 | cur.execute(updatesql, (sat,level,date,doy,year,stype)) 92 | 93 | conn.commit() 94 | cur.close() 95 | conn.close() 96 | -------------------------------------------------------------------------------- /grid.py: -------------------------------------------------------------------------------- 1 | from shapely.geometry import Point, box 2 | import psycopg2 3 | import numpy as np 4 | import sys 5 | from configobj import ConfigObj 6 | import os 7 | from geojson2aoi import geojson2aoi 8 | import argparse 9 | 10 | 11 | parser = argparse.ArgumentParser(description='created a grid in a database based on extent of geojson file. Requires fields: srs') 12 | parser.add_argument('site', help='config file for specified site') 13 | parser.add_argument('geojson', help="geojson file, which provides extent of AOI") 14 | parser.add_argument('tablename', help="database tablename") 15 | parser.add_argument('--tilesize', default=3000, type=int,help="Size of tiles in meter. Defaults to 3000m") 16 | parser.add_argument('--margin', default=0, type=int,help="size of margin between tiles in meter") 17 | args = parser.parse_args() 18 | 19 | cfg_path=args.site 20 | 21 | cfg = ConfigObj(cfg_path) 22 | 23 | def setdefault(key,value): 24 | if key not in cfg.keys(): 25 | print "WARNING: {} not specified in {}, defaulting to '{}'".format(key,cfg_path, value) 26 | cfg[key]=value 27 | # set defaults if not defined 28 | 29 | setdefault('dbhost','dbserver') 30 | setdefault('dbase','fields') 31 | setdefault('dbtable','test') 32 | 33 | conn = psycopg2.connect('postgres://{}:{}@{}/{}'.format(os.environ["POSTGIS_USER"],os.environ["POSTGIS_PASSWORD"],cfg['dbhost'],cfg['dbase'])) 34 | curs=conn.cursor() 35 | 36 | srs=cfg["srs"].replace("_",":").lower() 37 | 38 | aoi=geojson2aoi(args.geojson,srs=srs) 39 | coords = np.array([coord.split(',') for coord in aoi.split(" ")]).astype(float).astype(int) 40 | 41 | # AOI 42 | x_min = coords[:,0].min() 43 | y_min = coords[:,1].min() 44 | x_max = coords[:,0].max() 45 | y_max = coords[:,1].max() # 46 | srs = int(srs.split(':')[-1]) # e.g. 'epsg:32632' -> 32632 47 | 48 | rand = np.random.rand() 49 | 50 | table_name = args.tablename 51 | 52 | point_distance = args.tilesize#m 53 | offset = point_distance/2 54 | 55 | commit_every = 1000 56 | 57 | create_table_sql=""" 58 | CREATE TABLE IF NOT EXISTS %s( 59 | id SERIAL NOT NULL PRIMARY KEY, 60 | geom geometry) 61 | """ % table_name 62 | # Send it to PostGIS 63 | print create_table_sql 64 | curs.execute(create_table_sql) 65 | 66 | x_grid = range(x_min+offset, x_max, point_distance) 67 | y_grid = range(y_min+offset, y_max, point_distance) 68 | n_points = len(x_grid)*len(y_grid) 69 | 70 | boxoffset=offset-args.margin/2 71 | 72 | count = 0 73 | for x in x_grid: 74 | for y in y_grid: 75 | count += 1 76 | center = Point(x, y) 77 | 78 | geom = box(x-boxoffset, y-boxoffset, x+boxoffset, y+boxoffset) 79 | 80 | insert_sql = """ 81 | INSERT INTO {0}(geom) 82 | VALUES (ST_SetSRID(%(geom)s::geometry, %(srid)s)) 83 | """.format(table_name) 84 | 85 | curs.execute( 86 | insert_sql, 87 | {'geom': geom.wkb_hex, 'srid': srs}) 88 | 89 | if count % commit_every == 0: 90 | conn.commit() 91 | print "count: %d/%d (%d%%), committing to db..." % (count, n_points, float(count)/float(n_points)*100) 92 | 93 | print "creating spatial index..." 94 | conn.commit() 95 | 96 | curs.execute("CREATE INDEX point_idx ON %s USING GIST (geom);" % table_name) 97 | conn.commit() 98 | 99 | conn.close() 100 | curs.close() 101 | print "done..." 102 | -------------------------------------------------------------------------------- /cfg/L2A_GIPP.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | INFO 5 | 6 | AUTO 7 | 8 | DEFAULT 9 | 10 | NONE 11 | 12 | http://data_public:GDdci@data.cgiar-csi.org/srtm/tiles/GeoTIFF/ 13 | 14 | 15 | S2_User_Product_Level-1C_Metadata.xsd 16 | S2_User_Product_Level-2A_Metadata.xsd 17 | S2_PDI_Level-1C_Tile_Metadata.xsd 18 | S2_PDI_Level-2A_Tile_Metadata.xsd 19 | S2_PDI_Level-1C_Datastrip_Metadata.xsd 20 | S2_PDI_Level-2A_Datastrip_Metadata.xsd 21 | 22 | 23 | S2_User_Product_Level-1C_Metadata.xsd 24 | S2_User_Product_Level-2A_Metadata.xsd 25 | S2_PDI_Level-1C_Tile_Metadata.xsd 26 | S2_PDI_Level-2A_Tile_Metadata.xsd 27 | S2_PDI_Level-1C_Datastrip_Metadata.xsd 28 | S2_PDI_Level-2A_Datastrip_Metadata.xsd 29 | 30 | L2A_GIPP.xsd 31 | L2A_CAL_SC_GIPP.xsd 32 | L2A_CAL_AC_GIPP.xsd 33 | 34 | 35 | 36 | 0 37 | 38 | 39 | 40 | 41 | RURAL 42 | 43 | AUTO 44 | 45 | 0 46 | 47 | 66 | 67 | 68 | 1 69 | 70 | 1 71 | 72 | 1 73 | 74 | 0 75 | 76 | 0 77 | 78 | 0.22 79 | 80 | 81 | 82 | 0 83 | 84 | 1.000 85 | 86 | 40.0 87 | 88 | 0.100 89 | 90 | 100.0 91 | 92 | 0.25 93 | 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /addvrttoproduct.py: -------------------------------------------------------------------------------- 1 | import json 2 | import subprocess 3 | import argparse 4 | import os 5 | import sys 6 | 7 | """ 8 | Takes Sentinel 2 Product file and creates 10m, 20m and 60m VRT files in a predefined order 9 | """ 10 | 11 | def main(): 12 | 13 | parser = argparse.ArgumentParser(description='adds VRT bands to Sentinel SAFE images for easier data handling') 14 | parser.add_argument('product', type=str, 15 | help='product path (to SAFE file)') 16 | parser.add_argument('--bandconfigfile', default=None, 17 | help='config json file for band sequences') 18 | 19 | parser.add_argument('--gdalbuildvrt', default="gdalbuildvrt", 20 | help='path to gdalbuildvrt program. Defaults to program defined in PATH') 21 | 22 | parser.add_argument('--verbose', action='store_true', 23 | help='display additional text') 24 | 25 | args = parser.parse_args() 26 | 27 | if not os.path.exists(args.product): 28 | print "No folder found unter product path. Exit" 29 | sys.exit() 30 | 31 | 32 | # set bandconfig default 33 | if args.bandconfigfile is None: 34 | args.bandconfig = json.loads(""" 35 | { 36 | "L1C": { 37 | "10m": ["B04","B03","B02","B08"], 38 | "20m": ["B05","B06","B07","B8A","B11","B12"], 39 | "60m": ["B01","B09","B10"] 40 | }, 41 | "L2A": { 42 | "10m" : ["B04","B03","B02","B08"], 43 | "20m" : ["B05","B06","B07","B8A","B11","B12","AOT","CLD","SCL","SNW","WVP"], 44 | "60m" : ["B01","B09"] 45 | } 46 | } 47 | """) 48 | else: 49 | with open(args.bandconfigfile) as data_file: 50 | args.bandconfig = json.load(data_file) 51 | 52 | addVRTs(args) 53 | 54 | def addVRTs(args): 55 | 56 | bandcfg = args.bandconfig 57 | 58 | # "/home/russwurm/projects/preprocessing/conda/bin/gdalbuildvrt" 59 | buildvrtcommand=args.gdalbuildvrt 60 | 61 | #product="fails/S2A_MSIL2A_20170223T101021_N0204_R022_T32UQU_20170223T101550.SAFE" 62 | #level="L2A" 63 | 64 | #product="works/S2A_USER_PRD_MSIL2A_PDMC_20161115T190806_R022_V20161115T101302_20161115T101302.SAFE" 65 | #level="L2A" 66 | 67 | if "L1C" in args.product.split('/')[-1]: 68 | level="L1C" 69 | elif "L2A" in args.product.split('/')[-1]: 70 | level="L2A" 71 | 72 | product=args.product 73 | 74 | #product="fails/S2A_MSIL1C_20170223T101021_N0204_R022_T32UQU_20170223T101550.SAFE" 75 | #level="L1C" 76 | 77 | if not os.path.exists(os.path.join(args.product,"vrt")): 78 | os.makedirs(os.path.join(args.product,"vrt")) 79 | 80 | def createvrt(product,band,resolution,level): 81 | if level=="L2A": # B01_10m.jp2 82 | bashCommand = "find {product} -name *{band}*{res}.jp2".format(product=product, band=band, res=resolution) 83 | if level=="L1C": # B01.jp2 84 | bashCommand = "find {product} -name *{band}.jp2".format(product=product, band=band) 85 | 86 | if args.verbose: print bashCommand 87 | 88 | process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE) 89 | output, error = process.communicate() 90 | 91 | images = output.split('\n') 92 | 93 | bashCommand = "{gdalbuildvrt} {product}/vrt/{band}_{res}.vrt {images}".format(gdalbuildvrt=buildvrtcommand, 94 | product=product, band=band,res=resolution, 95 | images=' '.join(images)) 96 | process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE) 97 | return process.communicate() 98 | 99 | for resolution in bandcfg[level].keys(): 100 | 101 | bands = bandcfg[level][resolution] 102 | 103 | # build vrt for each band 104 | for band in bands: 105 | 106 | # create one vrt for each band 107 | createvrt(product,band,resolution,level) 108 | 109 | # merge bands to resolution vrts 110 | 111 | bandvrts=" ".join(["{product}/vrt/{band}_{res}.vrt".format(product=product,band=b,res=resolution) for b in bands]) # B02.vrt B03.vrt B04.vrt B08.vrt AOT.vrt WVP.vrt 112 | 113 | # bandarguments # e.g. 114 | bandarg=" ".join(["-b {}".format(b+1) for b in range(len(bands))]) # '-b 1 -b 2 -b 3 -b 4 -b 5 -b 6' 115 | 116 | # merge bands in proper sequence for 10m.vrt 117 | bashCommand = "{gdalbuildvrt} -separate {bandarg} {product}/{resolution}.vrt {bandvrts}".format(gdalbuildvrt=buildvrtcommand, 118 | bandarg=bandarg, 119 | product=product, 120 | bandvrts=bandvrts, 121 | resolution=resolution) 122 | 123 | process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE) 124 | output,error=process.communicate() 125 | 126 | if __name__ == "__main__": 127 | main() 128 | -------------------------------------------------------------------------------- /status.py: -------------------------------------------------------------------------------- 1 | 2 | from configobj import ConfigObj 3 | import os 4 | import glob 5 | from os.path import join 6 | import pandas as pd 7 | import argparse 8 | from progressbar import * # just a simple progress bar 9 | 10 | 11 | def main(): 12 | parser = argparse.ArgumentParser(description='Creates status csv and todo lists') 13 | parser.add_argument('site', type=str, default='sites/bavaria.site',help='config file describing site paramters') 14 | parser.add_argument('-noupdate',action='store_true',help='print info, but dont overwrite todo and status files') 15 | args = parser.parse_args() 16 | configpath = args.site 17 | 18 | cfg = ConfigObj(configpath) 19 | 20 | cfg["project"] = cfg["project"].replace('$HOME', os.environ["HOME"]) 21 | cfg["path"] = cfg["path"].replace('$project', cfg["project"]) 22 | cfg["tifpath"] = cfg["tifpath"].replace('$project', cfg["project"]) 23 | 24 | print "looking for products..." 25 | status_df = look_for_products(cfg) 26 | status_df = append_todos(status_df) 27 | 28 | print_status(cfg,status_df) 29 | 30 | print 31 | if not args.noupdate: 32 | print "updating status.csv and todo lists..." 33 | write_status_and_todos(cfg, status_df) 34 | else: 35 | print "no updates..." 36 | 37 | def print_status(cfg,status_df): 38 | print "Found {} queried products".format(status_df.shape[0]) 39 | print "{} SAFE products are without Granules".format(status_df[status_df["nogranules"]].shape[0]) 40 | print 41 | print "Processing Level L1C:" 42 | print " {} SAFE products".format(status_df[status_df["L1C"]].shape[0]) 43 | print " {} tif 10m products".format(status_df[status_df["L1Ctif10"]].shape[0]) 44 | print " {} tif 20m products".format(status_df[status_df["L1Ctif20"]].shape[0]) 45 | print " {} tif 60m products".format(status_df[status_df["L1Ctif60"]].shape[0]) 46 | print " {} products in cropL1C.todo".format(status_df[status_df["do_cropL1C"]].shape[0]) 47 | print 48 | print "Processing Level L2A:" 49 | print " {} SAFE products".format(status_df[status_df["L2A"]].shape[0]) 50 | print " {} products in sen2cor.todo".format(status_df[status_df["do_sen2cor"]].shape[0]) 51 | print " {} tif 10m products".format(status_df[status_df["L2Atif10"]].shape[0]) 52 | print " {} tif 20m products".format(status_df[status_df["L2Atif20"]].shape[0]) 53 | print " {} tif 60m products".format(status_df[status_df["L2Atif60"]].shape[0]) 54 | print " {} products in cropL2A.todo".format(status_df[status_df["do_cropL2A"]].shape[0]) 55 | print 56 | print "detailed information at:" 57 | print join(cfg["path"],"status.csv") 58 | print 59 | print "query" 60 | print join(cfg["path"],cfg["queryfile"]) 61 | print 62 | print "Todo lists:" 63 | print join(cfg["path"],"sen2cor.todo") 64 | print join(cfg["path"],"cropL1C.todo") 65 | print join(cfg["path"],"cropL2A.todo") 66 | print join(cfg["path"],"download.todo") 67 | 68 | def look_for_products(cfg): 69 | 70 | SAFE = glob.glob(cfg["path"] + "/*.SAFE") 71 | tif = glob.glob(cfg["tifpath"] + "/*.tif") 72 | 73 | with open(join(cfg["path"], cfg["queryfile"]), 'r') as f: 74 | queries = f.read().splitlines() 75 | 76 | def l1ctol2a(product): 77 | return product.replace("OPER", "USER").replace("L1C", "L2A") 78 | 79 | # create empty dataframe 80 | df = pd.DataFrame(index=queries) 81 | 82 | widgets = ['looking for products: ', Percentage(), ' ', Bar(marker='#', left='[', right=']'), 83 | ' ', ETA()] # see docs for other options 84 | 85 | pbar = ProgressBar(widgets=widgets, maxval=df.shape[0]) 86 | pbar.start() 87 | 88 | # for each product in queryfile 89 | i = 0 90 | for product, row in df.iterrows(): 91 | pbar.update(i) 92 | i+=1 93 | 94 | # check if product has no granules 95 | if os.path.exists(join(cfg["path"], product + ".SAFE", "GRANULE")): 96 | df.loc[product, "nogranules"] = len(os.listdir(join(cfg["path"], product + ".SAFE", "GRANULE")))==0 97 | else: # if no product in present give benefit of doubt 98 | df.loc[product, "nogranules"] = False 99 | 100 | # check if L1C product exists in $path 101 | df.loc[product, "L1C"] = os.path.exists( 102 | join(cfg["path"], product + ".SAFE")) and not df.loc[product, "nogranules"] 103 | 104 | # check if L2A product exists in $path 105 | df.loc[product, "L2A"] = os.path.exists( 106 | join(cfg["path"], l1ctol2a(product) + ".SAFE")) 107 | 108 | # L1C: check if 10,20,60m tifs exist in $tifpath 109 | df.loc[product, "L1Ctif10"] = os.path.exists( 110 | join(cfg["tifpath"], product + "_10m.tif")) 111 | df.loc[product, "L1Ctif20"] = os.path.exists( 112 | join(cfg["tifpath"], product + "_20m.tif")) 113 | df.loc[product, "L1Ctif60"] = os.path.exists( 114 | join(cfg["tifpath"], product + "_60m.tif")) 115 | 116 | # L2A: check if 10,20,60m tifs exist in $tifpath 117 | df.loc[product, "L2Atif10"] = os.path.exists( 118 | join(cfg["tifpath"], l1ctol2a(product) + "_10m.tif")) 119 | df.loc[product, "L2Atif20"] = os.path.exists( 120 | join(cfg["tifpath"], l1ctol2a(product) + "_20m.tif")) 121 | df.loc[product, "L2Atif60"] = os.path.exists( 122 | join(cfg["tifpath"], l1ctol2a(product) + "_60m.tif")) 123 | 124 | pbar.finish() 125 | return df 126 | 127 | 128 | def append_todos(df): 129 | 130 | for product, row in df.iterrows(): 131 | # do sen2cor if L1C exists and no L2A 132 | #df.loc[product,"do_sen2cor"]=row["L1C"] and not row["L2A"] 133 | 134 | 135 | tifmissingL2A=not row["L2Atif10"] or not row["L2Atif20"] or not row["L2Atif60"] 136 | tifmissingL1C=not row["L1Ctif10"] or not row["L1Ctif20"] or not row["L1Ctif60"] 137 | 138 | # do sen2cor if L1C exists and at least one tif missing and the product is not empty 139 | df.loc[product, "do_sen2cor"] = row["L1C"] and tifmissingL2A and not row["nogranules"] 140 | 141 | # do crop L1C 142 | df.loc[product, "do_cropL1C"] = row["L1C"] and tifmissingL1C 143 | 144 | # do crop L1C 145 | df.loc[product, "do_cropL2A"] = row["L2A"] and tifmissingL2A 146 | 147 | # do download if not already downloaded (in L1C) and if not marked as nogranules 148 | df.loc[product, "do_download"] = not row["L1C"] and not row["nogranules"] 149 | 150 | return df 151 | 152 | 153 | def write_status_and_todos(cfg, df): 154 | 155 | def write(products, filename): 156 | with open(filename, 'w') as f: 157 | for item in products: 158 | f.write("%s\n" % item) 159 | 160 | write(list(df[df['do_sen2cor']].index), join(cfg["path"], "sen2cor.todo")) 161 | write(list(df[df['do_cropL1C']].index), join(cfg["path"], "cropL1C.todo")) 162 | write(list(df[df['do_cropL2A']].index), join(cfg["path"], "cropL2A.todo")) 163 | write(list(df[df['do_download']].index), join(cfg["path"], "download.todo")) 164 | 165 | df.to_csv(join(cfg["path"], "status.csv")) 166 | 167 | 168 | if __name__ == '__main__': 169 | main() 170 | -------------------------------------------------------------------------------- /downloadtiles.py: -------------------------------------------------------------------------------- 1 | import psycopg2 2 | import os 3 | import gdal 4 | import json 5 | import numpy as np 6 | from configobj import ConfigObj 7 | import argparse 8 | import datetime 9 | 10 | import cPickle as pickle 11 | 12 | import matplotlib.pyplot as plt 13 | 14 | import pylab 15 | from pylab import imshow, show, get_cmap 16 | import pandas as pd 17 | 18 | RASTER_NODATA_VALUE=0 19 | 20 | with open('cfg/bands.json') as data_file: 21 | bandcfg = json.load(data_file) 22 | 23 | 24 | def main(): 25 | 26 | parser = argparse.ArgumentParser( 27 | description='created pickle files based on tiles in raster database') 28 | parser.add_argument('site', help='config file for specified site') 29 | parser.add_argument('level', help="either L1C or L2A") 30 | parser.add_argument('--subset', help="split 'n/m' tile ids in m subsets and only process the n subset e.g. 1/5 to 4/5") 31 | args = parser.parse_args() 32 | 33 | cfg_path = args.site 34 | 35 | cfg = ConfigObj(cfg_path) 36 | 37 | conn = psycopg2.connect('postgres://{}:{}@{}/{}'.format(os.environ["POSTGIS_USER"],os.environ["POSTGIS_PASSWORD"],cfg['dbhost'],cfg['dbase'])) 38 | 39 | level = args.level 40 | rastertable = cfg['dbtable'] 41 | tiletable = cfg['tiletable'] 42 | 43 | project = cfg["project"].replace('$HOME', os.environ["HOME"]) 44 | tilefolder = cfg["tilefolder"].replace('$project', project) 45 | 46 | outfolder=os.path.join(tilefolder,tiletable) 47 | if not os.path.exists(outfolder): 48 | os.makedirs(outfolder) 49 | 50 | ids = pd.read_sql("select id from {}".format(tiletable),conn)['id'] 51 | 52 | # split tileids in subsets 53 | if args.subset: 54 | n,m = [int(i) for i in args.subset.split('/')] 55 | 56 | c = pd.cut(ids, m,labels=range(m)) 57 | 58 | # n-1 as index starts from 0 59 | ids=ids.loc[c.loc[c==n-1].index] 60 | 61 | print "using subset {} of {}: length {}".format(n, m, len(ids)) 62 | 63 | tileids = ids.values 64 | for tileid in tileids: 65 | conn = psycopg2.connect( 66 | 'postgres://{}:{}@{}/{}'.format(os.environ["POSTGIS_USER"], os.environ["POSTGIS_PASSWORD"], cfg['dbhost'], 67 | cfg['dbase'])) 68 | 69 | print "{} downloading Tile {}".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), tileid) 70 | download_tile(conn, level, rastertable, tiletable, outfolder, tileid,False) 71 | 72 | 73 | def download_tile(conn, level, rastertable, tiletable, outfolder, tileid, verbose=False): 74 | def filename(tileid): 75 | return "{0:04d}.pkl".format(tileid) 76 | 77 | dates = pd.read_sql("select distinct date from {} where date is not null".format(rastertable),conn)["date"] 78 | querieddates=dates.sort_values().values 79 | 80 | def toidx(bands): 81 | return [bandcfg[level][rtype].index(b) + 1 for b in bands] 82 | 83 | # inspect tile if already exists 84 | outfile=os.path.join(outfolder,filename(tileid)) 85 | if os.path.exists(outfile): 86 | with open(outfile,'rb') as f: 87 | tile = pickle.load(f) 88 | else: 89 | tile=None 90 | 91 | # create new empty sample 92 | ts = TimeSeriesSample(tile) 93 | for date in querieddates: 94 | 95 | if tile is not None: 96 | if date in tile['added']: 97 | if verbose: print "skipping {}, alreay in pickle file".format(date) 98 | continue 99 | 100 | if verbose: print date 101 | 102 | try: 103 | labels=queryLabel(conn, tiletable, tileid) 104 | 105 | raster=[] 106 | for rtype in ["10m","20m","60m"]: 107 | allbands = bandcfg[level][rtype] 108 | raster.append(queryRaster(conn, rastertable, tiletable, tileid, date, rtype, level, toidx(allbands))) 109 | 110 | except psycopg2.InternalError as err: 111 | # TypeError: typically Label query returns None 112 | # InternalError: ?? happenes on some tiles. Maybe connection interupted 113 | print "Caught error {}".format(err) 114 | # add to the failed raster to avoid querying this file again... 115 | ts.addfailed(date) 116 | continue 117 | except: 118 | ts.addfailed(date) 119 | continue 120 | 121 | 122 | # add single time sample to timeseries 123 | ts.add(date, labels, raster[0],raster[1], raster[2]) 124 | 125 | if verbose: print "writing to {}".format(outfile) 126 | ts.write_to_file(outfile) 127 | 128 | plt.show() 129 | 130 | def queryRaster(conn, rastertable, tiletable, tileid, date, rtype, level, bands, verbose=False): 131 | curs = conn.cursor() 132 | # convert band names to band numbers using config file 133 | 134 | sql = """ 135 | select 136 | ST_astiff(ST_UNION(ST_CLIP(r.rast, t.geom)),ARRAY{bands}) 137 | from 138 | {rastertable} r, {tiletable} t 139 | where 140 | t.id={tileid} and 141 | ST_INTERSECTS(r.rast,t.geom) and 142 | r.type='{rtype}' and 143 | r.level='{level}' and 144 | date='{date}' 145 | """.format(rastertable=rastertable, 146 | tiletable=tiletable, 147 | tileid=tileid, 148 | rtype=rtype, 149 | level=level, 150 | date=date.strftime("%Y-%m-%d"), 151 | bands=bands) 152 | 153 | if verbose: print sql 154 | # following https://gis.stackexchange.com/questions/130139/downloading-raster-data-into-python-from-postgis-using-psycopg2 155 | 156 | # Use a virtual memory file, which is named like this 157 | vsipath = '/vsimem/from_postgis' 158 | 159 | # Download raster data into Python as GeoTIFF, and make a virtual file for GDAL 160 | curs.execute(sql) 161 | 162 | gdal.FileFromMemBuffer(vsipath, bytes(curs.fetchone()[0])) 163 | 164 | # Read first band of raster with GDAL 165 | ds = gdal.Open(vsipath) 166 | arrays = [] 167 | for b in range(len(bands)): 168 | band = ds.GetRasterBand(b + 1) 169 | arrays.append(band.ReadAsArray()) 170 | 171 | # Close and clean up virtual memory file 172 | ds = band = None 173 | gdal.Unlink(vsipath) 174 | curs.close() 175 | 176 | return np.stack(arrays, axis=2) 177 | 178 | 179 | def buff2rast(buff): 180 | # Use a virtual memory file, which is named like this 181 | vsipath = '/vsimem/from_postgis' 182 | 183 | gdal.FileFromMemBuffer(vsipath, bytes(buff)) 184 | 185 | # Read first band of raster with GDAL 186 | ds = gdal.Open(vsipath) 187 | band = ds.GetRasterBand(1) 188 | 189 | gdal.Unlink(vsipath) 190 | return np.flipud(band.ReadAsArray()) 191 | 192 | 193 | def queryLabel(conn, tiletable, tileid, verbose=False): 194 | curs = conn.cursor() 195 | # convert band names to band numbers using config file 196 | 197 | 198 | sql = """ 199 | select st_astiff( 200 | st_clip( 201 | st_union( 202 | st_asraster(st_intersection(t.geom,l.geom), 203 | 10::float,10::float,st_xmin(t.geom)::float, st_xmax(t.geom)::float,'8BUI',l.labelid,-9999) 204 | ), 205 | t.geom, -9999) 206 | ) 207 | from {tiletable} t, fields l 208 | where t.id={tileid} and ST_Intersects(t.geom, l.geom) 209 | group by t.geom 210 | 211 | """.format(tileid=tileid, tiletable=tiletable) 212 | 213 | if verbose: print sql 214 | # following https://gis.stackexchange.com/questions/130139/downloading-raster-data-into-python-from-postgis-using-psycopg2 215 | 216 | # Download raster data into Python as GeoTIFF, and make a virtual file for GDAL 217 | curs.execute(sql) 218 | rs = curs.fetchone()[0] 219 | 220 | arr = buff2rast(rs) 221 | 222 | # Close and clean up virtual memory file 223 | 224 | curs.close() 225 | 226 | return arr 227 | 228 | 229 | def drawlabels(conn, tiletable, tileid): 230 | labelmap = pd.read_sql('select distinct labelid, label from labelmap where labelid is not null order by labelid', 231 | conn) 232 | 233 | def id2name(labelmap, labelid): 234 | return labelmap.loc[labelmap["labelid"] == labelid]["label"].values[0] 235 | 236 | fig, ax = plt.subplots() 237 | arr = queryLabel(conn, tiletable, tileid) 238 | im = ax.imshow(np.flipud(arr), cmap=get_cmap("Spectral"), interpolation='none') 239 | ax.set_title("Labels") 240 | uniques = np.unique(arr) 241 | 242 | cbar = fig.colorbar(im, ax=ax) 243 | cbar.set_ticks(uniques) 244 | cbar.set_ticklabels([id2name(labelmap, t) for t in uniques]) 245 | 246 | 247 | def plot(tileid, date, level, conn, rastertable, tiletable): 248 | print date 249 | 250 | def m(rtype): 251 | 252 | def toidx(bands): 253 | return [bandcfg[level][rtype].index(b) + 1 for b in bands] 254 | 255 | allbands = bandcfg[level][rtype] 256 | return queryRaster(conn, rastertable, tiletable, tileid, date, rtype, level, toidx(allbands)), allbands 257 | 258 | arr, bands = m("10m") 259 | 260 | fig, axs = plt.subplots(1, 4, figsize=(14, 10)) 261 | i = 0 262 | for ax in axs.reshape(-1): 263 | ax.imshow(arr[:, :, i], interpolation='none') 264 | ax.set_title(bands[i]) 265 | i += 1 266 | 267 | arr, bands = m("20m") 268 | fig, axs = plt.subplots(2, 3, figsize=(10, 6)) 269 | i = 0 270 | for ax in axs.reshape(-1): 271 | ax.imshow(arr[:, :, i], interpolation='none') 272 | ax.set_title(bands[i]) 273 | i += 1 274 | 275 | arr, bands = m("60m") 276 | fig, axs = plt.subplots(1, 3, figsize=(9, 6)) 277 | i = 0 278 | for ax in axs.reshape(-1): 279 | ax.imshow(arr[:, :, i], cmap=pylab.gray(), interpolation='none') 280 | ax.set_title(bands[i]) 281 | i += 1 282 | 283 | def queryDates(conn,table): 284 | curs=conn.cursor() 285 | return pd.read_sql("select distinct date from {}".format(table),conn) 286 | 287 | class TimeSeriesSample(): 288 | def __init__(self,tile): 289 | # 10 GSD, 20 GSD, 60 GSD, meta doy, year 290 | 291 | if tile is None: 292 | self.x10 = np.array([]) 293 | self.x20 = np.array([]) 294 | self.x60 = np.array([]) 295 | self.y = np.array([]) 296 | self.meta=np.array([]) 297 | self.added = np.array([]) 298 | else: 299 | self.x10 = tile['x10'] 300 | self.x20 = tile['x20'] 301 | self.x60 = tile['x60'] 302 | self.y = tile['y'] 303 | self.meta = tile['meta'] 304 | 305 | # dates which did not pass validateInput 306 | self.added = tile['added'] 307 | 308 | def validateInput(self, date, y, x10, x20, x60): 309 | """ Reject samples if needed (e.g. all raste values: )""" 310 | 311 | if np.all(x10==RASTER_NODATA_VALUE) | np.all(x20==RASTER_NODATA_VALUE) | np.all(x60==RASTER_NODATA_VALUE): 312 | return False 313 | 314 | return True 315 | 316 | def as_dict(self): 317 | return {'meta':self.meta, 'x10':self.x10, 'x20':self.x20, 'x60':self.x60, 'y':self.y, 'added':self.added} 318 | 319 | # add a single time 320 | def add(self,date, y, x10,x20,x60): 321 | self.added = np.append(self.added, date) 322 | 323 | if not self.validateInput(date, y, x10, x20, x60): 324 | return None 325 | 326 | x10normalized = x10 * 1e-3 327 | x20normalized = x20 * 1e-3 328 | x60normalized = x60 * 1e-3 329 | 330 | # extract day of year and year from date 331 | #meta = np.array([date.timetuple().tm_yday / 365.,date.year - 2016]) 332 | 333 | self.x10=np.append(self.x10,x10normalized) 334 | self.x20=np.append(self.x20, x20normalized) 335 | self.x60=np.append(self.x60, x60normalized) 336 | self.meta=np.append(self.meta, date) 337 | self.y=np.append(self.y, y) 338 | 339 | def addfailed(self,date): 340 | self.added = np.append(self.added, date) 341 | 342 | def write_to_file(self,filename): 343 | 344 | x10 = self.x10 345 | x20 = self.x20 346 | x60 = self.x60 347 | 348 | meta = self.meta 349 | 350 | y = self.y 351 | 352 | with open(filename, "wb") as f: 353 | pickle.dump(self.as_dict(), f, protocol=2) 354 | 355 | """ 356 | def write_to_file(self,filename): 357 | 358 | def _int64_feature(value): 359 | return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) 360 | 361 | def _bytes_feature(value): 362 | return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) 363 | 364 | writer = tf.python_io.TFRecordWriter(filename) 365 | 366 | x10 = np.stack(self.x10) 367 | x20 = np.stack(self.x20) 368 | x60 = np.stack(self.x60) 369 | 370 | feature={} 371 | for data,text in zip([x10,x20,x60],['10m','20m','60m']): 372 | t,w,h,d = data.shape 373 | 374 | feature['{}dates'.format(text)] = _int64_feature(t) 375 | feature['{}width'.format(text)] = _int64_feature(w) 376 | feature['{}height'.format(text)] = _int64_feature(h) 377 | feature['{}depth'.format(text)] = _int64_feature(d) 378 | feature['{}image'.format(text)] = _bytes_feature(data.tostring()) 379 | 380 | 381 | 382 | features=tf.train.Features(feature=feature) 383 | 384 | example = tf.train.Example(features=features) 385 | 386 | writer.write(example.SerializeToString()) 387 | return 0 388 | """ 389 | 390 | 391 | 392 | if __name__ == "__main__": 393 | main() 394 | -------------------------------------------------------------------------------- /dbviz.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import psycopg2\n", 10 | "from configobj import ConfigObj\n", 11 | "import sys\n", 12 | "import os\n", 13 | "import gdal\n", 14 | "import json\n", 15 | "import datetime\n", 16 | "import numpy as np\n", 17 | "\n", 18 | "import matplotlib.pyplot as plt\n", 19 | "%matplotlib inline\n", 20 | "\n", 21 | "from ipyleaflet import (\n", 22 | " Map,\n", 23 | " Marker,\n", 24 | " TileLayer, ImageOverlay,\n", 25 | " Polyline, Polygon, Rectangle, Circle, CircleMarker,\n", 26 | " GeoJSON,\n", 27 | " DrawControl\n", 28 | ")\n", 29 | "import json" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "cfg = ConfigObj(\"sites/bavaria.site\")\n", 39 | "\n", 40 | "with open('cfg/bands.json') as data_file:\n", 41 | " bandcfg = json.load(data_file)\n", 42 | " \n", 43 | "conn = psycopg2.connect('postgres://{}:{}@{}/{}'.format(os.environ[\"POSTGIS_USER\"],os.environ[\"POSTGIS_PASSWORD\"],cfg['dbhost'],cfg['dbase']))" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": null, 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [ 52 | "from patchfactory import queryDates\n", 53 | "import pylab\n", 54 | "from pylab import imshow, show, get_cmap\n", 55 | "import pandas as pd\n", 56 | "\n", 57 | "def queryRaster(conn,rastertable,tiletable, tileid, date, rtype, level, bands, verbose=False):\n", 58 | " curs=conn.cursor()\n", 59 | " # convert band names to band numbers using config file\n", 60 | "\n", 61 | " sql=\"\"\"\n", 62 | " select\n", 63 | " ST_astiff(ST_UNION(ST_CLIP(r.rast, t.geom)),ARRAY{bands})\n", 64 | " from\n", 65 | " {rastertable} r, {tiletable} t\n", 66 | " where\n", 67 | " t.id={tileid} and\n", 68 | " ST_INTERSECTS(r.rast,t.geom) and\n", 69 | " r.type='{rtype}' and\n", 70 | " r.level='{level}' and\n", 71 | " date='{date}'\n", 72 | " \"\"\".format(rastertable=rastertable,\n", 73 | " tiletable=tiletable,\n", 74 | " tileid=tileid,\n", 75 | " rtype=rtype,\n", 76 | " level=level,\n", 77 | " date=date.strftime(\"%Y-%m-%d\"),\n", 78 | " bands=bands)\n", 79 | "\n", 80 | " if verbose: print sql\n", 81 | " # following https://gis.stackexchange.com/questions/130139/downloading-raster-data-into-python-from-postgis-using-psycopg2\n", 82 | "\n", 83 | " # Use a virtual memory file, which is named like this\n", 84 | " vsipath = '/vsimem/from_postgis'\n", 85 | "\n", 86 | " # Download raster data into Python as GeoTIFF, and make a virtual file for GDAL\n", 87 | " curs.execute(sql)\n", 88 | " gdal.FileFromMemBuffer(vsipath, bytes(curs.fetchone()[0]))\n", 89 | "\n", 90 | " # Read first band of raster with GDAL\n", 91 | " ds = gdal.Open(vsipath)\n", 92 | " arrays=[]\n", 93 | " for b in range(len(bands)):\n", 94 | " band = ds.GetRasterBand(b+1)\n", 95 | " arrays.append(band.ReadAsArray())\n", 96 | "\n", 97 | " # Close and clean up virtual memory file\n", 98 | " ds = band = None\n", 99 | " gdal.Unlink(vsipath)\n", 100 | " curs.close()\n", 101 | "\n", 102 | " return np.stack(arrays,axis=2)\n", 103 | "\n", 104 | "def buff2rast(buff):\n", 105 | " # Use a virtual memory file, which is named like this\n", 106 | " vsipath = '/vsimem/from_postgis'\n", 107 | " \n", 108 | " gdal.FileFromMemBuffer(vsipath, bytes(buff))\n", 109 | "\n", 110 | " # Read first band of raster with GDAL\n", 111 | " ds = gdal.Open(vsipath)\n", 112 | " band=ds.GetRasterBand(1)\n", 113 | " \n", 114 | " gdal.Unlink(vsipath)\n", 115 | " return band.ReadAsArray()\n", 116 | "\n", 117 | "def queryLabel(conn,tiletable, tileid, verbose=False):\n", 118 | " curs=conn.cursor()\n", 119 | " # convert band names to band numbers using config file\n", 120 | "\n", 121 | " \n", 122 | " sql=\"\"\"\n", 123 | " select st_astiff(\n", 124 | " st_clip(\n", 125 | " st_union(\n", 126 | " st_asraster(st_intersection(t.geom,l.geom), \n", 127 | " 10::float,10::float,st_xmin(t.geom)::float, st_xmax(t.geom)::float,'8BUI',l.labelid,-9999)\n", 128 | " ),\n", 129 | " t.geom, -9999)\n", 130 | " )\n", 131 | " from {tiletable} t, fields l \n", 132 | " where t.id={tileid} and ST_Intersects(t.geom, l.geom)\n", 133 | " group by t.geom\n", 134 | " \n", 135 | " \"\"\".format(tileid=tileid, tiletable=tiletable)\n", 136 | " \n", 137 | " if verbose: print sql\n", 138 | " # following https://gis.stackexchange.com/questions/130139/downloading-raster-data-into-python-from-postgis-using-psycopg2\n", 139 | "\n", 140 | " # Download raster data into Python as GeoTIFF, and make a virtual file for GDAL\n", 141 | " curs.execute(sql)\n", 142 | " rs = curs.fetchone()[0]\n", 143 | " \n", 144 | " arr = buff2rast(rs)\n", 145 | " \n", 146 | " # Close and clean up virtual memory file\n", 147 | "\n", 148 | " curs.close()\n", 149 | "\n", 150 | " return arr\n", 151 | "\n", 152 | "def drawlabels(conn,tiletable,tileid):\n", 153 | " \n", 154 | " labelmap = pd.read_sql('select distinct labelid, label from labelmap where labelid is not null order by labelid', conn)\n", 155 | " def id2name(labelmap,labelid):\n", 156 | " return labelmap.loc[labelmap[\"labelid\"]==labelid][\"label\"].values[0]\n", 157 | " \n", 158 | " fig, ax = plt.subplots()\n", 159 | " arr = queryLabel(conn,tiletable,tileid)\n", 160 | " im = ax.imshow(np.flipud(arr), cmap=get_cmap(\"Spectral\"), interpolation='none')\n", 161 | " ax.set_title(\"Labels\")\n", 162 | " uniques=np.unique(arr)\n", 163 | " \n", 164 | " cbar=fig.colorbar(im, ax=ax )\n", 165 | " cbar.set_ticks(uniques)\n", 166 | " cbar.set_ticklabels([id2name(labelmap,t) for t in uniques])\n", 167 | " \n", 168 | "def plot(tileid,date,level):\n", 169 | " print date\n", 170 | " \n", 171 | " def m(rtype):\n", 172 | "\n", 173 | " def toidx(bands):\n", 174 | " return[bandcfg[level][rtype].index(b)+1 for b in bands]\n", 175 | "\n", 176 | " allbands=bandcfg[level][rtype]\n", 177 | " return queryRaster(conn,rastertable,tiletable, tileid, date, rtype, level, toidx(allbands)),allbands\n", 178 | "\n", 179 | " arr,bands = m(\"10m\")\n", 180 | "\n", 181 | " fig,axs=plt.subplots(1,4,figsize=(14,10))\n", 182 | " i=0\n", 183 | " for ax in axs.reshape(-1):\n", 184 | " ax.imshow(arr[:,:,i],interpolation='none')\n", 185 | " ax.set_title(bands[i])\n", 186 | " i+=1\n", 187 | "\n", 188 | " arr,bands = m(\"20m\")\n", 189 | " fig,axs=plt.subplots(2,3,figsize=(10,6))\n", 190 | " i=0\n", 191 | " for ax in axs.reshape(-1):\n", 192 | " ax.imshow(arr[:,:,i],interpolation='none')\n", 193 | " ax.set_title(bands[i])\n", 194 | " i+=1\n", 195 | "\n", 196 | " arr,bands = m(\"60m\")\n", 197 | " fig,axs=plt.subplots(1,3,figsize=(9,6))\n", 198 | " i=0\n", 199 | " for ax in axs.reshape(-1):\n", 200 | " ax.imshow(arr[:,:,i],cmap=pylab.gray(),interpolation='none')\n", 201 | " ax.set_title(bands[i])\n", 202 | " i+=1\n", 203 | "\n", 204 | "from ipywidgets import Layout\n", 205 | "def drawmap(tileid):\n", 206 | "\n", 207 | " sql=\"\"\"\n", 208 | " select\n", 209 | " ST_asgeojson(ST_Transform(t.geom,4326)), \n", 210 | " ST_asgeojson(ST_Transform(st_collect(st_intersection(t.geom,l.geom)),4326))\n", 211 | " from {} t, fields l \n", 212 | " where t.id={} and st_intersects(t.geom, l.geom)\n", 213 | " group by t.geom\n", 214 | " \"\"\".format(tiletable,tileid)\n", 215 | " \n", 216 | " cur=conn.cursor()\n", 217 | " cur.execute(sql)\n", 218 | " tile,feat = cur.fetchone()\n", 219 | " tilegeojson = json.loads(tile)\n", 220 | " featgeojson = json.loads(feat)\n", 221 | " cur.close()\n", 222 | "\n", 223 | " lon,lat = tilegeojson[\"coordinates\"][0][0]\n", 224 | " center = [lat,lon]\n", 225 | " zoom = 15\n", 226 | "\n", 227 | " tilelayer = GeoJSON(data=tilegeojson)\n", 228 | " featlayer = GeoJSON(data=featgeojson)\n", 229 | "\n", 230 | " m = Map(center=center, zoom=zoom,layout=Layout(width='50%', height='300px'))\n", 231 | "\n", 232 | " #m.add_layer(tilelayer)\n", 233 | " m.add_layer(featlayer)\n", 234 | " \n", 235 | " return m" 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "execution_count": null, 241 | "metadata": {}, 242 | "outputs": [], 243 | "source": [ 244 | "dates = queryDates(conn,\"bavaria\")[\"date\"].sort_values().values\n", 245 | "\n", 246 | "level='L1C'\n", 247 | "rastertable=\"bavaria\"\n", 248 | "tiletable=\"tiles480\"\n", 249 | "\n", 250 | "tileid=7000\n", 251 | "\n", 252 | "drawlabels(conn, tiletable, tileid)\n", 253 | "#datetime.datetime(2016,1,3)\n", 254 | "plot(tileid,dates[10],level)\n", 255 | "\n", 256 | "drawmap(tileid)" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": null, 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [ 265 | "level='L1C'\n", 266 | "rastertable=\"bavaria\"\n", 267 | "tiletable=\"tiles480\"\n", 268 | "tileid=6001\n", 269 | "\n", 270 | "drawlabels(conn,tiletable,tileid)" 271 | ] 272 | }, 273 | { 274 | "cell_type": "code", 275 | "execution_count": null, 276 | "metadata": {}, 277 | "outputs": [], 278 | "source": [ 279 | "conn.close()\n", 280 | "cur.close()\n" 281 | ] 282 | }, 283 | { 284 | "cell_type": "markdown", 285 | "metadata": {}, 286 | "source": [ 287 | "raster ST_AsRaster(geometry geom, double precision scalex, double precision scaley, double precision gridx, double precision gridy, text pixeltype, double precision value=1, double precision nodataval=0, double precision skewx=0, double precision skewy=0, boolean touched=false);" 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": null, 293 | "metadata": {}, 294 | "outputs": [], 295 | "source": [ 296 | "\n", 297 | "\n", 298 | "sql=\"\"\"\n", 299 | "select st_astiff(\n", 300 | " st_tile(\n", 301 | " st_clip(\n", 302 | " st_union(\n", 303 | " st_asraster(st_intersection(t.geom,l.geom), \n", 304 | " 10::float,10::float,st_xmin(t.geom)::float, st_xmax(t.geom)::float,'32BUI',l.code,-9999)\n", 305 | " ),\n", 306 | " t.geom, -9999),\n", 307 | " 48,48, True\n", 308 | " )\n", 309 | " )\n", 310 | " from tiles480 t, osm l \n", 311 | " where t.id=6604 and ST_Intersects(t.geom, l.geom)\n", 312 | " group by t.geom\n", 313 | " \"\"\"\n", 314 | "\n", 315 | "\n", 316 | "curs=conn.cursor()\n", 317 | "curs.execute(sql)\n", 318 | "#a = curs.fetchall()\n", 319 | "buff = curs.fetchone()" 320 | ] 321 | }, 322 | { 323 | "cell_type": "code", 324 | "execution_count": null, 325 | "metadata": {}, 326 | "outputs": [], 327 | "source": [ 328 | "vsipath = '/vsimem/from_postgis'\n", 329 | "gdal.FileFromMemBuffer(vsipath, bytes(buff[0]))\n", 330 | "\n", 331 | "# Read first band of raster with GDAL\n", 332 | "ds = gdal.Open(vsipath)\n", 333 | "\n", 334 | "band=ds.GetRasterBand(1)\n", 335 | "print ds.GetGeoTransform()\n", 336 | "gdal.Unlink(vsipath)\n", 337 | "arr = band.ReadAsArray()" 338 | ] 339 | }, 340 | { 341 | "cell_type": "code", 342 | "execution_count": null, 343 | "metadata": {}, 344 | "outputs": [], 345 | "source": [ 346 | "labelmap" 347 | ] 348 | }, 349 | { 350 | "cell_type": "code", 351 | "execution_count": null, 352 | "metadata": {}, 353 | "outputs": [], 354 | "source": [ 355 | "plt.imshow(np.flipud(arr))\n", 356 | "plt.title(np.unique(arr))" 357 | ] 358 | }, 359 | { 360 | "cell_type": "code", 361 | "execution_count": null, 362 | "metadata": {}, 363 | "outputs": [], 364 | "source": [ 365 | "u = np.unique(arr)\n", 366 | "arr[u[0]]=0 for unique in uniquevalues" 367 | ] 368 | }, 369 | { 370 | "cell_type": "code", 371 | "execution_count": null, 372 | "metadata": {}, 373 | "outputs": [], 374 | "source": [ 375 | "labelmap = pd.read_sql('select distinct labelid, label from labelmap where labelid is not null order by labelid', conn)" 376 | ] 377 | }, 378 | { 379 | "cell_type": "code", 380 | "execution_count": null, 381 | "metadata": {}, 382 | "outputs": [], 383 | "source": [ 384 | "labelmap" 385 | ] 386 | }, 387 | { 388 | "cell_type": "code", 389 | "execution_count": null, 390 | "metadata": {}, 391 | "outputs": [], 392 | "source": [ 393 | "labelmap.shape" 394 | ] 395 | }, 396 | { 397 | "cell_type": "code", 398 | "execution_count": null, 399 | "metadata": {}, 400 | "outputs": [], 401 | "source": [] 402 | } 403 | ], 404 | "metadata": { 405 | "anaconda-cloud": {}, 406 | "kernelspec": { 407 | "display_name": "Python 2", 408 | "language": "python", 409 | "name": "python2" 410 | }, 411 | "language_info": { 412 | "codemirror_mode": { 413 | "name": "ipython", 414 | "version": 2 415 | }, 416 | "file_extension": ".py", 417 | "mimetype": "text/x-python", 418 | "name": "python", 419 | "nbconvert_exporter": "python", 420 | "pygments_lexer": "ipython2" 421 | } 422 | }, 423 | "nbformat": 4, 424 | "nbformat_minor": 2 425 | } 426 | --------------------------------------------------------------------------------