├── .gitignore ├── CHANGES.txt ├── LICENSE.txt ├── MANIFEST.in ├── README.rst ├── bin └── gtloader ├── gtloader ├── __init__.py ├── __main__.py ├── arg.py ├── catalog.py ├── commands.py ├── convert.py ├── datatypes.py ├── log.py ├── projection.py └── raster.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | .ensime 3 | TAGS 4 | \#*# 5 | *~ 6 | .#* 7 | .lib 8 | *.aux.xml 9 | dist 10 | build 11 | 12 | *.pyc 13 | .project 14 | .classpath 15 | .cache 16 | .settings 17 | .history -------------------------------------------------------------------------------- /CHANGES.txt: -------------------------------------------------------------------------------- 1 | v0.1.0, 2014-01-24 -- Initial release. 2 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Licensed under the Apache License, Version 2.0 (the "License"); 2 | you may not use this file except in compliance with the License. 3 | You may obtain a copy of the License at 4 | 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CHANGES.txt LICENSE.txt README.rst -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ================= 2 | Python-GeoTrellis 3 | ================= 4 | 5 | Python-GeoTrellis provides funcitonality for working with GeoTrellis raster data. 6 | 7 | The documentation is at `the GeoTrellis documentation site`_. 8 | 9 | Installation / Setup 10 | ******************** 11 | 12 | Use pip to install the library: 13 | 14 | pip install python-geotrellis 15 | 16 | Documentation 17 | ************* 18 | 19 | See the documentation at `the GeoTrellis documentation site`_. 20 | 21 | License 22 | ******* 23 | 24 | python-geotrellis is licensed under the Apache 2.0 license. See ``LICENSE.txt`` for 25 | more details. 26 | 27 | Contribute 28 | ********** 29 | 30 | See a bug? Want to improve the docs or provide more examples? Thank you! 31 | Please open a pull-request with your improvements and we'll work to respond 32 | to it in a timely manner. 33 | 34 | The repository can be found at `the github repository`_. 35 | 36 | .. _`The GeoTrellis documentation site`: http://geotrellis.io/documentation/0.9.0/python-geotrellis/ 37 | .. _`the github repository`: http://github.com/geotrellis/python-geotrellis 38 | -------------------------------------------------------------------------------- /bin/gtloader: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | from gtloader import main 4 | 5 | if __name__ == "__main__": 6 | main(sys.argv[1:]) 7 | -------------------------------------------------------------------------------- /gtloader/__init__.py: -------------------------------------------------------------------------------- 1 | import commands 2 | 3 | def main(args): 4 | commands.main(args) 5 | -------------------------------------------------------------------------------- /gtloader/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import commands 4 | 5 | if __name__ == '__main__': 6 | commands.main(sys.argv[1:]) 7 | -------------------------------------------------------------------------------- /gtloader/arg.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code dealing with the ARG format. 3 | """ 4 | import math 5 | import struct 6 | 7 | import log 8 | from datatypes import * 9 | 10 | class ArgWriter(): 11 | """ 12 | Writes an arg in the 13 | given input datatype (such as bit, int8, float32, etc) 14 | 15 | buf is a file-like object with a method write(bytearray) 16 | datatype 17 | verify is a boolean. If 'verify' is set to true and 18 | data would be truncated, the operation will stop 19 | If 'verify' is set to false the write operation 20 | will truncate all datavalues to fit into the given 21 | datatype 22 | """ 23 | def __init__(self, buf, datatype, verify = True): 24 | self.buf = buf 25 | self.datatype = datatype 26 | self.verify = verify 27 | 28 | self.sfmt = to_struct_fmt(self.datatype) 29 | if not self.verify: 30 | self.truncate = self.get_truncator() 31 | 32 | if not self.sfmt: 33 | log.error("Couldn't find a python format for '%s'" % self.datatype) 34 | 35 | def write(self,values): 36 | """ 37 | Writes values to the buffer. 38 | 39 | values is a list-like structure with the data to write 40 | """ 41 | endian = '>' 42 | outputfmt = "%s%d%s" % (endian,len(values),self.sfmt) 43 | try: 44 | self.buf.write(struct.pack(outputfmt, *values)) 45 | except Exception, e: 46 | if self.verify: 47 | print 'Verifying data...' 48 | for v in values: 49 | #TODO: Handle bit types specially 50 | 51 | # Pack and unpack, see if we get the same value 52 | failed = False 53 | nv = None 54 | try: 55 | nv = struct.unpack(endian + self.sfmt, 56 | struct.pack(endian + self.sfmt, v))[0] 57 | except struct.error, e: 58 | print e 59 | failed = True 60 | 61 | # Note, is both nv and v are nan, check will fail: 62 | # nan != nan, but in our case we want that to be 63 | # true so we explicitly check for it here 64 | if failed or \ 65 | (nv != v and 66 | (not math.isnan(nv) and math.isnan(v))): 67 | log.error('Verification failed. Trying to '\ 68 | 'convert to %s resuled in: %s -> %s' %\ 69 | (self.datatype, v, nv)) 70 | else: # Just make it work 71 | print 'Truncating values...' 72 | for i in xrange(0,len(values)): 73 | self.buf.write(struct.pack(endian + self.sfmt, self.truncate(values[i]))) 74 | 75 | def get_truncator(self): 76 | """ Return a function that is able to truncate data 77 | based on the input type.""" 78 | def truncate_pow(exp, val): 79 | """ Given a signed interger datatype with 'exp' bits, 80 | truncate val so that it is in the interval: 81 | -2**(exp-1) <= val <= 2**(exp-1) - 1 82 | """ 83 | if math.isnan(float(val)): 84 | return -2**(exp-1) 85 | 86 | val = int(val) 87 | if val >= 2**(exp-1) - 1: 88 | return 2**(exp -1) - 1 89 | elif val < -2**(exp - 1): 90 | return -2**(exp - 1) 91 | else: 92 | return val 93 | 94 | if self.datatype == 'bit': 95 | return lambda val: int(val) & 0x1 96 | elif self.datatype == 'int8': 97 | return lambda val: truncate_pow(8, val) 98 | elif self.datatype == 'int16': 99 | return lambda val: truncate_pow(16, val) 100 | elif self.datatype == 'int32': 101 | return lambda val: truncate_pow(32, val) 102 | elif self.datatype == 'int64': 103 | return lambda val: truncate_pow(64, val) 104 | else: 105 | return lambda val: val 106 | 107 | -------------------------------------------------------------------------------- /gtloader/catalog.py: -------------------------------------------------------------------------------- 1 | """ 2 | Handles creating and modifying a GeoTrellis catalog. 3 | """ 4 | 5 | import os 6 | import json 7 | 8 | import log 9 | 10 | def catalog_add_dir(args): 11 | (head, tail) = os.path.split(args.directory) 12 | aname = args.name 13 | 14 | if aname is None or len(aname) == 0: 15 | if len(tail) == 0: 16 | tail = os.path.split(head)[1] 17 | 18 | aname = "%s:fs" % tail 19 | 20 | add_dir_to_catalog(args.catalog, args.directory, aname, args.cache_all) 21 | 22 | def catalog_has_store(catalog, storename): 23 | stores = [s for s in catalog['stores'] if s['store'] == storename] 24 | return len(stores) == 1 25 | 26 | def catalog_get_store(catalog, storename): 27 | stores = [s for s in catalog['stores'] if s['store'] == storename] 28 | 29 | if len(stores) == 0: 30 | log.error("Could not find store '%s'" % storename) 31 | elif len(stores) > 1: 32 | log.error("Invalid catalog, multiple stores with the name '%s'" % storename) 33 | else: 34 | return stores[0] 35 | 36 | def add_dir_to_catalog(catalogf, directory, name, cacheAll): 37 | catalog = json.loads(catalogf.read()) 38 | stores = catalog['stores'] 39 | 40 | if catalog_has_store(catalog, name): 41 | log.error('A datastore named "%s" already exists' % name) 42 | 43 | store = { 'store': name, 44 | 'params': { 45 | 'cacheAll': str(cacheAll), 46 | 'path': os.path.abspath(directory), 47 | 'type': 'fs' 48 | } 49 | } 50 | 51 | stores.append(store) 52 | 53 | with open(catalogf.name,'w') as cat: 54 | cat.write(json.dumps(catalog, sort_keys=True, 55 | indent=4, separators=(',', ': '))) 56 | 57 | def catalog_update(args): 58 | catalog = json.loads(args.catalog.read()) 59 | datasource = args.store 60 | field = args.field 61 | value = args.value 62 | 63 | store = catalog_get_store(catalog, datasource) 64 | 65 | if field == 'name': 66 | store['store'] = value 67 | else: 68 | if field in store['params']: 69 | store['params'][field] = value 70 | else: 71 | valid = "valid params: name, %s" % ', '.join(store['params'].keys()) 72 | log.error('Param "%s" does not exist in this data store (%s)' % ( 73 | field, valid)) 74 | 75 | with open(args.catalog.name,'w') as cat: 76 | cat.write(json.dumps(catalog, sort_keys=True, 77 | indent=4, separators=(',', ': '))) 78 | 79 | def catalog_list(args): 80 | catalog = json.loads(args.catalog.read()) 81 | 82 | print "Catalog: %s" % catalog['catalog'] 83 | 84 | if not catalog['stores']: 85 | print "[Catalog is empty]" 86 | else: 87 | for store in catalog['stores']: 88 | print store['store'] 89 | for param in store['params'].iteritems(): 90 | print " %s: %s" % param 91 | 92 | def catalog_create(args): 93 | catalog_file = args.catalog 94 | name = args.name 95 | 96 | if os.path.exists(catalog_file): 97 | log.error('A file already exists at "%s"' % catalog_file) 98 | 99 | base = { 'catalog': name, 100 | 'stores': [] } 101 | 102 | with open(catalog_file,'w') as cat: 103 | cat.write(json.dumps(base, sort_keys=True, 104 | indent=4, separators=(',', ': '))) 105 | -------------------------------------------------------------------------------- /gtloader/commands.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import argparse, os 3 | 4 | import log 5 | import catalog 6 | from convert import * 7 | from datatypes import arg 8 | from raster import Layer 9 | 10 | class InfoCommand: 11 | @staticmethod 12 | def execute(args): 13 | layer = Layer.fromPath(args.input) 14 | 15 | for i in layer.arg_metadata().iteritems(): 16 | print "%s: %s" % i 17 | 18 | @staticmethod 19 | def add_parser(subparsers): 20 | parser = subparsers.add_parser('info') 21 | 22 | parser.add_argument('input', 23 | metavar='INPUT', 24 | help='Path to input file.') 25 | parser.set_defaults(func=InfoCommand.execute) 26 | 27 | class ConvertAllCommand: 28 | @staticmethod 29 | def execute(args): 30 | if os.path.isfile(args.input): 31 | log.error("Path %s exists, but is a file." % args.input) 32 | 33 | if not os.path.isdir(args.input): 34 | log.error("Path %s does not exist." % args.input) 35 | 36 | if (args.rows_per_tile and not args.cols_per_tile) or \ 37 | (args.cols_per_tile and not args.rows_per_tile): 38 | log.error('You must specifiy both --rows-per-tile and '\ 39 | '--cols-per-tile or neither') 40 | 41 | if args.data_type == 'bit': 42 | #TODO: Support bit types 43 | log.error('bit datatype not yet supported') 44 | 45 | flist = os.listdir(args.input) 46 | if args.extension: 47 | flist = filter(lambda x: x.endswith(args.extension), flist) 48 | 49 | if not args.rows_per_tile: 50 | get_output = lambda f: os.path.join(args.output,os.path.splitext(f)[0] + '.arg') 51 | else: 52 | get_output = lambda f: args.output 53 | 54 | for f in flist: 55 | path = os.path.join(args.input,f) 56 | if not os.path.isfile(path): 57 | continue 58 | 59 | print "Converting %s" % (path) 60 | convert(path, 61 | get_output(f), 62 | args.data_type, 63 | args.band, 64 | args.name, 65 | not args.no_verify, 66 | args.cols_per_tile, 67 | args.rows_per_tile, 68 | args.legacy, 69 | args.clobber) 70 | 71 | @staticmethod 72 | def add_parser(subparsers): 73 | parser = subparsers.add_parser('convert-all') 74 | parser.add_argument('-t', '--data-type', 75 | help='Arg data type. Defaults to converting '\ 76 | 'between whatever the input datatype is. '\ 77 | 'Since unsigned types are not supported '\ 78 | 'they will automatically be promoted to '\ 79 | 'the next highest signed type.', 80 | choices=arg.datatypes, 81 | default=None) 82 | 83 | parser.add_argument('-n', '--name', 84 | default=None, 85 | help='Each layer requires a name for the '\ 86 | 'metadata. By default the name is '\ 87 | 'input file without the extension.') 88 | 89 | parser.add_argument('-b', '--band', 90 | type=int, 91 | default=1, 92 | help='A specific band to extract') 93 | 94 | parser.add_argument('-e', '--extension', 95 | help="Extension of files in the input directory to convert.") 96 | 97 | parser.add_argument('--clobber', 98 | action='store_true', 99 | help='Clobber existing files or directories.') 100 | 101 | parser.add_argument('--no-verify', 102 | action='store_true', 103 | help="Don't verify input data falls in a given "\ 104 | 'range (just truncate)') 105 | 106 | # File names 107 | parser.add_argument('input', 108 | help='Name of the input directory.') 109 | parser.add_argument('output', 110 | help='Output directory to write converted ARGs to.') 111 | # Tiles 112 | tiles_group = parser.add_argument_group('tiles') 113 | tiles_group.add_argument('--rows-per-tile', 114 | help='Number of rows per tile', 115 | default=None, 116 | type=int) 117 | tiles_group.add_argument('--cols-per-tile', 118 | help='Number of cols per tile', 119 | default=None, 120 | type=int) 121 | tiles_group.add_argument('--legacy', 122 | help='Write out the tiles in old tile format ' \ 123 | '(to work with GeoTrellis 0.8.x)', 124 | action='store_true') 125 | 126 | parser.set_defaults(func=ConvertAllCommand.execute) 127 | 128 | class ConvertCommand: 129 | @staticmethod 130 | def execute(args): 131 | if not os.path.isfile(args.input): 132 | log.error("Path %s does not exist." % args.input) 133 | 134 | if (args.rows_per_tile and not args.cols_per_tile) or \ 135 | (args.cols_per_tile and not args.rows_per_tile): 136 | log.error('You must specifiy both --rows-per-tile and '\ 137 | '--cols-per-tile or neither') 138 | 139 | if args.data_type == 'bit': 140 | #TODO: Support bit types 141 | log.error('bit datatype not yet supported') 142 | 143 | convert(args.input, 144 | args.output, 145 | args.data_type, 146 | args.band, 147 | args.name, 148 | not args.no_verify, 149 | args.cols_per_tile, 150 | args.rows_per_tile, 151 | args.legacy, 152 | args.clobber) 153 | 154 | @staticmethod 155 | def add_parser(subparsers): 156 | convert_parser = subparsers.add_parser('convert') 157 | convert_parser.add_argument('-t', '--data-type', 158 | help='Arg data type. Defaults to converting '\ 159 | 'between whatever the input datatype is. '\ 160 | 'Since unsigned types are not supported '\ 161 | 'they will automatically be promoted to '\ 162 | 'the next highest signed type.', 163 | choices=arg.datatypes, 164 | default=None) 165 | 166 | convert_parser.add_argument('-n', '--name', 167 | default=None, 168 | help='Each layer requires a name for the '\ 169 | 'metadata. By default the name is '\ 170 | 'input file without the extension.') 171 | 172 | convert_parser.add_argument('-b', '--band', 173 | type=int, 174 | default=1, 175 | help='A specific band to extract') 176 | 177 | convert_parser.add_argument('--clobber', 178 | action='store_true', 179 | help='Clobber existing files or directories.') 180 | 181 | convert_parser.add_argument('--no-verify', 182 | action='store_true', 183 | help="Don't verify input data falls in a given "\ 184 | 'range (just truncate)') 185 | 186 | # File names 187 | convert_parser.add_argument('input', 188 | help='Name of the input file') 189 | convert_parser.add_argument('output', 190 | help='Output file to write') 191 | # Tiles 192 | tiles_group = convert_parser.add_argument_group('tiles') 193 | tiles_group.add_argument('--rows-per-tile', 194 | help='Number of rows per tile', 195 | default=None, 196 | type=int) 197 | tiles_group.add_argument('--cols-per-tile', 198 | help='Number of cols per tile', 199 | default=None, 200 | type=int) 201 | tiles_group.add_argument('--legacy', 202 | help='Write out the tiles in old tile format ' \ 203 | '(to work with GeoTrellis 0.8.x)', 204 | action='store_true') 205 | 206 | convert_parser.set_defaults(func=ConvertCommand.execute) 207 | 208 | class CatalogCommand: 209 | class ListCommand: 210 | @staticmethod 211 | def execute(args): 212 | catalog.catalog_list(args) 213 | 214 | @classmethod 215 | def add_parser(cls,subparsers): 216 | catalog_list_parser = subparsers.add_parser('list') 217 | catalog_list_parser.add_argument('catalog', 218 | metavar='CATALOG', 219 | help='Path to catalog file', 220 | type=argparse.FileType('r')) 221 | 222 | catalog_list_parser.set_defaults(func=cls.execute) 223 | 224 | class UpdateCommand: 225 | @staticmethod 226 | def execute(args): 227 | catalog.catalog_update(args) 228 | 229 | @classmethod 230 | def add_parser(cls,subparsers): 231 | catalog_upd_parser = subparsers.add_parser('update') 232 | catalog_upd_parser.add_argument('catalog', 233 | help='Path to catalog file', 234 | type=argparse.FileType('rw')) 235 | 236 | catalog_upd_parser.add_argument('store', 237 | help='Data store to update') 238 | catalog_upd_parser.add_argument('field', 239 | help='Field to update') 240 | catalog_upd_parser.add_argument('value', 241 | help='New value') 242 | 243 | catalog_upd_parser.set_defaults(func=cls.execute) 244 | 245 | class AddDirectoryCommand: 246 | @staticmethod 247 | def execute(args): 248 | catalog.catalog_add_dir(args) 249 | 250 | @classmethod 251 | def add_parser(cls,subparsers): 252 | parser = subparsers.add_parser('add-dir') 253 | parser.add_argument('catalog', 254 | help='Path to catalog file', 255 | type=argparse.FileType('rw')) 256 | 257 | parser.add_argument('directory', 258 | help='Directory to add') 259 | 260 | parser.add_argument('--name', 261 | help='Name of the datasource, defaults '\ 262 | 'to the name of the directory with :fs') 263 | 264 | parser.add_argument('--cache-all', 265 | help='Set the cache all field to true', 266 | action='store_true') 267 | parser.set_defaults(func=cls.execute) 268 | 269 | class CreateCommand: 270 | @staticmethod 271 | def execute(args): 272 | catalog.catalog_create(args) 273 | 274 | @classmethod 275 | def add_parser(cls,subparsers): 276 | parser = subparsers.add_parser('create') 277 | parser.add_argument('catalog', 278 | help='Path to catalog file') 279 | parser.add_argument('name', 280 | nargs='?', 281 | help='Name of the catalog', 282 | default='catalog') 283 | 284 | parser.set_defaults(func=cls.execute) 285 | 286 | @classmethod 287 | def add_parser(cls,subparsers): 288 | catalog_parser = subparsers.add_parser('catalog') 289 | catalog_subparsers = catalog_parser.add_subparsers() 290 | 291 | cls.ListCommand.add_parser(catalog_subparsers) 292 | cls.UpdateCommand.add_parser(catalog_subparsers) 293 | cls.AddDirectoryCommand.add_parser(catalog_subparsers) 294 | cls.CreateCommand.add_parser(catalog_subparsers) 295 | 296 | def main(args): 297 | parser = argparse.ArgumentParser() 298 | subparsers = parser.add_subparsers() 299 | 300 | InfoCommand.add_parser(subparsers) 301 | ConvertAllCommand.add_parser(subparsers) 302 | ConvertCommand.add_parser(subparsers) 303 | CatalogCommand.add_parser(subparsers) 304 | 305 | parsed_args = parser.parse_args(args) 306 | parsed_args.func(parsed_args) 307 | -------------------------------------------------------------------------------- /gtloader/convert.py: -------------------------------------------------------------------------------- 1 | """ 2 | Converts between GDAL and python rasters. 3 | """ 4 | import sys, os, shutil 5 | import math 6 | import json 7 | import log 8 | from raster import Layer, GdalLayer, Extent 9 | from datatypes import * 10 | 11 | def convert(inputPath, 12 | output_path, 13 | data_type = None, 14 | band = 1, 15 | layer_name = None, 16 | verify = True, 17 | rows_per_tile = None, 18 | cols_per_tile = None, 19 | legacy = False, 20 | clobber = False): 21 | if band != 1: 22 | layer = GdalLayer(inputPath,band) 23 | else: 24 | layer = Layer.fromPath(inputPath) 25 | layer.init_data() 26 | 27 | raster_extent = layer.raster_extent() 28 | 29 | log.notice("Loading raster with width %s, height %s" % 30 | (raster_extent.cols, raster_extent.rows)) 31 | 32 | # Can't process to regular arg if size is greater than 33 | # java array 34 | if raster_extent.cols * raster_extent.rows > 2**31: 35 | log.error('Size (%s) too big for standard arg. '\ 36 | 'Use the "--rows-per-tile" and "--cols-per-tile" options'\ 37 | 'to create a tiled raster instead') 38 | 39 | if data_type is None: 40 | data_type = layer.data_type() 41 | 42 | if to_struct_fmt(data_type) is None: 43 | log.error('Could not determine datatype') 44 | 45 | if not layer_name: 46 | layer_name = '.'.join(os.path.basename(inputPath).split('.')[:-1]) 47 | 48 | if rows_per_tile and cols_per_tile: 49 | if os.path.isfile(output_path): 50 | log.error('Output path %s is a file.' % output_path) 51 | 52 | # Check if output is ready for files 53 | tile_dir = os.path.join(output_path, layer_name) 54 | if legacy: 55 | # GeoTrellis 0.8.x puts the metadata file inside the 56 | # tile directory, and names it layout.json 57 | json_path = os.path.join(tile_dir,'layout.json') 58 | else: 59 | json_path = os.path.join(output_path,'%s.json' % layer_name) 60 | 61 | if os.path.exists(tile_dir): 62 | if clobber: 63 | shutil.rmtree(tile_dir) 64 | else: 65 | log.error('Output directory %s already exists' % tile_dir) 66 | 67 | if os.path.exists(json_path): 68 | if clobber: 69 | os.remove(json_path) 70 | else: 71 | log.error('File %s already exists' % json_path) 72 | 73 | layer.write_tiled(tile_dir, 74 | json_path, 75 | layer_name, 76 | rows_per_tile, 77 | cols_per_tile, 78 | data_type = data_type, verify = verify) 79 | 80 | print 'Tile conversion completed.' 81 | else: 82 | if os.path.isdir(output_path): 83 | output_path = os.path.join(output_path, layer_name + '.arg') 84 | elif output_path.endswith('.json'): 85 | output_path = output_path[:-5] + '.arg' 86 | elif not output_path.endswith('.arg'): 87 | output_path += '.arg' 88 | 89 | metadata_file = output_path[0:-4] + '.json' 90 | 91 | if os.path.exists(output_path): 92 | if clobber: 93 | os.remove(output_path) 94 | else: 95 | log.error("File %s already exists" % output_path) 96 | 97 | if os.path.exists(metadata_file): 98 | if clobber: 99 | os.remove(metadata_file) 100 | else: 101 | log.error("File %s already exists" % metadata_file) 102 | 103 | layer.write_arg(output_path, data_type = data_type, verify = verify) 104 | layer.write_metadata(metadata_file, output_path, layer_name, data_type) 105 | layer.close() 106 | -------------------------------------------------------------------------------- /gtloader/datatypes.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module deals with raster data types. 3 | """ 4 | 5 | from gdalconst import * 6 | 7 | class arg: 8 | datatypes = ['bit','int8','int16','int32','float32','float64'] 9 | 10 | # Convert between gdal datatypes 11 | # and python struct package format strings 12 | gdt_datatype_map = { 13 | GDT_Byte: 'b', 14 | GDT_CInt16: 'h', 15 | GDT_Int16: 'h', 16 | GDT_CInt32: 'i', 17 | GDT_Int32: 'i', 18 | GDT_UInt16: 'H', 19 | GDT_UInt32: 'I', 20 | GDT_Float32: 'f', 21 | GDT_CFloat32: 'f', 22 | GDT_Float64: 'd' 23 | } 24 | 25 | # Convert between gdal datatypes 26 | # and arg datatypes 27 | gdal_arg_datatype_map = { 28 | GDT_Byte: 'int8', 29 | GDT_CInt16: 'int16', 30 | GDT_Int16: 'int16', 31 | GDT_CInt32: 'int32', 32 | GDT_Int32: 'int32', 33 | GDT_UInt16: 'int32', 34 | GDT_UInt32: 'float32', 35 | GDT_Float32: 'float32', 36 | GDT_CFloat32: 'float32', 37 | GDT_Float64: 'float64' 38 | } 39 | 40 | # Convert between ARG datatypes 41 | # and python struct package format strings 42 | inp_datatype_map = { 43 | 'bit': 'bit', 44 | 'int8': 'b', 45 | 'int16': 'h', 46 | 'int32': 'i', 47 | 'float32': 'f', 48 | 'float64': 'd' 49 | } 50 | 51 | # Maps ARG datatypes to NoData values 52 | nodata_map = { 53 | 'bit': 0, 54 | 'int8': -2**7, 55 | 'int16': -2**15, 56 | 'int32': -2**31, 57 | 'float32': float('nan'), 58 | 'float64': float('nan') 59 | } 60 | 61 | datatype_size_map = { 62 | 'bit': 1, 63 | 'int8': 8, 64 | 'int16': 16, 65 | 'int32': 32, 66 | 'float32': 32, 67 | 'float64': 64 68 | } 69 | 70 | def to_datatype_str(n): 71 | """ Convert from integer GDAL datatypes to 72 | python struct format strings """ 73 | return gdt_datatype_map.get(n, None) 74 | 75 | def to_datatype_arg(n): 76 | """ Convert from integer GDAL datatypes to 77 | arg datatypes """ 78 | return gdal_arg_datatype_map.get(n, None) 79 | 80 | def to_struct_fmt(n): 81 | """ Convert between input datatypes (int8, float32, etc) 82 | to python struct format strings """ 83 | return inp_datatype_map.get(n, None) 84 | 85 | def nodata_for_fmt(n): 86 | """ Convert between input datatypes (int8, float32, etc) 87 | to python struct format strings """ 88 | return nodata_map.get(n, None) 89 | -------------------------------------------------------------------------------- /gtloader/log.py: -------------------------------------------------------------------------------- 1 | """ 2 | Cheap and easy logging 3 | probably should use python logging 4 | """ 5 | 6 | def error(m,code=1): 7 | print 'ERROR: %s' % m 8 | exit(code) 9 | 10 | def warn(m): 11 | print 'WARNING: %s' % m 12 | 13 | def notice(m): 14 | print 'NOTICE: %s' % m 15 | -------------------------------------------------------------------------------- /gtloader/projection.py: -------------------------------------------------------------------------------- 1 | import osr 2 | 3 | import log 4 | 5 | def get_epsg(raster): 6 | """ Get the EPSG code from a raster or quit if there is an error """ 7 | sr = osr.SpatialReference(raster.GetProjection()) 8 | sr.AutoIdentifyEPSG() 9 | 10 | auth = sr.GetAttrValue('AUTHORITY',0) 11 | epsg = sr.GetAttrValue('AUTHORITY',1) 12 | 13 | if auth is None or epsg is None or auth.lower() != 'epsg': 14 | log.warn('Could not get EPSG projection') 15 | return 0 16 | else: 17 | return epsg 18 | -------------------------------------------------------------------------------- /gtloader/raster.py: -------------------------------------------------------------------------------- 1 | """ 2 | Defines classes that represent Rasters. 3 | """ 4 | import os, sys 5 | import struct 6 | import array 7 | import json 8 | import gdal 9 | import math 10 | 11 | import log 12 | import projection 13 | from arg import ArgWriter 14 | from datatypes import * 15 | 16 | class Extent(): 17 | def __init__(self, xmin, ymin, xmax, ymax): 18 | self.xmin = xmin 19 | self.ymin = ymin 20 | self.xmax = xmax 21 | self.ymax = ymax 22 | 23 | def __str__(self): 24 | return "Extent(%f,%f,%f,%f)" % (self.xmin,self.ymin,self.xmax,self.ymax) 25 | 26 | class RasterExtent(): 27 | def __init__(self, extent, cellwidth, cellheight, cols, rows): 28 | self.extent = extent 29 | self.cellwidth = cellwidth 30 | self.cellheight = cellheight 31 | self.cols = cols 32 | self.rows = rows 33 | 34 | class Layer: 35 | @staticmethod 36 | def fromPath(path): 37 | #Determine how to read the layer 38 | if path.endswith(".arg"): 39 | path = path[:-4] + ".json" 40 | if not os.path.exists(path): 41 | log.error("Could not find arg metadata file at " + path) 42 | if path.endswith(".json"): 43 | #Read metadata 44 | jf = open(path) 45 | meta = json.load(jf) 46 | jf.close() 47 | 48 | layer_type = meta["type"] 49 | if "path" in meta: 50 | data_path = meta["path"] 51 | else: 52 | data_path = path[:-5] + ".arg" 53 | 54 | if layer_type == "geotiff": 55 | if not data_path: 56 | data_path = "%s.tif" % path[:-5] 57 | return GdalLayer(meta,data_path) 58 | # elif layer_type == "tiled": 59 | # if not data_path: 60 | # data_path = path[:-5] 61 | # return TiledLayer(meta,data_path) 62 | # elif layer_type == "constant": 63 | # return ConstantLayer(meta,data_path) 64 | elif layer_type == "arg": 65 | return ArgLayer(meta,path,data_path) 66 | else: 67 | log.error("Layer type %s in file %s is not support" % (layer_type,path)) 68 | else: 69 | return GdalLayer(path) 70 | 71 | def init_data(self): 72 | raise "Must override in child class" 73 | 74 | def close(self): 75 | raise "Must override in child class" 76 | 77 | def data_type(self): 78 | raise "Must override in child class" 79 | 80 | def raster_extent(self): 81 | raise "Must override in child class" 82 | 83 | def no_data_value(self): 84 | raise "Must override in child class" 85 | 86 | def read_row(self,row,offset=0,size=None): 87 | raise "Must override in child class" 88 | 89 | def arg_metadata(self): 90 | raise "Must override in child class" 91 | 92 | @staticmethod 93 | def get_layer_name(path): 94 | return '.'.join(os.path.basename(path).split('.')[:-1]) 95 | 96 | def write_metadata(self, path, data_path, name=None, data_type = None, additional = None): 97 | if not name: 98 | name = Layer.get_layer_name(data_path) 99 | 100 | m = self.arg_metadata() 101 | m['layer'] = name 102 | if data_type: 103 | m['datatype'] = data_type 104 | if additional: 105 | m = dict(m.items() + additional.items()) 106 | 107 | with file(path,'w') as mdf: 108 | mdf.write( 109 | json.dumps(m, 110 | sort_keys=True, 111 | indent=4, 112 | separators=(',',': ')) + '\n') 113 | 114 | def write_arg(self, 115 | data_path, 116 | data_type = None, 117 | window = None, 118 | printprg = False, 119 | verify = True): 120 | """ 121 | Writes this layer out as an ARG. 122 | 123 | Parameters: 124 | path: Path to the arg file to be generated. 125 | data_type: Data type of the output arg. 126 | window: 4-tuple of (col_min, row_min, col_max, row_max) 127 | of the window of the raster to be exported. 128 | Defaults to the entire raster. 129 | verify: Set to false to not verify data type conversion (just truncate) 130 | printprg: Set to false to suppress printing progress. 131 | """ 132 | this_data_type = self.data_type() 133 | raster_extent = self.raster_extent() 134 | 135 | if not data_type: 136 | data_type = this_data_type 137 | 138 | # Process the windows for clipping 139 | if window is None: 140 | window = (0, 0, raster_extent.cols, raster_extent.rows) 141 | 142 | if len(window) != 4: 143 | log.error('Invalid window: %s' % window) 144 | 145 | start_col, end_col = window[0], window[2] 146 | total_cols = end_col - start_col 147 | 148 | start_row, end_row = window[1], window[3] 149 | total_rows = end_row - start_row 150 | 151 | ndv = self.no_data_value() 152 | 153 | # if NoData is 128 and type is byte, 128 is read in as -128 154 | if this_data_type == 'int8' and ndv == 128: 155 | ndv = -128 156 | 157 | arg_no_data = nodata_for_fmt(data_type) 158 | 159 | psize = int(total_rows / 100) 160 | 161 | output = file(data_path, 'wb') 162 | writer = ArgWriter(output,data_type,verify) 163 | 164 | # If needed, add NoData values to the start or end of the row 165 | prerow = [arg_no_data] * (0 - start_col) 166 | start_col = max(0,start_col) 167 | postrow = [arg_no_data] * (end_col - raster_extent.cols) 168 | total_cols_to_scan = min(end_col, raster_extent.cols) - start_col 169 | 170 | for row in xrange(start_row,end_row): 171 | if printprg and psize != 0 and row % psize == 0: 172 | row_progress = row - start_row 173 | sys.stdout.write('%d/%d (%d%%) completed\r' % (row_progress, 174 | total_rows, 175 | row_progress*100/total_rows)) 176 | sys.stdout.flush() 177 | output.flush() 178 | 179 | if row < 0 or row >= raster_extent.rows: 180 | ar = [arg_no_data] * total_cols 181 | else: 182 | ar = self.read_row(row,start_col,total_cols_to_scan) 183 | if prerow: 184 | ar = prerow + ar 185 | if postrow: 186 | ar = ar + postrow 187 | 188 | # Replace nodata before verification 189 | data = array.array(to_struct_fmt(data_type), ar) 190 | for i in xrange(0,len(data)): 191 | if data[i] == ndv: 192 | data[i] = arg_no_data 193 | 194 | writer.write(data) 195 | 196 | output.flush() 197 | output.close() 198 | 199 | if printprg: 200 | print "%d/%d (100%%) completed" % (total_rows,total_rows) 201 | 202 | def write_tiled(self, 203 | tile_dir, 204 | metadata_path, 205 | layer_name, 206 | tile_row_size, 207 | tile_col_size, 208 | data_type = None, 209 | printprg = True, 210 | verify = True): 211 | raster_extent = self.raster_extent() 212 | 213 | ntilecols = int(math.ceil(raster_extent.cols / float(tile_col_size))) 214 | ntilerows = int(math.ceil(raster_extent.rows / float(tile_row_size))) 215 | 216 | nrows = ntilerows * tile_row_size 217 | ncols = ntilecols * tile_col_size 218 | 219 | metadata = self.arg_metadata() 220 | tile_metadata = metadata.copy() 221 | 222 | tile_metadata['type'] = 'tiled' 223 | tile_metadata['layer'] = layer_name 224 | tile_metadata['path'] = os.path.basename(tile_dir) 225 | 226 | tile_metadata['tile_base'] = layer_name 227 | tile_metadata['layout_cols'] = str(ntilecols) 228 | tile_metadata['layout_rows'] = str(ntilerows) 229 | tile_metadata['pixel_cols'] = str(tile_col_size) 230 | tile_metadata['pixel_rows'] = str(tile_row_size) 231 | tile_metadata['cols'] = str(ncols) 232 | tile_metadata['rows'] = str(nrows) 233 | 234 | maxy = raster_extent.extent.ymax 235 | minx = raster_extent.extent.xmin 236 | cellwidth = raster_extent.cellwidth 237 | cellheight = raster_extent.cellheight 238 | 239 | # Account for tile division might extend past east and south extent. 240 | tile_xmax = minx + (cellwidth * tile_col_size * ntilecols) 241 | tile_ymin = maxy - (cellheight * tile_row_size * ntilerows) 242 | 243 | tile_metadata['xmax'] = tile_xmax 244 | tile_metadata['ymin'] = tile_ymin 245 | 246 | # Make the directory that will hold the tiles 247 | os.makedirs(tile_dir) 248 | 249 | with file(metadata_path,'w') as mdf: 250 | mdf.write( 251 | json.dumps(tile_metadata, 252 | sort_keys=True, 253 | indent=4, 254 | separators=(',',': ')) + '\n') 255 | 256 | tile_name_ft = '%s_%%d_%%d' % layer_name 257 | tile_path_ft = os.path.join(tile_dir,'%s.arg' % tile_name_ft) 258 | 259 | total_tiles = ntilerows * ntilecols 260 | 261 | for row in xrange(0,ntilerows): 262 | ystart = row * tile_row_size 263 | yend = ystart + tile_row_size 264 | 265 | for col in xrange(0,ntilecols): 266 | tileindex = col + row*ntilecols + 1 267 | p = int(tileindex * 100.0 / total_tiles) 268 | 269 | sys.stdout.write('Tile %d/%d (%d%%)\n' % 270 | (tileindex,total_tiles,p)) 271 | sys.stdout.flush() 272 | 273 | xstart = col * tile_col_size 274 | xend = xstart + tile_col_size 275 | 276 | # Get tile extent 277 | xmin = minx + xstart * cellwidth 278 | xmax = xmin + (cellwidth * tile_col_size) 279 | ymax = maxy - (cellheight * ystart) 280 | ymin = ymax - (cellheight * tile_row_size) 281 | 282 | filename = tile_path_ft % (col,row) 283 | 284 | newmetadata = metadata.copy() 285 | # Shift xmin and recalculate xmax 286 | newmetadata['layer'] = tile_name_ft % (col,row) 287 | newmetadata['xmin'] = xmin 288 | newmetadata['ymin'] = ymin 289 | newmetadata['xmax'] = xmax 290 | newmetadata['ymax'] = ymax 291 | newmetadata['rows'] = tile_row_size 292 | newmetadata['cols'] = tile_col_size 293 | 294 | self.write_arg(filename, window = (xstart,ystart,xend,yend), verify = verify) 295 | 296 | metadata_file = filename[0:-4] + '.json' 297 | with file(metadata_file,'w') as mdf: 298 | mdf.write( 299 | json.dumps(newmetadata, 300 | sort_keys=True, 301 | indent=4, 302 | separators=(',',': ')) + '\n') 303 | 304 | class ArgLayer(Layer): 305 | def __init__(self,meta,meta_path,data_path): 306 | self.meta = meta 307 | self.meta_path = meta_path 308 | self.data_path = data_path 309 | 310 | self.layer_type = meta["type"] 311 | 312 | self._data_type = meta["datatype"] 313 | self._no_data_value = nodata_map[self._data_type] 314 | self.name = meta["layer"] 315 | 316 | if "path" in meta: 317 | self.path = meta["path"] 318 | else: 319 | self.path = meta_path[:-4] + ".arg" 320 | 321 | self.size = datatype_size_map[self._data_type] 322 | 323 | xmin = float(meta["xmin"]) 324 | ymin = float(meta["ymin"]) 325 | xmax = float(meta["xmax"]) 326 | ymax = float(meta["ymax"]) 327 | extent = Extent(xmin,ymin,xmax,ymax) 328 | 329 | cw = float(meta["cellwidth"]) 330 | ch = float(meta["cellheight"]) 331 | 332 | cols = int(meta["cols"]) 333 | rows = int(meta["rows"]) 334 | 335 | self._raster_extent = RasterExtent(extent,cw,ch,cols,rows) 336 | 337 | self.epsg = meta["epsg"] 338 | self.xskew = meta["xskew"] 339 | self.yskew = meta["yskew"] 340 | 341 | self.fchar = inp_datatype_map[self._data_type] 342 | 343 | def init_data(self): 344 | self.data_file = open(self.data_path, "rb") 345 | 346 | def close(self): 347 | self.data_file.close() 348 | 349 | def data_type(self): 350 | return self._data_type 351 | 352 | def raster_extent(self): 353 | return self._raster_extent 354 | 355 | def no_data_value(self): 356 | return self._no_data_value 357 | 358 | def read_row(self,row,offset=0,cols=None): 359 | if not cols: 360 | cols = self.raster_extent().cols 361 | read_length = int(cols*(math.ceil(self.size/8))) 362 | self.seek(offset,row) 363 | data = self.data_file.read(read_length) 364 | endian = '>' 365 | inputfmt = "%s%d%s" % (endian,cols,self.fchar) 366 | return list(struct.unpack(inputfmt,data)) 367 | 368 | def arg_metadata(self): 369 | re = self.raster_extent() 370 | extent = re.extent 371 | return { 372 | 'type': 'arg', 373 | 'datatype': self.data_type(), 374 | 'xmin': extent.xmin, 375 | 'ymin': extent.ymin, 376 | 'xmax': extent.xmax, 377 | 'ymax': extent.ymax, 378 | 'cellwidth': re.cellwidth, 379 | 'cellheight': re.cellheight, 380 | 'rows': re.rows, 381 | 'cols': re.cols, 382 | 'xskew': 0, 383 | 'yskew': 0, 384 | 'epsg': int(self.epsg) 385 | } 386 | 387 | def seek(self,col,row): 388 | pos = ((row * self.raster_extent().cols) + col) * math.ceil(self.size/8) 389 | self.data_file.seek(pos) 390 | 391 | class GdalLayer(Layer): 392 | """ 393 | Represents one band in a raster as read by GDAL. 394 | """ 395 | def __init__(self, path, band = 1): 396 | self.path = path 397 | self.band = band 398 | 399 | def init_data(self): 400 | self.dataset = gdal.Open(self.path, GA_ReadOnly) 401 | 402 | bands = self.dataset.RasterCount 403 | if bands < self.band: 404 | log.error("This raster does not contain a band number %d" % band) 405 | self.band = self.dataset.GetRasterBand(self.band) 406 | 407 | self.epsg = projection.get_epsg(self.dataset) 408 | xmin,xres,rot1,ymin,rot2,yres = self.dataset.GetGeoTransform() 409 | self.rot1 = rot1 410 | self.rot2 = rot2 411 | 412 | cols = self.dataset.RasterXSize 413 | rows = self.dataset.RasterYSize 414 | 415 | xmax = xmin + cols*xres 416 | ymax = ymin + rows*yres 417 | 418 | # Since xres and yres can be negative, 419 | # we simply use min/max to select the proper bounding 420 | # box 421 | extent = Extent(min(xmin,xmax), 422 | min(ymin,ymax), 423 | max(xmin,xmax), 424 | max(ymin,ymax)) 425 | 426 | self._raster_extent = RasterExtent(extent,abs(xres),abs(yres),cols,rows) 427 | 428 | self._data_type = to_datatype_arg(self.band.DataType) 429 | self.fchar = to_datatype_str(self.band.DataType) 430 | 431 | def close(self): 432 | self.dataset = None 433 | 434 | def data_type(self): 435 | return self._data_type 436 | 437 | def raster_extent(self): 438 | return self._raster_extent 439 | 440 | def no_data_value(self): 441 | ndv = self.band.GetNoDataValue() 442 | # if NoData is 128 and type is byte, 128 is read in as -128 443 | if self.data_type() == 'int8' and ndv == 128: 444 | ndv = -128 445 | return ndv 446 | 447 | def read_row(self,row,offset=0,size=None): 448 | """ 449 | Reads a row from this raster. 450 | Optionally you can specify an offset and size 451 | to read only a section of the row. 452 | """ 453 | # Network Byte Order (Big Endian) 454 | if size is None: 455 | size = self.cols 456 | 457 | unpack_str = '%d%s' % (size,self.fchar) 458 | 459 | scanline = self.band.ReadRaster( 460 | offset, row, size, 1, size, 1, self.band.DataType) 461 | 462 | return list(struct.unpack(unpack_str, scanline)) 463 | 464 | def arg_metadata(self): 465 | re = self.raster_extent() 466 | extent = re.extent 467 | 468 | return { 469 | 'type': 'arg', 470 | 'datatype': self.data_type(), 471 | 'xmin': extent.xmin, 472 | 'ymin': extent.ymin, 473 | 'xmax': extent.xmax, 474 | 'ymax': extent.ymax, 475 | 'cellwidth': re.cellwidth, 476 | 'cellheight': re.cellheight, 477 | 'rows': re.rows, 478 | 'cols': re.cols, 479 | 'xskew': 0, 480 | 'yskew': 0, 481 | 'epsg': int(self.epsg) 482 | } 483 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name='python-geotrellis', 5 | version='0.1.0', 6 | author='Azavea', 7 | author_email='info@azavea.com', 8 | maintainer='Rob Emanuele', 9 | maintainer_email='remanuele@azavea.com', 10 | packages=find_packages(), 11 | url=['http://github.com/geotrellis/python-geotrellis'], 12 | scripts=['bin/gtloader'], 13 | license='LICENSE.txt', 14 | description='GeoTrellis library for mananging raster data.', 15 | long_description=open('README.rst').read(), 16 | install_requires=["gdal >= 1.9.1"], 17 | classifiers=[ 18 | 'Development Status :: 4 - Beta', 19 | 'License :: OSI Approved :: Apache Software License', 20 | 'Topic :: Scientific/Engineering :: GIS', 21 | 'Programming Language :: Python :: 2.7' 22 | ], 23 | ) 24 | --------------------------------------------------------------------------------