├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── converter │ ├── converter.go │ └── testfiles │ │ ├── linestring.geojson │ │ └── polygon.geojson ├── inspect │ └── inspect.go ├── spatialize │ ├── spatialize.go │ └── vendor │ │ ├── github.com │ │ ├── golang │ │ │ └── protobuf │ │ │ │ ├── LICENSE │ │ │ │ └── proto │ │ │ │ ├── Makefile │ │ │ │ ├── clone.go │ │ │ │ ├── decode.go │ │ │ │ ├── encode.go │ │ │ │ ├── equal.go │ │ │ │ ├── extensions.go │ │ │ │ ├── lib.go │ │ │ │ ├── message_set.go │ │ │ │ ├── pointer_reflect.go │ │ │ │ ├── pointer_unsafe.go │ │ │ │ ├── properties.go │ │ │ │ ├── text.go │ │ │ │ └── text_parser.go │ │ └── thomersch │ │ │ └── gosmparse │ │ │ ├── LICENSE │ │ │ ├── OSMPBF │ │ │ ├── fileformat.pb.go │ │ │ ├── fileformat.proto │ │ │ ├── osmformat.pb.go │ │ │ └── osmformat.proto │ │ │ ├── README.md │ │ │ ├── decoder.go │ │ │ ├── doc.go │ │ │ ├── elements.go │ │ │ ├── interface.go │ │ │ └── tags.go │ │ └── vendor.json └── tiler │ ├── .gitignore │ ├── featurecache.go │ ├── featurecache_fs.go │ ├── featurecache_leveldb.go │ ├── s3.go │ ├── s3_nop.go │ ├── shuffle.go │ ├── shuffle_pre1_10.go │ └── tiler.go ├── fileformat ├── README.md ├── ff.xml.svg └── fileformat.proto ├── go.mod ├── go.sum ├── lib ├── csv │ ├── codec.go │ ├── codec_test.go │ ├── csv.go │ └── testfiles │ │ └── gn_excerpt.csv ├── geojson │ ├── codec.go │ ├── codec_test.go │ ├── ogc_srid.go │ └── testdata │ │ ├── 01.geojson │ │ ├── id.geojson │ │ └── multipolygon.geojson ├── geojsonseq │ ├── codec.go │ ├── codec_test.go │ └── testdata │ │ └── example.geojsonseq ├── mapping │ ├── README.md │ ├── condition.go │ ├── default.go │ ├── file.go │ ├── file_test.go │ ├── mapping.yml │ ├── ops.go │ └── types.go ├── mvt │ ├── README.md │ ├── codec.go │ ├── codec_test.go │ ├── project.go │ ├── project_test.go │ └── vector_tile │ │ ├── README.md │ │ ├── generate.go │ │ ├── vector_tile.pb.go │ │ └── vector_tile.proto ├── progressbar │ └── progressbar.go ├── spaten │ ├── chunks.go │ ├── codec.go │ ├── codec_test.go │ ├── fileformat │ │ ├── fileformat.pb.go │ │ ├── generate.go │ │ └── transform.go │ ├── fuzz.go │ ├── lowlevel.go │ └── lowlevel_test.go ├── spatial │ ├── README.md │ ├── bbox.go │ ├── bbox.svg │ ├── bbox_test.go │ ├── clip_geos.go │ ├── clip_geos_test.go │ ├── clip_golang.go │ ├── clip_golang_test.go │ ├── codec.go │ ├── conversion.go │ ├── conversion_test.go │ ├── fuzz.go │ ├── geom.go │ ├── geom_test.go │ ├── line.go │ ├── line_test.go │ ├── merge.go │ ├── merge_test.go │ ├── point.go │ ├── point_test.go │ ├── polygon.go │ ├── polygon_test.go │ ├── projectable.go │ ├── spatial.go │ ├── string_1_10.go │ ├── string_1_9.go │ ├── testfiles │ │ ├── featurecollection.geojson │ │ ├── mergable_lines.geojson │ │ ├── polygon.json │ │ ├── polygon.wkb │ │ ├── polygon_with_holes.geojson │ │ ├── self_intersect.geojson │ │ └── winding_wild.geojson │ ├── topology.go │ ├── twkb.go │ ├── twkb_test.go │ ├── wkb.go │ └── wkb_test.go └── tile │ ├── bbox.go │ ├── bbox_test.go │ ├── codec.go │ ├── tile.go │ └── tile_test.go └── viewer ├── .gitignore ├── README.md ├── index.html ├── package-lock.json ├── package.json └── style.json /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - "1.12" 4 | - "1.13" 5 | - "1.14" 6 | script: 7 | - make build 8 | - make test 9 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GO := go 2 | GOBUILDOPTS ?= -v 3 | GOTESTOPTS := -v 4 | BINPATH := bin 5 | CMDPREFIX := github.com/thomersch/grandine/cmd 6 | 7 | export CGO_CFLAGS=-I. -I/usr/local/include 8 | export CGO_LDFLAGS=-L/usr/local/lib 9 | 10 | export GO111MODULE=on 11 | 12 | build: build-converter build-inspect build-spatialize build-tiler 13 | 14 | build-converter: 15 | $(GO) build $(GOBUILDOPTS) -o "$(BINPATH)/grandine-converter" $(CMDPREFIX)/converter 16 | 17 | build-inspect: 18 | $(GO) build $(GOBUILDOPTS) -o "$(BINPATH)/grandine-inspect" $(CMDPREFIX)/inspect 19 | 20 | build-spatialize: 21 | $(GO) build $(GOBUILDOPTS) -o "$(BINPATH)/grandine-spatialize" $(CMDPREFIX)/spatialize 22 | 23 | build-tiler: 24 | $(GO) build $(GOBUILDOPTS) -o "$(BINPATH)/grandine-tiler" $(CMDPREFIX)/tiler 25 | 26 | clean: 27 | rm '$(BINPATH)'/* 28 | 29 | test: 30 | $(GO) test $(GOTESTOPTS) ./... 31 | 32 | # retrieves deps for tests 33 | test-deps: 34 | $(GO) get -t -u -v ./... 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Grandine 2 | 3 | [![GoDoc](https://godoc.org/github.com/thomersch/grandine?status.svg)](https://godoc.org/github.com/thomersch/grandine) [![Build Status](https://travis-ci.org/thomersch/grandine.svg?branch=master)](https://travis-ci.org/thomersch/grandine) 4 | 5 | This repository contains libraries and command line tools for working with geospatial data. It aims to streamline vector tile generation and provides tooling for standardized geo data serialization. 6 | 7 | Initial work has been funded by the [Prototype Fund](https://prototypefund.de), powered by Open Knowledge Foundation Germany and the German Ministry for Research and Education. 8 | 9 | ![Prototype Fund](https://files.skowron.eu/grandine/logo-prototype.svg) ![Bundesministerium für Bildung und Forschung](https://files.skowron.eu/grandine/logo-bmbf.svg) ![Open Knowledge Foundation Deutschland](https://files.skowron.eu/grandine/logo-okfn.svg) 10 | 11 | ## Spaten File Format 12 | 13 | If you are looking for documentation on the Spaten geo spatial file format, look [here](https://thomas.skowron.eu/spaten/). The Go implementation resides inside [lib/spaten](https://github.com/thomersch/grandine/tree/master/lib/spaten). 14 | 15 | ## Requirements 16 | 17 | * Go ≥ 1.11, Go 1.14 recommended 18 | * GEOS 19 | * LevelDB 20 | 21 | ## Quickstart 22 | 23 | If you have built a Go project before, you probably already know what to do. If not: 24 | 25 | * Make sure you have Go installed. Preferably version 1.9 or higher. (If you can, use 1.10 or newer as it allows for some impactful performance improvements) 26 | * Execute `go get -u github.com/thomersch/grandine`. This will checkout a current version of the code into `~/go/src/github.com/thomersch/grandine` 27 | * Go to the checkout directory. Execute `make build`, this will put all executables into the `bin` directory. 28 | * All the executables can be called with the `-help` flag which will print out basic usage info. 29 | 30 | ## Tips 31 | 32 | ### How to concatinate/merge multiple spaten files into one 33 | 34 | grandine-converter -in fileA,fileB,fileC -out outfile 35 | 36 | Alternatively, if you want to use a pipe: 37 | 38 | grandine-converter -in fileA,fileB,fileC | your-app-here 39 | 40 | ### How to render a tile set from a spaten file 41 | 42 | grandine-tiler -in some_geodata.spaten -zoom 9,10,11 -out tiles/ 43 | 44 | By default, all data will be on the `default` layer. 45 | 46 | ## Structure 47 | 48 | * `fileformat` contains a draft spec for a new geo data format that aims to be flexible, with a big focus on being very fast to serialize/deserialize. 49 | * In `lib` you'll find a few Go libraries that provide a few primitives for handling spatial data: 50 | * `lib/spatial` contains functionality for handling points/lines/polygons and basic transformation operations. If you miss functionality, feel free to send a Pull Request, it would be greatly appreciated. 51 | * `lib/mvt` contains code for serializing Mapbox Vector Tiles. 52 | * There are a few command line tools in `cmd`: 53 | * `converter` is a helper tool for converting and concatenating geo data files 54 | * `spatialize` converts OpenStreetMap data into a Spaten data file as defined in `fileformat` 55 | * `tiler` generates vector tiles from spatial data 56 | -------------------------------------------------------------------------------- /cmd/converter/converter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "log" 9 | "os" 10 | "strings" 11 | 12 | "github.com/thomersch/grandine/lib/csv" 13 | "github.com/thomersch/grandine/lib/geojson" 14 | "github.com/thomersch/grandine/lib/geojsonseq" 15 | "github.com/thomersch/grandine/lib/mapping" 16 | "github.com/thomersch/grandine/lib/spaten" 17 | "github.com/thomersch/grandine/lib/spatial" 18 | ) 19 | 20 | type filelist []string 21 | 22 | func (fl *filelist) String() string { 23 | return fmt.Sprintf("%s", *fl) 24 | } 25 | 26 | func (fl *filelist) Set(value string) error { 27 | for _, s := range strings.Split(value, ",") { 28 | *fl = append(*fl, strings.TrimSpace(s)) 29 | } 30 | return nil 31 | } 32 | 33 | func main() { 34 | var ( 35 | infiles filelist 36 | conds []mapping.Condition 37 | ) 38 | dest := flag.String("out", "", "") 39 | mapFilePath := flag.String("mapping", "", "Path to mapping file which will be used to transform data.") 40 | csvLatColumn := flag.Int("csv-lat", 1, "If parsing CSV, which column contains the Latitude. Zero-indexed.") 41 | csvLonColumn := flag.Int("csv-lon", 2, "If parsing CSV, which column contains the Longitude. Zero-indexed.") 42 | csvDelimiter := flag.String("csv-delim", ",", "If parsing CSV, what is the delimiter between values") 43 | inCodecName := flag.String("in-codec", "spaten", "Specify codec for in-files. Only used for read from stdin.") 44 | flag.Var(&infiles, "in", "infile(s)") 45 | flag.Parse() 46 | 47 | if len(*csvDelimiter) > 1 { 48 | log.Fatal("CSV Delimiter: only single character delimiters are allowed") 49 | } 50 | 51 | if len(*mapFilePath) != 0 { 52 | mf, err := os.Open(*mapFilePath) 53 | if err != nil { 54 | log.Fatal(err) 55 | } 56 | conds, err = mapping.ParseMapping(mf) 57 | if err != nil { 58 | log.Fatal(err) 59 | } 60 | 61 | if len(conds) > 0 { 62 | log.Printf("input file(s) will be filtered using %v conditions", len(conds)) 63 | } 64 | } 65 | 66 | availableCodecs := []spatial.Codec{ 67 | &geojson.Codec{}, 68 | &spaten.Codec{}, 69 | &csv.Codec{ 70 | LatCol: *csvLatColumn, 71 | LonCol: *csvLonColumn, 72 | Delim: rune((*csvDelimiter)[0]), 73 | }, 74 | &geojsonseq.Codec{}, 75 | } 76 | 77 | // Determining which codec we will be using for the output. 78 | var ( 79 | enc interface{} 80 | err error 81 | ) 82 | if len(*dest) == 0 { 83 | enc = &spaten.Codec{} 84 | } else { 85 | enc, err = guessCodec(*dest, availableCodecs) 86 | if err != nil { 87 | log.Fatalf("file type of %s is not supported (please check for correct file extension)", *dest) 88 | } 89 | } 90 | encoder, ok := enc.(spatial.Encoder) 91 | if !ok { 92 | log.Fatalf("%T codec does not support writing", enc) 93 | } 94 | 95 | // Determine whether we're writing to a stream or file. 96 | var out io.WriteCloser 97 | if len(*dest) == 0 { 98 | out = os.Stdout 99 | } else { 100 | out, err = os.Create(*dest) 101 | if err != nil { 102 | log.Fatal(err) 103 | } 104 | } 105 | defer out.Close() 106 | 107 | var ( 108 | fc spatial.FeatureCollection 109 | finished func() error 110 | ) 111 | 112 | if len(infiles) == 0 { 113 | log.Println("No input files specified. Reading from stdin.") 114 | incodec, err := guessCodec("."+*inCodecName, availableCodecs) 115 | if err != nil { 116 | log.Fatalf("could not use incodec: %v", err) 117 | } 118 | inc, ok := incodec.(spatial.ChunkedDecoder) 119 | if !ok { 120 | log.Fatal("codec cannot be used for decoding") 121 | } 122 | icd, err := inc.ChunkedDecode(os.Stdin) 123 | if err != nil { 124 | log.Fatal(err) 125 | } 126 | for icd.Next() { 127 | err = icd.Scan(&fc) 128 | if err != nil { 129 | log.Fatal(err) 130 | } 131 | finished, err = write(out, &fc, encoder, conds) 132 | fc.Reset() 133 | } 134 | finished() 135 | } 136 | 137 | for _, infileName := range infiles { 138 | dec, err := guessCodec(infileName, availableCodecs) 139 | if err != nil { 140 | log.Fatalf("file type of %s is not supported (please check for correct file extension)", infileName) 141 | } 142 | decoder, ok := dec.(spatial.Decoder) 143 | if !ok { 144 | log.Fatalf("%T codec does not support reading", decoder) 145 | } 146 | 147 | r, err := os.Open(infileName) 148 | if err != nil { 149 | log.Fatalf("could not open %v for reading: %v", infileName, err) 150 | } 151 | defer r.Close() 152 | 153 | var finished func() error 154 | switch d := dec.(type) { 155 | case spatial.ChunkedDecoder: 156 | chunks, err := d.ChunkedDecode(r) 157 | if err != nil { 158 | log.Fatalf("could not decode %v: %v", infileName, err) 159 | } 160 | for chunks.Next() { 161 | chunks.Scan(&fc) 162 | finished, err = write(out, &fc, encoder, conds) 163 | if err != nil { 164 | log.Fatal(err) 165 | } 166 | fc.Features = []spatial.Feature{} 167 | } 168 | err = finished() 169 | if err != nil { 170 | log.Fatal(err) 171 | } 172 | case spatial.Decoder: 173 | err = decoder.Decode(r, &fc) 174 | if err != nil { 175 | log.Fatalf("could not decode %v: %v", infileName, err) 176 | } 177 | finished, err = write(out, &fc, encoder, conds) 178 | if err != nil { 179 | log.Fatal(err) 180 | } 181 | finished() 182 | } 183 | } 184 | } 185 | 186 | var featBuf []spatial.FeatureCollection // TODO: this is not optimal, needs better wrapping 187 | 188 | func write(w io.Writer, fs *spatial.FeatureCollection, enc spatial.Encoder, conds []mapping.Condition) (flush func() error, err error) { 189 | if len(conds) > 0 { 190 | var filtered []spatial.Feature 191 | for _, ft := range fs.Features { 192 | for _, cond := range conds { 193 | if cond.Matches(ft.Props) { 194 | filtered = append(filtered, cond.Transform(ft)...) 195 | } 196 | } 197 | } 198 | fs.Features = filtered 199 | } 200 | 201 | if e, ok := enc.(spatial.ChunkedEncoder); ok { 202 | err = e.EncodeChunk(w, fs) 203 | if err != nil { 204 | return func() error { return nil }, err 205 | } 206 | return func() error { return e.Close(w) }, nil 207 | } 208 | 209 | featBuf = append(featBuf, *fs) 210 | return func() error { 211 | var flat spatial.FeatureCollection 212 | for _, ftc := range featBuf { 213 | flat.Features = append(flat.Features, ftc.Features...) 214 | flat.SRID = ftc.SRID 215 | } 216 | return enc.Encode(w, &flat) 217 | }, nil 218 | } 219 | 220 | func guessCodec(filename string, codecs []spatial.Codec) (spatial.Codec, error) { 221 | fn := strings.ToLower(filename) 222 | for _, cd := range codecs { 223 | for _, ext := range cd.Extensions() { 224 | if strings.HasSuffix(fn, "."+ext) { 225 | return cd, nil 226 | } 227 | } 228 | } 229 | return nil, errors.New("file type not supported") 230 | } 231 | -------------------------------------------------------------------------------- /cmd/converter/testfiles/linestring.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": {}, 7 | "geometry": { 8 | "type": "LineString", 9 | "coordinates": [ 10 | [ 11 | 8.525390625, 12 | 52.908902047770255 13 | ], 14 | [ 15 | 16.611328125, 16 | 46.86019101567027 17 | ], 18 | [ 19 | -2.98828125, 20 | 44.276671273775186 21 | ], 22 | [ 23 | 0.615234375, 24 | 47.57652571374621 25 | ] 26 | ] 27 | } 28 | } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /cmd/converter/testfiles/polygon.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": {}, 7 | "geometry": { 8 | "type": "Polygon", 9 | "coordinates": [ 10 | [ 11 | [ 12 | 2.900390625, 13 | 50.62507306341435 14 | ], 15 | [ 16 | 2.900390625, 17 | 38.13455657705411 18 | ], 19 | [ 20 | 25.839843749999996, 21 | 31.353636941500987 22 | ], 23 | [ 24 | 33.3984375, 25 | 45.767522962149876 26 | ], 27 | [ 28 | 27.59765625, 29 | 54.059387886623576 30 | ], 31 | [ 32 | 11.162109375, 33 | 58.95000823335702 34 | ], 35 | [ 36 | -2.021484375, 37 | 55.92458580482951 38 | ], 39 | [ 40 | 2.900390625, 41 | 50.62507306341435 42 | ] 43 | ] 44 | ] 45 | } 46 | } 47 | ] 48 | } 49 | -------------------------------------------------------------------------------- /cmd/inspect/inspect.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "os" 8 | "os/exec" 9 | "sort" 10 | "strings" 11 | 12 | "github.com/thomersch/grandine/lib/spaten" 13 | "github.com/thomersch/grandine/lib/spatial" 14 | ) 15 | 16 | func colorKV(s string) string { 17 | if strings.HasPrefix(s, "@") { 18 | return "\033[37m" + s + "\033[0m" 19 | } 20 | return s 21 | } 22 | 23 | func prettyPrint(m map[string]interface{}) string { 24 | var ( 25 | i int 26 | ol = make([]string, len(m)) 27 | o string 28 | ) 29 | 30 | for k := range m { 31 | ol[i] = k 32 | i++ 33 | } 34 | sort.Strings(ol) 35 | for _, k := range ol { 36 | o += colorKV(fmt.Sprintf("%v=%v ", k, m[k])) 37 | } 38 | return o 39 | } 40 | 41 | func main() { 42 | if len(os.Args) < 2 { 43 | fmt.Printf("Usage: %s filepath\n", os.Args[0]) 44 | os.Exit(1) 45 | } 46 | filepath := os.Args[len(os.Args)-1] 47 | f, err := os.Open(filepath) 48 | if err != nil { 49 | log.Fatalf("Could not open %v", filepath) 50 | } 51 | defer f.Close() 52 | 53 | pp := exec.Command("less", "-R") 54 | pp.Stdout = os.Stdout 55 | stdin, err := pp.StdinPipe() 56 | if err != nil { 57 | log.Fatalf("Could not open pipe for pager: %v", err) 58 | } 59 | pp.Start() 60 | 61 | go func() { 62 | defer stdin.Close() 63 | 64 | var ( 65 | codec spaten.Codec 66 | fc = spatial.NewFeatureCollection() 67 | ) 68 | hd, err := spaten.ReadFileHeader(f) 69 | if err != nil { 70 | io.WriteString(stdin, fmt.Sprintf("Invalid Header: %v", err)) 71 | return 72 | } 73 | fmt.Fprintf(stdin, "Spaten file, Version %v\n", hd.Version) 74 | _, err = f.Seek(0, 0) 75 | if err != nil { 76 | log.Fatal(err) 77 | } 78 | 79 | chunks, err := codec.ChunkedDecode(f) 80 | if err != nil { 81 | log.Fatalf("could not decode: %v", err) 82 | } 83 | for chunks.Next() { 84 | err = chunks.Scan(fc) 85 | if err != nil { 86 | log.Fatal(err) 87 | } 88 | for _, ft := range fc.Features { 89 | fmt.Fprintf(stdin, "\033[34m%v\033[0m ", ft.Geometry.Typ()) 90 | fmt.Fprintf(stdin, "%v", prettyPrint(ft.Props)) 91 | fmt.Fprintf(stdin, "\033[31m%v\033[0m\n", ft.Geometry) 92 | } 93 | fc.Features = []spatial.Feature{} 94 | } 95 | }() 96 | pp.Wait() 97 | } 98 | -------------------------------------------------------------------------------- /cmd/spatialize/vendor/github.com/golang/protobuf/LICENSE: -------------------------------------------------------------------------------- 1 | Go support for Protocol Buffers - Google's data interchange format 2 | 3 | Copyright 2010 The Go Authors. All rights reserved. 4 | https://github.com/golang/protobuf 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are 8 | met: 9 | 10 | * Redistributions of source code must retain the above copyright 11 | notice, this list of conditions and the following disclaimer. 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following disclaimer 14 | in the documentation and/or other materials provided with the 15 | distribution. 16 | * Neither the name of Google Inc. nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | -------------------------------------------------------------------------------- /cmd/spatialize/vendor/github.com/golang/protobuf/proto/Makefile: -------------------------------------------------------------------------------- 1 | # Go support for Protocol Buffers - Google's data interchange format 2 | # 3 | # Copyright 2010 The Go Authors. All rights reserved. 4 | # https://github.com/golang/protobuf 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following disclaimer 14 | # in the documentation and/or other materials provided with the 15 | # distribution. 16 | # * Neither the name of Google Inc. nor the names of its 17 | # contributors may be used to endorse or promote products derived from 18 | # this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | install: 33 | go install 34 | 35 | test: install generate-test-pbs 36 | go test 37 | 38 | 39 | generate-test-pbs: 40 | make install 41 | make -C testdata 42 | protoc --go_out=Mtestdata/test.proto=github.com/golang/protobuf/proto/testdata,Mgoogle/protobuf/any.proto=github.com/golang/protobuf/ptypes/any:. proto3_proto/proto3.proto 43 | make 44 | -------------------------------------------------------------------------------- /cmd/spatialize/vendor/github.com/golang/protobuf/proto/clone.go: -------------------------------------------------------------------------------- 1 | // Go support for Protocol Buffers - Google's data interchange format 2 | // 3 | // Copyright 2011 The Go Authors. All rights reserved. 4 | // https://github.com/golang/protobuf 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // * Neither the name of Google Inc. nor the names of its 17 | // contributors may be used to endorse or promote products derived from 18 | // this software without specific prior written permission. 19 | // 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | // Protocol buffer deep copy and merge. 33 | // TODO: RawMessage. 34 | 35 | package proto 36 | 37 | import ( 38 | "log" 39 | "reflect" 40 | "strings" 41 | ) 42 | 43 | // Clone returns a deep copy of a protocol buffer. 44 | func Clone(pb Message) Message { 45 | in := reflect.ValueOf(pb) 46 | if in.IsNil() { 47 | return pb 48 | } 49 | 50 | out := reflect.New(in.Type().Elem()) 51 | // out is empty so a merge is a deep copy. 52 | mergeStruct(out.Elem(), in.Elem()) 53 | return out.Interface().(Message) 54 | } 55 | 56 | // Merge merges src into dst. 57 | // Required and optional fields that are set in src will be set to that value in dst. 58 | // Elements of repeated fields will be appended. 59 | // Merge panics if src and dst are not the same type, or if dst is nil. 60 | func Merge(dst, src Message) { 61 | in := reflect.ValueOf(src) 62 | out := reflect.ValueOf(dst) 63 | if out.IsNil() { 64 | panic("proto: nil destination") 65 | } 66 | if in.Type() != out.Type() { 67 | // Explicit test prior to mergeStruct so that mistyped nils will fail 68 | panic("proto: type mismatch") 69 | } 70 | if in.IsNil() { 71 | // Merging nil into non-nil is a quiet no-op 72 | return 73 | } 74 | mergeStruct(out.Elem(), in.Elem()) 75 | } 76 | 77 | func mergeStruct(out, in reflect.Value) { 78 | sprop := GetProperties(in.Type()) 79 | for i := 0; i < in.NumField(); i++ { 80 | f := in.Type().Field(i) 81 | if strings.HasPrefix(f.Name, "XXX_") { 82 | continue 83 | } 84 | mergeAny(out.Field(i), in.Field(i), false, sprop.Prop[i]) 85 | } 86 | 87 | if emIn, ok := extendable(in.Addr().Interface()); ok { 88 | emOut, _ := extendable(out.Addr().Interface()) 89 | mIn, muIn := emIn.extensionsRead() 90 | if mIn != nil { 91 | mOut := emOut.extensionsWrite() 92 | muIn.Lock() 93 | mergeExtension(mOut, mIn) 94 | muIn.Unlock() 95 | } 96 | } 97 | 98 | uf := in.FieldByName("XXX_unrecognized") 99 | if !uf.IsValid() { 100 | return 101 | } 102 | uin := uf.Bytes() 103 | if len(uin) > 0 { 104 | out.FieldByName("XXX_unrecognized").SetBytes(append([]byte(nil), uin...)) 105 | } 106 | } 107 | 108 | // mergeAny performs a merge between two values of the same type. 109 | // viaPtr indicates whether the values were indirected through a pointer (implying proto2). 110 | // prop is set if this is a struct field (it may be nil). 111 | func mergeAny(out, in reflect.Value, viaPtr bool, prop *Properties) { 112 | if in.Type() == protoMessageType { 113 | if !in.IsNil() { 114 | if out.IsNil() { 115 | out.Set(reflect.ValueOf(Clone(in.Interface().(Message)))) 116 | } else { 117 | Merge(out.Interface().(Message), in.Interface().(Message)) 118 | } 119 | } 120 | return 121 | } 122 | switch in.Kind() { 123 | case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Int32, reflect.Int64, 124 | reflect.String, reflect.Uint32, reflect.Uint64: 125 | if !viaPtr && isProto3Zero(in) { 126 | return 127 | } 128 | out.Set(in) 129 | case reflect.Interface: 130 | // Probably a oneof field; copy non-nil values. 131 | if in.IsNil() { 132 | return 133 | } 134 | // Allocate destination if it is not set, or set to a different type. 135 | // Otherwise we will merge as normal. 136 | if out.IsNil() || out.Elem().Type() != in.Elem().Type() { 137 | out.Set(reflect.New(in.Elem().Elem().Type())) // interface -> *T -> T -> new(T) 138 | } 139 | mergeAny(out.Elem(), in.Elem(), false, nil) 140 | case reflect.Map: 141 | if in.Len() == 0 { 142 | return 143 | } 144 | if out.IsNil() { 145 | out.Set(reflect.MakeMap(in.Type())) 146 | } 147 | // For maps with value types of *T or []byte we need to deep copy each value. 148 | elemKind := in.Type().Elem().Kind() 149 | for _, key := range in.MapKeys() { 150 | var val reflect.Value 151 | switch elemKind { 152 | case reflect.Ptr: 153 | val = reflect.New(in.Type().Elem().Elem()) 154 | mergeAny(val, in.MapIndex(key), false, nil) 155 | case reflect.Slice: 156 | val = in.MapIndex(key) 157 | val = reflect.ValueOf(append([]byte{}, val.Bytes()...)) 158 | default: 159 | val = in.MapIndex(key) 160 | } 161 | out.SetMapIndex(key, val) 162 | } 163 | case reflect.Ptr: 164 | if in.IsNil() { 165 | return 166 | } 167 | if out.IsNil() { 168 | out.Set(reflect.New(in.Elem().Type())) 169 | } 170 | mergeAny(out.Elem(), in.Elem(), true, nil) 171 | case reflect.Slice: 172 | if in.IsNil() { 173 | return 174 | } 175 | if in.Type().Elem().Kind() == reflect.Uint8 { 176 | // []byte is a scalar bytes field, not a repeated field. 177 | 178 | // Edge case: if this is in a proto3 message, a zero length 179 | // bytes field is considered the zero value, and should not 180 | // be merged. 181 | if prop != nil && prop.proto3 && in.Len() == 0 { 182 | return 183 | } 184 | 185 | // Make a deep copy. 186 | // Append to []byte{} instead of []byte(nil) so that we never end up 187 | // with a nil result. 188 | out.SetBytes(append([]byte{}, in.Bytes()...)) 189 | return 190 | } 191 | n := in.Len() 192 | if out.IsNil() { 193 | out.Set(reflect.MakeSlice(in.Type(), 0, n)) 194 | } 195 | switch in.Type().Elem().Kind() { 196 | case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Int32, reflect.Int64, 197 | reflect.String, reflect.Uint32, reflect.Uint64: 198 | out.Set(reflect.AppendSlice(out, in)) 199 | default: 200 | for i := 0; i < n; i++ { 201 | x := reflect.Indirect(reflect.New(in.Type().Elem())) 202 | mergeAny(x, in.Index(i), false, nil) 203 | out.Set(reflect.Append(out, x)) 204 | } 205 | } 206 | case reflect.Struct: 207 | mergeStruct(out, in) 208 | default: 209 | // unknown type, so not a protocol buffer 210 | log.Printf("proto: don't know how to copy %v", in) 211 | } 212 | } 213 | 214 | func mergeExtension(out, in map[int32]Extension) { 215 | for extNum, eIn := range in { 216 | eOut := Extension{desc: eIn.desc} 217 | if eIn.value != nil { 218 | v := reflect.New(reflect.TypeOf(eIn.value)).Elem() 219 | mergeAny(v, reflect.ValueOf(eIn.value), false, nil) 220 | eOut.value = v.Interface() 221 | } 222 | if eIn.enc != nil { 223 | eOut.enc = make([]byte, len(eIn.enc)) 224 | copy(eOut.enc, eIn.enc) 225 | } 226 | 227 | out[extNum] = eOut 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /cmd/spatialize/vendor/github.com/thomersch/gosmparse/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015-2017, Thomas Skowron 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cmd/spatialize/vendor/github.com/thomersch/gosmparse/OSMPBF/fileformat.proto: -------------------------------------------------------------------------------- 1 | /** Copyright (c) 2010 Scott A. Crosby. 2 | 3 | This program is free software: you can redistribute it and/or modify 4 | it under the terms of the GNU Lesser General Public License as 5 | published by the Free Software Foundation, either version 3 of the 6 | License, or (at your option) any later version. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU Lesser General Public License for more details. 12 | 13 | You should have received a copy of the GNU Lesser General Public License 14 | along with this program. If not, see . 15 | 16 | */ 17 | 18 | option optimize_for = LITE_RUNTIME; 19 | option java_package = "crosby.binary"; 20 | package OSMPBF; 21 | 22 | //protoc --java_out=../.. fileformat.proto 23 | 24 | 25 | // 26 | // STORAGE LAYER: Storing primitives. 27 | // 28 | 29 | message Blob { 30 | optional bytes raw = 1; // No compression 31 | optional int32 raw_size = 2; // When compressed, the uncompressed size 32 | 33 | // Possible compressed versions of the data. 34 | optional bytes zlib_data = 3; 35 | 36 | // PROPOSED feature for LZMA compressed data. SUPPORT IS NOT REQUIRED. 37 | optional bytes lzma_data = 4; 38 | 39 | // Formerly used for bzip2 compressed data. Depreciated in 2010. 40 | optional bytes OBSOLETE_bzip2_data = 5 [deprecated=true]; // Don't reuse this tag number. 41 | } 42 | 43 | /* A file contains an sequence of fileblock headers, each prefixed by 44 | their length in network byte order, followed by a data block 45 | containing the actual data. types staring with a "_" are reserved. 46 | */ 47 | 48 | message BlobHeader { 49 | required string type = 1; 50 | optional bytes indexdata = 2; 51 | required int32 datasize = 3; 52 | } 53 | 54 | -------------------------------------------------------------------------------- /cmd/spatialize/vendor/github.com/thomersch/gosmparse/README.md: -------------------------------------------------------------------------------- 1 | # OpenStreetMap PBF Parser in Go 2 | 3 | [![GoDoc](https://godoc.org/github.com/thomersch/gosmparse?status.svg)](https://godoc.org/github.com/thomersch/gosmparse) [![Build Status](https://travis-ci.org/thomersch/gosmparse.svg?branch=master)](https://travis-ci.org/thomersch/gosmparse) 4 | 5 | Gosmparse works already, but the API may change ([Documentation](https://godoc.org/github.com/thomersch/gosmparse)). 6 | 7 | It has been designed with performance and maximum usage convenience in mind; on an Intel Core i7-6820HQ with NVMe flash it is able to process 67 MB/s, so a planet file can be processed in less than 10 minutes. If you find possible speed-ups or other improvements, let me know. 8 | 9 | ## Characteristics 10 | 11 | * fast 12 | * `panic`-free 13 | * tested with different files from different sources/generators 14 | * more than 80% test coverage and has benchmarks for all hot spots 15 | * one dependency only: [Go protobuf package](http://github.com/golang/protobuf) 16 | * can read from any io.Reader (e.g. for parsing during download) 17 | 18 | ## Install 19 | 20 | ``` 21 | go get -u github.com/thomersch/gosmparse 22 | ``` 23 | 24 | ## Example Usage 25 | 26 | ```go 27 | // Implement the gosmparser.OSMReader interface here. 28 | // Streaming data will call those functions. 29 | type dataHandler struct{} 30 | 31 | func (d *dataHandler) ReadNode(n gosmparse.Node) {} 32 | func (d *dataHandler) ReadWay(w gosmparse.Way) {} 33 | func (d *dataHandler) ReadRelation(r gosmparse.Relation) {} 34 | 35 | func ExampleNewDecoder() { 36 | r, err := os.Open("filename.pbf") 37 | if err != nil { 38 | panic(err) 39 | } 40 | dec := gosmparse.NewDecoder(r) 41 | // Parse will block until it is done or an error occurs. 42 | err = dec.Parse(&dataHandler{}) 43 | if err != nil { 44 | panic(err) 45 | } 46 | } 47 | ``` 48 | 49 | ## Download & Parse 50 | 51 | It is possible to parse during download, so you don't have to wait for a download to finish to be able to start the parsing/processing. You can simply use the standard Go `net/http` package and pass `resp.Body` to the decoder. 52 | 53 | ```go 54 | resp, err := http.Get("http://download.geofabrik.de/europe/germany/bremen-latest.osm.pbf") 55 | if err != nil { 56 | panic(err) 57 | } 58 | defer resp.Body.Close() 59 | dec := gosmparse.NewDecoder(resp.Body) 60 | err = dec.Parse(&dataHandler{}) 61 | if err != nil { 62 | panic(err) 63 | } 64 | ``` 65 | 66 | ## Did it break? 67 | 68 | If you found a case, where gosmparse broke, please report it and provide the file that caused the failure. 69 | -------------------------------------------------------------------------------- /cmd/spatialize/vendor/github.com/thomersch/gosmparse/decoder.go: -------------------------------------------------------------------------------- 1 | package gosmparse 2 | 3 | import ( 4 | "bytes" 5 | "compress/zlib" 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | "runtime" 10 | "sync" 11 | 12 | "github.com/thomersch/gosmparse/OSMPBF" 13 | 14 | "github.com/golang/protobuf/proto" 15 | ) 16 | 17 | // A Decoder reads and decodes OSM data from an input stream. 18 | type Decoder struct { 19 | // QueueSize allows to tune the memory usage vs. parse speed. 20 | // A larger QueueSize will consume more memory, but may speed up the parsing process. 21 | QueueSize int 22 | r io.Reader 23 | o OSMReader 24 | } 25 | 26 | // NewDecoder returns a new decoder that reads from r. 27 | func NewDecoder(r io.Reader) *Decoder { 28 | return &Decoder{ 29 | r: r, 30 | QueueSize: 200, 31 | } 32 | } 33 | 34 | // Parse starts the parsing process that will stream data into the given OSMReader. 35 | func (d *Decoder) Parse(o OSMReader) error { 36 | d.o = o 37 | header, _, err := d.block() 38 | if err != nil { 39 | return err 40 | } 41 | // TODO: parser checks 42 | if header.GetType() != "OSMHeader" { 43 | return fmt.Errorf("Invalid header of first data block. Wanted: OSMHeader, have: %s", header.GetType()) 44 | } 45 | 46 | errChan := make(chan error) 47 | // feeder 48 | blobs := make(chan *OSMPBF.Blob, d.QueueSize) 49 | go func() { 50 | defer close(blobs) 51 | for { 52 | _, blob, err := d.block() 53 | if err != nil { 54 | if err == io.EOF { 55 | return 56 | } 57 | errChan <- err 58 | return 59 | } 60 | blobs <- blob 61 | } 62 | }() 63 | 64 | consumerCount := runtime.GOMAXPROCS(0) 65 | var wg sync.WaitGroup 66 | for i := 0; i < consumerCount; i++ { 67 | wg.Add(1) 68 | go func() { 69 | defer wg.Done() 70 | for blob := range blobs { 71 | err := d.readElements(blob) 72 | if err != nil { 73 | errChan <- err 74 | return 75 | } 76 | } 77 | }() 78 | } 79 | 80 | finished := make(chan bool) 81 | go func() { 82 | wg.Wait() 83 | finished <- true 84 | }() 85 | select { 86 | case err = <-errChan: 87 | return err 88 | case <-finished: 89 | return nil 90 | } 91 | } 92 | 93 | func (d *Decoder) block() (*OSMPBF.BlobHeader, *OSMPBF.Blob, error) { 94 | // BlobHeaderLength 95 | headerSizeBuf := make([]byte, 4) 96 | if _, err := io.ReadFull(d.r, headerSizeBuf); err != nil { 97 | return nil, nil, err 98 | } 99 | headerSize := binary.BigEndian.Uint32(headerSizeBuf) 100 | 101 | // BlobHeader 102 | headerBuf := make([]byte, headerSize) 103 | if _, err := io.ReadFull(d.r, headerBuf); err != nil { 104 | return nil, nil, err 105 | } 106 | blobHeader := new(OSMPBF.BlobHeader) 107 | if err := proto.Unmarshal(headerBuf, blobHeader); err != nil { 108 | return nil, nil, err 109 | } 110 | 111 | // Blob 112 | blobBuf := make([]byte, blobHeader.GetDatasize()) 113 | _, err := io.ReadFull(d.r, blobBuf) 114 | if err != nil { 115 | return nil, nil, err 116 | } 117 | blob := new(OSMPBF.Blob) 118 | if err := proto.Unmarshal(blobBuf, blob); err != nil { 119 | return nil, nil, err 120 | } 121 | return blobHeader, blob, nil 122 | } 123 | 124 | func (d *Decoder) readElements(blob *OSMPBF.Blob) error { 125 | pb, err := d.blobData(blob) 126 | if err != nil { 127 | return err 128 | } 129 | 130 | for _, pg := range pb.Primitivegroup { 131 | switch { 132 | case pg.Dense != nil: 133 | if err := denseNode(d.o, pb, pg.Dense); err != nil { 134 | return err 135 | } 136 | case len(pg.Ways) != 0: 137 | if err := way(d.o, pb, pg.Ways); err != nil { 138 | return err 139 | } 140 | case len(pg.Relations) != 0: 141 | if err := relation(d.o, pb, pg.Relations); err != nil { 142 | return err 143 | } 144 | case len(pg.Nodes) != 0: 145 | return fmt.Errorf("Nodes are not supported") 146 | default: 147 | return fmt.Errorf("no supported data in primitive group") 148 | } 149 | } 150 | return nil 151 | } 152 | 153 | // should be concurrency safe 154 | func (d *Decoder) blobData(blob *OSMPBF.Blob) (*OSMPBF.PrimitiveBlock, error) { 155 | buf := make([]byte, blob.GetRawSize()) 156 | switch { 157 | case blob.Raw != nil: 158 | buf = blob.Raw 159 | case blob.ZlibData != nil: 160 | r, err := zlib.NewReader(bytes.NewReader(blob.GetZlibData())) 161 | if err != nil { 162 | return nil, err 163 | } 164 | defer r.Close() 165 | 166 | n, err := io.ReadFull(r, buf) 167 | if err != nil { 168 | return nil, err 169 | } 170 | if n != int(blob.GetRawSize()) { 171 | return nil, fmt.Errorf("expected %v bytes, read %v", blob.GetRawSize(), n) 172 | } 173 | default: 174 | return nil, fmt.Errorf("found block with unknown data") 175 | } 176 | var primitiveBlock = OSMPBF.PrimitiveBlock{} 177 | err := proto.Unmarshal(buf, &primitiveBlock) 178 | return &primitiveBlock, err 179 | } 180 | -------------------------------------------------------------------------------- /cmd/spatialize/vendor/github.com/thomersch/gosmparse/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package gosmparse is a library for parsing OpenStreetMap binary PBF files. 3 | 4 | It has been designed for very fast, flexible, streamed parsing of small and 5 | large files. 6 | */ 7 | package gosmparse 8 | -------------------------------------------------------------------------------- /cmd/spatialize/vendor/github.com/thomersch/gosmparse/elements.go: -------------------------------------------------------------------------------- 1 | package gosmparse 2 | 3 | import "github.com/thomersch/gosmparse/OSMPBF" 4 | 5 | // Node is an OSM data element with a position and tags (key/value pairs). 6 | type Node struct { 7 | ID int64 8 | Lat float64 9 | Lon float64 10 | Tags map[string]string 11 | } 12 | 13 | // Way is an OSM data element that consists of Nodes and tags (key/value pairs). 14 | // Ways can describe line strings or areas. 15 | type Way struct { 16 | ID int64 17 | NodeIDs []int64 18 | Tags map[string]string 19 | } 20 | 21 | // Relation is an OSM data element that contains multiple elements (RelationMember) 22 | // and has tags (key/value pairs). 23 | type Relation struct { 24 | ID int64 25 | Members []RelationMember 26 | Tags map[string]string 27 | } 28 | 29 | // MemberType describes the type of a relation member (node/way/relation). 30 | type MemberType int 31 | 32 | const ( 33 | NodeType MemberType = iota 34 | WayType 35 | RelationType 36 | ) 37 | 38 | // RelationMember refers to an element in a relation. It contains the ID of the element 39 | // (node/way/relation) and the role. 40 | type RelationMember struct { 41 | ID int64 42 | Type MemberType 43 | Role string 44 | } 45 | 46 | func denseNode(o OSMReader, pb *OSMPBF.PrimitiveBlock, dn *OSMPBF.DenseNodes) error { 47 | // dateGran := pb.GetDateGranularity() 48 | gran := int64(pb.GetGranularity()) 49 | latOffset := pb.GetLatOffset() 50 | lonOffset := pb.GetLonOffset() 51 | 52 | var ( 53 | n Node 54 | id, lat, lon int64 55 | kvPos int // position in kv slice 56 | ) 57 | for index := range dn.Id { 58 | id = dn.Id[index] + id 59 | lat = dn.Lat[index] + lat 60 | lon = dn.Lon[index] + lon 61 | 62 | n.ID = id 63 | n.Lat = 1e-9 * float64(latOffset+(gran*lat)) 64 | n.Lon = 1e-9 * float64(lonOffset+(gran*lon)) 65 | 66 | kvPos, n.Tags = unpackTags(pb.Stringtable.GetS(), kvPos, dn.KeysVals) 67 | o.ReadNode(n) 68 | } 69 | return nil 70 | } 71 | 72 | func way(o OSMReader, pb *OSMPBF.PrimitiveBlock, ways []*OSMPBF.Way) error { 73 | // dateGran := pb.GetDateGranularity() 74 | st := pb.Stringtable.GetS() 75 | 76 | var ( 77 | w Way 78 | nodeID int64 79 | ) 80 | for _, way := range ways { 81 | w.ID = way.GetId() 82 | nodeID = 0 83 | w.NodeIDs = make([]int64, len(way.Refs)) 84 | w.Tags = make(map[string]string) 85 | for pos, key := range way.Keys { 86 | keyString := string(st[int(key)]) 87 | w.Tags[keyString] = string(st[way.Vals[pos]]) 88 | } 89 | for index := range way.Refs { 90 | nodeID = way.Refs[index] + nodeID 91 | w.NodeIDs[index] = nodeID 92 | } 93 | o.ReadWay(w) 94 | } 95 | return nil 96 | } 97 | 98 | func relation(o OSMReader, pb *OSMPBF.PrimitiveBlock, relations []*OSMPBF.Relation) error { 99 | st := pb.Stringtable.GetS() 100 | // dateGran := pb.GetDateGranularity() 101 | var r Relation 102 | for _, rel := range relations { 103 | r.ID = *rel.Id 104 | r.Members = make([]RelationMember, len(rel.Memids)) 105 | var ( 106 | relMember RelationMember 107 | memID int64 108 | ) 109 | r.Tags = make(map[string]string) 110 | for pos, key := range rel.Keys { 111 | keyString := string(st[int(key)]) 112 | r.Tags[keyString] = string(st[rel.Vals[pos]]) 113 | } 114 | for memIndex := range rel.Memids { 115 | memID = rel.Memids[memIndex] + memID 116 | relMember.ID = memID 117 | switch rel.Types[memIndex] { 118 | case OSMPBF.Relation_NODE: 119 | relMember.Type = NodeType 120 | case OSMPBF.Relation_WAY: 121 | relMember.Type = WayType 122 | case OSMPBF.Relation_RELATION: 123 | relMember.Type = RelationType 124 | } 125 | relMember.Role = string(st[rel.RolesSid[memIndex]]) 126 | r.Members[memIndex] = relMember 127 | } 128 | o.ReadRelation(r) 129 | } 130 | return nil 131 | } 132 | -------------------------------------------------------------------------------- /cmd/spatialize/vendor/github.com/thomersch/gosmparse/interface.go: -------------------------------------------------------------------------------- 1 | package gosmparse 2 | 3 | // OSMReader is the interface that needs to be implemented in order to receive 4 | // Elements from the parsing process. 5 | type OSMReader interface { 6 | ReadNode(Node) 7 | ReadWay(Way) 8 | ReadRelation(Relation) 9 | } 10 | -------------------------------------------------------------------------------- /cmd/spatialize/vendor/github.com/thomersch/gosmparse/tags.go: -------------------------------------------------------------------------------- 1 | package gosmparse 2 | 3 | func unpackTags(st []string, pos int, kv []int32) (int, map[string]string) { 4 | tags := map[string]string{} 5 | for pos < len(kv) { 6 | if kv[pos] == 0 { 7 | pos++ 8 | break 9 | } 10 | tags[st[kv[pos]]] = st[kv[pos+1]] 11 | pos = pos + 2 12 | } 13 | return pos, tags 14 | } 15 | -------------------------------------------------------------------------------- /cmd/spatialize/vendor/vendor.json: -------------------------------------------------------------------------------- 1 | { 2 | "comment": "", 3 | "ignore": "test", 4 | "package": [ 5 | { 6 | "checksumSHA1": "yqF125xVSkmfLpIVGrLlfE05IUk=", 7 | "path": "github.com/golang/protobuf/proto", 8 | "revision": "748d386b5c1ea99658fd69fe9f03991ce86a90c1", 9 | "revisionTime": "2017-07-26T21:28:29Z" 10 | }, 11 | { 12 | "checksumSHA1": "qm6IKSVeo3TLZcAmNlmBm9uPliE=", 13 | "path": "github.com/thomersch/gosmparse", 14 | "revision": "868a6f151a1ca3f5fd950e618b86883d6055d462", 15 | "revisionTime": "2017-07-16T12:49:31Z" 16 | }, 17 | { 18 | "checksumSHA1": "AZUoZ2q4UB+2eyTu1WxUxwIKNTI=", 19 | "path": "github.com/thomersch/gosmparse/OSMPBF", 20 | "revision": "868a6f151a1ca3f5fd950e618b86883d6055d462", 21 | "revisionTime": "2017-07-16T12:49:31Z" 22 | } 23 | ], 24 | "rootPath": "github.com/thomersch/grandine/cmd/spatialize" 25 | } 26 | -------------------------------------------------------------------------------- /cmd/tiler/.gitignore: -------------------------------------------------------------------------------- 1 | tiles/ 2 | -------------------------------------------------------------------------------- /cmd/tiler/featurecache.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/thomersch/grandine/lib/spatial" 7 | "github.com/thomersch/grandine/lib/tile" 8 | ) 9 | 10 | type cacheInitFunc func([]int) (FeatureCache, error) 11 | 12 | var ( 13 | caches map[string]cacheInitFunc 14 | cacheinit sync.Once 15 | ) 16 | 17 | func init() { 18 | registerCache("table", func(zl []int) (FeatureCache, error) { 19 | nt, err := NewFeatureTable(zl) 20 | return nt, err 21 | }) 22 | registerCache("map", func(zl []int) (FeatureCache, error) { 23 | nm, err := NewFeatureMap(zl) 24 | return nm, err 25 | }) 26 | } 27 | 28 | func registerCache(name string, c cacheInitFunc) { 29 | cacheinit.Do(func() { 30 | caches = make(map[string]cacheInitFunc) 31 | }) 32 | caches[name] = c 33 | } 34 | 35 | func availableCaches() []string { 36 | var cc []string 37 | for k := range caches { 38 | cc = append(cc, k) 39 | } 40 | return cc 41 | } 42 | 43 | type FeatureCache interface { 44 | AddFeature(spatial.Feature) 45 | GetFeatures(tile.ID) []spatial.Feature 46 | 47 | BBox() spatial.BBox 48 | Count() int 49 | } 50 | 51 | type FeatureTable struct { 52 | Zoomlevels []int 53 | 54 | count int 55 | table map[int][][][]spatial.Feature 56 | bbox *spatial.BBox 57 | } 58 | 59 | func NewFeatureTable(zoomlevels []int) (*FeatureTable, error) { 60 | ftab := FeatureTable{Zoomlevels: zoomlevels} 61 | 62 | ftab.table = map[int][][][]spatial.Feature{} 63 | for _, zl := range ftab.Zoomlevels { 64 | l := pow(2, zl) 65 | ftab.table[zl] = make([][][]spatial.Feature, l) 66 | for x := range ftab.table[zl] { 67 | ftab.table[zl][x] = make([][]spatial.Feature, l) 68 | } 69 | } 70 | return &ftab, nil 71 | } 72 | 73 | func (ftab *FeatureTable) AddFeature(ft spatial.Feature) { 74 | for _, zl := range ftab.Zoomlevels { 75 | if !renderable(ft.Props, zl) { 76 | continue 77 | } 78 | for _, tid := range tile.Coverage(ft.Geometry.BBox(), zl) { 79 | ftab.table[zl][tid.X][tid.Y] = append(ftab.table[zl][tid.X][tid.Y], ft) 80 | } 81 | } 82 | if ftab.bbox == nil { 83 | var bb = ft.Geometry.BBox() 84 | ftab.bbox = &bb 85 | } else { 86 | ftab.bbox.ExtendWith(ft.Geometry.BBox()) 87 | } 88 | 89 | ftab.count++ 90 | } 91 | 92 | func (ftab *FeatureTable) GetFeatures(tid tile.ID) []spatial.Feature { 93 | return ftab.table[tid.Z][tid.X][tid.Y] 94 | } 95 | 96 | func (ftab *FeatureTable) Count() int { 97 | return ftab.count 98 | } 99 | 100 | func (ftab *FeatureTable) BBox() spatial.BBox { 101 | if ftab.bbox == nil { 102 | return spatial.BBox{} 103 | } 104 | return *ftab.bbox 105 | } 106 | 107 | type FeatureMap struct { 108 | Zoomlevels []int 109 | 110 | bbox *spatial.BBox 111 | count int 112 | m map[tile.ID][]spatial.Feature 113 | } 114 | 115 | func NewFeatureMap(zl []int) (*FeatureMap, error) { 116 | return &FeatureMap{ 117 | Zoomlevels: zl, 118 | m: map[tile.ID][]spatial.Feature{}, 119 | }, nil 120 | } 121 | 122 | func (fm *FeatureMap) AddFeature(ft spatial.Feature) { 123 | for _, zl := range fm.Zoomlevels { 124 | if !renderable(ft.Props, zl) { 125 | continue 126 | } 127 | for _, tid := range tile.Coverage(ft.Geometry.BBox(), zl) { 128 | _, ok := fm.m[tid] 129 | if !ok { 130 | fm.m[tid] = make([]spatial.Feature, 0, 1) 131 | } 132 | fm.m[tid] = append(fm.m[tid], ft) 133 | } 134 | } 135 | if fm.bbox == nil { 136 | var bb = ft.Geometry.BBox() 137 | fm.bbox = &bb 138 | } else { 139 | fm.bbox.ExtendWith(ft.Geometry.BBox()) 140 | } 141 | 142 | fm.count++ 143 | } 144 | 145 | func (fm *FeatureMap) GetFeatures(tid tile.ID) []spatial.Feature { 146 | return fm.m[tid] 147 | } 148 | 149 | func (fm *FeatureMap) BBox() spatial.BBox { 150 | return *fm.bbox 151 | } 152 | 153 | func (fm *FeatureMap) Count() int { 154 | return fm.count 155 | } 156 | 157 | func (fm *FeatureMap) Dump() map[tile.ID][]spatial.Feature { 158 | return fm.m 159 | } 160 | 161 | func (fm *FeatureMap) Clear() { 162 | fm.m = map[tile.ID][]spatial.Feature{} 163 | } 164 | -------------------------------------------------------------------------------- /cmd/tiler/featurecache_fs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "path" 9 | "strconv" 10 | "time" 11 | 12 | "github.com/thomersch/grandine/lib/spaten" 13 | "github.com/thomersch/grandine/lib/spatial" 14 | "github.com/thomersch/grandine/lib/tile" 15 | ) 16 | 17 | func init() { 18 | registerCache("filesystem", NewFileSystemCache) 19 | } 20 | 21 | type FileSystemCache struct { 22 | Zoomlevels []int 23 | CacheSize int 24 | 25 | basePath string 26 | fp map[tile.ID]*os.File 27 | 28 | cache *FeatureMap 29 | 30 | count int 31 | lastCheckpoint time.Time 32 | 33 | bbox *spatial.BBox 34 | } 35 | 36 | func NewFileSystemCache(zl []int) (FeatureCache, error) { 37 | basePath, err := ioutil.TempDir("", "tiler-fscache") 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | fm, err := NewFeatureMap(zl) 43 | if err != nil { 44 | return nil, err 45 | } 46 | 47 | return &FileSystemCache{ 48 | Zoomlevels: zl, 49 | CacheSize: 1000000, 50 | basePath: basePath, 51 | fp: make(map[tile.ID]*os.File), 52 | 53 | cache: fm, 54 | lastCheckpoint: time.Now(), 55 | }, nil 56 | } 57 | 58 | func (fsc *FileSystemCache) Close() error { 59 | os.RemoveAll(fsc.basePath) 60 | return nil 61 | } 62 | 63 | func (fsc *FileSystemCache) fpath(tid tile.ID) string { 64 | return path.Join(fsc.basePath, strconv.Itoa(tid.Z)+"-"+strconv.Itoa(tid.X)+"-"+strconv.Itoa(tid.Y)) 65 | } 66 | 67 | func (fsc *FileSystemCache) flush() error { 68 | var ( 69 | c spaten.Codec 70 | err error 71 | ) 72 | for tid, fts := range fsc.cache.Dump() { 73 | fd, ok := fsc.fp[tid] 74 | if !ok { 75 | var wbuf bytes.Buffer 76 | err := c.Encode(&wbuf, &spatial.FeatureCollection{Features: fts}) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | f, err := os.Create(fsc.fpath(tid)) 82 | if err != nil { 83 | return err 84 | } 85 | fsc.fp[tid] = f 86 | wbuf.WriteTo(f) 87 | } else { 88 | err = spaten.WriteBlock(fd, fts, nil) 89 | if err != nil { 90 | return err 91 | } 92 | } 93 | } 94 | fsc.cache.Clear() 95 | return nil 96 | } 97 | 98 | func (fsc *FileSystemCache) AddFeature(ft spatial.Feature) { 99 | fsc.count++ 100 | 101 | if fsc.count%fsc.CacheSize == 0 { 102 | showMemStats() 103 | err := fsc.flush() 104 | if err != nil { 105 | panic(err) 106 | } 107 | 108 | log.Printf("Written %v features to disk (%.0f/s)", fsc.count, float64(fsc.CacheSize)/time.Since(fsc.lastCheckpoint).Seconds()) 109 | fsc.lastCheckpoint = time.Now() 110 | } 111 | 112 | fsc.cache.AddFeature(ft) 113 | 114 | if fsc.bbox == nil { 115 | var bb = ft.Geometry.BBox() 116 | fsc.bbox = &bb 117 | } else { 118 | fsc.bbox.ExtendWith(ft.Geometry.BBox()) 119 | } 120 | } 121 | 122 | func (fsc *FileSystemCache) GetFeatures(tid tile.ID) []spatial.Feature { 123 | var ( 124 | c spaten.Codec 125 | fc spatial.FeatureCollection 126 | ) 127 | 128 | fp, ok := fsc.fp[tid] 129 | if !ok { 130 | return nil 131 | } 132 | 133 | _, err := fp.Seek(0, 0) 134 | if err != nil { 135 | panic(err) 136 | } 137 | err = c.Decode(fp, &fc) 138 | if err != nil { 139 | panic(err) 140 | } 141 | return fc.Features 142 | } 143 | 144 | func (fsc *FileSystemCache) BBox() spatial.BBox { 145 | return *fsc.bbox 146 | } 147 | 148 | func (fsc *FileSystemCache) Count() int { 149 | return fsc.count 150 | } 151 | -------------------------------------------------------------------------------- /cmd/tiler/featurecache_leveldb.go: -------------------------------------------------------------------------------- 1 | // +build !noleveldb 2 | 3 | package main 4 | 5 | import ( 6 | "bytes" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "time" 11 | 12 | "github.com/jmhodges/levigo" 13 | 14 | "github.com/thomersch/grandine/lib/spaten" 15 | "github.com/thomersch/grandine/lib/spatial" 16 | "github.com/thomersch/grandine/lib/tile" 17 | ) 18 | 19 | func init() { 20 | registerCache("leveldb", NewLevelDBCache) 21 | } 22 | 23 | type LevelDBCache struct { 24 | Zoomlevels []int 25 | CacheSize int 26 | 27 | db *levigo.DB 28 | dbpath string 29 | 30 | cache *FeatureMap 31 | 32 | count int 33 | lastCheckpoint time.Time 34 | 35 | bbox *spatial.BBox 36 | } 37 | 38 | func NewLevelDBCache(zl []int) (FeatureCache, error) { 39 | ldbopt := levigo.NewOptions() 40 | ldbopt.SetCreateIfMissing(true) 41 | ldbopt.SetWriteBufferSize(100000000) 42 | ldbopt.SetCompression(levigo.NoCompression) 43 | 44 | dbpath, err := ioutil.TempDir("", "tiler-leveldb") 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | leveldb, err := levigo.Open(dbpath, ldbopt) 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | fm, err := NewFeatureMap(zl) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | return &LevelDBCache{ 60 | Zoomlevels: zl, 61 | CacheSize: 1000000, 62 | db: leveldb, 63 | dbpath: dbpath, 64 | cache: fm, 65 | lastCheckpoint: time.Now(), 66 | }, nil 67 | } 68 | 69 | func (ldb *LevelDBCache) Close() error { 70 | ldb.flush() 71 | ldb.db.Close() 72 | os.RemoveAll(ldb.dbpath) 73 | return nil 74 | } 75 | 76 | func (ldb *LevelDBCache) flush() error { 77 | var c spaten.Codec 78 | for tid, fts := range ldb.cache.Dump() { 79 | tidb := []byte(tid.String()) 80 | 81 | rbuf, err := ldb.db.Get(levigo.NewReadOptions(), tidb) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | if len(rbuf) == 0 { 87 | var wbuf bytes.Buffer 88 | err := c.Encode(&wbuf, &spatial.FeatureCollection{Features: fts}) 89 | if err != nil { 90 | return err 91 | } 92 | err = ldb.db.Put(levigo.NewWriteOptions(), tidb, wbuf.Bytes()) 93 | if err != nil { 94 | return err 95 | } 96 | } else { 97 | var ( 98 | buf = bytes.NewReader(rbuf) 99 | coll spatial.FeatureCollection 100 | ) 101 | err := c.Decode(buf, &coll) 102 | if err != nil { 103 | return err 104 | } 105 | coll.Features = append(coll.Features, fts...) 106 | var ob bytes.Buffer 107 | err = c.Encode(&ob, &coll) 108 | if err != nil { 109 | return err 110 | } 111 | err = ldb.db.Put(levigo.NewWriteOptions(), tidb, ob.Bytes()) 112 | if err != nil { 113 | return err 114 | } 115 | } 116 | } 117 | ldb.cache.Clear() 118 | return nil 119 | } 120 | 121 | func (ldb *LevelDBCache) AddFeature(ft spatial.Feature) { 122 | ldb.count++ 123 | 124 | if ldb.count%ldb.CacheSize == 0 { 125 | showMemStats() 126 | err := ldb.flush() 127 | if err != nil { 128 | panic(err) 129 | } 130 | 131 | log.Printf("Written %v features to disk (%.0f/s)", ldb.count, float64(ldb.CacheSize)/time.Since(ldb.lastCheckpoint).Seconds()) 132 | ldb.lastCheckpoint = time.Now() 133 | } 134 | 135 | ldb.cache.AddFeature(ft) 136 | 137 | if ldb.bbox == nil { 138 | var bb = ft.Geometry.BBox() 139 | ldb.bbox = &bb 140 | } else { 141 | ldb.bbox.ExtendWith(ft.Geometry.BBox()) 142 | } 143 | } 144 | 145 | func (ldb *LevelDBCache) GetFeatures(tid tile.ID) []spatial.Feature { 146 | var ( 147 | c spaten.Codec 148 | fc spatial.FeatureCollection 149 | ) 150 | if ldb.cache.Count() != 0 { 151 | ldb.flush() 152 | } 153 | 154 | tidb := []byte(tid.String()) 155 | 156 | buf, err := ldb.db.Get(levigo.NewReadOptions(), tidb) 157 | if err != nil { 158 | panic(err) 159 | } 160 | if len(buf) == 0 { 161 | return nil 162 | } 163 | var b = bytes.NewReader(buf) 164 | err = c.Decode(b, &fc) 165 | if err != nil { 166 | panic(err) 167 | } 168 | return fc.Features 169 | } 170 | 171 | func (ldb *LevelDBCache) BBox() spatial.BBox { 172 | return *ldb.bbox 173 | } 174 | 175 | func (ldb *LevelDBCache) Count() int { 176 | return ldb.count 177 | } 178 | -------------------------------------------------------------------------------- /cmd/tiler/s3.go: -------------------------------------------------------------------------------- 1 | // +build go1.13 2 | 3 | package main 4 | 5 | import ( 6 | "bytes" 7 | "context" 8 | "fmt" 9 | 10 | "github.com/pkg/errors" 11 | "github.com/thomersch/grandine/lib/tile" 12 | 13 | minio "github.com/minio/minio-go/v7" 14 | "github.com/minio/minio-go/v7/pkg/credentials" 15 | ) 16 | 17 | const S3ProtoPrefix = "s3://" 18 | 19 | type s3Client struct { 20 | *minio.Client 21 | bucket string 22 | } 23 | 24 | type S3TileWriter struct { 25 | client s3Client 26 | } 27 | 28 | func NewS3TileWriter(endpoint, bucket, accessKeyID, secretAccessKey string) (*S3TileWriter, error) { 29 | s3tw := &S3TileWriter{} 30 | client, err := minio.New(endpoint, &minio.Options{ 31 | Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), 32 | }) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | ok, err := client.BucketExists(context.Background(), bucket) 38 | if err != nil { 39 | return nil, err 40 | } 41 | if !ok { 42 | return nil, fmt.Errorf("S3 bucket %s does not exist", bucket) 43 | } 44 | 45 | s3tw.client = s3Client{client, bucket} 46 | 47 | return s3tw, nil 48 | } 49 | 50 | func (s3 *S3TileWriter) WriteTile(tID tile.ID, buf []byte, ext string) error { 51 | r := bytes.NewReader(buf) 52 | 53 | _, err := s3.client.PutObject(context.Background(), s3.client.bucket, tID.String()+"."+ext, r, int64(r.Len()), minio.PutObjectOptions{}) 54 | 55 | return errors.Wrap(err, "S3 upload failed") 56 | } 57 | -------------------------------------------------------------------------------- /cmd/tiler/s3_nop.go: -------------------------------------------------------------------------------- 1 | // +build !go1.13 2 | 3 | package main 4 | 5 | import "github.com/thomersch/grandine/lib/tile" 6 | 7 | const S3ProtoPrefix = "s3://" 8 | 9 | type S3TileWriter struct{} 10 | 11 | func (s3 *S3TileWriter) WriteTile(tID tile.ID, buf []byte, ext string) error { 12 | panic("S3TileWriter needs at least Go 1.13") 13 | } 14 | 15 | func NewS3TileWriter(endpoint, bucket, accessKeyID, secretAccessKey string) (*S3TileWriter, error) { 16 | panic("S3TileWriter needs at least Go 1.13") 17 | } 18 | -------------------------------------------------------------------------------- /cmd/tiler/shuffle.go: -------------------------------------------------------------------------------- 1 | // +build go1.10 2 | 3 | package main 4 | 5 | import ( 6 | "math/rand" 7 | 8 | "github.com/thomersch/grandine/lib/tile" 9 | ) 10 | 11 | func shuffleWork(w []tile.ID) { 12 | rand.Shuffle(len(w), func(i, j int) { 13 | w[i], w[j] = w[j], w[i] 14 | }) 15 | } 16 | -------------------------------------------------------------------------------- /cmd/tiler/shuffle_pre1_10.go: -------------------------------------------------------------------------------- 1 | // +build !go1.10 2 | 3 | package main 4 | 5 | import "github.com/thomersch/grandine/lib/tile" 6 | 7 | func shuffleWork(w []tile.ID) {} 8 | -------------------------------------------------------------------------------- /fileformat/README.md: -------------------------------------------------------------------------------- 1 | # Spaten, a geodata format 2 | 3 | ![File Format Sketch](https://rawgit.com/thomersch/grandine/master/fileformat/ff.xml.svg) 4 | -------------------------------------------------------------------------------- /fileformat/fileformat.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | message Body { 4 | Meta meta = 1; 5 | repeated Feature feature = 2; 6 | } 7 | 8 | message Meta { 9 | repeated Tag tags = 1; 10 | } 11 | 12 | message Feature { 13 | enum GeomType { 14 | UNKNOWN = 0; 15 | POINT = 1; 16 | LINE = 2; 17 | POLYGON = 3; 18 | } 19 | enum GeomSerialization { 20 | WKB = 0; 21 | } 22 | GeomType geomtype = 1; 23 | GeomSerialization geomserial = 2; 24 | bytes geom = 3; 25 | 26 | // geometry bbox 27 | double left = 4; 28 | double right = 5; 29 | double top = 6; 30 | double bottom = 7; 31 | 32 | repeated Tag tags = 8; 33 | } 34 | 35 | message Tag { 36 | enum ValueType { 37 | STRING = 0; 38 | INT = 1; 39 | DOUBLE = 2; 40 | } 41 | string key = 1; 42 | bytes value = 2; 43 | ValueType type = 3; 44 | } 45 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/thomersch/grandine 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/ctessum/polyclip-go v1.0.1 7 | github.com/davecgh/go-spew v1.1.1 8 | github.com/dhconnelly/rtreego v1.0.0 9 | github.com/dustin/go-humanize v1.0.0 10 | github.com/golang/protobuf v1.4.2 11 | github.com/jmhodges/levigo v1.0.0 12 | github.com/minio/minio-go/v7 v7.0.2 13 | github.com/paulsmith/gogeos v0.1.2 // indirect 14 | github.com/pkg/errors v0.8.1 15 | github.com/pmezard/gogeos v0.1.2 16 | github.com/stretchr/testify v1.4.0 17 | github.com/thomersch/gosmparse v1.0.0 18 | github.com/twpayne/go-geom v1.0.5 19 | gopkg.in/yaml.v2 v2.2.8 20 | ) 21 | -------------------------------------------------------------------------------- /lib/csv/codec.go: -------------------------------------------------------------------------------- 1 | package csv 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "io" 7 | "strconv" 8 | 9 | "github.com/thomersch/grandine/lib/spatial" 10 | ) 11 | 12 | const defaultChunkSize = 100000 13 | 14 | type csvReader interface { 15 | Read() ([]string, error) 16 | } 17 | 18 | type Chunks struct { 19 | chunkSize int 20 | lineDecoder func(csvReader) (spatial.Feature, error) 21 | csvRdr csvReader 22 | 23 | endReached bool 24 | } 25 | 26 | func (c *Chunks) Next() bool { 27 | return !c.endReached 28 | } 29 | 30 | func (c *Chunks) Scan(fc *spatial.FeatureCollection) error { 31 | if c.chunkSize == 0 { 32 | c.chunkSize = defaultChunkSize 33 | } 34 | var ( 35 | fts = make([]spatial.Feature, c.chunkSize) 36 | err error 37 | read int 38 | ) 39 | for i := range fts { 40 | fts[i], err = c.lineDecoder(c.csvRdr) 41 | if err == io.EOF { 42 | c.endReached = true 43 | break 44 | } 45 | if err != nil { 46 | return err 47 | } 48 | read++ 49 | } 50 | fc.Features = append(fc.Features, fts[:read]...) 51 | return nil 52 | } 53 | 54 | type Codec struct { 55 | LatCol, LonCol int 56 | Delim rune 57 | 58 | keys []string 59 | } 60 | 61 | func (c *Codec) decodeLine(csvr csvReader) (spatial.Feature, error) { 62 | var ( 63 | ft = spatial.NewFeature() 64 | pt spatial.Point 65 | ) 66 | 67 | record, err := csvr.Read() 68 | if err != nil { 69 | return ft, err 70 | } 71 | 72 | if c.LonCol >= len(record) || c.LatCol >= len(record) { 73 | return ft, fmt.Errorf("there are not enough columns in: '%v'", record) 74 | } 75 | 76 | pt.X, err = strconv.ParseFloat(record[c.LonCol], 64) 77 | if err != nil { 78 | return ft, err 79 | } 80 | pt.Y, err = strconv.ParseFloat(record[c.LatCol], 64) 81 | if err != nil { 82 | return ft, err 83 | } 84 | ft.Geometry = spatial.MustNewGeom(pt) 85 | for i, val := range record { 86 | if i >= len(c.keys) { 87 | // there are more value in this line than header keys 88 | continue 89 | } 90 | ft.Props[c.keys[i]] = val 91 | } 92 | return ft, nil 93 | } 94 | 95 | func (c *Codec) newReader(r io.Reader) csvReader { 96 | csvRdr := csv.NewReader(r) 97 | if c.Delim == 0 { 98 | csvRdr.Comma = ' ' 99 | } else { 100 | csvRdr.Comma = c.Delim 101 | } 102 | csvRdr.LazyQuotes = false 103 | return csvRdr 104 | } 105 | 106 | func (c *Codec) Decode(r io.Reader, fs *spatial.FeatureCollection) error { 107 | csvRdr := c.newReader(r) 108 | 109 | var err error 110 | c.keys, err = csvRdr.Read() 111 | if err != nil { 112 | return err 113 | } 114 | 115 | for { 116 | ft, err := c.decodeLine(csvRdr) 117 | if err == io.EOF { 118 | break 119 | } 120 | if err != nil { 121 | return err 122 | } 123 | fs.Features = append(fs.Features, ft) 124 | } 125 | return nil 126 | } 127 | 128 | func (c *Codec) ChunkedDecode(r io.Reader) (spatial.Chunks, error) { 129 | csvRdr := c.newReader(r) 130 | 131 | var err error 132 | c.keys, err = csvRdr.Read() 133 | if err != nil { 134 | return nil, err 135 | } 136 | 137 | return &Chunks{ 138 | chunkSize: defaultChunkSize, 139 | lineDecoder: c.decodeLine, 140 | csvRdr: csvRdr, 141 | }, nil 142 | } 143 | 144 | func (c *Codec) Extensions() []string { 145 | return []string{"csv", "txt"} 146 | } 147 | -------------------------------------------------------------------------------- /lib/csv/codec_test.go: -------------------------------------------------------------------------------- 1 | package csv 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/thomersch/grandine/lib/spatial" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestCSVDecode(t *testing.T) { 13 | f, err := os.Open("testfiles/gn_excerpt.csv") 14 | assert.Nil(t, err) 15 | 16 | csvr := Codec{ 17 | LatCol: 4, 18 | LonCol: 5, 19 | } 20 | fcoll := spatial.FeatureCollection{} 21 | err = csvr.Decode(f, &fcoll) 22 | assert.Nil(t, err) 23 | assert.Equal(t, "les Escaldes", fcoll.Features[0].Props["name"]) 24 | pt, err := fcoll.Features[0].Geometry.Point() 25 | assert.Nil(t, err) 26 | assert.Equal(t, 1.53414, pt.X) 27 | assert.Equal(t, 42.50729, pt.Y) 28 | } 29 | -------------------------------------------------------------------------------- /lib/csv/csv.go: -------------------------------------------------------------------------------- 1 | package csv 2 | 3 | /* 4 | Package csv can be used for reading CSV files which contain spatial information, 5 | like for example Geonames data dumps. 6 | */ 7 | -------------------------------------------------------------------------------- /lib/csv/testfiles/gn_excerpt.csv: -------------------------------------------------------------------------------- 1 | geonameid name asciiname alternatenames latitude longitude feature class feature country cc2 admin1 admin2 admin3 admin4 population elevation dem timezone modification date 2 | 3040051 les Escaldes les Escaldes Ehskal'des-Ehndzhordani,Escaldes,Escaldes-Engordany,Les Escaldes,esukarudesu=engorudani jiao qu,lai sai si ka er de-en ge er da,Эскальдес-Энджордани,エスカルデス＝エンゴルダニ教区,萊塞斯卡爾德-恩戈爾達,萊塞斯卡爾德－恩戈爾達 42.50729 1.53414 P PPLA AD 08 15853 1033 Europe/Andorra 2008-10-15 3 | 3041563 Andorra la Vella Andorra la Vella ALV,Ando-la-Vyey,Andora,Andora la Vela,Andora la Velja,Andora lja Vehl'ja,Andoro Malnova,Andorra,Andorra Tuan,Andorra a Vella,Andorra la Biella,Andorra la Vella,Andorra la Vielha,Andorra-a-Velha,Andorra-la-Vel'ja,Andorra-la-Vielye,Andorre-la-Vieille,Andò-la-Vyèy,Andòrra la Vièlha,an dao er cheng,andolalabeya,andwra la fyla,Ανδόρρα,Андора ла Веля,Андора ла Веља,Андора ля Вэлья,Андорра-ла-Велья,אנדורה לה וולה,أندورا لا فيلا,አንዶራ ላ ቬላ,アンドラ・ラ・ヴェリャ,安道爾城,안도라라베야 42.50779 1.52109 P PPLC AD 07 20430 1037 Europe/Andorra 2010-05-30 4 | 290594 Umm al Qaywayn Umm al Qaywayn Oumm al Qaiwain,Oumm al Qaïwaïn,Um al Kawain,Um al Quweim,Umm al Qaiwain,Umm al Qawain,Umm al Qaywayn,Umm al-Quwain,Umm-ehl'-Kajvajn,Yumul al Quwain,am alqywyn,Умм-эль-Кайвайн,أم القيوين 25.56473 55.55517 P PPLA AE 07 44411 2 Asia/Dubai 2014-10-07 5 | 291074 Ras al-Khaimah Ras al-Khaimah Julfa,Khaimah,RKT,Ra's al Khaymah,Ra's al-Chaima,Ras al Khaimah,Ras al-Khaimah,Ras el Khaimah,Ras el Khaïmah,Ras el-Kheima,Ras-ehl'-Khajma,Ra’s al Khaymah,Ra’s al-Chaima,ras alkhymt,Рас-эль-Хайма,رأس الخيمة 25.78953 55.9432 P PPLA AE 05 115949 2 Asia/Dubai 2015-12-05 6 | -------------------------------------------------------------------------------- /lib/geojson/codec.go: -------------------------------------------------------------------------------- 1 | package geojson 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "io" 7 | "strings" 8 | 9 | "github.com/thomersch/grandine/lib/spatial" 10 | ) 11 | 12 | type Codec struct{} 13 | 14 | func (c *Codec) Decode(r io.Reader, fc *spatial.FeatureCollection) error { 15 | var gjfc featureColl 16 | err := json.NewDecoder(r).Decode(&gjfc) 17 | if err != nil { 18 | return err 19 | } 20 | fc.Features = append(fc.Features, gjfc.Features...) 21 | if srid, ok := ogcSRID[gjfc.CRS.Properties.Name]; ok { 22 | if len(fc.SRID) == 0 { 23 | fc.SRID = srid 24 | } 25 | if len(srid) != 0 && fc.SRID != srid { 26 | return errors.New("incompatible projections: ") 27 | } 28 | } 29 | return nil 30 | } 31 | 32 | func (c *Codec) Encode(w io.Writer, fc *spatial.FeatureCollection) error { 33 | if len(fc.SRID) != 0 { 34 | geojsonFC := featureColl{} 35 | geojsonFC.Type = "FeatureCollection" 36 | geojsonFC.Features = fc.Features 37 | geojsonFC.CRS.Properties.Name, _ = sridOGC[fc.SRID] 38 | geojsonFC.CRS.Type = "name" 39 | return json.NewEncoder(w).Encode(&geojsonFC) 40 | } 41 | geojsonFC := featureCollNoCRS{} 42 | geojsonFC.Type = "FeatureCollection" 43 | geojsonFC.Features = fc.Features 44 | return json.NewEncoder(w).Encode(&geojsonFC) 45 | } 46 | 47 | func (c *Codec) Extensions() []string { 48 | return []string{"geojson", "json"} 49 | } 50 | 51 | type featureCollNoCRS struct { 52 | Type string `json:"type"` 53 | Features FeatList `json:"features"` 54 | } 55 | 56 | type featureColl struct { 57 | featureCollNoCRS 58 | CRS struct { 59 | Type string `json:"type"` 60 | Properties struct { 61 | Name string `json:"name"` 62 | } `json:"properties"` 63 | } `json:"crs"` 64 | } 65 | 66 | type FeatureProto struct { 67 | Geometry struct { 68 | Type string `json:"type"` 69 | Coordinates json.RawMessage `json:"coordinates"` 70 | } 71 | ID string 72 | Properties map[string]interface{} `json:"properties"` 73 | } 74 | 75 | type FeatList []spatial.Feature 76 | 77 | func (fl *FeatList) UnmarshalJSON(buf []byte) error { 78 | var fts []FeatureProto 79 | err := json.Unmarshal(buf, &fts) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | *fl = make([]spatial.Feature, 0, len(fts)) 85 | for _, inft := range fts { 86 | if inft.ID != "" { 87 | inft.Properties["id"] = inft.ID 88 | } 89 | 90 | err = fl.UnmarshalJSONCoords(inft) 91 | if err != nil { 92 | return err 93 | } 94 | } 95 | return nil 96 | } 97 | 98 | func (fl *FeatList) UnmarshalJSONCoords(fp FeatureProto) error { 99 | var err error 100 | if singularType(fp.Geometry.Type) { 101 | var ft spatial.Feature 102 | ft.Props = fp.Properties 103 | err = ft.Geometry.UnmarshalJSONCoords(fp.Geometry.Type, fp.Geometry.Coordinates) 104 | if err == spatial.ErrorEmptyGeomType { 105 | // TODO: Shall we warn here somehow? 106 | return nil 107 | } 108 | if err != nil { 109 | return err 110 | } 111 | *fl = append(*fl, ft) 112 | } else { 113 | // Because the lib doesn't have native Multi* types, we split those into single geometries. 114 | var singles []json.RawMessage 115 | json.Unmarshal(fp.Geometry.Coordinates, &singles) 116 | for _, single := range singles { 117 | var ft spatial.Feature 118 | ft.Props = fp.Properties 119 | err = ft.Geometry.UnmarshalJSONCoords(fp.Geometry.Type[5:], single) 120 | if err != nil { 121 | return err 122 | } 123 | *fl = append(*fl, ft) 124 | } 125 | } 126 | return nil 127 | } 128 | 129 | func singularType(typ string) bool { 130 | return !strings.HasPrefix(strings.ToLower(typ), "multi") 131 | } 132 | -------------------------------------------------------------------------------- /lib/geojson/codec_test.go: -------------------------------------------------------------------------------- 1 | package geojson 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | "testing" 7 | 8 | "github.com/thomersch/grandine/lib/spatial" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestDecode(t *testing.T) { 14 | f, err := os.Open("testdata/01.geojson") 15 | assert.Nil(t, err) 16 | 17 | var ( 18 | c = &Codec{} 19 | fc = spatial.FeatureCollection{} 20 | ) 21 | err = c.Decode(f, &fc) 22 | assert.Nil(t, err) 23 | assert.Equal(t, "4326", fc.SRID) 24 | assert.Len(t, fc.Features, 1) 25 | } 26 | 27 | func TestDecodeID(t *testing.T) { 28 | f, err := os.Open("testdata/id.geojson") 29 | assert.Nil(t, err) 30 | 31 | var ( 32 | c = &Codec{} 33 | fc = spatial.FeatureCollection{} 34 | ) 35 | err = c.Decode(f, &fc) 36 | assert.Nil(t, err) 37 | assert.Len(t, fc.Features, 2) 38 | assert.Equal(t, fc.Features[0].Properties()["id"], "asdf") 39 | assert.NotContains(t, fc.Features[1].Properties(), "id") 40 | } 41 | 42 | func TestDecodeMultipolygon(t *testing.T) { 43 | f, err := os.Open("testdata/multipolygon.geojson") 44 | assert.Nil(t, err) 45 | 46 | var ( 47 | c = &Codec{} 48 | fc = spatial.FeatureCollection{} 49 | ) 50 | err = c.Decode(f, &fc) 51 | assert.Nil(t, err) 52 | assert.Len(t, fc.Features, 2) 53 | } 54 | 55 | func TestEncode(t *testing.T) { 56 | var ( 57 | fc spatial.FeatureCollection 58 | c Codec 59 | buf = bytes.NewBuffer(make([]byte, 1000)) 60 | ) 61 | 62 | fc.SRID = "4326" 63 | err := c.Encode(buf, &fc) 64 | assert.Nil(t, err) 65 | } 66 | -------------------------------------------------------------------------------- /lib/geojson/ogc_srid.go: -------------------------------------------------------------------------------- 1 | package geojson 2 | 3 | var ( 4 | ogcSRID = map[string]string{ 5 | "urn:ogc:def:crs:OGC:1.3:CRS84": "4326", 6 | } 7 | sridOGC = map[string]string{ 8 | "4326": "urn:ogc:def:crs:OGC:1.3:CRS84", 9 | } 10 | ) 11 | -------------------------------------------------------------------------------- /lib/geojson/testdata/01.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "crs": { 3 | "properties": { 4 | "name": "urn:ogc:def:crs:OGC:1.3:CRS84" 5 | }, 6 | "type": "name" 7 | }, 8 | "features": [ 9 | { 10 | "geometry": { 11 | "coordinates": [ 12 | [ 13 | [ 14 | 17.979785156250017, 15 | 59.329052734374997 16 | ], 17 | [ 18 | 17.876171875000011, 19 | 59.270800781250003 20 | ], 21 | [ 22 | 17.570507812500011, 23 | 59.267626953125003 24 | ], 25 | [ 26 | 17.474511718750023, 27 | 59.29150390625 28 | ], 29 | [ 30 | 17.370703125, 31 | 59.294921875 32 | ], 33 | [ 34 | 17.304589843750023, 35 | 59.272167968749997 36 | ], 37 | [ 38 | 17.175195312500023, 39 | 59.355810546874999 40 | ], 41 | [ 42 | 17.065625, 43 | 59.373242187499997 44 | ], 45 | [ 46 | 16.913867187500017, 47 | 59.445849609375003 48 | ], 49 | [ 50 | 16.742285156250006, 51 | 59.430615234374997 52 | ], 53 | [ 54 | 16.610449218750006, 55 | 59.45351562499999 56 | ], 57 | [ 58 | 16.144335937500017, 59 | 59.44775390625 60 | ], 61 | [ 62 | 16.044238281250017, 63 | 59.478466796874997 64 | ], 65 | [ 66 | 16.251757812500017, 67 | 59.493212890625003 68 | ], 69 | [ 70 | 16.47265625, 71 | 59.519384765624999 72 | ], 73 | [ 74 | 16.573828125, 75 | 59.611669921874999 76 | ], 77 | [ 78 | 16.646875, 79 | 59.559277343749997 80 | ], 81 | [ 82 | 16.752343750000023, 83 | 59.543310546874999 84 | ], 85 | [ 86 | 16.841015625000011, 87 | 59.5875 88 | ], 89 | [ 90 | 16.9775390625, 91 | 59.550683593749994 92 | ], 93 | [ 94 | 17.062695312500011, 95 | 59.569238281249994 96 | ], 97 | [ 98 | 17.3720703125, 99 | 59.495751953124994 100 | ], 101 | [ 102 | 17.390527343750023, 103 | 59.58447265625 104 | ], 105 | [ 106 | 17.534472656250017, 107 | 59.539404296874999 108 | ], 109 | [ 110 | 17.687304687500017, 111 | 59.541601562499999 112 | ], 113 | [ 114 | 17.671582031250011, 115 | 59.594775390625003 116 | ], 117 | [ 118 | 17.760058593750017, 119 | 59.620507812499994 120 | ], 121 | [ 122 | 17.785937500000017, 123 | 59.597998046874999 124 | ], 125 | [ 126 | 17.80859375, 127 | 59.55322265625 128 | ], 129 | [ 130 | 17.772851562500023, 131 | 59.414111328125003 132 | ], 133 | [ 134 | 17.829296875000011, 135 | 59.379003906249999 136 | ], 137 | [ 138 | 17.964257812500023, 139 | 59.359375 140 | ], 141 | [ 142 | 17.979785156250017, 143 | 59.329052734374997 144 | ] 145 | ] 146 | ], 147 | "type": "Polygon" 148 | }, 149 | "properties": { 150 | "admin": null, 151 | "featurecla": "Lake", 152 | "name": "Mlaren", 153 | "name_alt": null, 154 | "note": null, 155 | "scalerank": 2 156 | }, 157 | "type": "Feature" 158 | } 159 | ], 160 | "type": "FeatureCollection" 161 | } 162 | -------------------------------------------------------------------------------- /lib/geojson/testdata/id.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "id": "asdf", 7 | "properties": {}, 8 | "geometry": { 9 | "type": "Point", 10 | "coordinates": [ 11 | 10.546875, 12 | 51.39920565355378 13 | ] 14 | } 15 | }, 16 | { 17 | "type": "Feature", 18 | "properties": {}, 19 | "geometry": { 20 | "type": "Point", 21 | "coordinates": [ 22 | 11.546875, 23 | 51.39920565355378 24 | ] 25 | } 26 | } 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /lib/geojson/testdata/multipolygon.geojson: -------------------------------------------------------------------------------- 1 | {"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"MultiPolygon","coordinates":[[[[14.0625,42.5530802889558],[49.5703125,42.5530802889558],[49.5703125,63.39152174400882],[14.0625,63.39152174400882],[14.0625,42.5530802889558]]],[[[-86.484375,20.3034175184893],[-44.29687499999999,20.3034175184893],[-44.29687499999999,48.45835188280866],[-86.484375,48.45835188280866],[-86.484375,20.3034175184893]]]]}}]} 2 | -------------------------------------------------------------------------------- /lib/geojsonseq/codec.go: -------------------------------------------------------------------------------- 1 | package geojsonseq 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | 9 | "github.com/thomersch/grandine/lib/geojson" 10 | "github.com/thomersch/grandine/lib/spatial" 11 | ) 12 | 13 | const resourceSep = byte('\x1E') 14 | 15 | type Codec struct{} 16 | 17 | func (c *Codec) Decode(io.Reader, *spatial.FeatureCollection) error { 18 | panic("not implemented yet, please use ChunkedDecode") 19 | } 20 | 21 | func (c *Codec) ChunkedDecode(r io.Reader) (spatial.Chunks, error) { 22 | var rs = make([]byte, 1) 23 | _, err := r.Read(rs) 24 | if rs[0] != resourceSep || err != nil { 25 | return nil, fmt.Errorf("a geojson sequence must start with a resource separator, got: %v", rs[0]) 26 | } 27 | return &chunk{reader: bufio.NewReader(r)}, nil 28 | } 29 | 30 | func (c *Codec) Extensions() []string { 31 | return []string{"geojsonseq"} 32 | } 33 | 34 | type chunk struct { 35 | endReached bool 36 | reader *bufio.Reader 37 | buf []byte 38 | err error 39 | } 40 | 41 | func (ch *chunk) Next() bool { 42 | if ch.endReached { 43 | return false 44 | } 45 | var err error 46 | ch.buf, err = ch.reader.ReadBytes(resourceSep) 47 | if err == io.EOF { 48 | ch.endReached = true 49 | } else if err != nil { 50 | ch.err = err // we cannot return the value here, so it will pop up on the next scan 51 | return true 52 | } 53 | ch.buf = ch.buf[:len(ch.buf)-1] 54 | return true 55 | } 56 | 57 | func (ch *chunk) Scan(fc *spatial.FeatureCollection) error { 58 | if ch.err != nil { 59 | return ch.err 60 | } 61 | var fts geojson.FeatList 62 | err := json.Unmarshal(append(append([]byte(`[`), ch.buf...), ']'), &fts) 63 | if err != nil { 64 | return err 65 | } 66 | fc.Features = append(fc.Features, fts...) 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /lib/geojsonseq/codec_test.go: -------------------------------------------------------------------------------- 1 | package geojsonseq 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/thomersch/grandine/lib/spatial" 10 | ) 11 | 12 | func TestChunkedDecode(t *testing.T) { 13 | f, err := os.Open("testdata/example.geojsonseq") 14 | assert.Nil(t, err) 15 | defer f.Close() 16 | 17 | c := Codec{} 18 | chunks, err := c.ChunkedDecode(f) 19 | assert.Nil(t, err) 20 | 21 | var fcoll spatial.FeatureCollection 22 | for chunks.Next() { 23 | err := chunks.Scan(&fcoll) 24 | assert.Nil(t, err) 25 | } 26 | assert.Len(t, fcoll.Features, 10) 27 | } 28 | 29 | type gjProducer struct { 30 | Data []byte 31 | NRecords int 32 | pos int 33 | } 34 | 35 | func (p *gjProducer) Read(buf []byte) (int, error) { 36 | var read int 37 | 38 | if p.pos == p.NRecords*len(p.Data) { 39 | return 0, io.EOF 40 | } 41 | for i := 0; i < len(buf); i++ { 42 | buf[i] = p.Data[p.pos%len(p.Data)] 43 | p.pos++ 44 | read++ 45 | if p.pos == p.NRecords*len(p.Data) { 46 | break 47 | } 48 | } 49 | return read, nil 50 | } 51 | 52 | func BenchmarkChunkedDecode(b *testing.B) { 53 | var g = gjProducer{ 54 | NRecords: b.N, 55 | Data: []byte(`{"type":"Feature","geometry":{"type":"Point","coordinates":[8.9026586,53.100125]},"properties":{}} 56 | `), 57 | } 58 | 59 | c := Codec{} 60 | 61 | b.ReportAllocs() 62 | b.ResetTimer() 63 | 64 | chunks, err := c.ChunkedDecode(&g) 65 | assert.Nil(b, err) 66 | 67 | var fcoll spatial.FeatureCollection 68 | 69 | for chunks.Next() { 70 | err := chunks.Scan(&fcoll) 71 | assert.Nil(b, err) 72 | } 73 | assert.Len(b, fcoll.Features, b.N) 74 | } 75 | -------------------------------------------------------------------------------- /lib/geojsonseq/testdata/example.geojsonseq: -------------------------------------------------------------------------------- 1 | {"type":"Feature","geometry":{"type":"Point","coordinates":[8.7819939,53.070685]},"properties":{"name":"Bremen-Neustadt","highway":"motorway_junction","TMC:cid_58:tabcd_1:Class":"Point","TMC:cid_58:tabcd_1:Direction":"negative","TMC:cid_58:tabcd_1:LCLversion":"10.1","TMC:cid_58:tabcd_1:LocationCode":"25041"}} 2 | {"type":"Feature","geometry":{"type":"Point","coordinates":[8.8710006,53.0984451]},"properties":{"bicycle":"yes","highway":"traffic_signals","crossing":"traffic_signals"}} 3 | {"type":"Feature","geometry":{"type":"Point","coordinates":[8.8740043,53.101282]},"properties":{"bicycle":"yes","highway":"traffic_signals","crossing":"traffic_signals"}} 4 | {"type":"Feature","geometry":{"type":"Point","coordinates":[8.87012,53.0971022]},"properties":{"bicycle":"yes","highway":"traffic_signals","crossing":"traffic_signals"}} 5 | {"type":"Feature","geometry":{"type":"Point","coordinates":[8.8698447,53.0971582]},"properties":{"bicycle":"yes","highway":"traffic_signals","crossing":"traffic_signals"}} 6 | {"type":"Feature","geometry":{"type":"Point","coordinates":[8.8858872,53.099723]},"properties":{"highway":"traffic_signals"}} 7 | {"type":"Feature","geometry":{"type":"Point","coordinates":[8.9026586,53.100125]},"properties":{"highway":"traffic_signals"}} 8 | {"type":"Feature","geometry":{"type":"Point","coordinates":[8.8842552,53.1053179]},"properties":{"crossing":"traffic_signals","highway":"traffic_signals"}} 9 | {"type":"Feature","geometry":{"type":"Point","coordinates":[8.8164368,53.1165751]},"properties":{"created_by":"nope"}} 10 | {"type":"Feature","geometry":{"type":"Point","coordinates":[8.7429251,53.1395794]},"properties":{"alt_name":"Bremen-Industriehäfen","highway":"motorway_junction","name":"Dreieck Bremen-Industriehäfen","ref":"17","TMC:cid_58:tabcd_1:Class":"Point","TMC:cid_58:tabcd_1:Direction":"positive","TMC:cid_58:tabcd_1:LCLversion":"9.00","TMC:cid_58:tabcd_1:LocationCode":"10715","TMC:cid_58:tabcd_1:NextLocationCode":"29672","TMC:cid_58:tabcd_1:PrevLocationCode":"43210"}} 11 | -------------------------------------------------------------------------------- /lib/mapping/README.md: -------------------------------------------------------------------------------- 1 | # lib/mapping 2 | 3 | [![GoDoc](https://godoc.org/github.com/thomersch/grandine?status.svg)](https://godoc.org/github.com/thomersch/grandine/lib/mapping) 4 | 5 | This package provides facilities for parsing and applying mapping files. Mapping files allow to convert existing, freely tagged geodata into a stricter subset, as defined by a mapping file. This approach can be very useful if you want to rename keys, want to convert data types or just reduce the dataset without having to write code. 6 | 7 | A mapping file is a yaml file (thus JSON is fully compatible, if you prefer to use that), which defines a list of projections, consisting of an input condition and one or more output statements. 8 | 9 | ## Condition 10 | 11 | The key and value specified in `src` is the input condition. Any element that matches the given combination will be transformed using the `dest` parameters. 12 | 13 | ### Examples 14 | 15 | * `{key: "highway", value: "primary"}` captures all elements with `highway=primary` 16 | * `{key: "highway", value: "*"}` captures all elements that have any tag with the key `highway`. 17 | 18 | ## Output 19 | 20 | In `dest` one can specify one or more output tags. Those can be either static for any matched element or have dynamic, typed values, which are inherited from the original element. 21 | 22 | ### Examples 23 | 24 | * `- {key: "class", value: "railway"}` inserts a `class=highway` into all matched elements, as defined in `src`. 25 | * `- {key: "v-max", value: "$maxspeed", type: int}` retrieves the maxspeed value from the source element, converts it to an integer and inserts it into `v-max`. 26 | 27 | ### Types 28 | 29 | Any output element can have a `type` element, which defines the data type. Currently supported: 30 | 31 | * `int`, casting to integer 32 | * `string`, a series of bytes, no conversion, equivalent to not specifying any type 33 | * no type, just interpreting as string 34 | 35 | ## Full Example 36 | 37 | - src: 38 | key: highway 39 | value: primary 40 | dest: 41 | - {key: "@layer", value: "transportation"} 42 | - {key: "class", value: "$highway"} 43 | 44 | - src: 45 | key: building 46 | value: "*" 47 | dest: 48 | - {key: "@layer", value: "building"} 49 | - {key: "@zoom:min", value: 14} 50 | 51 | - src: 52 | key: railway 53 | value: "*" 54 | dest: 55 | - {key: "@layer", value: "transportation"} 56 | - {key: "class", value: "railway"} 57 | - {key: "maxspeed", value: "$maxspeed", type: int} 58 | -------------------------------------------------------------------------------- /lib/mapping/condition.go: -------------------------------------------------------------------------------- 1 | package mapping 2 | 3 | import "github.com/thomersch/grandine/lib/spatial" 4 | 5 | type tagMapFn func(map[string]interface{}) map[string]interface{} 6 | type geomOp func(spatial.Geom) []spatial.Geom 7 | 8 | type Condition struct { 9 | // TODO: make it possible to specify Condition type (node/way/rel) 10 | key string 11 | value []string 12 | mapper tagMapFn 13 | op geomOp 14 | } 15 | 16 | func (c *Condition) Matches(kv map[string]interface{}) bool { 17 | if v, ok := kv[c.key]; ok { 18 | if len(c.value) == 0 || (len(c.value) == 0 && c.value[0] == v) { 19 | return true 20 | } 21 | for _, pv := range c.value { 22 | if pv == v { 23 | return true 24 | } 25 | } 26 | } 27 | return false 28 | } 29 | 30 | // Map converts an incoming key-value map using the given mapping. 31 | func (c *Condition) Map(kv map[string]interface{}) map[string]interface{} { 32 | return c.mapper(kv) 33 | } 34 | 35 | // Transform applies property mapping and performs geometry operations. 36 | // Can emit multiple features, depending on the operation. 37 | func (c *Condition) Transform(f spatial.Feature) []spatial.Feature { 38 | if c.op == nil { 39 | return []spatial.Feature{ 40 | {Props: c.Map(f.Props), Geometry: f.Geometry}, 41 | } 42 | } 43 | 44 | var ( 45 | fts []spatial.Feature 46 | props = c.Map(f.Props) 47 | ) 48 | for _, ng := range c.op(f.Geometry) { 49 | fts = append(fts, spatial.Feature{Props: props, Geometry: ng}) 50 | } 51 | return fts 52 | } 53 | -------------------------------------------------------------------------------- /lib/mapping/default.go: -------------------------------------------------------------------------------- 1 | package mapping 2 | 3 | var ( 4 | transportationMapFn = func(kv map[string]interface{}) map[string]interface{} { 5 | var cl string 6 | if class, ok := kv["highway"]; ok { 7 | cl = class.(string) 8 | } 9 | return map[string]interface{}{ 10 | "@layer": "transportation", 11 | "class": cl, 12 | } 13 | } 14 | 15 | landuseMapFn = func(kv map[string]interface{}) map[string]interface{} { 16 | return map[string]interface{}{ 17 | "__type": "area", 18 | "@layer": "landcover", 19 | "class": "wood", 20 | } 21 | } 22 | 23 | aerowayMapFn = func(kv map[string]interface{}) map[string]interface{} { 24 | var cl string 25 | if class, ok := kv["aeroway"]; ok { 26 | cl = class.(string) 27 | } 28 | return map[string]interface{}{ 29 | "@layer": "aeroway", 30 | "class": cl, 31 | } 32 | } 33 | 34 | buildingMapFn = func(kv map[string]interface{}) map[string]interface{} { 35 | return map[string]interface{}{ 36 | "@layer": "building", 37 | "@zoom:min": 14, 38 | } 39 | } 40 | 41 | waterwayMapFn = func(kv map[string]interface{}) map[string]interface{} { 42 | var cl string 43 | if class, ok := kv["waterway"]; ok { 44 | cl = class.(string) 45 | } 46 | return map[string]interface{}{ 47 | "@layer": "waterway", 48 | "class": cl, 49 | } 50 | } 51 | 52 | Default = []Condition{ 53 | {"aeroway", []string{"aerodrome"}, aerowayMapFn, nil}, 54 | {"aeroway", []string{"apron"}, aerowayMapFn, nil}, 55 | {"aeroway", []string{"heliport"}, aerowayMapFn, nil}, 56 | {"aeroway", []string{"runway"}, aerowayMapFn, nil}, 57 | {"aeroway", []string{"helipad"}, aerowayMapFn, nil}, 58 | {"aeroway", []string{"taxiway"}, aerowayMapFn, nil}, 59 | {"highway", []string{"motorway"}, transportationMapFn, nil}, 60 | {"highway", []string{"primary"}, transportationMapFn, nil}, 61 | {"highway", []string{"trunk"}, transportationMapFn, nil}, 62 | {"highway", []string{"secondary"}, transportationMapFn, nil}, 63 | {"highway", []string{"tertiary"}, transportationMapFn, nil}, 64 | {"building", []string{""}, buildingMapFn, nil}, 65 | {"landuse", []string{"forest"}, landuseMapFn, nil}, 66 | {"railway", []string{"rail"}, transportationMapFn, nil}, 67 | {"waterway", []string{"river"}, waterwayMapFn, nil}, 68 | } 69 | ) 70 | -------------------------------------------------------------------------------- /lib/mapping/file.go: -------------------------------------------------------------------------------- 1 | package mapping 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "strconv" 9 | 10 | yaml "gopkg.in/yaml.v2" 11 | ) 12 | 13 | type fileMapKV struct { 14 | Key string `yaml:"key"` 15 | Value interface{} `yaml:"value"` 16 | Typ mapType `yaml:"type"` 17 | } 18 | 19 | type fileMap struct { 20 | Src fileMapKV `yaml:"src"` 21 | Dest []fileMapKV `yaml:"dest"` 22 | Op string `yaml:"op"` 23 | } 24 | 25 | type fileMappings []fileMap 26 | 27 | type typedField struct { 28 | Name string 29 | Typ mapType 30 | } 31 | 32 | func ParseMapping(r io.Reader) ([]Condition, error) { 33 | buf, err := ioutil.ReadAll(r) 34 | if err != nil { 35 | return nil, err 36 | } 37 | 38 | var fms fileMappings 39 | err = yaml.UnmarshalStrict(buf, &fms) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | var conds []Condition 45 | for _, fm := range fms { 46 | var sv []string 47 | 48 | svi, ok := fm.Src.Value.([]interface{}) 49 | if ok { 50 | // List of strings 51 | for _, svii := range svi { 52 | if s, ok := svii.(string); ok { 53 | sv = append(sv, s) 54 | } else { 55 | return nil, fmt.Errorf("source key %s must be of type string (has: %v)", fm.Src.Key, fm.Src.Value) 56 | } 57 | } 58 | } else { 59 | svs, ok := fm.Src.Value.(string) 60 | if !ok { 61 | return nil, fmt.Errorf("source key %s must be of type string (has: %v)", fm.Src.Key, fm.Src.Value) 62 | } 63 | if svs == "*" { 64 | sv = []string{} 65 | } else { 66 | sv = []string{svs} 67 | } 68 | } 69 | 70 | var ( 71 | staticKV = map[string]interface{}{} 72 | dynamicKV = map[string]typedField{} 73 | ) 74 | for _, kvm := range fm.Dest { 75 | if dv, ok := kvm.Value.(string); !ok { 76 | staticKV[kvm.Key] = kvm.Value 77 | } else { 78 | if dv[0:1] != "$" { 79 | staticKV[kvm.Key] = dv 80 | } else { 81 | // TODO: this can probably be optimized by generating more specific methods at parse time 82 | dynamicKV[kvm.Key] = typedField{Name: dv[1:], Typ: kvm.Typ} 83 | } 84 | } 85 | } 86 | cond := Condition{ 87 | key: fm.Src.Key, 88 | value: sv, 89 | } 90 | if len(dynamicKV) == 0 { 91 | sm := staticMapper{staticElems: staticKV} 92 | cond.mapper = sm.Map 93 | } else { 94 | dm := dynamicMapper{staticElems: staticKV, dynamicElems: dynamicKV} 95 | cond.mapper = dm.Map 96 | } 97 | 98 | switch fm.Op { 99 | case "lines": 100 | cond.op = polyToLines 101 | } 102 | 103 | conds = append(conds, cond) 104 | } 105 | return conds, nil 106 | } 107 | 108 | type staticMapper struct { 109 | staticElems map[string]interface{} 110 | } 111 | 112 | func (sm *staticMapper) Map(_ map[string]interface{}) map[string]interface{} { 113 | return sm.staticElems 114 | } 115 | 116 | type dynamicMapper struct { 117 | staticElems map[string]interface{} 118 | dynamicElems map[string]typedField 119 | } 120 | 121 | func (dm *dynamicMapper) Map(src map[string]interface{}) map[string]interface{} { 122 | var ( 123 | vals = map[string]interface{}{} 124 | err error 125 | ) 126 | for k, v := range dm.staticElems { 127 | vals[k] = v 128 | } 129 | for keyName, field := range dm.dynamicElems { 130 | if srcV, ok := src[field.Name]; ok { 131 | switch field.Typ { 132 | case mapTypeInt: 133 | vals[keyName], err = dm.toInt(srcV) 134 | default: 135 | vals[keyName] = srcV 136 | } 137 | if err != nil { 138 | log.Println(err) // Not sure if this won't get too verbose. Let's keep it here for some time. 139 | vals[keyName] = srcV 140 | err = nil 141 | } 142 | } 143 | } 144 | return vals 145 | } 146 | 147 | func (dm *dynamicMapper) toInt(i interface{}) (int, error) { 148 | switch v := i.(type) { 149 | case string: 150 | k, err := strconv.Atoi(v) 151 | if err == nil { 152 | return k, nil 153 | } 154 | if v == "yes" { 155 | return 1, nil 156 | } 157 | if v == "no" { 158 | return 0, nil 159 | } 160 | return 0, err 161 | case int: 162 | return v, nil 163 | default: 164 | return 0, fmt.Errorf("cannot convert %v (type %T) to int", v, v) 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /lib/mapping/file_test.go: -------------------------------------------------------------------------------- 1 | package mapping 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestParseMapping(t *testing.T) { 11 | f, err := os.Open("mapping.yml") 12 | assert.Nil(t, err) 13 | 14 | conds, err := ParseMapping(f) 15 | assert.Nil(t, err) 16 | 17 | srcKV := map[string]interface{}{ 18 | "building": "yes", 19 | } 20 | assert.True(t, conds[1].Matches(srcKV)) 21 | assert.Equal(t, map[string]interface{}{ 22 | "@layer": "building", 23 | "@zoom:min": 14, 24 | }, conds[1].Map(srcKV)) 25 | 26 | srcKV = map[string]interface{}{ 27 | "highway": "primary", 28 | } 29 | assert.True(t, conds[0].Matches(srcKV)) 30 | assert.Equal(t, map[string]interface{}{ 31 | "@layer": "transportation", 32 | "class": "primary", 33 | }, conds[0].Map(srcKV)) 34 | 35 | srcKV = map[string]interface{}{ 36 | "railway": "rail", 37 | "maxspeed": "300", 38 | } 39 | assert.True(t, conds[2].Matches(srcKV)) 40 | assert.Equal(t, map[string]interface{}{ 41 | "@layer": "transportation", 42 | "class": "railway", 43 | "maxspeed": 300, 44 | }, conds[2].Map(srcKV)) 45 | 46 | srcKV = map[string]interface{}{ 47 | "foo": "c", 48 | } 49 | assert.False(t, conds[3].Matches(srcKV)) 50 | 51 | srcKV = map[string]interface{}{ 52 | "foo": "b", 53 | } 54 | assert.True(t, conds[3].Matches(srcKV)) 55 | 56 | srcKV = map[string]interface{}{ 57 | "foo": "c", 58 | } 59 | assert.False(t, conds[3].Matches(srcKV)) 60 | 61 | srcKV = map[string]interface{}{ 62 | "foo": "a", 63 | } 64 | assert.True(t, conds[3].Matches(srcKV)) 65 | } 66 | -------------------------------------------------------------------------------- /lib/mapping/mapping.yml: -------------------------------------------------------------------------------- 1 | - src: 2 | key: highway 3 | value: primary 4 | dest: 5 | - {key: "@layer", value: "transportation"} 6 | - {key: "class", value: "$highway"} 7 | 8 | - src: 9 | key: building 10 | value: "*" 11 | dest: 12 | - {key: "@layer", value: "building"} 13 | - {key: "@zoom:min", value: 14} 14 | 15 | - src: 16 | key: railway 17 | value: "*" 18 | dest: 19 | - {key: "@layer", value: "transportation"} 20 | - {key: "class", value: "railway"} 21 | - {key: "maxspeed", value: "$maxspeed", type: int} 22 | op: lines 23 | 24 | - src: 25 | key: foo 26 | value: [a, b] 27 | dest: 28 | - {key: "bar", value: "baz"} 29 | -------------------------------------------------------------------------------- /lib/mapping/ops.go: -------------------------------------------------------------------------------- 1 | package mapping 2 | 3 | import "github.com/thomersch/grandine/lib/spatial" 4 | 5 | func polyToLines(g spatial.Geom) []spatial.Geom { 6 | poly, err := g.Polygon() 7 | if err != nil { 8 | return nil 9 | } 10 | var lines = make([]spatial.Geom, 0, len(poly)) 11 | for _, ring := range poly { 12 | lines = append(lines, spatial.MustNewGeom(ring)) 13 | } 14 | return lines 15 | } 16 | -------------------------------------------------------------------------------- /lib/mapping/types.go: -------------------------------------------------------------------------------- 1 | package mapping 2 | 3 | import "fmt" 4 | 5 | type mapType uint8 6 | 7 | const ( 8 | mapTypeString mapType = iota 9 | mapTypeInt 10 | ) 11 | 12 | func (mt *mapType) UnmarshalYAML(unmarshal func(interface{}) error) error { 13 | var ts string 14 | err := unmarshal(&ts) 15 | if err != nil { 16 | return err 17 | } 18 | switch ts { 19 | case "int": 20 | *mt = mapTypeInt 21 | case "string": 22 | *mt = mapTypeString 23 | default: 24 | return fmt.Errorf("unknown type: %s (allowed values: int, string)", ts) 25 | } 26 | return nil 27 | } 28 | 29 | func InterfaceMap(i map[string]string) (om map[string]interface{}) { 30 | om = make(map[string]interface{}) 31 | for k, v := range i { 32 | om[k] = v 33 | } 34 | return 35 | } 36 | -------------------------------------------------------------------------------- /lib/mvt/README.md: -------------------------------------------------------------------------------- 1 | ## Snippets 2 | 3 | * `cat testtile.mvt | protoc --decode=vector_tile.Tile vector_tile/vector_tile.proto` 4 | -------------------------------------------------------------------------------- /lib/mvt/codec.go: -------------------------------------------------------------------------------- 1 | package mvt 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | vt "github.com/thomersch/grandine/lib/mvt/vector_tile" 8 | "github.com/thomersch/grandine/lib/spatial" 9 | "github.com/thomersch/grandine/lib/tile" 10 | 11 | "github.com/golang/protobuf/proto" 12 | ) 13 | 14 | type cmd uint32 15 | 16 | const ( 17 | cmdMoveTo cmd = 1 18 | cmdLineTo cmd = 2 19 | cmdClosePath cmd = 7 20 | 21 | defaultExtent = 4096 22 | ) 23 | 24 | var ( 25 | vtPoint = vt.Tile_POINT 26 | vtLine = vt.Tile_LINESTRING 27 | vtPoly = vt.Tile_POLYGON 28 | vtLayerVersion = uint32(2) 29 | 30 | skipAtKeys = true // if enabled keys that start with "@" will be ignored 31 | 32 | errNoGeom = errors.New("no valid geometries") 33 | ) 34 | 35 | type Codec struct{} 36 | 37 | func (c *Codec) EncodeTile(features map[string][]spatial.Feature, tid tile.ID) ([]byte, error) { 38 | return EncodeTile(features, tid) 39 | } 40 | 41 | func (c *Codec) Extension() string { 42 | return "mvt" 43 | } 44 | 45 | func encodeCommandInt(c cmd, count uint32) uint32 { 46 | return (uint32(c) & 0x7) | (count << 3) 47 | } 48 | 49 | func decodeCommandInt(cmdInt uint32) (cmd, uint32) { 50 | return cmd(cmdInt & 0x7), cmdInt >> 3 51 | } 52 | 53 | func encodeZigZag(i int) uint32 { 54 | return uint32((i << 1) ^ (i >> 31)) 55 | } 56 | 57 | func EncodeTile(features map[string][]spatial.Feature, tid tile.ID) ([]byte, error) { 58 | vtile, err := assembleTile(features, tid) 59 | if err != nil { 60 | return nil, err 61 | } 62 | if len(vtile.Layers) == 0 { 63 | return nil, nil 64 | } 65 | return proto.Marshal(&vtile) 66 | } 67 | 68 | func assembleTile(features map[string][]spatial.Feature, tid tile.ID) (vt.Tile, error) { 69 | var vtile vt.Tile 70 | for layerName, layerFeats := range features { 71 | layer, err := assembleLayer(layerFeats, tid) 72 | if err != nil { 73 | return vtile, err 74 | } 75 | if len(layer.Features) == 0 { 76 | continue 77 | } 78 | var ln = layerName 79 | layer.Name = &ln // &layerName can't be used directly, because pointers are reused in for range loops 80 | layer.Version = &vtLayerVersion 81 | vtile.Layers = append(vtile.Layers, &layer) 82 | } 83 | return vtile, nil 84 | } 85 | 86 | // tagElems is an intermediate data structure for serializing keys or values into flat 87 | // index referenced lists as used by MVT 88 | type tagElems map[interface{}]int 89 | 90 | func (te tagElems) Index(v interface{}) int { 91 | if pos, ok := te[v]; ok { 92 | return pos 93 | } 94 | pos := len(te) 95 | te[v] = pos 96 | return pos 97 | } 98 | 99 | func (te tagElems) Strings() []string { 100 | var l = make([]string, len(te)) 101 | for elem, pos := range te { 102 | l[pos] = elem.(string) 103 | } 104 | return l 105 | } 106 | 107 | func (te tagElems) Values() []*vt.Tile_Value { 108 | var l = make([]*vt.Tile_Value, len(te)) 109 | for val, pos := range te { 110 | var tv vt.Tile_Value 111 | switch v := val.(type) { 112 | case string: 113 | tv.StringValue = &v 114 | case float32: 115 | tv.FloatValue = &v 116 | case float64: 117 | tv.DoubleValue = &v 118 | case int: 119 | i := int64(v) 120 | tv.SintValue = &i 121 | case int64: 122 | tv.SintValue = &v 123 | case uint: 124 | i := uint64(v) 125 | tv.UintValue = &i 126 | case uint64: 127 | tv.UintValue = &v 128 | case bool: 129 | tv.BoolValue = &v 130 | default: 131 | s := fmt.Sprintf("%s", v) 132 | tv.StringValue = &s 133 | } 134 | l[pos] = &tv 135 | } 136 | return l 137 | } 138 | 139 | func assembleLayer(features []spatial.Feature, tid tile.ID) (vt.Tile_Layer, error) { 140 | var ( 141 | tl vt.Tile_Layer 142 | err error 143 | tp = newTileParams(tid, defaultExtent) 144 | ext = uint32(defaultExtent) 145 | keys = tagElems{} 146 | vals = tagElems{} 147 | clipbbox = spatial.BBox{SW: spatial.Point{0, 0}, NE: spatial.Point{defaultExtent, defaultExtent}} // clipping mask in tile coordinate system 148 | ) 149 | 150 | var clippedFts = make([]spatial.Feature, 0, len(features)) 151 | for _, ft := range features { 152 | ng := ft.Geometry.Copy() 153 | ng.Project(func(pt spatial.Point) spatial.Point { 154 | return tilePoint(pt, tp) 155 | }) 156 | for _, geom := range ng.ClipToBBox(clipbbox) { 157 | clippedFts = append(clippedFts, spatial.Feature{Props: ft.Props, Geometry: geom}) 158 | } 159 | } 160 | 161 | for _, feat := range spatial.MergeFeatures(clippedFts) { 162 | var tileFeat vt.Tile_Feature 163 | 164 | for k, v := range feat.Properties() { 165 | if skipAtKeys && k[0] == '@' { 166 | continue 167 | } 168 | kpos := keys.Index(k) 169 | vpos := vals.Index(v) 170 | tileFeat.Tags = append(tileFeat.Tags, uint32(kpos), uint32(vpos)) 171 | } 172 | 173 | tileFeat.Geometry, err = encodeGeometry([]spatial.Geom{feat.Geometry}, tid) 174 | if len(tileFeat.Geometry) == 0 || err == errNoGeom { 175 | continue 176 | } 177 | if err != nil { 178 | return tl, err 179 | } 180 | switch feat.Geometry.Typ() { 181 | case spatial.GeomTypePoint: 182 | tileFeat.Type = &vtPoint 183 | case spatial.GeomTypeLineString: 184 | tileFeat.Type = &vtLine 185 | case spatial.GeomTypePolygon: 186 | tileFeat.Type = &vtPoly 187 | default: 188 | return tl, errors.New("unknown geometry type") 189 | } 190 | 191 | tl.Features = append(tl.Features, &tileFeat) 192 | tl.Extent = &ext //TODO: configurable? 193 | } 194 | 195 | tl.Keys = keys.Strings() 196 | tl.Values = vals.Values() 197 | return tl, nil 198 | } 199 | 200 | // Encodes one or more geometries of the same type into one (multi-)geometry. 201 | // Geometry coordinates must be in tile coordinate system. 202 | func encodeGeometry(geoms []spatial.Geom, tid tile.ID) (commands []uint32, err error) { 203 | var ( 204 | cur [2]int 205 | dx, dy int 206 | ) 207 | 208 | var typ spatial.GeomType 209 | for _, geom := range geoms { 210 | if typ != 0 && typ != geom.Typ() { 211 | return nil, errors.New("encodeGeometry only accepts uniform geoms") 212 | } 213 | 214 | cur[0] = 0 215 | cur[1] = 0 216 | switch geom.Typ() { 217 | case spatial.GeomTypePoint: 218 | pt := geom.MustPoint() 219 | dx = int(pt.X) - cur[0] 220 | dy = int(pt.Y) - cur[1] 221 | // TODO: support multipoint 222 | commands = append(commands, encodeCommandInt(cmdMoveTo, 1), encodeZigZag(dx), encodeZigZag(dy)) 223 | case spatial.GeomTypeLineString: 224 | commands = append(commands, encodeLine(geom.MustLineString(), &cur)...) 225 | case spatial.GeomTypePolygon: 226 | for _, ring := range geom.MustPolygon() { 227 | l := encodeLine(ring, &cur) 228 | if l == nil { 229 | return nil, errNoGeom 230 | } 231 | commands = append(commands, l...) 232 | commands = append(commands, encodeCommandInt(cmdClosePath, 1)) 233 | } 234 | } 235 | } 236 | return commands, nil 237 | } 238 | 239 | func encodeLine(ln spatial.Line, cur *[2]int) []uint32 { 240 | var ( 241 | commands = make([]uint32, len(ln)*2+2) // len=number of coordinates + initial move to + size 242 | dx, dy int 243 | ) 244 | commands[0] = encodeCommandInt(cmdMoveTo, 1) 245 | commands[3] = encodeCommandInt(cmdLineTo, uint32(len(commands)-4)/2) 246 | 247 | for i, tc := range ln { 248 | dx = int(tc.X) - cur[0] 249 | dy = int(tc.Y) - cur[1] 250 | cur[0] = int(tc.X) 251 | cur[1] = int(tc.Y) 252 | if i == 0 { 253 | commands[1] = encodeZigZag(int(dx)) 254 | commands[2] = encodeZigZag(int(dy)) 255 | } else { 256 | commands[i+i+2] = encodeZigZag(int(dx)) 257 | commands[i+i+3] = encodeZigZag(int(dy)) 258 | } 259 | } 260 | return commands 261 | } 262 | -------------------------------------------------------------------------------- /lib/mvt/codec_test.go: -------------------------------------------------------------------------------- 1 | package mvt 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/thomersch/grandine/lib/spatial" 9 | "github.com/thomersch/grandine/lib/tile" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestEncodeGeometry(t *testing.T) { 15 | tcs := []struct { 16 | geom []interface{} 17 | expectedResult []uint32 18 | }{ 19 | { 20 | geom: []interface{}{ 21 | spatial.Point{1, 1}, 22 | }, 23 | // TODO: validate coordinates 24 | expectedResult: []uint32{9, 2, 2}, 25 | }, 26 | { 27 | geom: []interface{}{ 28 | spatial.Point{25, 17}, 29 | }, 30 | // TODO: validate coordinates 31 | expectedResult: []uint32{9, 50, 34}, 32 | }, 33 | } 34 | 35 | for n, tc := range tcs { 36 | t.Run(fmt.Sprintf("%v", n), func(t *testing.T) { 37 | var geoms []spatial.Geom 38 | for _, g := range tc.geom { 39 | geom, err := spatial.NewGeom(g) 40 | assert.Nil(t, err) 41 | geoms = append(geoms, geom) 42 | } 43 | res, err := encodeGeometry(geoms, tile.ID{X: 1, Y: 0, Z: 1}) 44 | assert.Nil(t, err) 45 | assert.Equal(t, tc.expectedResult, res) 46 | }) 47 | } 48 | } 49 | 50 | func TestEncodeTile(t *testing.T) { 51 | var features []spatial.Feature 52 | geoms := []interface{}{ 53 | // Point 54 | spatial.Point{45, 45}, 55 | spatial.Point{50, 47}, 56 | spatial.Point{100, 40}, 57 | spatial.Point{179, 40}, 58 | // LineString 59 | spatial.Line{ 60 | spatial.Point{ 61 | -1.0546875, 62 | 55.97379820507658, 63 | }, 64 | spatial.Point{ 65 | 14.765625, 66 | 44.08758502824516, 67 | }, 68 | spatial.Point{ 69 | 39.7265625, 70 | 67.7427590666639, 71 | }, 72 | spatial.Point{ 73 | 16.875, 74 | 67.06743335108297, 75 | }, 76 | spatial.Point{ 77 | 16.171875, 78 | 58.07787626787517, 79 | }, 80 | }, 81 | spatial.Polygon{spatial.Line{ 82 | spatial.Point{ 83 | 2.8125, 84 | 54.77534585936447, 85 | }, 86 | spatial.Point{ 87 | 1.23046875, 88 | 47.87214396888731, 89 | }, 90 | spatial.Point{ 91 | 7.207031249999999, 92 | 37.020098201368114, 93 | }, 94 | spatial.Point{ 95 | 21.26953125, 96 | 40.97989806962013, 97 | }, 98 | spatial.Point{ 99 | 29.8828125, 100 | 48.69096039092549, 101 | }, 102 | spatial.Point{ 103 | 31.113281249999996, 104 | 53.12040528310657, 105 | }, 106 | spatial.Point{ 107 | 23.90625, 108 | 60.413852350464914, 109 | }, 110 | spatial.Point{ 111 | 10.01953125, 112 | 60.84491057364915, 113 | }, 114 | spatial.Point{ 115 | 2.8125, 116 | 54.77534585936447, 117 | }, 118 | }}, 119 | } 120 | 121 | for _, geom := range geoms { 122 | g, err := spatial.NewGeom(geom) 123 | assert.Nil(t, err) 124 | features = append(features, spatial.Feature{Geometry: g}) 125 | } 126 | 127 | features[0].Props = map[string]interface{}{ 128 | "highway": "primary", 129 | "oneway": 1, 130 | } 131 | features[1].Props = map[string]interface{}{ 132 | "highway": "secondary", 133 | "oneway": -1, 134 | } 135 | features[2].Props = map[string]interface{}{ 136 | "ignorance": "strength", 137 | } 138 | 139 | layers := map[string][]spatial.Feature{ 140 | "main": features, 141 | } 142 | 143 | buf, err := EncodeTile(layers, tile.ID{X: 1, Y: 0, Z: 1}) 144 | assert.Nil(t, err) 145 | 146 | var b bytes.Buffer 147 | b.Write(buf) 148 | } 149 | 150 | func BenchmarkEncodeLine(b *testing.B) { 151 | b.ReportAllocs() 152 | 153 | ln := spatial.Line{{0, 0}, {3, 5}, {1, 2}, {3, 4}, {10, 9}} 154 | for i := 0; i < b.N; i++ { 155 | var cur [2]int 156 | encodeLine(ln, &cur) 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /lib/mvt/project.go: -------------------------------------------------------------------------------- 1 | package mvt 2 | 3 | import ( 4 | "math" 5 | 6 | "github.com/thomersch/grandine/lib/spatial" 7 | "github.com/thomersch/grandine/lib/tile" 8 | ) 9 | 10 | const earthRadius = 6378137 11 | 12 | // flip axis 13 | func flip(v int, extent int) int { 14 | return extent - v 15 | } 16 | 17 | func flipFloat(v float64, extent int) float64 { 18 | return float64(flip(int(v), extent)) 19 | } 20 | 21 | func tileOffset(bb spatial.BBox) (xOffset, yOffset float64) { 22 | p := proj4326To3857(bb.SW) 23 | return p.X, p.Y 24 | } 25 | 26 | func tileScalingFactor(bb spatial.BBox, extent int) (xScale, yScale float64) { 27 | var ( 28 | pSW = proj4326To3857(bb.SW) 29 | pNE = proj4326To3857(bb.NE) 30 | deltaX = math.Abs(pSW.X - pNE.X) 31 | deltaY = math.Abs(pSW.Y - pNE.Y) 32 | ) 33 | return deltaX * float64(extent), deltaY * float64(extent) 34 | } 35 | 36 | func proj4326To3857(pt spatial.Point) spatial.Point { 37 | return spatial.Point{ 38 | /* Lon/X */ degToRad(pt.X) * earthRadius, 39 | /* Lat/Y */ math.Log(math.Tan(degToRad(pt.Y)/2+math.Pi/4)) * earthRadius, 40 | } 41 | } 42 | 43 | func tileCoord(p spatial.Point, tp tileParams) (x, y int) { 44 | newPt := tilePoint(p, tp) 45 | return int(newPt.X), int(newPt.Y) 46 | } 47 | 48 | func tilePoint(p spatial.Point, tp tileParams) spatial.Point { 49 | pt := proj4326To3857(p) 50 | return spatial.Point{ 51 | X: (pt.X - tp.xOffset) / (tp.xScale / float64(tp.extent)) * float64(tp.extent), 52 | Y: flipFloat((pt.Y-tp.yOffset)/(tp.yScale/float64(tp.extent))*float64(tp.extent), tp.extent), 53 | } 54 | } 55 | 56 | func degToRad(v float64) float64 { 57 | return v / (180 / math.Pi) 58 | } 59 | 60 | func radToDeg(v float64) float64 { 61 | return v * (180 / math.Pi) 62 | } 63 | 64 | type tileParams struct { 65 | xScale, yScale float64 66 | xOffset, yOffset float64 67 | extent int 68 | } 69 | 70 | func newTileParams(tid tile.ID, ext int) tileParams { 71 | var tp tileParams 72 | tp.xScale, tp.yScale = tileScalingFactor(tid.BBox(), ext) 73 | tp.xOffset, tp.yOffset = tileOffset(tid.BBox()) 74 | tp.extent = ext 75 | return tp 76 | } 77 | -------------------------------------------------------------------------------- /lib/mvt/project_test.go: -------------------------------------------------------------------------------- 1 | package mvt 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/thomersch/grandine/lib/spatial" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestScalePoint(t *testing.T) { 12 | bb := spatial.BBox{ 13 | spatial.Point{50, 10}, 14 | spatial.Point{52, 12}, 15 | } 16 | var tp tileParams 17 | tp.extent = 4096 18 | tp.xScale, tp.yScale = tileScalingFactor(bb, tp.extent) 19 | tp.xOffset, tp.yOffset = tileOffset(bb) 20 | 21 | tX, tY := tileCoord(spatial.Point{50, 10}, tp) 22 | assert.Equal(t, 0, tX) 23 | assert.Equal(t, tp.extent, tY) 24 | 25 | tX, tY = tileCoord(spatial.Point{51, 10}, tp) 26 | assert.Equal(t, tp.extent/2, tX) 27 | assert.Equal(t, tp.extent, tY) 28 | 29 | tX, tY = tileCoord(spatial.Point{52, 12}, tp) 30 | assert.Equal(t, tp.extent, tX) 31 | assert.Equal(t, 0, tY) 32 | } 33 | 34 | func TestProj4326To3857(t *testing.T) { 35 | assert.Equal(t, spatial.Point{4.57523107160354e+06, 2.28488107006733e+06}, proj4326To3857(spatial.Point{41.1, 20.1}).RoundedCoords()) 36 | assert.Equal(t, spatial.Point{4.57523107160354e+06, -2.28488107006733e+06}, proj4326To3857(spatial.Point{41.1, -20.1}).RoundedCoords()) 37 | } 38 | 39 | func scalePointToTileBarePoint(pt spatial.Point, extent int, xScale, yScale float64, xOffset, yOffset float64) spatial.Point { 40 | return spatial.Point{ 41 | (pt.X - xOffset) / (xScale / float64(extent)) * float64(extent), 42 | (pt.Y - yOffset) / (yScale / float64(extent)) * float64(extent), 43 | } 44 | } 45 | 46 | func BenchmarkPointBare(b *testing.B) { 47 | pt := spatial.Point{1, 2} 48 | b.ReportAllocs() 49 | b.ResetTimer() 50 | 51 | for i := 0; i < b.N; i++ { 52 | scalePointToTileBarePoint(pt, 4096, 1000, 1000, 3, 6) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /lib/mvt/vector_tile/README.md: -------------------------------------------------------------------------------- 1 | The contents of those package are dervied from https://github.com/mapbox/vector-tile-spec/tree/master/2.1 2 | -------------------------------------------------------------------------------- /lib/mvt/vector_tile/generate.go: -------------------------------------------------------------------------------- 1 | package vector_tile 2 | 3 | //go:generate protoc --go_out=. --proto_path . vector_tile.proto 4 | -------------------------------------------------------------------------------- /lib/mvt/vector_tile/vector_tile.proto: -------------------------------------------------------------------------------- 1 | package vector_tile; 2 | 3 | option optimize_for = LITE_RUNTIME; 4 | 5 | message Tile { 6 | 7 | // GeomType is described in section 4.3.4 of the specification 8 | enum GeomType { 9 | UNKNOWN = 0; 10 | POINT = 1; 11 | LINESTRING = 2; 12 | POLYGON = 3; 13 | } 14 | 15 | // Variant type encoding 16 | // The use of values is described in section 4.1 of the specification 17 | message Value { 18 | // Exactly one of these values must be present in a valid message 19 | optional string string_value = 1; 20 | optional float float_value = 2; 21 | optional double double_value = 3; 22 | optional int64 int_value = 4; 23 | optional uint64 uint_value = 5; 24 | optional sint64 sint_value = 6; 25 | optional bool bool_value = 7; 26 | 27 | extensions 8 to max; 28 | } 29 | 30 | // Features are described in section 4.2 of the specification 31 | message Feature { 32 | optional uint64 id = 1 [ default = 0 ]; 33 | 34 | // Tags of this feature are encoded as repeated pairs of 35 | // integers. 36 | // A detailed description of tags is located in sections 37 | // 4.2 and 4.4 of the specification 38 | repeated uint32 tags = 2 [ packed = true ]; 39 | 40 | // The type of geometry stored in this feature. 41 | optional GeomType type = 3 [ default = UNKNOWN ]; 42 | 43 | // Contains a stream of commands and parameters (vertices). 44 | // A detailed description on geometry encoding is located in 45 | // section 4.3 of the specification. 46 | repeated uint32 geometry = 4 [ packed = true ]; 47 | } 48 | 49 | // Layers are described in section 4.1 of the specification 50 | message Layer { 51 | // Any compliant implementation must first read the version 52 | // number encoded in this message and choose the correct 53 | // implementation for this version number before proceeding to 54 | // decode other parts of this message. 55 | required uint32 version = 15 [ default = 1 ]; 56 | 57 | required string name = 1; 58 | 59 | // The actual features in this tile. 60 | repeated Feature features = 2; 61 | 62 | // Dictionary encoding for keys 63 | repeated string keys = 3; 64 | 65 | // Dictionary encoding for values 66 | repeated Value values = 4; 67 | 68 | // Although this is an "optional" field it is required by the specification. 69 | // See https://github.com/mapbox/vector-tile-spec/issues/47 70 | optional uint32 extent = 5 [ default = 4096 ]; 71 | 72 | extensions 16 to max; 73 | } 74 | 75 | repeated Layer layers = 3; 76 | 77 | extensions 16 to 8191; 78 | } 79 | -------------------------------------------------------------------------------- /lib/progressbar/progressbar.go: -------------------------------------------------------------------------------- 1 | package progressbar 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "strconv" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | type bar struct { 13 | curPos int 14 | taskCount int 15 | workers int 16 | } 17 | 18 | const line = " " 19 | 20 | func NewBar(taskCount int, workers int) (chan<- struct{}, func()) { 21 | var ch = make(chan struct{}, 5000) 22 | b := &bar{ 23 | taskCount: taskCount, 24 | workers: workers, 25 | } 26 | go b.consume(ch) 27 | go func() { 28 | for range time.NewTicker(100 * time.Millisecond).C { 29 | b.draw() 30 | } 31 | }() 32 | return ch, b.done 33 | } 34 | 35 | func (b *bar) done() { 36 | b.draw() 37 | fmt.Println(" ✅") 38 | } 39 | 40 | func (b *bar) consume(c <-chan struct{}) { 41 | for range c { 42 | b.curPos++ 43 | } 44 | } 45 | 46 | func (b *bar) clearLine() { 47 | fmt.Print("\r" + line + "\r") 48 | } 49 | 50 | func (b *bar) draw() { 51 | b.clearLine() 52 | fmt.Printf("%v/%v", b.curPos, b.taskCount) 53 | } 54 | 55 | func maxWidth() int { 56 | cmd := exec.Command("stty", "size") 57 | cmd.Stdin = os.Stdin 58 | sttyOut, err := cmd.Output() 59 | if err != nil { 60 | return 80 61 | } 62 | width, err := strconv.Atoi(strings.Split(string(sttyOut), " ")[0]) 63 | if err != nil { 64 | return 80 65 | } 66 | return width 67 | } 68 | -------------------------------------------------------------------------------- /lib/spaten/chunks.go: -------------------------------------------------------------------------------- 1 | package spaten 2 | 3 | import ( 4 | "io" 5 | "sync" 6 | 7 | "github.com/thomersch/grandine/lib/spatial" 8 | ) 9 | 10 | type Chunks struct { 11 | endReached bool 12 | reader io.Reader 13 | // Parallel reading of a file is not allowed, could be theoretically improved by reading from 14 | // stream and passing the buffer into the decoder, but this needs underlying changes. 15 | readerMtx sync.Mutex 16 | } 17 | 18 | func (c *Chunks) Next() bool { 19 | return !c.endReached 20 | } 21 | 22 | func (c *Chunks) Scan(fc *spatial.FeatureCollection) error { 23 | c.readerMtx.Lock() 24 | defer c.readerMtx.Unlock() 25 | err := readBlock(c.reader, fc) 26 | if err == io.EOF { 27 | c.endReached = true 28 | } 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /lib/spaten/codec.go: -------------------------------------------------------------------------------- 1 | package spaten 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/thomersch/grandine/lib/spatial" 7 | ) 8 | 9 | type Codec struct { 10 | headerWritten bool 11 | writeQueue []spatial.Feature 12 | } 13 | 14 | const blockSize = 1000 15 | 16 | func (c *Codec) Encode(w io.Writer, fc *spatial.FeatureCollection) error { 17 | err := WriteFileHeader(w) 18 | if err != nil { 19 | return err 20 | } 21 | 22 | for _, ftBlk := range featureBlocks(blockSize, fc.Features) { 23 | var meta map[string]interface{} 24 | if len(fc.SRID) != 0 { 25 | meta = map[string]interface{}{ 26 | "@srid": fc.SRID, 27 | } 28 | } 29 | 30 | err = WriteBlock(w, ftBlk, meta) 31 | if err != nil { 32 | return err 33 | } 34 | } 35 | return nil 36 | } 37 | 38 | // EncodeChunk enqueues features to be written out. Call Close when done with the stream. 39 | func (c *Codec) EncodeChunk(w io.Writer, fc *spatial.FeatureCollection) error { 40 | if !c.headerWritten { 41 | err := WriteFileHeader(w) 42 | if err != nil { 43 | return err 44 | } 45 | c.headerWritten = true 46 | } 47 | 48 | var newQueue []spatial.Feature 49 | c.writeQueue = append(c.writeQueue, fc.Features...) 50 | for _, ftBlk := range featureBlocks(blockSize, c.writeQueue) { 51 | if len(ftBlk) < blockSize { 52 | // the block is not full, so let's schedule for next write 53 | newQueue = append(newQueue, ftBlk...) 54 | } else { 55 | err := WriteBlock(w, ftBlk, nil) 56 | if err != nil { 57 | return err 58 | } 59 | } 60 | } 61 | c.writeQueue = newQueue 62 | return nil 63 | } 64 | 65 | func (c *Codec) Close(w io.Writer) error { 66 | if len(c.writeQueue) > 0 { 67 | return WriteBlock(w, c.writeQueue, nil) 68 | } 69 | return nil 70 | } 71 | 72 | // ChunkedDecode is the preferred method for reading large datasets. It retrieves a file block 73 | // at a time, making it possible to traverse the file in a streaming manner without allocating 74 | // enough memory to fit the whole file. 75 | func (c *Codec) ChunkedDecode(r io.Reader) (spatial.Chunks, error) { 76 | _, err := ReadFileHeader(r) 77 | if err != nil { 78 | return nil, err 79 | } 80 | return &Chunks{ 81 | reader: r, 82 | }, nil 83 | } 84 | 85 | func (c *Codec) Decode(r io.Reader, fc *spatial.FeatureCollection) error { 86 | _, err := ReadFileHeader(r) 87 | if err != nil { 88 | return err 89 | } 90 | err = ReadBlocks(r, fc) 91 | return err 92 | } 93 | 94 | func (c *Codec) Extensions() []string { 95 | return []string{"spaten"} 96 | } 97 | 98 | // featureBlocks slices a slice of geometries into slices with a max size 99 | func featureBlocks(size int, src []spatial.Feature) [][]spatial.Feature { 100 | if len(src) <= size { 101 | return [][]spatial.Feature{src} 102 | } 103 | 104 | var ( 105 | i int 106 | res [][]spatial.Feature 107 | end int 108 | ) 109 | for end < len(src) { 110 | end = (i + 1) * size 111 | if end > len(src) { 112 | end = len(src) 113 | } 114 | res = append(res, src[i*size:end]) 115 | i++ 116 | } 117 | return res 118 | } 119 | -------------------------------------------------------------------------------- /lib/spaten/codec_test.go: -------------------------------------------------------------------------------- 1 | package spaten 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/thomersch/grandine/lib/geojson" 8 | "github.com/thomersch/grandine/lib/spatial" 9 | ) 10 | 11 | func BenchmarkCodecThroughput(b *testing.B) { 12 | var ( 13 | fc = &spatial.FeatureCollection{Features: []spatial.Feature{}} 14 | sc = &Codec{} 15 | gjc = &geojson.Codec{} 16 | buf = bytes.NewBuffer(make([]byte, 0, 200000000)) 17 | ) 18 | for i := 0; i < 50000; i++ { 19 | fc.Features = append(fc.Features, spatial.Feature{ 20 | Geometry: spatial.MustNewGeom(spatial.Point{1, 2}), 21 | Props: map[string]interface{}{"weight": 0}, 22 | }) 23 | } 24 | for i := 0; i < 50000; i++ { 25 | fc.Features = append(fc.Features, spatial.Feature{ 26 | Geometry: spatial.MustNewGeom([]spatial.Point{{1, 2}, {3, 5}, {9, 0}, {2, 9}}), 27 | Props: map[string]interface{}{"value": 14, "description": "i am a line"}, 28 | }) 29 | } 30 | 31 | var sb, gjb []byte 32 | 33 | b.Run("Spaten Encode", func(b *testing.B) { 34 | b.ReportAllocs() 35 | for i := 0; i < b.N; i++ { 36 | buf.Reset() 37 | sc.Encode(buf, fc) 38 | } 39 | sb = buf.Bytes() 40 | }) 41 | b.Run("Spaten Decode", func(b *testing.B) { 42 | b.ReportAllocs() 43 | decColl := spatial.FeatureCollection{Features: []spatial.Feature{}} 44 | for i := 0; i < b.N; i++ { 45 | sc.Decode(bytes.NewBuffer(sb), &decColl) 46 | } 47 | }) 48 | 49 | b.Run("GeoJSON Encode", func(b *testing.B) { 50 | b.ReportAllocs() 51 | for i := 0; i < b.N; i++ { 52 | buf.Reset() 53 | gjc.Encode(buf, fc) 54 | } 55 | gjb = buf.Bytes() 56 | }) 57 | b.Run("GeoJSON Decode", func(b *testing.B) { 58 | b.ReportAllocs() 59 | decColl := spatial.FeatureCollection{Features: []spatial.Feature{}} 60 | for i := 0; i < b.N; i++ { 61 | gjc.Decode(bytes.NewBuffer(gjb), &decColl) 62 | } 63 | }) 64 | } 65 | -------------------------------------------------------------------------------- /lib/spaten/fileformat/generate.go: -------------------------------------------------------------------------------- 1 | package fileformat 2 | 3 | //go:generate protoc --gofast_out=. --proto_path ../../../fileformat ../../../fileformat/fileformat.proto 4 | -------------------------------------------------------------------------------- /lib/spaten/fileformat/transform.go: -------------------------------------------------------------------------------- 1 | package fileformat 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "math" 8 | ) 9 | 10 | var vtypers map[string]valueTyper 11 | 12 | func init() { 13 | vtypers = map[string]valueTyper{ 14 | "string": func(s interface{}) ([]byte, Tag_ValueType, error) { 15 | return []byte(s.(string)), Tag_STRING, nil 16 | }, 17 | "int": func(s interface{}) ([]byte, Tag_ValueType, error) { 18 | var buf = make([]byte, 8) 19 | binary.LittleEndian.PutUint64(buf, uint64(s.(int))) 20 | return buf, Tag_INT, nil 21 | }, 22 | "float64": func(f interface{}) ([]byte, Tag_ValueType, error) { 23 | var buf = make([]byte, 8) 24 | binary.LittleEndian.PutUint64(buf, math.Float64bits(f.(float64))) 25 | return buf, Tag_DOUBLE, nil 26 | }, 27 | "": func(f interface{}) ([]byte, Tag_ValueType, error) { 28 | return []byte{}, Tag_STRING, nil 29 | }, 30 | } 31 | } 32 | 33 | type valueTyper func(interface{}) ([]byte, Tag_ValueType, error) 34 | 35 | func ValueType(i interface{}) ([]byte, Tag_ValueType, error) { 36 | t := fmt.Sprintf("%T", i) 37 | vt, ok := vtypers[t] 38 | if !ok { 39 | return nil, Tag_STRING, fmt.Errorf("unknown type: %s (value: %v)", t, i) 40 | } 41 | return vt(i) 42 | } 43 | 44 | // KeyValue retrieves key and value from a Tag. 45 | func KeyValue(t *Tag) (string, interface{}, error) { 46 | switch t.GetType() { 47 | case Tag_STRING: 48 | return t.Key, string(t.GetValue()), nil 49 | case Tag_INT: 50 | return t.Key, int(binary.LittleEndian.Uint64(t.GetValue())), nil 51 | case Tag_DOUBLE: 52 | var ( 53 | buf = bytes.NewBuffer(t.GetValue()) 54 | f float64 55 | ) 56 | err := binary.Read(buf, binary.LittleEndian, &f) 57 | return t.Key, f, err 58 | default: 59 | // TODO 60 | return t.Key, nil, nil 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /lib/spaten/fuzz.go: -------------------------------------------------------------------------------- 1 | // +build gofuzz 2 | 3 | package spaten 4 | 5 | import ( 6 | "bytes" 7 | 8 | "github.com/thomersch/grandine/lib/spatial" 9 | ) 10 | 11 | func Fuzz(data []byte) int { 12 | var ( 13 | c Codec 14 | fc = spatial.NewFeatureCollection() 15 | ) 16 | 17 | err := c.Decode(bytes.NewReader(data), fc) 18 | if err != nil { 19 | return 0 20 | } 21 | return 1 22 | } 23 | -------------------------------------------------------------------------------- /lib/spaten/lowlevel.go: -------------------------------------------------------------------------------- 1 | package spaten 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "sync" 10 | 11 | "github.com/thomersch/grandine/lib/spaten/fileformat" 12 | "github.com/thomersch/grandine/lib/spatial" 13 | 14 | "github.com/golang/protobuf/proto" 15 | ) 16 | 17 | const ( 18 | cookie = "SPAT" 19 | version = 0 20 | ) 21 | 22 | type Header struct { 23 | Version int 24 | } 25 | 26 | var encoding = binary.LittleEndian 27 | 28 | func WriteFileHeader(w io.Writer) error { 29 | const headerSize = 8 30 | 31 | buf := make([]byte, headerSize) 32 | buf = append([]byte(cookie), buf[:4]...) 33 | binary.LittleEndian.PutUint32(buf[4:], version) 34 | 35 | n, err := w.Write(buf) 36 | if n != headerSize { 37 | return io.EOF 38 | } 39 | return err 40 | } 41 | 42 | func ReadFileHeader(r io.Reader) (Header, error) { 43 | var ( 44 | ck = make([]byte, 4) 45 | vers uint32 46 | hd Header 47 | ) 48 | if _, err := r.Read(ck); err != nil { 49 | return hd, fmt.Errorf("could not read file header cookie: %s", err) 50 | } 51 | if string(ck) != cookie { 52 | return hd, errors.New("invalid cookie") 53 | } 54 | 55 | if err := binary.Read(r, binary.LittleEndian, &vers); err != nil { 56 | return hd, err 57 | } 58 | hd.Version = int(vers) 59 | if vers > version { 60 | return hd, errors.New("invalid file version") 61 | } 62 | return hd, nil 63 | } 64 | 65 | // WriteBlock writes a block of spatial data (note that every valid Spaten file needs a file header in front). 66 | // meta may be nil, if you don't wish to add any block meta. 67 | func WriteBlock(w io.Writer, fs []spatial.Feature, meta map[string]interface{}) error { 68 | blockBody := &fileformat.Body{} 69 | props, err := propertiesToTags(meta) 70 | if err != nil { 71 | return err 72 | } 73 | blockBody.Meta = &fileformat.Meta{ 74 | Tags: props, 75 | } 76 | 77 | for _, f := range fs { 78 | nf, err := PackFeature(f) 79 | if err != nil { 80 | return err 81 | } 82 | 83 | blockBody.Feature = append(blockBody.Feature, &nf) 84 | } 85 | bodyBuf, err := proto.Marshal(blockBody) 86 | if err != nil { 87 | return err 88 | } 89 | 90 | blockHeaderBuf := make([]byte, 8) 91 | // Body Length 92 | binary.LittleEndian.PutUint32(blockHeaderBuf[:4], uint32(len(bodyBuf))) 93 | // Flags 94 | binary.LittleEndian.PutUint16(blockHeaderBuf[4:6], 0) 95 | // Compression 96 | blockHeaderBuf[6] = 0 97 | // Message Type 98 | blockHeaderBuf[7] = 0 99 | 100 | w.Write(append(blockHeaderBuf, bodyBuf...)) 101 | return nil 102 | } 103 | 104 | // PackFeature encapusaltes a spatial feature into an encodable Spaten feature. 105 | // This is a low level interface and not guaranteed to be stable. 106 | func PackFeature(f spatial.Feature) (fileformat.Feature, error) { 107 | var ( 108 | nf fileformat.Feature 109 | err error 110 | ) 111 | nf.Tags, err = propertiesToTags(f.Properties()) 112 | if err != nil { 113 | return nf, err 114 | } 115 | 116 | // TODO: make encoder configurable 117 | nf.Geom, err = f.MarshalWKB() 118 | if err != nil { 119 | return nf, err 120 | } 121 | return nf, nil 122 | } 123 | 124 | var featureBufPool = sync.Pool{ 125 | New: func() interface{} { 126 | return bytes.NewBuffer(make([]byte, 0, 16)) 127 | }, 128 | } 129 | 130 | // UnpackFeature unpacks a Spaten feature into a usable spatial feature. 131 | // This is a low level interface and not guaranteed to be stable. 132 | func UnpackFeature(pf *fileformat.Feature) (spatial.Feature, error) { 133 | var geomBuf = featureBufPool.Get().(*bytes.Buffer) 134 | geomBuf.Reset() 135 | geomBuf.Write(pf.GetGeom()) 136 | geom, err := spatial.GeomFromWKB(geomBuf) 137 | if err != nil { 138 | return spatial.Feature{}, err 139 | } 140 | featureBufPool.Put(geomBuf) 141 | feature := spatial.Feature{ 142 | Props: map[string]interface{}{}, 143 | Geometry: geom, 144 | } 145 | 146 | for _, tag := range pf.Tags { 147 | k, v, err := fileformat.KeyValue(tag) 148 | if err != nil { 149 | // TODO 150 | return feature, err 151 | } 152 | feature.Props[k] = v 153 | } 154 | return feature, nil 155 | } 156 | 157 | func propertiesToTags(props map[string]interface{}) ([]*fileformat.Tag, error) { 158 | var tags []*fileformat.Tag = make([]*fileformat.Tag, 0, len(props)) 159 | if props == nil { 160 | return tags, nil 161 | } 162 | for k, v := range props { 163 | val, typ, err := fileformat.ValueType(v) 164 | if err != nil { 165 | return nil, err 166 | } 167 | tags = append(tags, &fileformat.Tag{ 168 | Key: k, 169 | Value: val, 170 | Type: typ, 171 | }) 172 | } 173 | return tags, nil 174 | } 175 | 176 | type blockHeader struct { 177 | bodyLen uint32 178 | flags uint16 179 | compression uint8 180 | messageType uint8 181 | } 182 | 183 | var blockBodyPool = sync.Pool{ 184 | New: func() interface{} { 185 | return &fileformat.Body{} 186 | }, 187 | } 188 | 189 | func readBlock(r io.Reader, fs *spatial.FeatureCollection) error { 190 | var hd blockHeader 191 | 192 | headerBuf := make([]byte, 8) 193 | n, err := r.Read(headerBuf) 194 | if n == 0 { 195 | return io.EOF 196 | } 197 | if err != nil { 198 | return fmt.Errorf("could not read block header: %v", err) 199 | } 200 | 201 | hd.bodyLen = binary.LittleEndian.Uint32(headerBuf[0:4]) 202 | hd.flags = binary.LittleEndian.Uint16(headerBuf[4:6]) 203 | hd.compression = uint8(headerBuf[6]) 204 | if hd.compression != 0 { 205 | return errors.New("compression is not supported") 206 | } 207 | 208 | hd.messageType = uint8(headerBuf[7]) 209 | if hd.messageType != 0 { 210 | return errors.New("message type is not supported") 211 | } 212 | 213 | var ( 214 | buf = make([]byte, hd.bodyLen) 215 | blockBody = blockBodyPool.Get().(*fileformat.Body) 216 | ) 217 | blockBody.Reset() 218 | n, err = io.ReadFull(r, buf) 219 | if n != int(hd.bodyLen) { 220 | return fmt.Errorf("incomplete block: expected %v bytes, %v available", hd.bodyLen, n) 221 | } 222 | if err != nil { 223 | return err 224 | } 225 | if err := blockBody.Unmarshal(buf); err != nil { 226 | return err 227 | } 228 | if len(fs.Features) == 0 { 229 | // only prealloc if empty, so no user data gets truncated 230 | fs.Features = make([]spatial.Feature, 0, len(blockBody.GetFeature())) 231 | } 232 | for _, f := range blockBody.GetFeature() { 233 | feature, err := UnpackFeature(f) 234 | if err != nil { 235 | return err 236 | } 237 | fs.Features = append(fs.Features, feature) 238 | } 239 | blockBodyPool.Put(blockBody) 240 | return nil 241 | } 242 | 243 | // ReadBlocks is a function for reading all features from a file at once. 244 | func ReadBlocks(r io.Reader, fs *spatial.FeatureCollection) error { 245 | var err error 246 | for { 247 | err = readBlock(r, fs) 248 | if err == io.EOF { 249 | break 250 | } 251 | if err != nil { 252 | return err 253 | } 254 | } 255 | return nil 256 | } 257 | -------------------------------------------------------------------------------- /lib/spaten/lowlevel_test.go: -------------------------------------------------------------------------------- 1 | package spaten 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "encoding/hex" 7 | "fmt" 8 | "strconv" 9 | "testing" 10 | "time" 11 | 12 | "github.com/stretchr/testify/assert" 13 | "github.com/thomersch/grandine/lib/spatial" 14 | ) 15 | 16 | // TestReadFileHeader is here in order to prevent regressions in header parsing. 17 | func TestReadFileHeader(t *testing.T) { 18 | buf, err := hex.DecodeString("5350415400000000") 19 | assert.Nil(t, err) 20 | r := bytes.NewBuffer(buf) 21 | 22 | hd, err := ReadFileHeader(r) 23 | assert.Nil(t, err) 24 | assert.Equal(t, Header{Version: 0}, hd) 25 | } 26 | 27 | func TestHeaderSelfTest(t *testing.T) { 28 | var buf bytes.Buffer 29 | err := WriteFileHeader(&buf) 30 | assert.Nil(t, err) 31 | 32 | _, err = ReadFileHeader(&buf) 33 | assert.Nil(t, err) 34 | } 35 | 36 | func TestBlockSelfTest(t *testing.T) { 37 | var ( 38 | buf bytes.Buffer 39 | fcoll = spatial.FeatureCollection{ 40 | Features: []spatial.Feature{ 41 | { 42 | Props: map[string]interface{}{ 43 | "key1": 1, 44 | "key2": "string", 45 | "key3": -12.981, 46 | }, 47 | Geometry: spatial.MustNewGeom(spatial.Point{24, 1}), 48 | }, 49 | { 50 | Props: map[string]interface{}{ 51 | "yes": "NO", 52 | }, 53 | Geometry: spatial.MustNewGeom(spatial.Line{{24, 1}, {25, 0}, {9, -4}}), 54 | }, 55 | { 56 | Props: map[string]interface{}{ 57 | "name": "RichardF Box", 58 | }, 59 | Geometry: spatial.MustNewGeom(spatial.Polygon{{{24, 1}, {25, 0}, {9, -4}}}), 60 | }, 61 | }, 62 | } 63 | ) 64 | 65 | err := WriteBlock(&buf, fcoll.Features, nil) 66 | assert.Nil(t, err) 67 | 68 | var fcollRead spatial.FeatureCollection 69 | err = ReadBlocks(&buf, &fcollRead) 70 | assert.Nil(t, err) 71 | assert.Equal(t, fcoll, fcollRead) 72 | } 73 | 74 | func TestBlockHeaderEncoding(t *testing.T) { 75 | var ( 76 | buf bytes.Buffer 77 | fs = []spatial.Feature{ 78 | { 79 | Geometry: spatial.MustNewGeom(spatial.Point{1, 2}), 80 | }, 81 | } 82 | ) 83 | 84 | err := WriteBlock(&buf, fs, nil) 85 | assert.Nil(t, err) 86 | 87 | const headerLength = 8 // TODO: consider exporting this 88 | // Compare buffer size with size written in header. 89 | assert.Equal(t, buf.Len()-headerLength, int(binary.LittleEndian.Uint32(buf.Bytes()[:4]))) 90 | assert.Equal(t, "00000000", fmt.Sprintf("%x", buf.Bytes()[4:8])) 91 | } 92 | 93 | func TestInvalidBlockSize(t *testing.T) { 94 | buf, err := hex.DecodeString("FFFFFFFF00000000AAAA") 95 | assert.Nil(t, err) 96 | 97 | fc := spatial.NewFeatureCollection() 98 | err = readBlock(bytes.NewBuffer(buf), fc) 99 | assert.NotNil(t, err) 100 | } 101 | 102 | func TestWeirdFiles(t *testing.T) { 103 | var fls = []struct { 104 | buf string 105 | shouldErr bool 106 | }{ 107 | {"53504154000000000000000000000a0012171a15010100000000000000002440e523e8ca28c5517c1df8aa9998c44a40", true}, 108 | {"53504154000000000000000000000000", false}, 109 | {"53504154000000001b00000030303030303012171a15010300000030303000000000003030303030303030", true}, 110 | {"53504154000000001010101000000000", true}, 111 | } 112 | 113 | for i, f := range fls { 114 | t.Run(strconv.Itoa(i), func(t *testing.T) { 115 | var c Codec 116 | buf, err := hex.DecodeString(f.buf) 117 | assert.Nil(t, err) 118 | 119 | fc := spatial.NewFeatureCollection() 120 | err = c.Decode(bytes.NewBuffer(buf), fc) 121 | if f.shouldErr { 122 | assert.NotNil(t, err) 123 | } else { 124 | assert.Nil(t, err) 125 | } 126 | }) 127 | } 128 | } 129 | 130 | func BenchmarkReadBlock(b *testing.B) { 131 | var ( 132 | buf = bytes.NewBuffer([]byte{}) 133 | fs = spatial.NewFeatureCollection() 134 | ) 135 | err := WriteBlock(buf, fs.Features, nil) 136 | assert.Nil(b, err) 137 | r := bytes.NewReader(buf.Bytes()) 138 | 139 | b.ResetTimer() 140 | b.ReportAllocs() 141 | for i := 0; i < b.N; i++ { 142 | r.Seek(0, 0) 143 | err := readBlock(r, fs) 144 | assert.Nil(b, err) 145 | } 146 | } 147 | 148 | func BenchmarkReadBlockThroughput(b *testing.B) { 149 | var ( 150 | wBuf = bytes.NewBuffer([]byte{}) 151 | fs = spatial.NewFeatureCollection() 152 | ) 153 | 154 | err := WriteBlock(wBuf, []spatial.Feature{ 155 | { 156 | Geometry: spatial.MustNewGeom(spatial.Point{2, 3}), 157 | Props: map[string]interface{}{ 158 | "highway": "primary", 159 | "number": 1, 160 | }, 161 | }, 162 | }, nil) 163 | assert.Nil(b, err) 164 | 165 | var ( 166 | ptBuf = wBuf.Bytes()[8:] 167 | fullBuf = make([]byte, 8) 168 | ) 169 | for n := 0; n < 100000; n++ { 170 | fullBuf = append(fullBuf, ptBuf...) 171 | } 172 | binary.LittleEndian.PutUint32(fullBuf[:4], uint32(len(fullBuf)-8)) 173 | 174 | b.ReportAllocs() 175 | b.ResetTimer() 176 | 177 | t := time.Now() 178 | for n := 0; n < b.N; n++ { 179 | r := bytes.NewBuffer(fullBuf) 180 | err = readBlock(r, fs) 181 | assert.Nil(b, err) 182 | fs.Features = []spatial.Feature{} 183 | } 184 | b.Logf("%v bytes read, in %v blocks, throughput: %v B/s", len(fullBuf)*b.N, b.N, int(float64(len(fullBuf)*b.N)/time.Since(t).Seconds())) 185 | } 186 | -------------------------------------------------------------------------------- /lib/spatial/README.md: -------------------------------------------------------------------------------- 1 | # lib/spatial 2 | 3 | lib/spatial is a small Golang library for handling geospatial data. 4 | 5 | Its focus is on serializing/deserializing into standardized data formats: Currently it supports GeoJSON, WKB and TinyWKB (TWKB). 6 | -------------------------------------------------------------------------------- /lib/spatial/bbox.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import "math" 4 | 5 | type BBox struct { 6 | SW, NE Point 7 | } 8 | 9 | func (b1 *BBox) ExtendWith(b2 BBox) { 10 | b1.SW = Point{math.Min(b1.SW.X, b2.SW.X), math.Min(b1.SW.Y, b2.SW.Y)} 11 | b1.NE = Point{math.Max(b1.NE.X, b2.NE.X), math.Max(b1.NE.Y, b2.NE.Y)} 12 | } 13 | 14 | // In determines if the bboxes overlap. 15 | func (b BBox) Overlaps(b2 BBox) bool { 16 | return b.SW.InBBox(b2) || b.NE.InBBox(b2) || b2.SW.InBBox(b) || b2.NE.InBBox(b) 17 | } 18 | 19 | func (b BBox) FullyIn(b2 BBox) bool { 20 | return b.SW.InBBox(b2) && b.NE.InBBox(b2) 21 | } 22 | 23 | func (b BBox) Segments() []Segment { 24 | return []Segment{ 25 | { 26 | {b.SW.X, b.SW.Y}, 27 | {b.SW.X, b.NE.Y}, 28 | }, 29 | { 30 | {b.SW.X, b.NE.Y}, 31 | {b.NE.X, b.NE.Y}, 32 | }, 33 | { 34 | {b.NE.X, b.NE.Y}, 35 | {b.NE.X, b.SW.Y}, 36 | }, 37 | { 38 | {b.NE.X, b.SW.Y}, 39 | {b.SW.X, b.SW.Y}, 40 | }, 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /lib/spatial/bbox.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | -------------------------------------------------------------------------------- /lib/spatial/bbox_test.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestBBoxIn(t *testing.T) { 11 | for n, tc := range []struct { 12 | inner BBox 13 | outer BBox 14 | in bool 15 | }{ 16 | { 17 | inner: BBox{ 18 | Point{20, 20}, Point{30, 30}, 19 | }, 20 | outer: BBox{ 21 | Point{10, 10}, Point{50, 50}, 22 | }, 23 | in: true, 24 | }, 25 | { 26 | inner: BBox{ 27 | Point{10, 10}, Point{50, 50}, 28 | }, 29 | outer: BBox{ 30 | Point{20, 20}, Point{30, 30}, 31 | }, 32 | in: true, 33 | }, 34 | { 35 | inner: BBox{ 36 | Point{30, 10}, Point{40, 20}, 37 | }, 38 | outer: BBox{ 39 | Point{45, 10}, Point{95, 60}, 40 | }, 41 | in: false, 42 | }, 43 | { 44 | inner: BBox{ 45 | Point{70, 10}, Point{80, 20}, 46 | }, 47 | outer: BBox{ 48 | Point{10, 10}, Point{60, 60}, 49 | }, 50 | in: false, 51 | }, 52 | { 53 | inner: BBox{ 54 | Point{70, 80}, Point{95, 95}, 55 | }, 56 | outer: BBox{ 57 | Point{10, 10}, Point{60, 60}, 58 | }, 59 | in: false, 60 | }, 61 | } { 62 | t.Run(fmt.Sprintf("%v", n), func(t *testing.T) { 63 | assert.Equal(t, tc.in, tc.inner.Overlaps(tc.outer)) 64 | }) 65 | } 66 | } 67 | 68 | func TestBBoxFullyIn(t *testing.T) { 69 | b1 := BBox{SW: Point{1, 1}, NE: Point{2, 2}} 70 | b2 := BBox{SW: Point{0, 0}, NE: Point{3, 3}} 71 | 72 | assert.True(t, b1.FullyIn(b2)) 73 | assert.False(t, b2.FullyIn(b1)) 74 | } 75 | -------------------------------------------------------------------------------- /lib/spatial/clip_geos.go: -------------------------------------------------------------------------------- 1 | // +build !golangclip 2 | 3 | package spatial 4 | 5 | import ( 6 | "log" 7 | 8 | "github.com/pmezard/gogeos/geos" 9 | ) 10 | 11 | func (p Polygon) clipToBBox(b BBox) []Geom { 12 | gpoly := p.geos() 13 | if gpoly == nil { 14 | return nil 15 | } 16 | 17 | var bboxLine = make([]geos.Coord, 0, 4) 18 | for _, pt := range NewLinesFromSegments(BBoxBorders(b.SW, b.NE))[0] { 19 | bboxLine = append(bboxLine, geos.NewCoord(pt.X, pt.Y)) 20 | } 21 | 22 | bboxPoly := geos.Must(geos.NewPolygon(bboxLine)) 23 | res, err := bboxPoly.Intersection(gpoly) 24 | if err != nil { 25 | // Sometimes there is a minor topology problem, a zero buffer helps often. 26 | gpolyBuffed, err := gpoly.Buffer(0) 27 | if err != nil { 28 | panic(err) 29 | } 30 | res, err = bboxPoly.Intersection(gpolyBuffed) 31 | if err != nil { 32 | panic(err) 33 | } 34 | } 35 | 36 | var resGeoms []Geom 37 | for _, poly := range geosToPolygons(res) { 38 | resGeoms = append(resGeoms, MustNewGeom(poly)) 39 | } 40 | return resGeoms 41 | } 42 | 43 | func (p Polygon) geos() *geos.Geometry { 44 | var rings = make([][]geos.Coord, 0, len(p)) 45 | for _, ring := range p { 46 | var rg = make([]geos.Coord, 0, len(ring)) 47 | for _, pt := range ring { 48 | rg = append(rg, geos.NewCoord(pt.X, pt.Y)) 49 | } 50 | rg = append(rg, rg[0]) 51 | rings = append(rings, rg) 52 | } 53 | var gpoly *geos.Geometry 54 | if len(rings) == 0 { 55 | return nil 56 | } 57 | if len(rings) > 1 { 58 | return geos.Must(geos.NewPolygon(rings[0], rings[1:]...)) 59 | } 60 | gpoly, err := geos.NewPolygon(rings[0]) 61 | if err != nil { 62 | log.Printf("invalid polygon: %v", err) 63 | return nil 64 | } 65 | return gpoly 66 | } 67 | 68 | func geosToPolygons(g *geos.Geometry) []Polygon { 69 | ty, _ := g.Type() 70 | if ty == geos.POLYGON { 71 | return []Polygon{geosToPolygon(g)} 72 | } 73 | nmax, err := g.NGeometry() 74 | if err != nil { 75 | panic(err) 76 | } 77 | var polys = make([]Polygon, 0, nmax) 78 | for n := 0; n < nmax; n++ { 79 | polys = append(polys, geosToPolygon(geos.Must(g.Geometry(n)))) 80 | } 81 | return polys 82 | } 83 | 84 | func geosToPolygon(g *geos.Geometry) Polygon { 85 | sh, err := g.Shell() 86 | if err != nil { 87 | return Polygon{} 88 | } 89 | crds, err := sh.Coords() 90 | if err != nil { 91 | panic(err) 92 | } 93 | if len(crds) == 0 { // we got an empty polygon 94 | return Polygon{} 95 | } 96 | var ( 97 | p = make(Polygon, 0, 8) 98 | ring = make([]Point, 0, len(crds)) 99 | ) 100 | for _, crd := range crds { 101 | ring = append(ring, Point{crd.X, crd.Y}) 102 | } 103 | p = append(p, ring[:len(ring)-1]) 104 | 105 | holes, _ := g.Holes() 106 | for _, hole := range holes { 107 | crds, err = hole.Coords() 108 | if err != nil { 109 | panic(err) 110 | } 111 | ring = make([]Point, 0, len(crds)) 112 | for _, crd := range crds { 113 | ring = append(ring, Point{crd.X, crd.Y}) 114 | } 115 | p = append(p, ring[:len(ring)-1]) 116 | } 117 | return p 118 | } 119 | -------------------------------------------------------------------------------- /lib/spatial/clip_geos_test.go: -------------------------------------------------------------------------------- 1 | // +build !golangclip 2 | 3 | package spatial 4 | 5 | import ( 6 | "encoding/json" 7 | "os" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestGeosConversion(t *testing.T) { 14 | ply := Polygon{{{0, 1}, {2.5, 1}, {3, 2}, {2.5, 3}, {1, 3}}} 15 | gs := ply.geos() 16 | assert.Equal(t, ply, geosToPolygon(gs)) 17 | } 18 | 19 | func TestGeosSelfIntersect(t *testing.T) { 20 | f, err := os.Open("testfiles/self_intersect.geojson") 21 | assert.Nil(t, err) 22 | defer f.Close() 23 | 24 | fc := NewFeatureCollection() 25 | err = json.NewDecoder(f).Decode(&fc) 26 | assert.Nil(t, err) 27 | 28 | res := fc.Features[0].Geometry.MustPolygon().clipToBBox(BBox{Point{0, 0}, Point{2000, 2000}}) 29 | assert.Len(t, res, 2) // TODO: some more testing might be useful 30 | } 31 | -------------------------------------------------------------------------------- /lib/spatial/clip_golang.go: -------------------------------------------------------------------------------- 1 | // +build golangclip 2 | 3 | package spatial 4 | 5 | import polyclip "github.com/ctessum/polyclip-go" 6 | 7 | func (p Polygon) clipToBBox(b BBox) []Geom { 8 | pcp := p.polyclipPolygon() 9 | bboxPoly := polyclip.Polygon{polyclip.Contour{ 10 | polyclip.Point{b.SW.X, b.SW.Y}, 11 | polyclip.Point{b.NE.X, b.SW.Y}, 12 | polyclip.Point{b.NE.X, b.NE.Y}, 13 | polyclip.Point{b.SW.X, b.NE.Y}, 14 | }} 15 | return []Geom{ 16 | MustNewGeom( 17 | polyClipToPolygon(pcp.Construct(polyclip.INTERSECTION, bboxPoly)), 18 | ), 19 | } 20 | } 21 | 22 | func (p Polygon) polyclipPolygon() polyclip.Polygon { 23 | // TODO: polyclip has the same data structure, check if there is any possiblity for some speed-up hack 24 | var pcp polyclip.Polygon 25 | for _, ring := range p { 26 | var cnt polyclip.Contour 27 | for _, pt := range ring { 28 | cnt = append(cnt, polyclip.Point{pt.X, pt.Y}) 29 | } 30 | pcp = append(pcp, cnt) 31 | } 32 | return pcp 33 | } 34 | 35 | func polyClipToPolygon(pcp polyclip.Polygon) Polygon { 36 | var p Polygon 37 | for _, countour := range pcp { 38 | var ring Line 39 | for _, pt := range countour { 40 | ring = append(ring, Point{pt.X, pt.Y}) 41 | } 42 | p = append(p, ring) 43 | } 44 | return p 45 | } 46 | -------------------------------------------------------------------------------- /lib/spatial/clip_golang_test.go: -------------------------------------------------------------------------------- 1 | // +build golangclip 2 | 3 | package spatial 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestClipPolygon(t *testing.T) { 12 | poly1, err := NewGeom(Polygon{ 13 | { 14 | {0, 1}, {0, 0}, {1, 0}, {1, 1}, 15 | }, 16 | }) 17 | poly2, err := NewGeom(Polygon{ 18 | { 19 | {0, 0}, {0, 0.2}, {0.8, 0.2}, {0.8, 0.8}, {0, 0.8}, {0, 1}, {1, 1}, {1, 0}, 20 | }, 21 | }) 22 | poly3, err := NewGeom(Polygon{ 23 | { 24 | {0, 10}, {0, 0}, {10, 0}, 25 | }, 26 | }) 27 | assert.Nil(t, err) 28 | t.Run("uncut", func(t *testing.T) { 29 | assert.Equal(t, []Geom{poly1}, poly1.ClipToBBox(BBox{Point{0, 0}, Point{1, 1}})) 30 | }) 31 | t.Run("single ring cut", func(t *testing.T) { 32 | polyCut, err := NewGeom(Polygon{ 33 | { 34 | {0, 1}, 35 | {0, 0}, 36 | {0.5, 0}, 37 | {0.5, 1}, 38 | }, 39 | }) 40 | assert.Nil(t, err) 41 | assert.Equal(t, []Geom{polyCut}, poly1.ClipToBBox(BBox{Point{0, 0}, Point{0.5, 1}})) 42 | }) 43 | t.Run("single ring into two subpolygons", func(t *testing.T) { 44 | polyCut1, err := NewGeom(Polygon{ 45 | { 46 | {0, 0.2}, 47 | {0, 0}, 48 | {0.5, 0}, 49 | {0.5, 0.2}, 50 | }, 51 | { 52 | {0, 1}, 53 | {0, 0.8}, 54 | {0.5, 0.8}, 55 | {0.5, 1}, 56 | }, 57 | }) 58 | assert.Nil(t, err) 59 | assert.Equal(t, polyCut1, poly2.ClipToBBox(BBox{Point{-0.1, -0.1}, Point{0.5, 1.1}})[0]) 60 | }) 61 | t.Run("triangle cut", func(t *testing.T) { 62 | poly3.ClipToBBox(BBox{Point{5, -5}, Point{20, 20}}) 63 | }) 64 | 65 | // TODO: test cut where the bbox of the polygon overlaps with cut bbox, but isn't actually inside 66 | } 67 | -------------------------------------------------------------------------------- /lib/spatial/codec.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import "io" 4 | 5 | type Chunks interface { 6 | Next() bool 7 | Scan(fc *FeatureCollection) error 8 | } 9 | 10 | type Decoder interface { 11 | Decode(io.Reader, *FeatureCollection) error 12 | } 13 | 14 | type ChunkedDecoder interface { 15 | ChunkedDecode(io.Reader) (Chunks, error) 16 | } 17 | 18 | type Encoder interface { 19 | Encode(io.Writer, *FeatureCollection) error 20 | } 21 | 22 | type ChunkedEncoder interface { 23 | EncodeChunk(io.Writer, *FeatureCollection) error 24 | Close(io.Writer) error 25 | } 26 | 27 | // A Codec needs to be able to tell which file extensions (e.g. "geojson") 28 | // are commonly used to persist files. Moreover a Codec SHOULD either implement 29 | // a Decoder or Encoder. 30 | type Codec interface { 31 | Extensions() []string 32 | } 33 | -------------------------------------------------------------------------------- /lib/spatial/conversion.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | // LineSegmentToCarthesian converts a line segment into a carthesian representation. 4 | // Possible improvement: normalization of values 5 | func LineSegmentToCarthesian(p1, p2 Point) (a, b, c float64) { 6 | a = p1.Y - p2.Y 7 | b = p2.X - p1.X 8 | c = p2.X*p1.Y - p1.X*p2.Y 9 | return 10 | } 11 | -------------------------------------------------------------------------------- /lib/spatial/conversion_test.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestLineSegmentToCarthesian(t *testing.T) { 10 | a, b, c := LineSegmentToCarthesian(Point{0, 3}, Point{4, 1}) 11 | assert.Equal(t, float64(2), a) 12 | assert.Equal(t, float64(4), b) 13 | assert.Equal(t, float64(12), c) 14 | } 15 | -------------------------------------------------------------------------------- /lib/spatial/fuzz.go: -------------------------------------------------------------------------------- 1 | // +build gofuzz 2 | 3 | package spatial 4 | 5 | import "bytes" 6 | 7 | func Fuzz(data []byte) int { 8 | var g Geom 9 | 10 | err := g.UnmarshalWKB(bytes.NewReader(data)) 11 | if err != nil { 12 | return 0 13 | } 14 | return 1 15 | } 16 | -------------------------------------------------------------------------------- /lib/spatial/geom_test.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "os" 10 | "testing" 11 | 12 | "github.com/stretchr/testify/assert" 13 | "github.com/twpayne/go-geom/encoding/wkb" 14 | ) 15 | 16 | func TestMarshalWKBPoint(t *testing.T) { 17 | spt := Point{-21.123456, 0.981231} 18 | g, err := NewGeom(spt) 19 | assert.Nil(t, err) 20 | buf, err := g.MarshalWKB() 21 | assert.Nil(t, err) 22 | 23 | // test against third party implementation 24 | tpt, err := wkb.Unmarshal(buf) 25 | assert.Nil(t, err) 26 | assert.Equal(t, tpt.FlatCoords()[0], spt.X) 27 | assert.Equal(t, tpt.FlatCoords()[1], spt.Y) 28 | 29 | // test against own implementation 30 | rp := &Geom{} 31 | err = rp.UnmarshalWKB(bytes.NewReader(buf)) 32 | assert.Nil(t, err) 33 | pt, err := rp.Point() 34 | assert.Nil(t, err) 35 | assert.Equal(t, spt, *pt) 36 | } 37 | 38 | func TestWKBUnmarshal(t *testing.T) { 39 | f, err := os.Open("testfiles/polygon.wkb") 40 | assert.Nil(t, err) 41 | 42 | buf, err := ioutil.ReadAll(f) 43 | assert.Nil(t, err) 44 | 45 | var g Geom 46 | err = g.UnmarshalWKB(bytes.NewBuffer(buf)) 47 | assert.Nil(t, err) 48 | assert.Equal(t, g.Typ(), GeomTypePolygon) 49 | 50 | bufOut, err := g.MarshalWKB() 51 | assert.Nil(t, err) 52 | assert.Equal(t, buf, bufOut) 53 | 54 | // TODO: fix the test, which broke because of winding order reconstruction 55 | // gj, err := os.Open("testfiles/polygon.json") 56 | // assert.Nil(t, err) 57 | // gjbuf, err := ioutil.ReadAll(gj) 58 | 59 | // var g2 Geom 60 | // g2.UnmarshalJSON(gjbuf) 61 | // assert.Equal(t, g, g2) 62 | } 63 | 64 | func TestUnmarshalWKBEOF(t *testing.T) { 65 | var buf []byte 66 | fmt.Sscanf("09000000000000000000f03f00000000000000400000000000000840000000000000104000000000000014400000000000001040", "%x", &buf) 67 | 68 | _, err := wkbReadLineString(bytes.NewReader(buf)) 69 | assert.Equal(t, io.EOF, err) 70 | } 71 | 72 | func TestMarshalWKBLineString(t *testing.T) { 73 | sls := Line{{1, 2}, {3, 4}, {5, 4}} 74 | g, err := NewGeom(sls) 75 | assert.Nil(t, err) 76 | buf, err := g.MarshalWKB() 77 | assert.Nil(t, err) 78 | 79 | tls, err := wkb.Unmarshal(buf) 80 | assert.Nil(t, err) 81 | assert.Equal(t, tls.FlatCoords()[0], sls[0].X) 82 | assert.Equal(t, tls.FlatCoords()[1], sls[0].Y) 83 | assert.Equal(t, tls.FlatCoords()[2], sls[1].X) 84 | assert.Equal(t, tls.FlatCoords()[3], sls[1].Y) 85 | assert.Equal(t, tls.FlatCoords()[4], sls[2].X) 86 | assert.Equal(t, tls.FlatCoords()[5], sls[2].Y) 87 | 88 | rp := &Geom{} 89 | err = rp.UnmarshalWKB(bytes.NewReader(buf)) 90 | assert.Nil(t, err) 91 | ls, err := rp.LineString() 92 | assert.Nil(t, err) 93 | assert.Equal(t, sls, ls) 94 | } 95 | 96 | func TestMarshalWKBPolygon(t *testing.T) { 97 | spoly := Polygon{ 98 | { 99 | {1, 2}, {3, 4}, {5, 4}, 100 | }, 101 | { 102 | {2, 2}, {3, 4}, {2, 2}, 103 | }, 104 | } 105 | g, err := NewGeom(spoly) 106 | assert.Nil(t, err) 107 | buf, err := g.MarshalWKB() 108 | assert.Nil(t, err) 109 | 110 | _, err = wkb.Unmarshal(buf) 111 | assert.Nil(t, err) 112 | 113 | rp := &Geom{} 114 | err = rp.UnmarshalWKB(bytes.NewReader(buf)) 115 | assert.Nil(t, err) 116 | poly, err := rp.Polygon() 117 | assert.Nil(t, err) 118 | assert.Equal(t, spoly, poly) 119 | } 120 | 121 | func TestGeoJSON(t *testing.T) { 122 | f, err := os.Open("testfiles/featurecollection.geojson") 123 | assert.Nil(t, err) 124 | defer f.Close() 125 | 126 | fc := FeatureCollection{} 127 | err = json.NewDecoder(f).Decode(&fc) 128 | assert.Nil(t, err) 129 | 130 | p, err := fc.Features[0].Geometry.Point() 131 | assert.Nil(t, err) 132 | assert.NotNil(t, p) 133 | 134 | ls, err := fc.Features[1].Geometry.LineString() 135 | assert.Nil(t, err) 136 | assert.NotNil(t, ls) 137 | 138 | poly, err := fc.Features[2].Geometry.Polygon() 139 | assert.Nil(t, err) 140 | assert.NotNil(t, poly) 141 | 142 | buf, err := json.Marshal(fc) 143 | assert.Nil(t, err) 144 | assert.NotNil(t, buf) 145 | } 146 | 147 | func BenchmarkWKBMarshalPoint(b *testing.B) { 148 | g, err := NewGeom(Point{2, 3}) 149 | assert.Nil(b, err) 150 | b.ReportAllocs() 151 | b.ResetTimer() 152 | 153 | for i := 0; i < b.N; i++ { 154 | g.MarshalWKB() 155 | } 156 | } 157 | 158 | func BenchmarkWKBMarshalRawPoint(b *testing.B) { 159 | var buf bytes.Buffer 160 | p := Point{2, 3} 161 | b.ReportAllocs() 162 | b.ResetTimer() 163 | 164 | for i := 0; i < b.N; i++ { 165 | wkbWritePoint(&buf, p) 166 | } 167 | } 168 | 169 | func BenchmarkWKBMarshalLineString(b *testing.B) { 170 | ls := []Point{{2, 3}, {5, 6}, {10, 15}, {20, 50}} 171 | g, err := NewGeom(ls) 172 | assert.Nil(b, err) 173 | b.ReportAllocs() 174 | b.ResetTimer() 175 | 176 | for i := 0; i < b.N; i++ { 177 | g.MarshalWKB() 178 | } 179 | } 180 | 181 | func BenchmarkWKBMarshalPoly(b *testing.B) { 182 | poly := [][]Point{{{2, 3}, {5, 6}, {10, 15}, {2, 3}}, {{10, 15}, {5, 6}, {10, 15}}} 183 | g, err := NewGeom(poly) 184 | assert.Nil(b, err) 185 | b.ReportAllocs() 186 | b.ResetTimer() 187 | 188 | for i := 0; i < b.N; i++ { 189 | g.MarshalWKB() 190 | } 191 | } 192 | 193 | func BenchmarkWKBMarshalRawPoly(b *testing.B) { 194 | var buf bytes.Buffer 195 | poly := Polygon{{{2, 3}, {5, 6}, {10, 15}, {2, 3}}, {{10, 15}, {5, 6}, {10, 15}}} 196 | b.ReportAllocs() 197 | b.ResetTimer() 198 | 199 | for i := 0; i < b.N; i++ { 200 | wkbWritePolygon(&buf, poly) 201 | } 202 | } 203 | 204 | func BenchmarkWKBUnmarshalPoint(b *testing.B) { 205 | var rawPt []byte 206 | _, err := fmt.Sscanf("b77efacf9a1f35c0b648da8d3e66ef3f", "%x", &rawPt) 207 | assert.Nil(b, err) 208 | r := bytes.NewReader(rawPt) 209 | 210 | b.ReportAllocs() 211 | b.ResetTimer() 212 | 213 | for i := 0; i < b.N; i++ { 214 | r.Reset(rawPt) 215 | wkbReadPoint(r) 216 | } 217 | } 218 | 219 | func BenchmarkWKBUnmarshalLineString(b *testing.B) { 220 | var rawLine []byte 221 | _, err := fmt.Sscanf("03000000000000000000f03f00000000000000400000000000000840000000000000104000000000000014400000000000001040", "%x", &rawLine) 222 | assert.Nil(b, err) 223 | r := bytes.NewReader(rawLine) 224 | 225 | b.ReportAllocs() 226 | b.ResetTimer() 227 | 228 | for i := 0; i < b.N; i++ { 229 | r.Reset(rawLine) 230 | wkbReadLineString(r) 231 | } 232 | } 233 | 234 | func BenchmarkWKBUnmarshalPoly(b *testing.B) { 235 | var rawPoly []byte 236 | _, err := fmt.Sscanf("0200000003000000000000000000f03f0000000000000040000000000000084000000000000010400000000000001440000000000000104003000000000000000000004000000000000000400000000000000840000000000000104000000000000000400000000000000040", "%x", &rawPoly) 237 | assert.Nil(b, err) 238 | r := bytes.NewReader(rawPoly) 239 | 240 | b.ReportAllocs() 241 | b.ResetTimer() 242 | 243 | for i := 0; i < b.N; i++ { 244 | r.Reset(rawPoly) 245 | wkbReadPolygon(r) 246 | } 247 | } 248 | 249 | func TestClipPoint(t *testing.T) { 250 | p, err := NewGeom(Point{1, 1}) 251 | assert.Nil(t, err) 252 | t.Run("inside", func(t *testing.T) { 253 | assert.Equal(t, []Geom{p}, p.ClipToBBox(BBox{Point{0, 0}, Point{5, 5}})) 254 | }) 255 | t.Run("outside", func(t *testing.T) { 256 | assert.Equal(t, []Geom{}, p.ClipToBBox(BBox{Point{5, 0}, Point{5, 5}})) 257 | }) 258 | t.Run("on SW edge", func(t *testing.T) { 259 | assert.Equal(t, []Geom{p}, p.ClipToBBox(BBox{Point{0, 0}, Point{1, 1}})) 260 | }) 261 | t.Run("on NE edge", func(t *testing.T) { 262 | assert.Equal(t, []Geom{p}, p.ClipToBBox(BBox{Point{1, 1}, Point{2, 2}})) 263 | }) 264 | } 265 | 266 | func TestMarshalJSONMutation(t *testing.T) { 267 | original := Polygon{Line{{1, 2}, {3, 5}, {4, 9}}} 268 | g := MustNewGeom(original) 269 | g.MarshalJSON() 270 | 271 | assert.Equal(t, 3, len(original[0])) 272 | } 273 | -------------------------------------------------------------------------------- /lib/spatial/merge.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import "sort" 4 | 5 | // MergeFeatures aggregates features that have the same properties, if possible. 6 | func MergeFeatures(fts []Feature) []Feature { 7 | if len(fts) == 1 { 8 | return fts 9 | } 10 | 11 | out := fts[:0] 12 | buckets := tagBuckets(fts) 13 | for bid := range buckets { 14 | for { 15 | startLen := len(buckets[bid]) 16 | if startLen == 1 { 17 | break 18 | } 19 | buckets[bid] = searchAndMerge(buckets[bid]) 20 | if len(buckets[bid]) == startLen { 21 | break 22 | } 23 | } 24 | } 25 | 26 | for _, bucket := range buckets { 27 | out = append(out, bucket...) 28 | } 29 | return out 30 | } 31 | 32 | func tagBuckets(fts []Feature) [][]Feature { 33 | var buckets [][]Feature 34 | 35 | Outer: 36 | for _, ft := range fts { 37 | for bID := range buckets { 38 | if equalProps(buckets[bID][0].Props, ft.Props) { 39 | buckets[bID] = append(buckets[bID], ft) 40 | continue Outer 41 | } 42 | } 43 | buckets = append(buckets, []Feature{ft}) 44 | } 45 | return buckets 46 | } 47 | 48 | type ignoreList []int 49 | 50 | func (il ignoreList) search(i int) int { 51 | return sort.Search(len(il), func(pos int) bool { return il[pos] >= i }) 52 | } 53 | 54 | func (il ignoreList) Has(i int) bool { 55 | res := il.search(i) 56 | if res < len(il) && il[res] == i { 57 | return true 58 | } 59 | return false 60 | } 61 | 62 | func (il *ignoreList) Add(i int) { 63 | r := il.search(i) 64 | iil := *il 65 | *il = append(iil[:r], append(ignoreList{i}, iil[r:]...)...) 66 | } 67 | 68 | func searchAndMerge(fts []Feature) []Feature { 69 | if len(fts) == 0 { 70 | return fts 71 | } 72 | var ignore = make(ignoreList, 0, len(fts)/10) 73 | 74 | for refID := range fts { 75 | if ignore.Has(refID) { 76 | continue 77 | } 78 | for i, ft := range fts { 79 | if ignore.Has(i) || i == refID { 80 | continue 81 | } 82 | if ft.Geometry.typ != fts[refID].Geometry.typ { 83 | continue 84 | } 85 | switch ft.Geometry.typ { 86 | case GeomTypeLineString: 87 | l, merged := mergeLines(fts[refID].Geometry.g.(Line), ft.Geometry.g.(Line)) 88 | if merged { 89 | fts[refID].Geometry.set(l) 90 | ignore.Add(i) 91 | } 92 | } 93 | } 94 | } 95 | 96 | var out = make([]Feature, 0, len(fts)-len(ignore)) 97 | for pos, ft := range fts { 98 | if ignore.Has(pos) { 99 | continue 100 | } 101 | out = append(out, ft) 102 | } 103 | return out 104 | } 105 | 106 | func mergeLines(l1, l2 Line) (Line, bool) { 107 | if l1[len(l1)-1] == l2[0] { 108 | return append(l1, l2[1:]...), true 109 | } 110 | if l2[len(l2)-1] == l1[0] { 111 | return append(l2, l1[1:]...), true 112 | } 113 | return l1, false 114 | } 115 | 116 | func equalProps(p1, p2 map[string]interface{}) bool { 117 | if len(p1) != len(p2) { 118 | return false 119 | } 120 | for k, v1 := range p1 { 121 | if v2, ok := p2[k]; !ok { 122 | return false 123 | } else { 124 | if v1 != v2 { 125 | return false 126 | } 127 | } 128 | } 129 | return true 130 | } 131 | -------------------------------------------------------------------------------- /lib/spatial/merge_test.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestMerge(t *testing.T) { 12 | t.Run("different key-value", func(t *testing.T) { 13 | props1 := map[string]interface{}{ 14 | "1": 2, 15 | } 16 | props2 := map[string]interface{}{ 17 | "1": 3.1, 18 | } 19 | feats := []Feature{ 20 | { 21 | Props: props1, 22 | Geometry: MustNewGeom(Line{{1, 2}, {3, 4}}), 23 | }, 24 | { 25 | Props: props2, 26 | Geometry: MustNewGeom(Line{{3, 4}, {5, 6}}), 27 | }, 28 | } 29 | assert.Equal(t, feats, MergeFeatures(feats)) 30 | }) 31 | 32 | t.Run("continuous", func(t *testing.T) { 33 | props := map[string]interface{}{ 34 | "a": 1, 35 | "b": "foo", 36 | "c": 1.234, 37 | } 38 | feat1 := Feature{ 39 | Props: props, 40 | Geometry: MustNewGeom(Line{{1, 0}, {1, 1}, {2, 3}, {5, 6}}), 41 | } 42 | feat2 := Feature{ 43 | Props: props, 44 | Geometry: MustNewGeom(Line{{5, 6}, {7, 8}, {6, 6}, {4, 5}}), 45 | } 46 | assert.Equal(t, []Feature{ 47 | { 48 | Props: props, 49 | Geometry: MustNewGeom(Line{{1, 0}, {1, 1}, {2, 3}, {5, 6}, {7, 8}, {6, 6}, {4, 5}}), 50 | }, 51 | }, MergeFeatures([]Feature{feat1, feat2})) 52 | }) 53 | } 54 | 55 | func TestMergeFoo(t *testing.T) { 56 | f, err := os.Open("testfiles/mergable_lines.geojson") 57 | assert.Nil(t, err) 58 | 59 | var fcoll = NewFeatureCollection() 60 | err = json.NewDecoder(f).Decode(fcoll) 61 | assert.Nil(t, err) 62 | 63 | fcoll.Features = searchAndMerge(fcoll.Features) 64 | 65 | assert.Len(t, fcoll.Features, 1) 66 | assert.True(t, len(fcoll.Features[0].Geometry.MustLineString()) > 7) 67 | } 68 | 69 | func TestBuckets(t *testing.T) { 70 | f1 := Feature{Props: map[string]interface{}{"1": 2}} 71 | f2 := Feature{Props: map[string]interface{}{"1": 2}} 72 | f3 := Feature{Props: map[string]interface{}{"1": 3}} 73 | 74 | bucks := tagBuckets([]Feature{f1, f2, f3}) 75 | assert.Len(t, bucks, 2) 76 | } 77 | 78 | func TestIgnoreList(t *testing.T) { 79 | il := ignoreList{1, 5, 9} 80 | il.Add(3) 81 | assert.Equal(t, ignoreList{1, 3, 5, 9}, il) 82 | il.Add(19) 83 | assert.Equal(t, ignoreList{1, 3, 5, 9, 19}, il) 84 | assert.True(t, il.Has(1)) 85 | assert.False(t, il.Has(4)) 86 | } 87 | -------------------------------------------------------------------------------- /lib/spatial/point.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "math" 7 | "strconv" 8 | ) 9 | 10 | type Point struct{ X, Y float64 } 11 | 12 | func (p *Point) Project(proj ConvertFunc) { 13 | np := proj(*p) 14 | p.X = np.X 15 | p.Y = np.Y 16 | } 17 | 18 | func (p Point) InBBox(b BBox) bool { 19 | return b.SW.X <= p.X && b.NE.X >= p.X && 20 | b.SW.Y <= p.Y && b.NE.Y >= p.Y 21 | } 22 | 23 | func (p Point) ClipToBBox(b BBox) []Geom { 24 | if p.InBBox(b) { 25 | g := MustNewGeom(p) 26 | return []Geom{g} 27 | } 28 | return []Geom{} 29 | } 30 | 31 | func (p Point) String() string { 32 | return fmt.Sprintf("(X: %v, Y: %v)", p.X, p.Y) 33 | } 34 | 35 | func (p Point) MarshalJSON() ([]byte, error) { 36 | var b = make([]byte, 1, 30) 37 | b[0] = '[' 38 | // TODO: Have a possibility to reduce precision to a limited number of decimals. 39 | b = strconv.AppendFloat(b, p.X, 'f', -1, 64) 40 | b = append(b, ',') 41 | b = strconv.AppendFloat(b, p.Y, 'f', -1, 64) 42 | return append(b, ']'), nil 43 | } 44 | 45 | func (p *Point) SetX(x float64) { 46 | p.X = x 47 | } 48 | 49 | func (p *Point) SetY(y float64) { 50 | p.Y = y 51 | } 52 | 53 | func (p *Point) UnmarshalJSON(buf []byte) error { 54 | var arrayPt [2]float64 55 | if err := json.Unmarshal(buf, &arrayPt); err != nil { 56 | return err 57 | } 58 | p.X = arrayPt[0] 59 | p.Y = arrayPt[1] 60 | return nil 61 | } 62 | 63 | const pointPrecision = 8 64 | 65 | func (p Point) RoundedCoords() Point { 66 | return Point{ 67 | roundWithPrecision(p.X, pointPrecision), 68 | roundWithPrecision(p.Y, pointPrecision), 69 | } 70 | } 71 | 72 | func (p Point) InPolygon(poly Polygon) bool { 73 | bbox := poly[0].BBox() 74 | if !p.InBBox(bbox) { 75 | return false 76 | } 77 | 78 | var allsegs = make([]Segment, 0, len(poly)*10) // preallocating for 10 segments per ring 79 | for _, ln := range poly { 80 | allsegs = append(allsegs, ln.Segments()...) 81 | allsegs = append(allsegs, Segment{ln[len(ln)-1], ln[0]}) // closing segment 82 | } 83 | 84 | var ( 85 | outTestPoint = Point{bbox.SW.X - 1, bbox.SW.Y - 1} 86 | l = Line{p, outTestPoint} 87 | ) 88 | 89 | intersections := l.Intersections(allsegs) 90 | if len(intersections)%2 == 0 { 91 | for _, itsct := range intersections { 92 | // if the intersection is exactly the tested point, the point is actually inside 93 | // TODO: test more 94 | if itsct == p { 95 | return true 96 | } 97 | } 98 | return false 99 | } 100 | return true 101 | } 102 | 103 | func (p Point) Copy() Projectable { 104 | return &Point{X: p.X, Y: p.Y} 105 | } 106 | 107 | const earthRadiusMeters = 6371000 // WGS84, optimized for minimal square relative error 108 | 109 | // DistanceTo returns the distance in meters between the points. 110 | func (p *Point) HaversineDistance(p2 *Point) float64 { 111 | lat1 := degToRad(p.Y) 112 | lon1 := degToRad(p.X) 113 | lat2 := degToRad(p2.Y) 114 | lon2 := degToRad(p2.X) 115 | 116 | dLat := lat2 - lat1 117 | dLon := lon2 - lon1 118 | 119 | a := math.Pow(math.Sin(dLat/2), 2) + math.Cos(lat1)*math.Cos(lat2)* 120 | math.Pow(math.Sin(dLon/2), 2) 121 | c := 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a)) 122 | return c * earthRadiusMeters 123 | } 124 | 125 | func degToRad(deg float64) float64 { 126 | return deg * math.Pi / 180 127 | } 128 | 129 | func round(v float64) float64 { 130 | if v < 0 { 131 | return math.Ceil(v - 0.5) 132 | } 133 | return math.Floor(v + 0.5) 134 | } 135 | 136 | func roundWithPrecision(v float64, decimals int) float64 { 137 | s := math.Pow(10, float64(decimals)) 138 | return round(v*s) / s 139 | } 140 | -------------------------------------------------------------------------------- /lib/spatial/point_test.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import ( 4 | "encoding/json" 5 | "math" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestRoundWithPrecision(t *testing.T) { 12 | pt1 := Point{-5.4213000001, 10.9874000001} 13 | assert.Equal(t, Point{-5.4213, 10.9874}, pt1.RoundedCoords()) 14 | } 15 | 16 | func TestPointInPolygon(t *testing.T) { 17 | square := Polygon{ 18 | { 19 | {-1, 1}, {-1, -1}, {1, -1}, {1, 1}, 20 | }, 21 | } 22 | triangle := Polygon{ 23 | { 24 | {0, 0}, {1, 2}, {2, 0}, 25 | }, 26 | } 27 | squareWithHole := Polygon{ 28 | { 29 | {0, 0}, {0, 10}, {10, 10}, {10, 0}, 30 | }, 31 | { 32 | {2.5, 2.5}, {2.5, 7.5}, {7.5, 7.5}, {7.5, 2.5}, 33 | }, 34 | } 35 | 36 | t.Run("simple in", func(t *testing.T) { 37 | pt := Point{0, 0} 38 | assert.True(t, pt.InPolygon(square)) 39 | }) 40 | 41 | t.Run("simple out 1", func(t *testing.T) { 42 | pt := Point{-2, -2} 43 | assert.False(t, pt.InPolygon(square)) 44 | }) 45 | 46 | t.Run("simple out 2", func(t *testing.T) { 47 | pt := Point{3, 3} 48 | assert.False(t, pt.InPolygon(square)) 49 | }) 50 | 51 | t.Run("triangle in", func(t *testing.T) { 52 | pt := Point{1, 1} 53 | assert.True(t, pt.InPolygon(triangle)) 54 | }) 55 | t.Run("triangle out", func(t *testing.T) { 56 | pt := Point{0.5, 1.1} 57 | assert.False(t, pt.InPolygon(triangle)) 58 | }) 59 | 60 | t.Run("holed in", func(t *testing.T) { 61 | pt := Point{1, 1} 62 | assert.True(t, pt.InPolygon(squareWithHole)) 63 | }) 64 | t.Run("holed out", func(t *testing.T) { 65 | pt := Point{5, 5} 66 | assert.False(t, pt.InPolygon(squareWithHole)) 67 | }) 68 | 69 | t.Run("closing segment", func(t *testing.T) { 70 | pt := Point{25.48828125, -18.312810846425432} 71 | poly := Polygon{Line{Point{X: 7.3828125, Y: -23.241346102386135}, Point{X: 28.4765625, Y: -8.05922962720018}, Point{X: 55.1953125, Y: -11.178401873711772}, Point{X: 22.148437499999996, Y: -33.137551192346145}}} 72 | assert.True(t, pt.InPolygon(poly)) 73 | }) 74 | } 75 | 76 | func TestPointJSONMarshal(t *testing.T) { 77 | p := Point{-12.00000000001, 179.1} 78 | buf, err := json.Marshal(p) 79 | assert.Nil(t, err) 80 | assert.Equal(t, []byte("179.1]"), buf[len(buf)-6:]) 81 | } 82 | 83 | func BenchmarkPointJSONMarshal(b *testing.B) { 84 | p := Point{12.00000000001, 13.000000000001} 85 | 86 | b.ReportAllocs() 87 | for i := 0; i < b.N; i++ { 88 | json.Marshal(p) 89 | } 90 | } 91 | 92 | func TestHaversineDistance(t *testing.T) { 93 | p1 := &Point{X: 7.06659, Y: 50.88354} 94 | p2 := &Point{X: 6.96299, Y: 50.93846} 95 | assert.Equal(t, 9490.0, math.Round(p1.HaversineDistance(p2))) 96 | } 97 | -------------------------------------------------------------------------------- /lib/spatial/polygon.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | // Polygon is a data type for storing simple polygons. 4 | type Polygon []Line 5 | 6 | func (p Polygon) Project(proj ConvertFunc) { 7 | for ri := range p { 8 | for i := range p[ri] { 9 | p[ri][i] = proj(p[ri][i]) 10 | } 11 | } 12 | } 13 | 14 | func (p Polygon) Copy() Projectable { 15 | var np Polygon 16 | for _, ring := range p { 17 | np = append(np, ring.Copy().(Line)) 18 | } 19 | return np 20 | } 21 | 22 | func (p Polygon) String() string { 23 | return p.string() 24 | } 25 | 26 | func (p Polygon) ClipToBBox(bbox BBox) []Geom { 27 | // Speed-ups for common cases to eliminate the need for calling geos. 28 | if len(p) == 1 && len(p[0].Intersections(bbox.Segments())) == 0 { 29 | if bbox.FullyIn(p[0].BBox()) { 30 | return []Geom{MustNewGeom(Polygon{Line{ 31 | bbox.SW, {bbox.NE.X, bbox.SW.Y}, bbox.NE, {bbox.SW.X, bbox.NE.Y}, 32 | }})} 33 | } 34 | if p[0].BBox().FullyIn(bbox) { 35 | return []Geom{MustNewGeom(p)} 36 | } 37 | } 38 | 39 | return p.clipToBBox(bbox) 40 | } 41 | 42 | func (p Polygon) Rewind() { 43 | for _, ring := range p { 44 | ring.Reverse() 45 | } 46 | } 47 | 48 | func (p Polygon) FixWinding() { 49 | for n, ring := range p { 50 | if n == 0 { 51 | // First ring must be outer and therefore clockwise. 52 | if !ring.Clockwise() { 53 | ring.Reverse() 54 | } 55 | continue 56 | } 57 | // Compare in how many rings the point is located. 58 | // If the number is odd, it's a hole. 59 | var inrings int 60 | for ninner, cring := range p { 61 | if n == ninner { 62 | continue 63 | } 64 | if ring[0].InPolygon(Polygon{cring}) { 65 | inrings++ 66 | } 67 | } 68 | if (inrings%2 == 0 && !ring.Clockwise()) || (inrings%2 == 1 && ring.Clockwise()) { 69 | ring.Reverse() 70 | } 71 | } 72 | } 73 | 74 | func (p Polygon) ValidTopology() bool { 75 | return len(p.topologyErrors()) == 0 76 | } 77 | 78 | func (p Polygon) MustJSON() []byte { 79 | g := MustNewGeom(p) 80 | j, err := g.MarshalJSON() 81 | if err != nil { 82 | panic(err) 83 | } 84 | return j 85 | } 86 | 87 | type segErr struct { 88 | Ring int 89 | Seg int 90 | } 91 | 92 | func (p Polygon) topologyErrors() (errSegments []segErr) { 93 | for nRing, ring := range p { 94 | for nSeg, seg := range ring.SegmentsWithClosing() { 95 | for nSegCmp, segCmp := range ring.SegmentsWithClosing() { 96 | if nSeg == nSegCmp { 97 | continue 98 | } 99 | ipt, has := seg.Intersection(segCmp) 100 | if has && (ipt != seg[0] && ipt != seg[1]) { 101 | errSegments = append(errSegments, segErr{Ring: nRing, Seg: nSeg}) 102 | } 103 | } 104 | } 105 | } 106 | return 107 | } 108 | -------------------------------------------------------------------------------- /lib/spatial/polygon_test.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestRewind(t *testing.T) { 12 | p := Polygon{ 13 | {{1, 2}, {8, 9}, {10, 12}, {1, 2}}, 14 | {{0, 1}, {7, 9}, {0, 1}, {2, 12}, {0, 1}}, 15 | } 16 | p.Rewind() 17 | assert.Equal(t, Polygon{ 18 | {{1, 2}, {10, 12}, {8, 9}, {1, 2}}, 19 | {{0, 1}, {2, 12}, {0, 1}, {7, 9}, {0, 1}}, 20 | }, p) 21 | } 22 | 23 | func TestWinding(t *testing.T) { 24 | f, err := os.Open("testfiles/winding_wild.geojson") 25 | assert.Nil(t, err) 26 | defer f.Close() 27 | 28 | var fc FeatureCollection 29 | err = json.NewDecoder(f).Decode(&fc) 30 | assert.Nil(t, err) 31 | 32 | var outOrder []bool 33 | for _, ring := range fc.Features[0].Geometry.MustPolygon() { 34 | outOrder = append(outOrder, ring.Clockwise()) 35 | } 36 | assert.Equal(t, []bool{true, false, true, true, false}, outOrder) // correct order 37 | } 38 | 39 | func TestFixWinding(t *testing.T) { 40 | g := Geom{typ: GeomTypePolygon, g: Polygon{ 41 | Line{Point{X: -2.109375, Y: 11.178401873711785}, Point{X: -16.875, Y: -43.06888777416961}, Point{X: 62.57812500000001, Y: -43.580390855607845}, Point{X: 81.5625, Y: 8.407168163601076}}, 42 | Line{Point{X: 7.3828125, Y: -23.241346102386135}, Point{X: 28.4765625, Y: -8.05922962720018}, Point{X: 55.1953125, Y: -11.178401873711772}, Point{X: 22.148437499999996, Y: -33.137551192346145}}, 43 | Line{Point{X: 25.48828125, Y: -18.312810846425432}, Point{X: 33.22265625, Y: -16.720385051693988}, Point{X: 34.013671875, Y: -21.207458730482642}, Point{X: 23.466796875, Y: -24.766784522874428}}, 44 | Line{Point{X: 27.5537109375, Y: -12.618897304044012}, Point{X: 29.02587890625, Y: -12.146745814539685}, Point{X: 29.377441406249996, Y: -14.604847155053898}, Point{X: 26.3671875, Y: -15.855673509998681}}, 45 | Line{Point{X: 27.0703125, Y: -20.3034175184893}, Point{X: 27.509765625, Y: -21.616579336740593}, Point{X: 31.113281249999996, Y: -19.559790136497398}}}} 46 | 47 | poly := g.MustPolygon() 48 | var inOrder []bool 49 | for _, ring := range poly { 50 | inOrder = append(inOrder, ring.Clockwise()) 51 | } 52 | assert.Equal(t, []bool{true, false, false, false, true}, inOrder) // wild order 53 | 54 | poly.FixWinding() 55 | 56 | var outOrder []bool 57 | for _, ring := range poly { 58 | outOrder = append(outOrder, ring.Clockwise()) 59 | } 60 | assert.Equal(t, []bool{true, false, true, true, false}, outOrder) // correct order 61 | } 62 | 63 | func BenchmarkClipToBBox(b *testing.B) { 64 | f, err := os.Open("testfiles/polygon_with_holes.geojson") 65 | assert.Nil(b, err) 66 | 67 | var fc FeatureCollection 68 | err = json.NewDecoder(f).Decode(&fc) 69 | assert.Nil(b, err) 70 | assert.Equal(b, 1, len(fc.Features)) 71 | 72 | poly, err := fc.Features[0].Geometry.Polygon() 73 | assert.Nil(b, err) 74 | bbox := BBox{SW: Point{27.377929, 60.930432}, NE: Point{29.53125, 62.754725}} 75 | 76 | b.ReportAllocs() 77 | b.ResetTimer() 78 | 79 | for i := 0; i < b.N; i++ { 80 | poly.ClipToBBox(bbox) 81 | } 82 | } 83 | 84 | func BenchmarkStringRepr(b *testing.B) { 85 | p := Polygon{ 86 | Line{ 87 | Point{1, 2}, Point{3, 4}, 88 | }, 89 | Line{ 90 | Point{1, 2}, Point{3, 4}, 91 | }, 92 | Line{ 93 | Point{1, 2}, Point{3, 4}, 94 | }, 95 | } // this is probably not valid, but this is not important for that benchmark 96 | 97 | b.ReportAllocs() 98 | for i := 0; i < b.N; i++ { 99 | p.String() 100 | } 101 | } 102 | 103 | func BenchmarkFixWinding(b *testing.B) { 104 | b.ReportAllocs() 105 | for i := 0; i < b.N; i++ { 106 | Polygon{ 107 | Line{Point{X: -2.109375, Y: 11.178401873711785}, Point{X: -16.875, Y: -43.06888777416961}, Point{X: 62.57812500000001, Y: -43.580390855607845}, Point{X: 81.5625, Y: 8.407168163601076}}, 108 | Line{Point{X: 7.3828125, Y: -23.241346102386135}, Point{X: 28.4765625, Y: -8.05922962720018}, Point{X: 55.1953125, Y: -11.178401873711772}, Point{X: 22.148437499999996, Y: -33.137551192346145}}, 109 | Line{Point{X: 25.48828125, Y: -18.312810846425432}, Point{X: 33.22265625, Y: -16.720385051693988}, Point{X: 34.013671875, Y: -21.207458730482642}, Point{X: 23.466796875, Y: -24.766784522874428}}, 110 | Line{Point{X: 27.5537109375, Y: -12.618897304044012}, Point{X: 29.02587890625, Y: -12.146745814539685}, Point{X: 29.377441406249996, Y: -14.604847155053898}, Point{X: 26.3671875, Y: -15.855673509998681}}, 111 | Line{Point{X: 27.0703125, Y: -20.3034175184893}, Point{X: 27.509765625, Y: -21.616579336740593}, Point{X: 31.113281249999996, Y: -19.559790136497398}}}.FixWinding() 112 | } 113 | } 114 | 115 | func TestPolygonValidTopology(t *testing.T) { 116 | p := Polygon{Line{{3, 4}, {2, 9}, {1, 4}}} 117 | assert.True(t, p.ValidTopology()) 118 | 119 | p = Polygon{Line{{3, 4}, {2, 9}, {1, 4}, {1, 5}}} 120 | assert.False(t, p.ValidTopology()) 121 | } 122 | 123 | func TestPolygonClipBBoxShortCircuit(t *testing.T) { 124 | t.Run("completely inside bbox", func(t *testing.T) { 125 | p := Polygon{Line{{1, 1}, {2, 1}, {2, 2}, {1, 2}}} 126 | bbox := BBox{SW: Point{0, 0}, NE: Point{3, 3}} 127 | 128 | assert.Equal(t, 129 | []Geom{MustNewGeom(Polygon{Line{ 130 | {1, 1}, {2, 1}, {2, 2}, {1, 2}, 131 | }})}, 132 | p.ClipToBBox(bbox), 133 | ) 134 | }) 135 | 136 | t.Run("fit to bbox", func(t *testing.T) { 137 | p := Polygon{Line{{0, 0}, {3, 0}, {3, 3}, {0, 3}}} 138 | bbox := BBox{SW: Point{1, 1}, NE: Point{2, 2}} 139 | 140 | assert.Equal(t, 141 | []Geom{MustNewGeom(Polygon{Line{ 142 | {1, 1}, {2, 1}, {2, 2}, {1, 2}, 143 | }})}, 144 | p.ClipToBBox(bbox), 145 | ) 146 | }) 147 | 148 | t.Run("no speedup", func(t *testing.T) { 149 | p := Polygon{Line{{0, 0}, {3, 0}, {0, 3}}} 150 | bbox := BBox{SW: Point{1, 1}, NE: Point{2, 2}} 151 | 152 | assert.Equal(t, 153 | []Geom{MustNewGeom(Polygon{Line{ 154 | {1, 1}, {1, 2}, {2, 1}, 155 | }})}, 156 | p.ClipToBBox(bbox), 157 | ) 158 | }) 159 | } 160 | -------------------------------------------------------------------------------- /lib/spatial/projectable.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | type ConvertFunc func(Point) Point 4 | 5 | type Projectable interface { 6 | Project(ConvertFunc) 7 | Copy() Projectable 8 | } 9 | -------------------------------------------------------------------------------- /lib/spatial/spatial.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import ( 4 | "encoding/json" 5 | "math" 6 | 7 | "github.com/dhconnelly/rtreego" 8 | ) 9 | 10 | type PropertyRetriever interface { 11 | Properties() map[string]interface{} 12 | } 13 | 14 | type Filterable interface { 15 | Filter(BBox) []Feature 16 | } 17 | 18 | // Feature is a data structure which holds geometry and tags/properties of a geographical feature. 19 | type Feature struct { 20 | Props map[string]interface{} `json:"properties"` 21 | Geometry Geom 22 | } 23 | 24 | func NewFeature() Feature { 25 | return Feature{ 26 | Props: map[string]interface{}{}, 27 | } 28 | } 29 | 30 | func (f *Feature) Properties() map[string]interface{} { 31 | return f.Props 32 | } 33 | 34 | func (f *Feature) MarshalWKB() ([]byte, error) { 35 | return f.Geometry.MarshalWKB() 36 | } 37 | 38 | func (f Feature) MarshalJSON() ([]byte, error) { 39 | tfc := struct { 40 | Type string `json:"type"` 41 | Props map[string]interface{} `json:"properties"` 42 | Geometry Geom `json:"geometry"` 43 | }{ 44 | Type: "Feature", 45 | Props: f.Props, 46 | Geometry: f.Geometry, 47 | } 48 | return json.Marshal(tfc) 49 | } 50 | 51 | func bboxToRect(bbox BBox) *rtreego.Rect { 52 | dist := []float64{bbox.NE.X - bbox.SW.X, bbox.NE.Y - bbox.SW.Y} 53 | // rtreego doesn't allow zero sized bboxes 54 | if dist[0] == 0 { 55 | dist[0] = math.SmallestNonzeroFloat64 56 | } 57 | if dist[1] == 0 { 58 | dist[1] = math.SmallestNonzeroFloat64 59 | } 60 | r, err := rtreego.NewRect(rtreego.Point{bbox.SW.X, bbox.SW.Y}, dist) 61 | if err != nil { 62 | panic(err) 63 | } 64 | return r 65 | } 66 | 67 | type FeatureCollection struct { 68 | Features []Feature `json:"features"` 69 | SRID string `json:"-"` 70 | } // TODO: consider adding properties field 71 | 72 | func NewFeatureCollection() *FeatureCollection { 73 | return &FeatureCollection{Features: []Feature{}} 74 | } 75 | 76 | // Deprecated. Please use geojson.Codec. 77 | func (fc FeatureCollection) MarshalJSON() ([]byte, error) { 78 | wfc := struct { 79 | Type string `json:"type"` 80 | Features []Feature `json:"features"` 81 | }{ 82 | Type: "FeatureCollection", 83 | Features: fc.Features, 84 | } 85 | return json.Marshal(wfc) 86 | } 87 | 88 | func (fc *FeatureCollection) Filter(bbox BBox) []Feature { 89 | var filtered []Feature 90 | 91 | for _, feat := range fc.Features { 92 | if feat.Geometry.Overlaps(bbox) { 93 | filtered = append(filtered, feat) 94 | } 95 | } 96 | return filtered 97 | } 98 | 99 | // Reset removes all features. 100 | func (fc *FeatureCollection) Reset() { 101 | fc.Features = []Feature{} 102 | } 103 | 104 | type rtreeFeat Feature 105 | 106 | func (ft rtreeFeat) Bounds() *rtreego.Rect { 107 | return bboxToRect(ft.Geometry.BBox()) 108 | } 109 | 110 | // RTreeCollection is a FeatureCollection which is backed by a rtree. 111 | type RTreeCollection struct { 112 | rt *rtreego.Rtree 113 | } 114 | 115 | func NewRTreeCollection(features ...Feature) *RTreeCollection { 116 | var fts []rtreego.Spatial 117 | for _, ft := range features { 118 | fts = append(fts, rtreeFeat(ft)) 119 | } 120 | 121 | return &RTreeCollection{ 122 | // TODO: find out optimal branching factor 123 | rt: rtreego.NewTree(2, 32, 64, fts...), 124 | } 125 | } 126 | 127 | func (rt *RTreeCollection) Add(feature Feature) { 128 | rt.rt.Insert(rtreeFeat(feature)) 129 | } 130 | 131 | func (rt *RTreeCollection) Filter(bbox BBox) []Feature { 132 | var fts []Feature 133 | for _, ft := range rt.rt.SearchIntersect(bboxToRect(bbox)) { 134 | fts = append(fts, Feature(ft.(rtreeFeat))) 135 | } 136 | return fts 137 | } 138 | -------------------------------------------------------------------------------- /lib/spatial/string_1_10.go: -------------------------------------------------------------------------------- 1 | // +build go1.10 2 | 3 | package spatial 4 | 5 | import ( 6 | "fmt" 7 | "strings" 8 | ) 9 | 10 | func (p Polygon) string() string { 11 | var b strings.Builder 12 | b.WriteByte('(') 13 | for pos, line := range p { 14 | fmt.Fprintf(&b, "%v", line) 15 | if pos != len(p)-1 { 16 | b.WriteString(", ") 17 | } 18 | } 19 | b.WriteByte(')') 20 | return b.String() 21 | } 22 | 23 | func (l Line) string() string { 24 | var b strings.Builder 25 | for pos, point := range l { 26 | fmt.Fprintf(&b, "%v", point) 27 | if pos != len(l)-1 { 28 | b.WriteString(", ") 29 | } 30 | } 31 | return b.String() 32 | } 33 | -------------------------------------------------------------------------------- /lib/spatial/string_1_9.go: -------------------------------------------------------------------------------- 1 | // +build !go1.10 2 | 3 | package spatial 4 | 5 | import "fmt" 6 | 7 | func (p Polygon) string() string { 8 | s := "(" 9 | for _, line := range p { 10 | s += fmt.Sprintf("%v, ", line) 11 | } 12 | return s[:len(s)-2] + ")" 13 | } 14 | 15 | func (l Line) string() string { 16 | s := "" 17 | for _, point := range l { 18 | s += fmt.Sprintf("%v, ", point) 19 | } 20 | return s[:len(s)-2] 21 | } 22 | -------------------------------------------------------------------------------- /lib/spatial/testfiles/featurecollection.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": {}, 7 | "geometry": { 8 | "type": "Point", 9 | "coordinates": [ 10 | 8.7890625, 11 | 50.93073802371819 12 | ] 13 | } 14 | }, 15 | { 16 | "type": "Feature", 17 | "properties": {}, 18 | "geometry": { 19 | "type": "LineString", 20 | "coordinates": [ 21 | [ 22 | 9.052734375, 23 | 52.908902047770255 24 | ], 25 | [ 26 | 11.953125, 27 | 53.225768435790194 28 | ], 29 | [ 30 | 11.9091796875, 31 | 49.866316729538674 32 | ], 33 | [ 34 | 9.31640625, 35 | 49.75287993415023 36 | ] 37 | ] 38 | } 39 | }, 40 | { 41 | "type": "Feature", 42 | "properties": {}, 43 | "geometry": { 44 | "type": "Polygon", 45 | "coordinates": [ 46 | [ 47 | [ 48 | 3.69140625, 49 | 48.45835188280866 50 | ], 51 | [ 52 | 3.69140625, 53 | 48.45835188280866 54 | ], 55 | [ 56 | 3.69140625, 57 | 48.45835188280866 58 | ], 59 | [ 60 | 3.69140625, 61 | 48.45835188280866 62 | ] 63 | ] 64 | ] 65 | } 66 | }, 67 | { 68 | "type": "Feature", 69 | "properties": {}, 70 | "geometry": { 71 | "type": "Polygon", 72 | "coordinates": [ 73 | [ 74 | [ 75 | 0.7470703125, 76 | 45.42929873257377 77 | ], 78 | [ 79 | 2.28515625, 80 | 42.779275360241904 81 | ], 82 | [ 83 | 17.666015625, 84 | 39.50404070558415 85 | ], 86 | [ 87 | 17.885742187499996, 88 | 49.95121990866204 89 | ], 90 | [ 91 | 14.1943359375, 92 | 57.11238500793401 93 | ], 94 | [ 95 | 4.1748046875, 96 | 53.51418452077113 97 | ], 98 | [ 99 | 0.7470703125, 100 | 45.42929873257377 101 | ] 102 | ] 103 | ] 104 | } 105 | } 106 | ] 107 | } 108 | -------------------------------------------------------------------------------- /lib/spatial/testfiles/mergable_lines.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": {}, 7 | "geometry": { 8 | "type": "LineString", 9 | "coordinates": [ 10 | [ 11 | 10.513916015625, 12 | 48.436489955944154 13 | ], 14 | [ 15 | 10.821533203125, 16 | 48.71271258145237 17 | ], 18 | [ 19 | 10.667724609375, 20 | 48.8936153614802 21 | ], 22 | [ 23 | 10.733642578125, 24 | 49.21759710517596 25 | ], 26 | [ 27 | 10.535888671875, 28 | 49.532339195028115 29 | ], 30 | [ 31 | 10.623779296875, 32 | 49.89463439573421 33 | ], 34 | [ 35 | 10.65673828125, 36 | 50.04302974380058 37 | ] 38 | ] 39 | } 40 | }, 41 | { 42 | "type": "Feature", 43 | "properties": {}, 44 | "geometry": { 45 | "type": "LineString", 46 | "coordinates": [ 47 | [ 48 | 10.65673828125, 49 | 50.04302974380058 50 | ], 51 | [ 52 | 10.70068359375, 53 | 50.54136296522161 54 | ], 55 | [ 56 | 10.272216796875, 57 | 51.04830113331224 58 | ], 59 | [ 60 | 11.00830078125, 61 | 51.549751017014195 62 | ], 63 | [ 64 | 10.810546875, 65 | 51.896833883012484 66 | ], 67 | [ 68 | 10.799560546875, 69 | 52.20760667286523 70 | ], 71 | [ 72 | 10.799560546875, 73 | 52.221069523572794 74 | ] 75 | ] 76 | } 77 | }, 78 | { 79 | "type": "Feature", 80 | "properties": {}, 81 | "geometry": { 82 | "type": "LineString", 83 | "coordinates": [ 84 | [ 85 | 10.799560546875, 86 | 52.221069523572794 87 | ], 88 | [ 89 | 10.590820312499998, 90 | 52.395715477302076 91 | ], 92 | [ 93 | 10.2392578125, 94 | 52.619725272670266 95 | ], 96 | [ 97 | 10.458984375, 98 | 52.872445481488825 99 | ], 100 | [ 101 | 9.898681640625, 102 | 53.03130376554964 103 | ], 104 | [ 105 | 10.0140380859375, 106 | 53.19616119954287 107 | ], 108 | [ 109 | 9.1900634765625, 110 | 53.20274235350723 111 | ] 112 | ] 113 | } 114 | } 115 | ] 116 | } 117 | -------------------------------------------------------------------------------- /lib/spatial/testfiles/polygon.json: -------------------------------------------------------------------------------- 1 | {"type":"Polygon","coordinates":[[[0.8349609375,45.78284835197676],[4.350585937499999,45.78284835197676],[4.350585937499999,49.023461463214126],[0.8349609375,49.023461463214126],[0.8349609375,45.78284835197676]]]} 2 | -------------------------------------------------------------------------------- /lib/spatial/testfiles/polygon.wkb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomersch/grandine/85aa7e7bb2a5b7a6f7085822a984906eef67c2b8/lib/spatial/testfiles/polygon.wkb -------------------------------------------------------------------------------- /lib/spatial/testfiles/winding_wild.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": {}, 7 | "geometry": { 8 | "type": "Polygon", 9 | "coordinates": [ 10 | [ 11 | [ 12 | -2.109375, 13 | 11.178401873711785 14 | ], 15 | [ 16 | -16.875, 17 | -43.06888777416961 18 | ], 19 | [ 20 | 62.57812500000001, 21 | -43.580390855607845 22 | ], 23 | [ 24 | 81.5625, 25 | 8.407168163601076 26 | ], 27 | [ 28 | -2.109375, 29 | 11.178401873711785 30 | ] 31 | ], 32 | [ 33 | [ 34 | 7.3828125, 35 | -23.241346102386135 36 | ], 37 | [ 38 | 28.4765625, 39 | -8.05922962720018 40 | ], 41 | [ 42 | 55.1953125, 43 | -11.178401873711772 44 | ], 45 | [ 46 | 22.148437499999996, 47 | -33.137551192346145 48 | ], 49 | [ 50 | 7.3828125, 51 | -23.241346102386135 52 | ] 53 | ], 54 | [ 55 | [ 56 | 25.48828125, 57 | -18.312810846425432 58 | ], 59 | [ 60 | 33.22265625, 61 | -16.720385051693988 62 | ], 63 | [ 64 | 34.013671875, 65 | -21.207458730482642 66 | ], 67 | [ 68 | 23.466796875, 69 | -24.766784522874428 70 | ], 71 | [ 72 | 25.48828125, 73 | -18.312810846425432 74 | ] 75 | ], 76 | [ 77 | [ 78 | 27.5537109375, 79 | -12.618897304044012 80 | ], 81 | [ 82 | 29.02587890625, 83 | -12.146745814539685 84 | ], 85 | [ 86 | 29.377441406249996, 87 | -14.604847155053898 88 | ], 89 | [ 90 | 26.3671875, 91 | -15.855673509998681 92 | ], 93 | [ 94 | 27.5537109375, 95 | -12.618897304044012 96 | ] 97 | ], 98 | [ 99 | [ 100 | 27.0703125, 101 | -20.3034175184893 102 | ], 103 | [ 104 | 27.509765625, 105 | -21.616579336740593 106 | ], 107 | [ 108 | 31.113281249999996, 109 | -19.559790136497398 110 | ], 111 | [ 112 | 27.0703125, 113 | -20.3034175184893 114 | ] 115 | ] 116 | ] 117 | } 118 | } 119 | ] 120 | } 121 | -------------------------------------------------------------------------------- /lib/spatial/topology.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | type Validatable interface { 4 | ValidTopology() bool 5 | } 6 | -------------------------------------------------------------------------------- /lib/spatial/twkb.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import ( 4 | "encoding/binary" 5 | "io" 6 | "math" 7 | ) 8 | 9 | func twkbWriteHeader(w io.Writer, gt GeomType, precision int) error { 10 | var buf = make([]byte, 2) 11 | buf[0] = byte(precision<<4) ^ byte(gt) 12 | _, err := w.Write(buf) 13 | return err 14 | } 15 | 16 | func twkbWritePoint(w io.Writer, p Point, previous Point, precision int) error { 17 | var ( 18 | xi = int(p.X * math.Pow10(precision)) 19 | yi = int(p.Y * math.Pow10(precision)) 20 | xpi = int(previous.X * math.Pow10(precision)) 21 | ypi = int(previous.Y * math.Pow10(precision)) 22 | 23 | buf = make([]byte, 20) // up to 10 bytes per varint 24 | ) 25 | 26 | dx := int64(xi - xpi) 27 | dy := int64(yi - ypi) 28 | bwx := binary.PutVarint(buf, dx) 29 | bwy := binary.PutVarint(buf[bwx:], dy) 30 | 31 | _, err := w.Write(buf[:bwx+bwy]) 32 | return err 33 | } 34 | 35 | type combinedReader interface { 36 | io.Reader 37 | io.ByteReader 38 | } 39 | 40 | type wrappedReader struct { 41 | io.Reader 42 | } 43 | 44 | func (wr *wrappedReader) ReadByte() (c byte, err error) { 45 | var b = make([]byte, 1) 46 | n, err := wr.Read(b) 47 | if err != nil { 48 | return b[0], err 49 | } 50 | if n != 1 { 51 | return b[0], io.EOF 52 | } 53 | return b[0], nil 54 | } 55 | 56 | type twkbHeader struct { 57 | typ GeomType 58 | precision int 59 | // metadata attributes 60 | bbox, size, idList, extendedPrecision, emptyGeom bool 61 | } 62 | 63 | func unzigzag(nVal int) int { 64 | if (nVal & 1) == 0 { 65 | return nVal >> 1 66 | } 67 | return -(nVal >> 1) - 1 68 | } 69 | 70 | func twkbReadHeader(r io.Reader) (twkbHeader, error) { 71 | var ( 72 | // BIT USAGE 73 | // 1-4 type 74 | // 5-8 precision 75 | // 9 bbox 76 | // 10 size attribute 77 | // 11 id list 78 | // 12 extended precision 79 | // 13 empty geom 80 | // 14-16 unused 81 | by = make([]byte, 2) 82 | hd twkbHeader 83 | ) 84 | _, err := r.Read(by) 85 | hd.typ = GeomType(by[0] & 15) 86 | hd.precision = int(by[0] >> 4) 87 | hd.bbox = int(by[1])&1 == 1 88 | hd.size = int(by[1])&2 == 2 89 | hd.idList = int(by[1])&4 == 4 90 | hd.extendedPrecision = int(by[1])&8 == 8 91 | hd.emptyGeom = int(by[1])&16 == 16 92 | return hd, err 93 | } 94 | 95 | func twkbReadPoint(r io.Reader, previous Point, precision int) (Point, error) { 96 | wr, ok := r.(io.ByteReader) 97 | if !ok { 98 | wr = &wrappedReader{r} 99 | } 100 | var pt Point 101 | xe, err := binary.ReadVarint(wr) 102 | if err != nil { 103 | return pt, err 104 | } 105 | ye, err := binary.ReadVarint(wr) 106 | if err != nil { 107 | return pt, err 108 | } 109 | xΔ := float64(xe) / math.Pow10(precision) 110 | yΔ := float64(ye) / math.Pow10(precision) 111 | 112 | pt.X = xΔ + previous.X 113 | pt.Y = yΔ + previous.Y 114 | return pt, nil 115 | } 116 | 117 | func twkbReadLineString(r io.Reader, precision int) ([]Point, error) { 118 | wr, ok := r.(combinedReader) 119 | if !ok { 120 | wr = &wrappedReader{r} 121 | } 122 | npoints, err := binary.ReadUvarint(wr) 123 | if err != nil { 124 | return nil, err 125 | } 126 | 127 | var ( 128 | ls = make([]Point, npoints) 129 | lastPt Point 130 | ) 131 | for i := 0; i < int(npoints); i++ { 132 | lastPt, err = twkbReadPoint(wr, lastPt, precision) 133 | if err != nil { 134 | return ls, err 135 | } 136 | ls[i] = lastPt 137 | } 138 | return ls, nil 139 | } 140 | 141 | func twkbWriteLineString(w io.Writer, ls []Point, precision int) error { 142 | buf := make([]byte, 10) 143 | bWritten := binary.PutUvarint(buf, uint64(len(ls))) 144 | _, err := w.Write(buf[:bWritten-1]) 145 | if err != nil { 146 | return err 147 | } 148 | var previous Point 149 | for _, pt := range ls { 150 | if err = twkbWritePoint(w, pt, previous, precision); err != nil { 151 | return err 152 | } 153 | } 154 | return nil 155 | } 156 | -------------------------------------------------------------------------------- /lib/spatial/twkb_test.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import ( 4 | "bytes" 5 | "encoding/hex" 6 | "fmt" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestTWKBReadHeader(t *testing.T) { 13 | buf, err := hex.DecodeString("24FF") 14 | assert.Nil(t, err) 15 | r := bytes.NewBuffer(buf) 16 | hd, err := twkbReadHeader(r) 17 | assert.Nil(t, err) 18 | assert.True(t, hd.bbox) 19 | } 20 | 21 | func TestTWKBWriteHeader(t *testing.T) { 22 | w := &bytes.Buffer{} 23 | typ := GeomTypeLineString 24 | precision := 4 25 | twkbWriteHeader(w, typ, precision) 26 | 27 | hd, err := twkbReadHeader(w) 28 | assert.Nil(t, err) 29 | assert.Equal(t, twkbHeader{typ: typ, precision: precision}, hd) 30 | } 31 | 32 | func TestTWKBWritePoint(t *testing.T) { 33 | precision := 6 34 | origPt := Point{-212, 12.3} 35 | buf := bytes.Buffer{} 36 | err := twkbWritePoint(&buf, origPt, Point{}, precision) 37 | assert.Nil(t, err) 38 | 39 | pt, err := twkbReadPoint(&buf, Point{}, precision) 40 | assert.Nil(t, err) 41 | assert.Equal(t, origPt, pt) 42 | } 43 | 44 | func TestTWKBReadPoint(t *testing.T) { 45 | buf, err := hex.DecodeString("01000204") 46 | assert.Nil(t, err) 47 | r := bytes.NewBuffer(buf) 48 | 49 | hd, err := twkbReadHeader(r) 50 | assert.Nil(t, err) 51 | pt, err := twkbReadPoint(r, Point{}, hd.precision) 52 | assert.Nil(t, err) 53 | assert.Equal(t, Point{1, 2}, pt) 54 | } 55 | 56 | func TestTWKBReadLine(t *testing.T) { 57 | buf, err := hex.DecodeString("02000202020808") 58 | assert.Nil(t, err) 59 | r := bytes.NewBuffer(buf) 60 | 61 | hd, err := twkbReadHeader(r) 62 | assert.Nil(t, err) 63 | ls, err := twkbReadLineString(r, hd.precision) 64 | assert.Nil(t, err) 65 | assert.Equal(t, []Point{{1, 1}, {5, 5}}, ls) 66 | } 67 | 68 | func BenchmarkTWKBWriteRawPoint(b *testing.B) { 69 | precision := 2 70 | p := Point{2, 3} 71 | buf := bytes.Buffer{} 72 | b.ReportAllocs() 73 | b.ResetTimer() 74 | 75 | for i := 0; i < b.N; i++ { 76 | err := twkbWritePoint(&buf, p, Point{}, precision) 77 | assert.Nil(b, err) 78 | } 79 | } 80 | 81 | func BenchmarkTWKBReadRawPoint(b *testing.B) { 82 | var rawPt []byte 83 | _, err := fmt.Sscanf("fff396ca01c0bbdd0b", "%x", &rawPt) 84 | assert.Nil(b, err) 85 | r := bytes.NewReader(rawPt) 86 | 87 | b.ReportAllocs() 88 | b.ResetTimer() 89 | 90 | for i := 0; i < b.N; i++ { 91 | r.Reset(rawPt) 92 | twkbReadPoint(r, Point{}, 0) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /lib/spatial/wkb.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "math" 8 | ) 9 | 10 | type WKBable interface { 11 | MarshalWKB() ([]byte, error) 12 | } 13 | 14 | type WKBableWithProps interface { 15 | WKBable 16 | PropertyRetriever 17 | } 18 | 19 | const ( 20 | wkbRawPointSize = 16 21 | wkbLittleEndian = 1 22 | ) 23 | 24 | func GeomFromWKB(r io.Reader) (Geom, error) { 25 | var ( 26 | g Geom 27 | wkbEndianness = make([]byte, 1) 28 | ) 29 | _, err := r.Read(wkbEndianness) 30 | if err != nil { 31 | return g, err 32 | } 33 | if wkbEndianness[0] != wkbLittleEndian { 34 | return g, errors.New("only little endian is supported") 35 | } 36 | 37 | g.typ, err = wkbReadHeader(r) 38 | if err != nil { 39 | return g, err 40 | } 41 | switch g.typ { 42 | case GeomTypePoint: 43 | var pt Point 44 | pt, err = wkbReadPoint(r) 45 | g.g = &pt 46 | case GeomTypeLineString: 47 | g.g, err = wkbReadLineString(r) 48 | case GeomTypePolygon: 49 | g.g, err = wkbReadPolygon(r) 50 | default: 51 | return g, fmt.Errorf("unsupported GeomType: %v", g.typ) 52 | } 53 | return g, err 54 | } 55 | 56 | func wkbReadHeader(r io.Reader) (GeomType, error) { 57 | var buf = make([]byte, 4) 58 | _, err := r.Read(buf) 59 | gt := endianness.Uint32(buf) 60 | return GeomType(gt), err 61 | } 62 | 63 | func wkbWritePoint(w io.Writer, p Point) error { 64 | var ( 65 | err error 66 | buf = make([]byte, wkbRawPointSize) 67 | ) 68 | endianness.PutUint64(buf[:8], math.Float64bits(p.X)) 69 | endianness.PutUint64(buf[8:16], math.Float64bits(p.Y)) 70 | _, err = w.Write(buf) 71 | if err != nil { 72 | return err 73 | } 74 | return nil 75 | } 76 | 77 | func wkbWriteLineString(w io.Writer, ls []Point) error { 78 | // write number of points 79 | var ln = make([]byte, 4) 80 | endianness.PutUint32(ln, uint32(len(ls))) 81 | _, err := w.Write(ln) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | for _, pt := range ls { 87 | err = wkbWritePoint(w, pt) 88 | if err != nil { 89 | return err 90 | } 91 | } 92 | return nil 93 | } 94 | 95 | func wkbWritePolygon(w io.Writer, poly Polygon) error { 96 | // write number of rings 97 | var lnr = make([]byte, 4) 98 | endianness.PutUint32(lnr, uint32(len(poly))) 99 | _, err := w.Write(lnr) 100 | if err != nil { 101 | return err 102 | } 103 | 104 | for _, ring := range poly { 105 | err = wkbWriteLineString(w, append(ring, ring[0])) // wkb closes rings with the first element, the internal implementation doesn't 106 | if err != nil { 107 | return err 108 | } 109 | } 110 | return nil 111 | } 112 | 113 | // TODO: evaluate returning Geom instead of Point 114 | func wkbReadPoint(r io.Reader) (p Point, err error) { 115 | var buf = make([]byte, wkbRawPointSize) 116 | n, err := r.Read(buf) 117 | if n != wkbRawPointSize { 118 | return p, io.EOF 119 | } 120 | if err != nil { 121 | return 122 | } 123 | p.X = math.Float64frombits(endianness.Uint64(buf[:8])) 124 | p.Y = math.Float64frombits(endianness.Uint64(buf[8:16])) 125 | return 126 | } 127 | 128 | // TODO: evaluate returning Geom instead of Point 129 | func wkbReadLineString(r io.Reader) (Line, error) { 130 | var buf = make([]byte, 4) 131 | _, err := r.Read(buf) 132 | if err != nil { 133 | return nil, err 134 | } 135 | nop := endianness.Uint32(buf) 136 | if nop == 0 { 137 | return nil, errors.New("a linestring needs to have at least one point") 138 | } 139 | 140 | var ls = make(Line, nop) 141 | for i := 0; i < int(nop); i++ { 142 | ls[i], err = wkbReadPoint(r) 143 | if err != nil { 144 | return ls, err 145 | } 146 | } 147 | return ls, nil 148 | } 149 | 150 | func wkbReadPolygon(r io.Reader) (Polygon, error) { 151 | var buf = make([]byte, 4) 152 | _, err := r.Read(buf) 153 | if err != nil { 154 | return nil, err 155 | } 156 | nor := endianness.Uint32(buf) 157 | if nor == 0 { 158 | return nil, errors.New("a polygon needs to have at least one ring") 159 | } 160 | 161 | var rings = make(Polygon, nor) 162 | for i := 0; i < int(nor); i++ { 163 | rings[i], err = wkbReadLineString(r) 164 | if err != nil { 165 | return rings, err 166 | } 167 | rings[i] = rings[i][:len(rings[i])-1] // wkb closes rings with the first element, the internal implementation doesn't 168 | } 169 | return rings, err 170 | } 171 | -------------------------------------------------------------------------------- /lib/spatial/wkb_test.go: -------------------------------------------------------------------------------- 1 | package spatial 2 | 3 | import ( 4 | "bytes" 5 | "encoding/hex" 6 | "os" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestDecodeWKBNullLineString(t *testing.T) { 13 | b, _ := hex.DecodeString("010300000030303000000000003030303030303030") 14 | buf := bytes.NewBuffer(b) 15 | 16 | var g Geom 17 | err := g.UnmarshalWKB(buf) 18 | assert.NotNil(t, err) 19 | } 20 | 21 | func TestGeomFromWKB(t *testing.T) { 22 | f, err := os.Open("testfiles/polygon.wkb") 23 | assert.Nil(t, err) 24 | defer f.Close() 25 | 26 | g, err := GeomFromWKB(f) 27 | assert.Nil(t, err) 28 | assert.Equal(t, g.Typ(), GeomTypePolygon) 29 | } 30 | 31 | func BenchmarkUnmarshalWKB(b *testing.B) { 32 | buf, _ := hex.DecodeString("03000000000000000000f03f00000000000000400000000000000840000000000000104000000000000014400000000000001040") 33 | 34 | b.Run("old style", func(b *testing.B) { 35 | b.ReportAllocs() 36 | for i := 0; i < b.N; i++ { 37 | var g Geom 38 | g.UnmarshalWKB(bytes.NewBuffer(buf)) 39 | } 40 | }) 41 | 42 | b.Run("new style", func(b *testing.B) { 43 | b.ReportAllocs() 44 | for i := 0; i < b.N; i++ { 45 | GeomFromWKB(bytes.NewBuffer(buf)) 46 | } 47 | }) 48 | } 49 | -------------------------------------------------------------------------------- /lib/tile/bbox.go: -------------------------------------------------------------------------------- 1 | package tile 2 | 3 | import "github.com/thomersch/grandine/lib/spatial" 4 | 5 | func Coverage(bb spatial.BBox, zoom int) []ID { 6 | // Tiles are counted from top-left to bottom-right 7 | tl := spatial.Point{bb.SW.X, bb.NE.Y} 8 | br := spatial.Point{bb.NE.X, bb.SW.Y} 9 | 10 | p1 := TileName(tl, zoom) 11 | p2 := TileName(br, zoom) 12 | 13 | var tiles []ID 14 | 15 | for x := p1.X; x <= p2.X; x++ { 16 | for y := p1.Y; y <= p2.Y; y++ { 17 | tiles = append(tiles, ID{X: x, Y: y, Z: zoom}) 18 | } 19 | } 20 | return tiles 21 | } 22 | -------------------------------------------------------------------------------- /lib/tile/bbox_test.go: -------------------------------------------------------------------------------- 1 | package tile 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/thomersch/grandine/lib/spatial" 7 | ) 8 | 9 | func TestCoverage(t *testing.T) { 10 | Coverage(spatial.BBox{spatial.Point{-5, -5}, spatial.Point{10, 10}}, 7) 11 | } 12 | -------------------------------------------------------------------------------- /lib/tile/codec.go: -------------------------------------------------------------------------------- 1 | package tile 2 | 3 | import ( 4 | "bytes" 5 | 6 | "github.com/thomersch/grandine/lib/geojson" 7 | "github.com/thomersch/grandine/lib/spatial" 8 | ) 9 | 10 | type Codec interface { 11 | EncodeTile(features map[string][]spatial.Feature, tid ID) ([]byte, error) 12 | Extension() string 13 | } 14 | 15 | type GeoJSONCodec struct{} 16 | 17 | func (g *GeoJSONCodec) EncodeTile(features map[string][]spatial.Feature, tid ID) ([]byte, error) { 18 | var buf bytes.Buffer 19 | var fts []spatial.Feature 20 | for _, ly := range features { 21 | for _, ft := range ly { 22 | fts = append(fts, ft) 23 | } 24 | } 25 | gjc := &geojson.Codec{} 26 | err := gjc.Encode(&buf, &spatial.FeatureCollection{Features: fts}) 27 | if err != nil { 28 | return nil, err 29 | } 30 | return buf.Bytes(), nil 31 | } 32 | 33 | func (g *GeoJSONCodec) Extension() string { 34 | return "geojson" 35 | } 36 | -------------------------------------------------------------------------------- /lib/tile/tile.go: -------------------------------------------------------------------------------- 1 | package tile 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | 7 | "github.com/thomersch/grandine/lib/spatial" 8 | ) 9 | 10 | const ( 11 | wgs84LatMax = 85.0511287 12 | wgs84LonMax = 180 13 | ) 14 | 15 | type ID struct { 16 | X, Y, Z int 17 | } 18 | 19 | func (t ID) BBox() spatial.BBox { 20 | nw := t.NW() 21 | se := ID{X: t.X + 1, Y: t.Y + 1, Z: t.Z}.NW() 22 | return spatial.BBox{SW: spatial.Point{nw.X, se.Y}, NE: spatial.Point{se.X, nw.Y}} 23 | } 24 | 25 | func (t ID) NW() spatial.Point { 26 | n := math.Pow(2, float64(t.Z)) 27 | lonDeg := float64(t.X)/n*360 - 180 28 | latRad := math.Atan(math.Sinh(math.Pi * (1 - 2*float64(t.Y)/n))) 29 | latDeg := latRad * 180 / math.Pi 30 | return spatial.Point{lonDeg, latDeg} 31 | } 32 | 33 | func (t ID) String() string { 34 | return fmt.Sprintf("%v/%v/%v", t.Z, t.X, t.Y) 35 | } 36 | 37 | func TileName(p spatial.Point, zoom int) ID { 38 | var ( 39 | zf = float64(zoom) 40 | latDeg = float64(floatBetween(-1*wgs84LatMax, wgs84LatMax, p.Y) * math.Pi / 180) 41 | ) 42 | return ID{ 43 | X: between(0, int(math.Exp2(zf)-1), 44 | int(math.Floor((float64(p.X)+180)/360*math.Exp2(zf)))), 45 | Y: between(0, int(math.Exp2(zf)-1), 46 | int(math.Floor((1-math.Log(math.Tan(latDeg)+1/math.Cos(latDeg))/math.Pi)/2*math.Exp2(zf)))), 47 | Z: zoom, 48 | } 49 | } 50 | 51 | // Resolution determines the minimal describable value inside a tile. 52 | func Resolution(zoomlevel, extent int) float64 { 53 | return 360 / (math.Pow(2, float64(zoomlevel)) * float64(extent)) 54 | } 55 | 56 | func between(min, max, v int) int { 57 | return int(math.Max(math.Min(float64(v), float64(max)), float64(min))) 58 | } 59 | 60 | func floatBetween(min, max, v float64) float64 { 61 | return math.Max(math.Min(v, max), min) 62 | } 63 | -------------------------------------------------------------------------------- /lib/tile/tile_test.go: -------------------------------------------------------------------------------- 1 | package tile 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/thomersch/grandine/lib/spatial" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestTileName(t *testing.T) { 13 | for _, tc := range []struct { 14 | p spatial.Point 15 | zl int 16 | expected ID 17 | }{ 18 | { 19 | p: spatial.Point{13.73630, 51.05377}, 20 | zl: 14, 21 | expected: ID{X: 8817, Y: 5481, Z: 14}, 22 | }, 23 | { 24 | p: spatial.Point{18.39856, -33.90184}, 25 | zl: 14, 26 | expected: ID{X: 9029, Y: 9833, Z: 14}, 27 | }, 28 | { 29 | p: spatial.Point{-54.59123, -25.59547}, 30 | zl: 14, 31 | expected: ID{X: 5707, Y: 9397, Z: 14}, 32 | }, 33 | { 34 | p: spatial.Point{-21.94073, 64.14607}, 35 | zl: 14, 36 | expected: ID{X: 7193, Y: 4354, Z: 14}, 37 | }, 38 | { 39 | p: spatial.Point{-31.16580, 83.65691}, 40 | zl: 14, 41 | expected: ID{X: 6773, Y: 648, Z: 14}, 42 | }, 43 | { 44 | p: spatial.Point{-64.45649, -85.04438}, 45 | zl: 14, 46 | expected: ID{X: 5258, Y: 16380, Z: 14}, 47 | }, 48 | { 49 | p: spatial.Point{180, -90}, 50 | zl: 1, 51 | expected: ID{X: 1, Y: 1, Z: 1}, 52 | }, 53 | { 54 | p: spatial.Point{-180, 90}, 55 | zl: 1, 56 | expected: ID{X: 0, Y: 0, Z: 1}, 57 | }, 58 | } { 59 | t.Run(fmt.Sprintf("%v_%v", tc.expected.X, tc.expected.Y), func(t *testing.T) { 60 | var fail bool 61 | ti := TileName(tc.p, tc.zl) 62 | if ti.X != tc.expected.X { 63 | fail = true 64 | } 65 | if ti.Y != tc.expected.Y { 66 | fail = true 67 | } 68 | if ti.Z != tc.expected.Z { 69 | fail = true 70 | } 71 | if fail { 72 | t.Fatalf("invalid conversion, expected %v, got %v", tc.expected, ti) 73 | } 74 | }) 75 | } 76 | } 77 | 78 | func TestTileBBox(t *testing.T) { 79 | for _, tc := range []struct { 80 | tid ID 81 | expected spatial.BBox 82 | }{ 83 | { 84 | tid: ID{0, 0, 0}, 85 | expected: spatial.BBox{spatial.Point{-180, -85.05112878}, spatial.Point{180, 85.05112878}}, 86 | }, 87 | { 88 | tid: ID{0, 0, 1}, 89 | expected: spatial.BBox{spatial.Point{-180, 0}, spatial.Point{0, 85.05112878}}, 90 | }, 91 | { 92 | tid: ID{0, 1, 1}, 93 | expected: spatial.BBox{spatial.Point{-180, -85.05112878}, spatial.Point{0, 0}}, 94 | }, 95 | { 96 | tid: ID{1, 2, 2}, 97 | expected: spatial.BBox{spatial.Point{-90, -66.51326044}, spatial.Point{0, 0}}, 98 | }, 99 | } { 100 | t.Run(fmt.Sprintf("%v_%v_%v", tc.tid.X, tc.tid.Y, tc.tid.Z), func(t *testing.T) { 101 | bbox := tc.tid.BBox() 102 | bbox.NE = bbox.NE.RoundedCoords() 103 | bbox.SW = bbox.SW.RoundedCoords() 104 | assert.Equal(t, tc.expected, bbox) 105 | }) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /viewer/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | tiles/* 3 | fonts/* 4 | -------------------------------------------------------------------------------- /viewer/README.md: -------------------------------------------------------------------------------- 1 | # Debug Vector Tile Viewer 2 | 3 | ## Usage 4 | 5 | npm install # installs dependencies 6 | npm run server # starts an HTTP server on localhost:8080 7 | 8 | Please note that the code assumes that tiles are placed in the `tiles/` directory. Consider placing a symlink. 9 | 10 | 11 | ### Generating Fonts 12 | 13 | You can build the fonts using [fontnik](https://github.com/mapbox/node-fontnik), which can be installed via `npm install fontnik`. The executable will be located in `node_modules/fontnik/bin/build-glyphs`. 14 | -------------------------------------------------------------------------------- /viewer/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Viewer 6 | 7 | 8 | 9 | 10 | 11 | 55 | 56 | 57 | 58 |
59 | 101 | 102 | 103 | -------------------------------------------------------------------------------- /viewer/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "http-server": "^0.10.0", 4 | "mapbox-gl": "^0.40.1", 5 | "mapbox-gl-inspect": "^1.2.4" 6 | }, 7 | "scripts": { 8 | "server": "node_modules/http-server/bin/http-server -a 127.0.0.1" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /viewer/style.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Points", 3 | "metadata": {}, 4 | "version": 8, 5 | "sources": { 6 | "mvt": { 7 | "type": "vector", 8 | "tiles": ["http://localhost:8080/tiles/{z}/{x}/{y}.mvt"], 9 | "maxzoom": 14 10 | } 11 | }, 12 | "glyphs": "http://localhost:8080/fonts/{fontstack}/{range}.pbf", 13 | "layers": [ 14 | { 15 | "id": "lines", 16 | "source": "mvt", 17 | "source-layer": "default", 18 | "type": "line", 19 | "paint": { 20 | "line-width": 2, 21 | "line-color": "grey" 22 | }, 23 | "filter": ["==", "$type", "LineString"] 24 | }, { 25 | "id": "transportation-primary", 26 | "source": "mvt", 27 | "source-layer": "transportation", 28 | "type": "line", 29 | "paint": { 30 | "line-width": 3, 31 | "line-color": "orange" 32 | }, 33 | "filter": ["==", "class", "primary"] 34 | }, { 35 | "id": "transportation-secondary", 36 | "source": "mvt", 37 | "source-layer": "transportation", 38 | "type": "line", 39 | "paint": { 40 | "line-width": 1, 41 | "line-color": "orange" 42 | }, 43 | "filter": ["==", "class", "secondary"] 44 | }, { 45 | "id": "transportation-tertiary", 46 | "source": "mvt", 47 | "source-layer": "transportation", 48 | "type": "line", 49 | "paint": { 50 | "line-width": 1, 51 | "line-color": "#CFCFCF" 52 | }, 53 | "filter": ["==", "class", "tertiary"] 54 | }, { 55 | "id": "poly", 56 | "source": "mvt", 57 | "source-layer": "default", 58 | "type": "fill", 59 | "paint": { 60 | "fill-color": "brown" 61 | }, 62 | "filter": ["==", "$type", "Polygon"] 63 | }, { 64 | "id": "pts", 65 | "source": "mvt", 66 | "source-layer": "default", 67 | "type": "circle", 68 | "paint": { 69 | "circle-radius": 1 70 | }, 71 | "filter": ["==", "$type", "Point"] 72 | }] 73 | } 74 | --------------------------------------------------------------------------------