├── .github └── workflows │ ├── golangci-lint.yml │ └── main.yml ├── .gitignore ├── .golangci.yml ├── CHANGELOG.md ├── LICENSE.md ├── README.md ├── annotate ├── change.go ├── change_test.go ├── datasource.go ├── edgecases_test.go ├── errors.go ├── errors_test.go ├── geo.go ├── geo_test.go ├── internal │ └── core │ │ ├── compute.go │ │ ├── compute_test.go │ │ ├── datasourcer_test.go │ │ ├── errors.go │ │ ├── types.go │ │ └── types_test.go ├── options.go ├── order.go ├── order_test.go ├── relation.go ├── relation_test.go ├── shared │ └── child.go ├── testdata │ ├── relation_2714790.osm │ ├── relation_2714790_expected.osm │ ├── relation_4017808.osm │ ├── relation_4017808_expected.osm │ ├── way_230391153.osm │ ├── way_230391153_expected.osm │ ├── way_6394949.osm │ └── way_6394949_expected.osm ├── way.go └── way_test.go ├── bounds.go ├── bounds_test.go ├── change.go ├── change_test.go ├── changeset.go ├── changeset_test.go ├── codecov.yml ├── datasource.go ├── datasource_test.go ├── diff.go ├── diff_test.go ├── element.go ├── element_test.go ├── feature.go ├── feature_test.go ├── go.mod ├── go.sum ├── internal └── mputil │ ├── join.go │ ├── join_test.go │ ├── mputil.go │ └── mputil_test.go ├── json.go ├── node.go ├── node_test.go ├── note.go ├── note_test.go ├── object.go ├── object_test.go ├── osm.go ├── osm_test.go ├── osmapi ├── README.md ├── changeset.go ├── changeset_test.go ├── datasource.go ├── datasource_test.go ├── live_test.go ├── map.go ├── map_test.go ├── node.go ├── node_test.go ├── note.go ├── note_test.go ├── options.go ├── relation.go ├── relation_test.go ├── user.go ├── user_test.go ├── way.go └── way_test.go ├── osmgeojson ├── README.md ├── benchmarks_test.go ├── build_polygon.go ├── build_polygon_test.go ├── convert.go ├── convert_test.go ├── options.go ├── options_test.go └── testdata │ └── benchmark.osm ├── osmpbf ├── README.md ├── decode.go ├── decode_data.go ├── decode_test.go ├── example_stats_test.go ├── internal │ └── osmpbf │ │ ├── README.md │ │ ├── fileformat.pb.go │ │ ├── fileformat.proto │ │ ├── generate.go │ │ ├── osmformat.pb.go │ │ └── osmformat.proto ├── scanner.go ├── scanner_test.go ├── zlib_cgo.go └── zlib_go.go ├── osmtest ├── scanner.go └── scanner_test.go ├── osmxml ├── example_test.go ├── scanner.go └── scanner_test.go ├── polygon.go ├── polygon_test.go ├── relation.go ├── relation_test.go ├── replication ├── README.md ├── changesets.go ├── changesets_test.go ├── datasource.go ├── interval.go ├── interval_test.go ├── live_test.go ├── search.go └── search_test.go ├── tag.go ├── tag_test.go ├── testdata ├── andorra-latest.osm.bz2 ├── annotate_diff.xml ├── annotated_diff.xml ├── changeset_38162206.osc ├── changeset_38162210.osc ├── compare_scanners.go ├── delaware-latest.osm.pbf ├── minute_871.osc ├── relation-updates.osm └── way-updates.osm ├── update.go ├── update_test.go ├── user.go ├── user_test.go ├── way.go └── way_test.go /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | 8 | permissions: 9 | contents: read 10 | 11 | jobs: 12 | golangci: 13 | name: lint 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v3 17 | - uses: actions/setup-go@v3 18 | with: 19 | go-version: '1.21' 20 | 21 | - name: golangci-lint 22 | uses: golangci/golangci-lint-action@v3 23 | with: 24 | version: v1.55 25 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | 8 | jobs: 9 | build-and-test: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | 14 | - name: Setup Go 15 | uses: actions/setup-go@v3 16 | with: 17 | go-version: '1.21' 18 | 19 | - name: Run build 20 | run: go build . 21 | 22 | - name: Run vet 23 | run: | 24 | go vet . 25 | 26 | - name: Run tests 27 | run: go test -v -coverprofile=profile.cov ./... 28 | 29 | - name: codecov 30 | uses: codecov/codecov-action@v1 31 | with: 32 | file: ./profile.cov 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.osm.bz2 2 | *.pbf 3 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | issues: 2 | exclude-rules: 3 | - path: '(.+)_test\.go' 4 | linters: 5 | - errcheck 6 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | ## [v0.8.0](https://github.com/paulmach/osm/compare/v0.7.1...v0.8.0) - 2024-01-08 6 | 7 | ### Changed 8 | 9 | - go 1.16 is required, updated usages of `ioutil` for similar functions in `io` and `os` 10 | 11 | ### Fixed 12 | 13 | - correctly JSON unmarshal elements with a type tag by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/53 14 | 15 | ## [v0.7.1](https://github.com/paulmach/osm/compare/v0.7.0...v0.7.1) - 2022-11-29 16 | 17 | ### Added 18 | 19 | - osm: add Tags.FindTag and Tags.HasTag methods by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/45 20 | 21 | ### Fixed 22 | 23 | - osm: support version as json number or string by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/46 24 | 25 | ## [v0.7.0](https://github.com/paulmach/osm/compare/v0.6.0...v0.7.0) - 2022-08-17 26 | 27 | ### Changed 28 | 29 | - remove node/ways/relations marshaling into this packages custom binary format by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/40 30 | 31 | ## [v0.6.0](https://github.com/paulmach/osm/compare/v0.5.0...v0.6.0) - 2022-08-16 32 | 33 | ### Added 34 | 35 | - json: ability to unmarshal osmjson by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/39 36 | - json: add support for external json implementations by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/39 37 | 38 | ## [v0.5.0](https://github.com/paulmach/osm/compare/v0.4.0...v0.5.0) - 2022-06-07 39 | 40 | ### Added 41 | 42 | - replication: ability to get changeset state by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/37 43 | - replication: search for state/sequence number by timestamp by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/38 44 | 45 | ## [v0.4.0](https://github.com/paulmach/osm/compare/v0.3.0...v0.4.0) - 2022-05-26 46 | 47 | ### Changed 48 | 49 | - protobuf: port to google protobuf by [@OlafFlebbeBosch](https://github.com/OlafFlebbeBoch) in https://github.com/paulmach/osm/pull/36 50 | 51 | ## [v0.3.0](https://github.com/paulmach/osm/compare/v0.2.2...v0.3.0) - 2022-04-21 52 | 53 | ### Added 54 | 55 | - osmpbf: preallocation node tags array by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/33 56 | - osmpbf: support "sparse" dense nodes by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/32 57 | - osmpbf: add filter functions by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/30 58 | 59 | ## [v0.2.2](https://github.com/paulmach/osm/compare/v0.2.1...v0.2.2) - 2021-04-27 60 | 61 | ### Fixed 62 | 63 | - osmpbf: fixed memory allocation issues by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/26 64 | 65 | ## [v0.2.1](https://github.com/paulmach/osm/compare/v0.2.0...v0.2.1) - 2021-02-04 66 | 67 | ### Changed 68 | 69 | - osmpbf: reduces memory usage when decoding by [@oflebbe](https://github.com/oflebbe) in https://github.com/paulmach/osm/pull/22 70 | - Fix some more typos by [@meyermarcel](https://github.com/meyermarcel) in https://github.com/paulmach/osm/pull/23 71 | 72 | ## [v0.2.0](https://github.com/paulmach/osm/compare/v0.1.1...v0.2.0) - 2021-01-09 73 | 74 | ### Changed 75 | 76 | - osmpbf: ability to efficiently skip types when decoding by [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/18 77 | - osmpbf: use [protoscan](https://github.com/paulmach/protoscan) for a 10%ish performance improvement 78 | - osmpbf: use cgo/czlib to decode protobufs (if cgo enabled), 20% faster on benchmarks [@paulmach](https://github.com/paulmach) in https://github.com/paulmach/osm/pull/19 79 | - deprecated node/ways/relations marshaling into this packages custom binary format [`8fcda5`](https://github.com/paulmach/osm/commit/8fcda5dc49b4767df63eccb5a25f3e63d5b17f4d) 80 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Paul Mach 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /annotate/datasource.go: -------------------------------------------------------------------------------- 1 | package annotate 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/paulmach/osm" 7 | "github.com/paulmach/osm/annotate/internal/core" 8 | "github.com/paulmach/osm/annotate/shared" 9 | 10 | "github.com/paulmach/orb" 11 | "github.com/paulmach/orb/planar" 12 | ) 13 | 14 | type wayDatasource struct { 15 | NodeHistoryDatasourcer 16 | } 17 | 18 | type wayChildDatasource struct { 19 | NodeHistoryAsChildrenDatasourcer 20 | } 21 | 22 | func newWayDatasourcer(ds NodeHistoryDatasourcer) core.Datasourcer { 23 | if d, ok := ds.(NodeHistoryAsChildrenDatasourcer); ok { 24 | return &wayChildDatasource{d} 25 | } 26 | 27 | return &wayDatasource{ds} 28 | } 29 | 30 | func (wds *wayDatasource) Get(ctx context.Context, id osm.FeatureID) (core.ChildList, error) { 31 | if id.Type() != osm.TypeNode { 32 | panic("only node types supported") 33 | } 34 | 35 | nodes, err := wds.NodeHistory(ctx, id.NodeID()) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | return nodesToChildList(nodes), nil 41 | } 42 | 43 | func (wds *wayChildDatasource) Get(ctx context.Context, id osm.FeatureID) (core.ChildList, error) { 44 | if id.Type() != osm.TypeNode { 45 | panic("only node types supported") 46 | } 47 | 48 | return wds.NodeHistoryAsChildren(ctx, id.NodeID()) 49 | } 50 | 51 | type relationDatasource struct { 52 | osm.HistoryDatasourcer 53 | } 54 | 55 | type relationChildDatasource struct { 56 | HistoryAsChildrenDatasourcer 57 | } 58 | 59 | func newRelationDatasourcer(ds osm.HistoryDatasourcer) core.Datasourcer { 60 | if d, ok := ds.(HistoryAsChildrenDatasourcer); ok { 61 | return &relationChildDatasource{d} 62 | } 63 | 64 | return &relationDatasource{ds} 65 | } 66 | 67 | func (rds *relationDatasource) Get(ctx context.Context, id osm.FeatureID) (core.ChildList, error) { 68 | 69 | switch id.Type() { 70 | case osm.TypeNode: 71 | nodes, err := rds.NodeHistory(ctx, id.NodeID()) 72 | if err != nil { 73 | return nil, err 74 | } 75 | 76 | return nodesToChildList(nodes), nil 77 | case osm.TypeWay: 78 | ways, err := rds.WayHistory(ctx, id.WayID()) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | return waysToChildList(ways), nil 84 | case osm.TypeRelation: 85 | relations, err := rds.RelationHistory(ctx, id.RelationID()) 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | return relationsToChildList(relations), nil 91 | } 92 | 93 | return nil, &UnsupportedMemberTypeError{ 94 | MemberType: id.Type(), 95 | } 96 | } 97 | 98 | func (rds *relationChildDatasource) Get(ctx context.Context, id osm.FeatureID) (core.ChildList, error) { 99 | 100 | switch id.Type() { 101 | case osm.TypeNode: 102 | return rds.NodeHistoryAsChildren(ctx, id.NodeID()) 103 | case osm.TypeWay: 104 | return rds.WayHistoryAsChildren(ctx, id.WayID()) 105 | case osm.TypeRelation: 106 | return rds.RelationHistoryAsChildren(ctx, id.RelationID()) 107 | } 108 | 109 | return nil, &UnsupportedMemberTypeError{ 110 | MemberType: id.Type(), 111 | } 112 | } 113 | 114 | func nodesToChildList(nodes osm.Nodes) core.ChildList { 115 | if len(nodes) == 0 { 116 | return nil 117 | } 118 | 119 | list := make(core.ChildList, len(nodes)) 120 | nodes.SortByIDVersion() 121 | for i, n := range nodes { 122 | c := shared.FromNode(n) 123 | c.VersionIndex = i 124 | list[i] = c 125 | } 126 | 127 | return list 128 | } 129 | 130 | func waysToChildList(ways osm.Ways) core.ChildList { 131 | if len(ways) == 0 { 132 | return nil 133 | } 134 | 135 | list := make(core.ChildList, len(ways)) 136 | ways.SortByIDVersion() 137 | for i, w := range ways { 138 | c := shared.FromWay(w) 139 | c.VersionIndex = i 140 | 141 | if i != 0 { 142 | c.ReverseOfPrevious = IsReverse(w, ways[i-1]) 143 | } 144 | 145 | list[i] = c 146 | } 147 | 148 | return list 149 | } 150 | 151 | // IsReverse checks to see if this way update was a "reversal". It is very tricky 152 | // to generally answer this question but easier for a relation minor update. 153 | // Since the relation wasn't updated we assume things are still connected and 154 | // can just check the endpoints. 155 | func IsReverse(w1, w2 *osm.Way) bool { 156 | if len(w1.Nodes) < 2 || len(w2.Nodes) < 2 { 157 | return false 158 | } 159 | 160 | // check if either is a ring 161 | if w1.Nodes[0].ID == w1.Nodes[len(w1.Nodes)-1].ID || 162 | w2.Nodes[0].ID == w2.Nodes[len(w2.Nodes)-1].ID { 163 | 164 | r1 := orb.Ring(w1.LineString()) 165 | r2 := orb.Ring(w2.LineString()) 166 | return planar.Area(r1)*planar.Area(r2) < 0 167 | } 168 | 169 | // not a ring so see if endpoint were flipped 170 | return w1.Nodes[0].ID == w2.Nodes[len(w2.Nodes)-1].ID && 171 | w2.Nodes[0].ID == w1.Nodes[len(w1.Nodes)-1].ID 172 | } 173 | 174 | func relationsToChildList(relations osm.Relations) core.ChildList { 175 | if len(relations) == 0 { 176 | return nil 177 | } 178 | 179 | list := make(core.ChildList, len(relations)) 180 | relations.SortByIDVersion() 181 | for i, r := range relations { 182 | c := shared.FromRelation(r) 183 | c.VersionIndex = i 184 | list[i] = c 185 | } 186 | 187 | return list 188 | } 189 | -------------------------------------------------------------------------------- /annotate/errors.go: -------------------------------------------------------------------------------- 1 | package annotate 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/paulmach/osm" 8 | "github.com/paulmach/osm/annotate/internal/core" 9 | ) 10 | 11 | // NoHistoryError is returned if there is no entry in the history 12 | // map for a specific child. 13 | type NoHistoryError struct { 14 | ID osm.FeatureID 15 | } 16 | 17 | // Error returns a pretty string of the error. 18 | func (e *NoHistoryError) Error() string { 19 | return fmt.Sprintf("element history not found for %v", e.ID) 20 | } 21 | 22 | // NoVisibleChildError is returned if there are no visible children 23 | // for a parent at a given time. 24 | type NoVisibleChildError struct { 25 | ID osm.FeatureID 26 | Timestamp time.Time 27 | } 28 | 29 | // Error returns a pretty string of the error. 30 | func (e *NoVisibleChildError) Error() string { 31 | return fmt.Sprintf("no visible child for %v at %v", e.ID, e.Timestamp) 32 | } 33 | 34 | // UnsupportedMemberTypeError is returned if a relation member is not a 35 | // node, way or relation. 36 | type UnsupportedMemberTypeError struct { 37 | RelationID osm.RelationID 38 | MemberType osm.Type 39 | Index int 40 | } 41 | 42 | // Error returns a pretty string of the error. 43 | func (e *UnsupportedMemberTypeError) Error() string { 44 | return fmt.Sprintf("unsupported member type %v for relation %d at %d", e.MemberType, e.RelationID, e.Index) 45 | } 46 | 47 | func mapErrors(err error) error { 48 | switch t := err.(type) { 49 | case *core.NoHistoryError: 50 | return &NoHistoryError{ 51 | ID: t.ChildID, 52 | } 53 | case *core.NoVisibleChildError: 54 | return &NoVisibleChildError{ 55 | ID: t.ChildID, 56 | Timestamp: t.Timestamp, 57 | } 58 | } 59 | 60 | return err 61 | } 62 | -------------------------------------------------------------------------------- /annotate/errors_test.go: -------------------------------------------------------------------------------- 1 | package annotate 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/paulmach/osm/annotate/internal/core" 8 | ) 9 | 10 | func TestMapErrors(t *testing.T) { 11 | e := mapErrors(&core.NoHistoryError{}) 12 | if _, ok := e.(*NoHistoryError); !ok { 13 | t.Errorf("should map NoHistoryError: %+v", e) 14 | } 15 | 16 | e = mapErrors(&core.NoVisibleChildError{}) 17 | if _, ok := e.(*NoVisibleChildError); !ok { 18 | t.Errorf("should map NoVisibleChildError: %+v", e) 19 | } 20 | 21 | err := errors.New("some error") 22 | if e := mapErrors(err); e != err { 23 | t.Errorf("should pass through other errors: %v", e) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /annotate/geo.go: -------------------------------------------------------------------------------- 1 | package annotate 2 | 3 | import ( 4 | "math" 5 | "time" 6 | 7 | "github.com/paulmach/orb" 8 | "github.com/paulmach/orb/geo" 9 | "github.com/paulmach/osm" 10 | "github.com/paulmach/osm/internal/mputil" 11 | ) 12 | 13 | func wayPointOnSurface(w *osm.Way) orb.Point { 14 | centroid := wayCentroid(w) 15 | 16 | // find closest node to centroid. 17 | // This is how ST_PointOnSurface is implemented. 18 | min := math.MaxFloat64 19 | index := 0 20 | for i, n := range w.Nodes { 21 | d := geo.Distance(centroid, n.Point()) 22 | if d < min { 23 | index = i 24 | min = d 25 | } 26 | } 27 | 28 | return w.Nodes[index].Point() 29 | } 30 | 31 | func wayCentroid(w *osm.Way) orb.Point { 32 | dist := 0.0 33 | point := orb.Point{} 34 | 35 | seg := [2]orb.Point{} 36 | 37 | for i := 0; i < len(w.Nodes)-1; i++ { 38 | seg[0] = w.Nodes[i].Point() 39 | seg[1] = w.Nodes[i+1].Point() 40 | 41 | d := geo.Distance(seg[0], seg[1]) 42 | 43 | point[0] += (seg[0][0] + seg[1][0]) / 2.0 * d 44 | point[1] += (seg[0][1] + seg[1][1]) / 2.0 * d 45 | 46 | dist += d 47 | } 48 | 49 | point[0] /= dist 50 | point[1] /= dist 51 | 52 | return point 53 | } 54 | 55 | // orientation will annotate the orientation of multipolygon relation members. 56 | // This makes it possible to reconstruct relations with partial data in the right direction. 57 | // Return value indicates if the result is 'tainted', e.g. not all way members were present. 58 | func orientation(members osm.Members, ways map[osm.WayID]*osm.Way, at time.Time) bool { 59 | outer, inner, tainted := mputil.Group(members, ways, at) 60 | 61 | outers := mputil.Join(outer) 62 | inners := mputil.Join(inner) 63 | 64 | for _, outer := range outers { 65 | annotateOrientation(members, outer, orb.CCW) 66 | } 67 | 68 | for _, inner := range inners { 69 | annotateOrientation(members, inner, orb.CW) 70 | } 71 | 72 | return tainted 73 | } 74 | 75 | func annotateOrientation(members osm.Members, ms mputil.MultiSegment, o orb.Orientation) { 76 | factor := orb.Orientation(1) 77 | if ms.Orientation() != o { 78 | factor = -1 79 | } 80 | 81 | for _, segment := range ms { 82 | if segment.Reversed { 83 | members[segment.Index].Orientation = -1 * factor * o 84 | } else { 85 | members[segment.Index].Orientation = factor * o 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /annotate/geo_test.go: -------------------------------------------------------------------------------- 1 | package annotate 2 | 3 | import ( 4 | "encoding/xml" 5 | "testing" 6 | 7 | "github.com/paulmach/orb" 8 | "github.com/paulmach/osm" 9 | ) 10 | 11 | func TestWayPointOnSurface(t *testing.T) { 12 | data := ` 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | ` 21 | 22 | var w *osm.Way 23 | if err := xml.Unmarshal([]byte(data), &w); err != nil { 24 | t.Fatalf("failed to unmarshal: %v", err) 25 | } 26 | 27 | sp := wayPointOnSurface(w) 28 | 29 | expected := orb.Point{w.Nodes[3].Lon, w.Nodes[3].Lat} 30 | if !sp.Equal(expected) { 31 | t.Errorf("incorrect centroid: %v", sp) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /annotate/internal/core/datasourcer_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/paulmach/osm" 8 | ) 9 | 10 | var _ Datasourcer = &TestDS{} 11 | 12 | var ErrNotFound = errors.New("not found") 13 | 14 | // TestDS implements a datasource for testing. 15 | type TestDS struct { 16 | data map[osm.FeatureID]ChildList 17 | } 18 | 19 | // Get returns the history in ChildList form. 20 | func (tds *TestDS) Get(ctx context.Context, id osm.FeatureID) (ChildList, error) { 21 | if tds.data == nil { 22 | return nil, ErrNotFound 23 | } 24 | 25 | v := tds.data[id] 26 | if v == nil { 27 | return nil, ErrNotFound 28 | } 29 | 30 | return v, nil 31 | } 32 | 33 | // MustGet is used by tests only to simplify some code. 34 | func (tds *TestDS) MustGet(id osm.FeatureID) ChildList { 35 | v, err := tds.Get(context.TODO(), id) 36 | if err != nil { 37 | panic(err) 38 | } 39 | 40 | return v 41 | } 42 | 43 | func (tds *TestDS) NotFound(err error) bool { 44 | return err == ErrNotFound 45 | } 46 | 47 | // Set sets the element history into the map. 48 | // The element is deleted if list is nil. 49 | func (tds *TestDS) Set(id osm.FeatureID, list ChildList) { 50 | if tds.data == nil { 51 | tds.data = make(map[osm.FeatureID]ChildList) 52 | } 53 | 54 | if list == nil { 55 | delete(tds.data, id) 56 | } 57 | 58 | tds.data[id] = list 59 | } 60 | -------------------------------------------------------------------------------- /annotate/internal/core/errors.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/paulmach/osm" 8 | ) 9 | 10 | // NoHistoryError is returned if there is no entry in the history 11 | // map for a specific child. 12 | type NoHistoryError struct { 13 | ChildID osm.FeatureID 14 | } 15 | 16 | // Error returns a pretty string of the error. 17 | func (e *NoHistoryError) Error() string { 18 | return fmt.Sprintf("element history not found for %v", e.ChildID) 19 | } 20 | 21 | // NoVisibleChildError is returned if there are no visible children 22 | // for a parent at a given time. 23 | type NoVisibleChildError struct { 24 | ChildID osm.FeatureID 25 | Timestamp time.Time 26 | } 27 | 28 | // Error returns a pretty string of the error. 29 | func (e *NoVisibleChildError) Error() string { 30 | return fmt.Sprintf("no visible child for %v at %v", e.ChildID, e.Timestamp) 31 | } 32 | -------------------------------------------------------------------------------- /annotate/internal/core/types.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/paulmach/osm" 7 | "github.com/paulmach/osm/annotate/shared" 8 | ) 9 | 10 | // A Parent is something that holds children. ie. ways have nodes as children 11 | // and relations can have nodes, ways and relations as children. 12 | type Parent interface { 13 | ID() osm.FeatureID // used for logging 14 | ChangesetID() osm.ChangesetID 15 | 16 | Version() int 17 | Visible() bool 18 | Timestamp() time.Time 19 | Committed() time.Time 20 | 21 | // Refs returns normalized information about the children. 22 | // Currently this is the feature ids and if it is already annotated. 23 | // Note: we auto-annotate all unannotated children if they would have 24 | // been filtered out. 25 | Refs() (osm.FeatureIDs, []bool) 26 | SetChild(idx int, c *shared.Child) 27 | } 28 | 29 | // A ChildList is a set 30 | type ChildList []*shared.Child 31 | 32 | // FindVisible locates the child visible at the given time. 33 | // If 'at' is on or after osm.CommitInfoStart the committed 34 | // time is used to determine visiblity. If 'at' is before, a range +-eps 35 | // around the give time. Will return the closes visible node within that 36 | // range, or the previous node if visible. Children after 'at' but within 37 | // the eps must have the same changeset id as provided (the parent's). 38 | // If the previous node is not visible, or does not exits will return nil. 39 | func (cl ChildList) FindVisible(cid osm.ChangesetID, at time.Time, eps time.Duration) *shared.Child { 40 | var ( 41 | diff time.Duration = -1 42 | nearest *shared.Child 43 | ) 44 | 45 | start := at.Add(-eps) 46 | for _, c := range cl { 47 | 48 | if c.Committed.Before(osm.CommitInfoStart) { 49 | // more complicated logic for early data. 50 | offset := c.Timestamp.Sub(start) 51 | visible := c.Visible 52 | 53 | // if this node is after the end then it's over 54 | if offset > 2*eps { 55 | break 56 | } 57 | 58 | // if we're before the start set with the latest node 59 | if offset < 0 { 60 | if visible { 61 | nearest = c 62 | } else { 63 | nearest = nil 64 | } 65 | 66 | continue 67 | } 68 | 69 | // we're in the range!!! 70 | d := absDuration(offset - eps) 71 | if diff < 0 || (d <= diff) { 72 | // first within range, set if not visible 73 | if diff == -1 && !visible && offset == 0 { 74 | nearest = nil 75 | } 76 | 77 | // only update nearest if visible since we want 78 | // the closest visible within the range. 79 | if visible { 80 | if offset <= eps { 81 | // if we're before at, pick it 82 | nearest = c 83 | } else if c.ChangesetID == cid { 84 | // if we're after at, changeset must be same 85 | nearest = c 86 | } else { 87 | // after at, not same changeset, ignore. 88 | continue 89 | } 90 | } 91 | 92 | diff = d 93 | } 94 | } else { 95 | // simpler logic, if committed is on or before 'at' 96 | // consider that element. 97 | if c.Committed.After(at) { 98 | break 99 | } 100 | 101 | if c.Visible { 102 | nearest = c 103 | } else { 104 | nearest = nil 105 | } 106 | } 107 | } 108 | 109 | return nearest 110 | } 111 | 112 | // VersionBefore finds the last child before a given time. 113 | func (cl ChildList) VersionBefore(end time.Time) *shared.Child { 114 | var latest *shared.Child 115 | 116 | for _, c := range cl { 117 | if !timeThreshold(c, 0).Before(end) { 118 | break 119 | } 120 | 121 | latest = c 122 | } 123 | 124 | return latest 125 | } 126 | 127 | func timeThreshold(c *shared.Child, esp time.Duration) time.Time { 128 | if c.Committed.Before(osm.CommitInfoStart) { 129 | return c.Timestamp.Add(esp) 130 | } 131 | 132 | return c.Committed 133 | } 134 | 135 | func timeThresholdParent(p Parent, esp time.Duration) time.Time { 136 | if p.Committed().Before(osm.CommitInfoStart) { 137 | return p.Timestamp().Add(esp) 138 | } 139 | 140 | return p.Committed() 141 | } 142 | 143 | func absDuration(d time.Duration) time.Duration { 144 | if d < 0 { 145 | return -d 146 | } 147 | 148 | return d 149 | } 150 | -------------------------------------------------------------------------------- /annotate/options.go: -------------------------------------------------------------------------------- 1 | package annotate 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/paulmach/osm" 7 | "github.com/paulmach/osm/annotate/internal/core" 8 | ) 9 | 10 | // Option is a parameter that can be used for annotating. 11 | type Option func(*core.Options) error 12 | 13 | const defaultThreshold = 30 * time.Minute 14 | 15 | // Threshold is used if the "committed at" time is unknown and deals with 16 | // the flexibility of commit orders, e.g. nodes in the same commit as 17 | // the way can have a timestamp after the way. Threshold defines the time 18 | // range to "forward group" these changes. 19 | // Default 30 minutes. 20 | func Threshold(t time.Duration) Option { 21 | return func(o *core.Options) error { 22 | o.Threshold = t 23 | return nil 24 | } 25 | } 26 | 27 | // IgnoreInconsistency will try to match children even if they are missing. 28 | // This should be used when you want to gracefully handle the weird data in OSM. 29 | // 30 | // Nodes with unclear/inconsistent data will not be annotated. Causes include: 31 | // - redacted data: In 2012 due to the license change data had to be removed. 32 | // This could be some nodes of a way. There exist ways for which some nodes have 33 | // just a single delete version, e.g. way 159081205, node 376130526 34 | // - data pre element versioning: pre-2012(?) data versions were not kept, so 35 | // for old ways there many be no information about some nodes. For example, 36 | // a node may be updated after a way and there is no way to get the original 37 | // version of the node and way. 38 | // - bad editors: sometimes a node is edited 7 times in a single changeset 39 | // and version 5 is a delete. See node 321452894, part of way 28831147. 40 | func IgnoreInconsistency(yes bool) Option { 41 | return func(o *core.Options) error { 42 | o.IgnoreInconsistency = yes 43 | return nil 44 | } 45 | } 46 | 47 | // IgnoreMissingChildren will ignore children for which the datasource returns 48 | // datasource.ErrNotFound. This can be useful for partial history extracts where 49 | // there may be relations for which the way was not included, e.g. a relation has 50 | // a way inside the extract bounds and other ways outside the bounds. 51 | func IgnoreMissingChildren(yes bool) Option { 52 | return func(o *core.Options) error { 53 | o.IgnoreMissingChildren = yes 54 | return nil 55 | } 56 | } 57 | 58 | // ChildFilter allows for only a subset of children to be annotated on the parent. 59 | // This can greatly improve update speed by only worrying about the children 60 | // updated in the same batch. All unannotated children will be annotated regardless 61 | // of the results of the filter function. 62 | func ChildFilter(filter func(osm.FeatureID) bool) Option { 63 | return func(o *core.Options) error { 64 | o.ChildFilter = filter 65 | return nil 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /annotate/order.go: -------------------------------------------------------------------------------- 1 | package annotate 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "github.com/paulmach/osm" 8 | ) 9 | 10 | // RelationHistoryDatasourcer is an more strict interface for when we only need the relation history. 11 | type RelationHistoryDatasourcer interface { 12 | RelationHistory(context.Context, osm.RelationID) (osm.Relations, error) 13 | NotFound(error) bool 14 | } 15 | 16 | var _ RelationHistoryDatasourcer = &osm.HistoryDatasource{} 17 | 18 | // A ChildFirstOrdering is a struct that allows for a set of relations to be 19 | // processed in a dept first order. Since relations can reference other 20 | // relations we need to make sure children are added before parents. 21 | type ChildFirstOrdering struct { 22 | // CompletedIndex is the number of relation ids in the provided 23 | // array that have been finished. This can be used as a good restart position. 24 | CompletedIndex int 25 | 26 | ctx context.Context 27 | done context.CancelFunc 28 | ds RelationHistoryDatasourcer 29 | visited map[osm.RelationID]struct{} 30 | out chan osm.RelationID 31 | wg sync.WaitGroup 32 | 33 | id osm.RelationID 34 | err error 35 | } 36 | 37 | // NewChildFirstOrdering creates a new ordering object. It is used to provided 38 | // a child before parent ordering for relations. This order must be used when 39 | // inserting+annotating relations into the datastore. 40 | func NewChildFirstOrdering( 41 | ctx context.Context, 42 | ids []osm.RelationID, 43 | ds RelationHistoryDatasourcer, 44 | ) *ChildFirstOrdering { 45 | ctx, done := context.WithCancel(ctx) 46 | o := &ChildFirstOrdering{ 47 | ctx: ctx, 48 | done: done, 49 | ds: ds, 50 | visited: make(map[osm.RelationID]struct{}, len(ids)), 51 | out: make(chan osm.RelationID), 52 | } 53 | 54 | o.wg.Add(1) 55 | go func() { 56 | defer o.wg.Done() 57 | defer close(o.out) 58 | 59 | path := make([]osm.RelationID, 0, 100) 60 | for i, id := range ids { 61 | err := o.walk(id, path) 62 | if err != nil { 63 | o.err = err 64 | return 65 | } 66 | 67 | o.CompletedIndex = i 68 | } 69 | }() 70 | 71 | return o 72 | } 73 | 74 | // Err returns a non-nil error if something went wrong with search, 75 | // like a cycle, or a datasource error. 76 | func (o *ChildFirstOrdering) Err() error { 77 | if o.err != nil { 78 | return o.err 79 | } 80 | 81 | return o.ctx.Err() 82 | } 83 | 84 | // Next locates the next relation id that can be used. 85 | // Returns false if the context is closed, something went wrong 86 | // or the full tree has been walked. 87 | func (o *ChildFirstOrdering) Next() bool { 88 | if o.err != nil || o.ctx.Err() != nil { 89 | return false 90 | } 91 | 92 | select { 93 | case id := <-o.out: 94 | if id == 0 { 95 | return false 96 | } 97 | o.id = id 98 | return true 99 | case <-o.ctx.Done(): 100 | return false 101 | } 102 | } 103 | 104 | // RelationID is the id found by the previous scan. 105 | func (o *ChildFirstOrdering) RelationID() osm.RelationID { 106 | return o.id 107 | } 108 | 109 | // Close can be used to terminate the scanning process before 110 | // all ids have been walked. 111 | func (o *ChildFirstOrdering) Close() { 112 | o.done() 113 | o.wg.Wait() 114 | } 115 | 116 | func (o *ChildFirstOrdering) walk(id osm.RelationID, path []osm.RelationID) error { 117 | if _, ok := o.visited[id]; ok { 118 | return nil 119 | } 120 | 121 | relations, err := o.ds.RelationHistory(o.ctx, id) 122 | if o.ds.NotFound(err) { 123 | return nil 124 | } 125 | 126 | if err != nil { 127 | return err 128 | } 129 | 130 | for _, r := range relations { 131 | for _, m := range r.Members { 132 | if m.Type != osm.TypeRelation { 133 | continue 134 | } 135 | 136 | mid := osm.RelationID(m.Ref) 137 | for _, pid := range path { 138 | if pid == mid { 139 | // circular relations are allowed, 140 | // source: https://github.com/openstreetmap/openstreetmap-website/issues/1465#issuecomment-282323187 141 | 142 | // since this relation is already being worked through higher 143 | // up the stack, we can just return here. 144 | return nil 145 | } 146 | } 147 | 148 | err := o.walk(mid, append(path, mid)) 149 | if err != nil { 150 | return err 151 | } 152 | } 153 | } 154 | 155 | if o.ctx.Err() != nil { 156 | return o.ctx.Err() 157 | } 158 | 159 | o.visited[id] = struct{}{} 160 | select { 161 | case o.out <- id: 162 | case <-o.ctx.Done(): 163 | return o.ctx.Err() 164 | } 165 | 166 | return nil 167 | } 168 | -------------------------------------------------------------------------------- /annotate/relation.go: -------------------------------------------------------------------------------- 1 | package annotate 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/paulmach/osm" 8 | "github.com/paulmach/osm/annotate/internal/core" 9 | "github.com/paulmach/osm/annotate/shared" 10 | ) 11 | 12 | // HistoryAsChildrenDatasourcer is an advanced data source that 13 | // returns the needed elements as children directly. 14 | type HistoryAsChildrenDatasourcer interface { 15 | osm.HistoryDatasourcer 16 | NodeHistoryAsChildren(context.Context, osm.NodeID) ([]*shared.Child, error) 17 | WayHistoryAsChildren(context.Context, osm.WayID) ([]*shared.Child, error) 18 | RelationHistoryAsChildren(context.Context, osm.RelationID) ([]*shared.Child, error) 19 | } 20 | 21 | // Relations computes the updates for the given relations 22 | // and annotate members with stuff like changeset and lon/lat data. 23 | // The input relations are modified to include this information. 24 | func Relations( 25 | ctx context.Context, 26 | relations osm.Relations, 27 | datasource osm.HistoryDatasourcer, 28 | opts ...Option, 29 | ) error { 30 | computeOpts := &core.Options{ 31 | Threshold: defaultThreshold, 32 | } 33 | for _, o := range opts { 34 | err := o(computeOpts) 35 | if err != nil { 36 | return err 37 | } 38 | } 39 | 40 | parents := make([]core.Parent, len(relations)) 41 | for i, r := range relations { 42 | parents[i] = &parentRelation{Relation: r} 43 | } 44 | 45 | rds := newRelationDatasourcer(datasource) 46 | updatesForParents, err := core.Compute(ctx, parents, rds, computeOpts) 47 | if err != nil { 48 | return mapErrors(err) 49 | } 50 | 51 | for _, p := range parents { 52 | r := p.(*parentRelation) 53 | if r.Relation.Polygon() { 54 | orientation(r.Relation.Members, r.ways, r.Relation.CommittedAt()) 55 | } 56 | } 57 | 58 | for i, updates := range updatesForParents { 59 | relations[i].Updates = updates 60 | } 61 | 62 | return nil 63 | } 64 | 65 | // A parentRelation wraps a osm.Relation into the core.Parent interface 66 | // so that updates can be computed. 67 | type parentRelation struct { 68 | Relation *osm.Relation 69 | ways map[osm.WayID]*osm.Way 70 | } 71 | 72 | func (r *parentRelation) ID() osm.FeatureID { 73 | return r.Relation.FeatureID() 74 | } 75 | 76 | func (r *parentRelation) ChangesetID() osm.ChangesetID { 77 | return r.Relation.ChangesetID 78 | } 79 | 80 | func (r *parentRelation) Version() int { 81 | return r.Relation.Version 82 | } 83 | 84 | func (r *parentRelation) Visible() bool { 85 | return r.Relation.Visible 86 | } 87 | 88 | func (r *parentRelation) Timestamp() time.Time { 89 | return r.Relation.Timestamp 90 | } 91 | 92 | func (r *parentRelation) Committed() time.Time { 93 | if r.Relation.Committed == nil { 94 | return time.Time{} 95 | } 96 | 97 | return *r.Relation.Committed 98 | } 99 | 100 | func (r *parentRelation) Refs() (osm.FeatureIDs, []bool) { 101 | ids := make(osm.FeatureIDs, len(r.Relation.Members)) 102 | annotated := make([]bool, len(r.Relation.Members)) 103 | 104 | for i := range r.Relation.Members { 105 | ids[i] = r.Relation.Members[i].FeatureID() 106 | annotated[i] = r.Relation.Members[i].Version != 0 107 | } 108 | 109 | return ids, annotated 110 | } 111 | 112 | func (r *parentRelation) SetChild(idx int, child *shared.Child) { 113 | if r.Relation.Polygon() && r.ways == nil { 114 | r.ways = make(map[osm.WayID]*osm.Way, len(r.Relation.Members)) 115 | } 116 | 117 | if child == nil { 118 | return 119 | } 120 | 121 | r.Relation.Members[idx].Version = child.Version 122 | r.Relation.Members[idx].ChangesetID = child.ChangesetID 123 | r.Relation.Members[idx].Lat = child.Lat 124 | r.Relation.Members[idx].Lon = child.Lon 125 | 126 | if r.ways != nil && child.Way != nil { 127 | r.ways[child.Way.ID] = child.Way 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /annotate/shared/child.go: -------------------------------------------------------------------------------- 1 | // Package shared is used by annotate and the internal core. 2 | // External usage of this package is for advanced use only. 3 | package shared 4 | 5 | import ( 6 | "time" 7 | 8 | "github.com/paulmach/osm" 9 | ) 10 | 11 | // A Child represents a node, way or relation that is a dependent for 12 | // annotating ways or relations. 13 | type Child struct { 14 | ID osm.FeatureID 15 | Version int 16 | ChangesetID osm.ChangesetID 17 | 18 | // VersionIndex is the index of the version if sorted from lowest to highest. 19 | // This is necessary since version don't have to start at 1 or be sequential. 20 | VersionIndex int 21 | Timestamp time.Time 22 | Committed time.Time 23 | 24 | // for nodes 25 | Lon, Lat float64 26 | 27 | // for ways 28 | Way *osm.Way 29 | ReverseOfPrevious bool 30 | 31 | // moving the visible bool here decreases the struct size from 32 | // size 120 (size class 128) to 112 (size class 112). 33 | Visible bool 34 | } 35 | 36 | // Update generates an update from this child. 37 | func (c *Child) Update() osm.Update { 38 | return osm.Update{ 39 | Version: c.Version, 40 | Timestamp: updateTimestamp(c.Timestamp, c.Committed), 41 | ChangesetID: c.ChangesetID, 42 | 43 | Lat: c.Lat, 44 | Lon: c.Lon, 45 | 46 | Reverse: c.ReverseOfPrevious, 47 | } 48 | } 49 | 50 | // FromNode converts a node to a child. 51 | func FromNode(n *osm.Node) *Child { 52 | c := &Child{ 53 | ID: n.FeatureID(), 54 | Version: n.Version, 55 | ChangesetID: n.ChangesetID, 56 | Visible: n.Visible, 57 | Timestamp: n.Timestamp, 58 | 59 | Lon: n.Lon, 60 | Lat: n.Lat, 61 | } 62 | 63 | if n.Committed != nil { 64 | c.Committed = *n.Committed 65 | } 66 | 67 | return c 68 | } 69 | 70 | // FromWay converts a way to a child. 71 | func FromWay(w *osm.Way) *Child { 72 | c := &Child{ 73 | ID: w.FeatureID(), 74 | Version: w.Version, 75 | ChangesetID: w.ChangesetID, 76 | Visible: w.Visible, 77 | Timestamp: w.Timestamp, 78 | Way: w, 79 | } 80 | 81 | if w.Committed != nil { 82 | c.Committed = *w.Committed 83 | } 84 | 85 | return c 86 | } 87 | 88 | // FromRelation converts a way to a child. 89 | func FromRelation(r *osm.Relation) *Child { 90 | c := &Child{ 91 | ID: r.FeatureID(), 92 | Version: r.Version, 93 | ChangesetID: r.ChangesetID, 94 | Visible: r.Visible, 95 | Timestamp: r.Timestamp, 96 | } 97 | 98 | if r.Committed != nil { 99 | c.Committed = *r.Committed 100 | } 101 | 102 | return c 103 | } 104 | 105 | func updateTimestamp(timestamp, committed time.Time) time.Time { 106 | if timestamp.Before(osm.CommitInfoStart) || committed.IsZero() { 107 | return timestamp 108 | } 109 | 110 | return committed 111 | } 112 | -------------------------------------------------------------------------------- /annotate/testdata/way_230391153.osm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /annotate/testdata/way_230391153_expected.osm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /annotate/way.go: -------------------------------------------------------------------------------- 1 | package annotate 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/paulmach/osm" 8 | "github.com/paulmach/osm/annotate/internal/core" 9 | "github.com/paulmach/osm/annotate/shared" 10 | ) 11 | 12 | // NodeHistoryDatasourcer is an more strict interface for when we only need node history. 13 | type NodeHistoryDatasourcer interface { 14 | NodeHistory(context.Context, osm.NodeID) (osm.Nodes, error) 15 | NotFound(error) bool 16 | } 17 | 18 | // NodeHistoryAsChildrenDatasourcer is an advanced data source that 19 | // returns the needed nodes as children directly. 20 | type NodeHistoryAsChildrenDatasourcer interface { 21 | NodeHistoryDatasourcer 22 | NodeHistoryAsChildren(context.Context, osm.NodeID) ([]*shared.Child, error) 23 | } 24 | 25 | var _ NodeHistoryDatasourcer = &osm.HistoryDatasource{} 26 | 27 | // Ways computes the updates for the given ways 28 | // and annotate the way nodes with changeset and lon/lat data. 29 | // The input ways are modified to include this information. 30 | func Ways( 31 | ctx context.Context, 32 | ways osm.Ways, 33 | datasource NodeHistoryDatasourcer, 34 | opts ...Option, 35 | ) error { 36 | computeOpts := &core.Options{ 37 | Threshold: defaultThreshold, 38 | } 39 | for _, o := range opts { 40 | err := o(computeOpts) 41 | if err != nil { 42 | return err 43 | } 44 | } 45 | 46 | parents := make([]core.Parent, len(ways)) 47 | for i, w := range ways { 48 | parents[i] = &parentWay{Way: w} 49 | } 50 | 51 | wds := newWayDatasourcer(datasource) 52 | updatesForParents, err := core.Compute(ctx, parents, wds, computeOpts) 53 | if err != nil { 54 | return mapErrors(err) 55 | } 56 | 57 | // fill in updates 58 | for i, updates := range updatesForParents { 59 | ways[i].Updates = updates 60 | } 61 | 62 | return nil 63 | } 64 | 65 | // A parentWay wraps a osm.Way into the core.Parent interface 66 | // so that updates can be computed. 67 | type parentWay struct { 68 | Way *osm.Way 69 | } 70 | 71 | func (w *parentWay) ID() osm.FeatureID { 72 | return w.Way.FeatureID() 73 | } 74 | 75 | func (w *parentWay) ChangesetID() osm.ChangesetID { 76 | return w.Way.ChangesetID 77 | } 78 | 79 | func (w *parentWay) Version() int { 80 | return w.Way.Version 81 | } 82 | 83 | func (w *parentWay) Visible() bool { 84 | return w.Way.Visible 85 | } 86 | 87 | func (w *parentWay) Timestamp() time.Time { 88 | return w.Way.Timestamp 89 | } 90 | 91 | func (w *parentWay) Committed() time.Time { 92 | if w.Way.Committed == nil { 93 | return time.Time{} 94 | } 95 | 96 | return *w.Way.Committed 97 | } 98 | 99 | func (w *parentWay) Refs() (osm.FeatureIDs, []bool) { 100 | ids := make(osm.FeatureIDs, len(w.Way.Nodes)) 101 | annotated := make([]bool, len(w.Way.Nodes)) 102 | 103 | for i := range w.Way.Nodes { 104 | ids[i] = w.Way.Nodes[i].FeatureID() 105 | annotated[i] = w.Way.Nodes[i].Version != 0 106 | } 107 | 108 | return ids, annotated 109 | } 110 | 111 | func (w *parentWay) SetChild(idx int, child *shared.Child) { 112 | if child == nil { 113 | return 114 | } 115 | 116 | w.Way.Nodes[idx].Version = child.Version 117 | w.Way.Nodes[idx].ChangesetID = child.ChangesetID 118 | w.Way.Nodes[idx].Lat = child.Lat 119 | w.Way.Nodes[idx].Lon = child.Lon 120 | } 121 | -------------------------------------------------------------------------------- /annotate/way_test.go: -------------------------------------------------------------------------------- 1 | package annotate 2 | 3 | import ( 4 | "context" 5 | "encoding/xml" 6 | "fmt" 7 | "os" 8 | "reflect" 9 | "testing" 10 | 11 | "github.com/paulmach/osm" 12 | ) 13 | 14 | func TestWays(t *testing.T) { 15 | ids := []osm.WayID{ 16 | 6394949, 17 | 230391153, 18 | } 19 | 20 | for _, id := range ids { 21 | o := loadTestdata(t, fmt.Sprintf("testdata/way_%d.osm", id)) 22 | 23 | ds := (&osm.OSM{Nodes: o.Nodes}).HistoryDatasource() 24 | err := Ways(context.Background(), o.Ways, ds) 25 | if err != nil { 26 | t.Fatalf("compute error: %v", err) 27 | } 28 | 29 | filename := fmt.Sprintf("testdata/way_%d_expected.osm", id) 30 | expected := loadTestdata(t, filename) 31 | 32 | if !reflect.DeepEqual(o.Ways, expected.Ways) { 33 | t.Errorf("expected way for id %d not equal", id) 34 | 35 | filename := fmt.Sprintf("testdata/way_%d_got.osm", id) 36 | t.Errorf("expected way not equal, file saved to %s", filename) 37 | 38 | data, _ := xml.MarshalIndent(&osm.OSM{Ways: o.Ways}, "", " ") 39 | err := os.WriteFile(filename, data, 0644) 40 | if err != nil { 41 | t.Fatalf("write error: %v", err) 42 | } 43 | } 44 | } 45 | } 46 | 47 | func TestWays_childFilter(t *testing.T) { 48 | nodes := osm.Nodes{ 49 | {ID: 1, Version: 1, Lat: 1, Lon: 1, Visible: true}, 50 | {ID: 1, Version: 2, Lat: 2, Lon: 2, Visible: true}, 51 | {ID: 1, Version: 3, Lat: 3, Lon: 3, Visible: true}, 52 | {ID: 2, Version: 1, Lat: 1, Lon: 1, Visible: true}, 53 | {ID: 2, Version: 2, Lat: 2, Lon: 2, Visible: true}, 54 | {ID: 2, Version: 3, Lat: 3, Lon: 3, Visible: true}, 55 | {ID: 3, Version: 1, Lat: 1, Lon: 1, Visible: true}, 56 | } 57 | 58 | ways := osm.Ways{ 59 | { 60 | ID: 1, 61 | Version: 1, 62 | Visible: true, 63 | Nodes: osm.WayNodes{ 64 | {ID: 1, Version: 1}, 65 | {ID: 2, Version: 1}, // filter says no annotate 66 | {ID: 3}, // annotate because not 67 | }, 68 | }, 69 | } 70 | 71 | ds := (&osm.OSM{Nodes: nodes}).HistoryDatasource() 72 | err := Ways( 73 | context.Background(), 74 | ways, 75 | ds, 76 | Threshold(0), 77 | ChildFilter(func(fid osm.FeatureID) bool { 78 | return fid == osm.NodeID(1).FeatureID() 79 | }), 80 | ) 81 | if err != nil { 82 | t.Fatalf("compute error: %v", err) 83 | } 84 | 85 | if ways[0].Nodes[0].Lat == 0 { 86 | t.Errorf("should annotate first node") 87 | } 88 | 89 | if ways[0].Nodes[1].Lat != 0 { 90 | t.Errorf("should not annotate second node") 91 | } 92 | 93 | if ways[0].Nodes[2].Lat == 0 { 94 | t.Errorf("should annotate third node") 95 | } 96 | } 97 | 98 | func BenchmarkWay(b *testing.B) { 99 | o := loadTestdata(b, "testdata/way_6394949.osm") 100 | ds := (&osm.OSM{Nodes: o.Nodes}).HistoryDatasource() 101 | 102 | b.ReportAllocs() 103 | b.ResetTimer() 104 | for n := 0; n < b.N; n++ { 105 | err := Ways(context.Background(), o.Ways, ds) 106 | if err != nil { 107 | b.Fatalf("compute error: %v", err) 108 | } 109 | } 110 | } 111 | 112 | func BenchmarkWays(b *testing.B) { 113 | o := loadTestdata(b, "testdata/relation_2714790.osm") 114 | ds := o.HistoryDatasource() 115 | 116 | b.ReportAllocs() 117 | b.ResetTimer() 118 | for n := 0; n < b.N; { 119 | for id, ways := range ds.Ways { 120 | err := Ways(context.Background(), ways, ds) 121 | if err != nil { 122 | b.Fatalf("compute error for way %d: %v", id, err) 123 | } 124 | 125 | n++ 126 | if n >= b.N { 127 | break 128 | } 129 | } 130 | } 131 | } 132 | 133 | func loadTestdata(tb testing.TB, filename string) *osm.OSM { 134 | data, err := os.ReadFile(filename) 135 | if err != nil { 136 | tb.Fatalf("unable to open file: %v", err) 137 | } 138 | 139 | o := &osm.OSM{} 140 | err = xml.Unmarshal(data, o) 141 | if err != nil { 142 | tb.Fatalf("unable to unmarshal data: %v", err) 143 | } 144 | 145 | return o 146 | } 147 | -------------------------------------------------------------------------------- /bounds.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/paulmach/orb/maptile" 7 | ) 8 | 9 | // Bounds are the bounds of osm data as defined in the xml file. 10 | type Bounds struct { 11 | MinLat float64 `xml:"minlat,attr"` 12 | MaxLat float64 `xml:"maxlat,attr"` 13 | MinLon float64 `xml:"minlon,attr"` 14 | MaxLon float64 `xml:"maxlon,attr"` 15 | } 16 | 17 | // NewBoundsFromTile creates a bound given an online map tile index. 18 | func NewBoundsFromTile(t maptile.Tile) (*Bounds, error) { 19 | maxIndex := uint32(1 << t.Z) 20 | if t.X >= maxIndex { 21 | return nil, errors.New("osm: x index out of range for this zoom") 22 | } 23 | if t.Y >= maxIndex { 24 | return nil, errors.New("osm: y index out of range for this zoom") 25 | } 26 | 27 | b := t.Bound() 28 | return &Bounds{ 29 | MinLat: b.Min.Lat(), 30 | MaxLat: b.Max.Lat(), 31 | MinLon: b.Min.Lon(), 32 | MaxLon: b.Max.Lon(), 33 | }, nil 34 | } 35 | 36 | // ContainsNode returns true if the node is within the bound. 37 | // Uses inclusive intervals, ie. returns true if on the boundary. 38 | func (b *Bounds) ContainsNode(n *Node) bool { 39 | if n.Lat < b.MinLat || n.Lat > b.MaxLat { 40 | return false 41 | } 42 | 43 | if n.Lon < b.MinLon || n.Lon > b.MaxLon { 44 | return false 45 | } 46 | 47 | return true 48 | } 49 | 50 | // ObjectID returns the bounds type but with 0 id. Since id doesn't make sense. 51 | // This is here to implement the Object interface since it technically is an 52 | // osm object type. It also allows bounds to be returned via the osmxml.Scanner. 53 | func (b *Bounds) ObjectID() ObjectID { 54 | return ObjectID(boundsMask) 55 | } 56 | -------------------------------------------------------------------------------- /bounds_test.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/paulmach/orb/maptile" 8 | ) 9 | 10 | func TestNewBoundFromTile(t *testing.T) { 11 | if _, err := NewBoundsFromTile(maptile.New(1000, 1, 3)); err == nil { 12 | t.Errorf("should return error for x out of bound") 13 | } 14 | 15 | if _, err := NewBoundsFromTile(maptile.New(1, 1000, 3)); err == nil { 16 | t.Errorf("should return error for y out of bound") 17 | } 18 | 19 | bounds, _ := NewBoundsFromTile(maptile.New(7, 8, 9)) 20 | 21 | // check 9 tiles around bounds 22 | for i := -1; i <= 1; i++ { 23 | for j := -1; j <= 1; j++ { 24 | t.Run(fmt.Sprintf("i %d j %d", i, j), func(t *testing.T) { 25 | n := centroid(mustBounds(t, uint32(7+i), uint32(8+j), 9)) 26 | if i == 0 && j == 0 { 27 | if !bounds.ContainsNode(n) { 28 | t.Errorf("should contain point") 29 | } 30 | } else { 31 | if bounds.ContainsNode(n) { 32 | t.Errorf("should not contain point") 33 | } 34 | } 35 | }) 36 | } 37 | } 38 | } 39 | 40 | func TestBounds_ContainsNode(t *testing.T) { 41 | b := &Bounds{} 42 | 43 | if v := b.ContainsNode(&Node{}); !v { 44 | t.Errorf("should contain node on boundary") 45 | } 46 | 47 | if v := b.ContainsNode(&Node{Lat: -1}); v { 48 | t.Errorf("should not contain node outside bounds") 49 | } 50 | if v := b.ContainsNode(&Node{Lat: 1}); v { 51 | t.Errorf("should not contain node outside bounds") 52 | } 53 | if v := b.ContainsNode(&Node{Lon: -1}); v { 54 | t.Errorf("should not contain node outside bounds") 55 | } 56 | 57 | if v := b.ContainsNode(&Node{Lon: 1}); v { 58 | t.Errorf("should not contain node outside bounds") 59 | } 60 | } 61 | 62 | func mustBounds(t *testing.T, x, y uint32, z maptile.Zoom) *Bounds { 63 | bounds, err := NewBoundsFromTile(maptile.New(x, y, z)) 64 | if err != nil { 65 | t.Fatalf("invalid bounds: %v", err) 66 | } 67 | 68 | return bounds 69 | } 70 | 71 | func centroid(b *Bounds) *Node { 72 | return &Node{ 73 | Lon: (b.MinLon + b.MaxLon) / 2, 74 | Lat: (b.MinLat + b.MaxLat) / 2, 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /change.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "encoding/xml" 5 | ) 6 | 7 | // Change is the structure of a changeset to be 8 | // uploaded or downloaded from the osm api server. 9 | // See: http://wiki.openstreetmap.org/wiki/OsmChange 10 | type Change struct { 11 | Version string `xml:"version,attr,omitempty" json:"version,omitempty"` 12 | Generator string `xml:"generator,attr,omitempty" json:"generator,omitempty"` 13 | 14 | // to indicate the origin of the data 15 | Copyright string `xml:"copyright,attr,omitempty" json:"copyright,omitempty"` 16 | Attribution string `xml:"attribution,attr,omitempty" json:"attribution,omitempty"` 17 | License string `xml:"license,attr,omitempty" json:"license,omitempty"` 18 | 19 | Create *OSM `xml:"create" json:"create,omitempty"` 20 | Modify *OSM `xml:"modify" json:"modify,omitempty"` 21 | Delete *OSM `xml:"delete" json:"delete,omitempty"` 22 | } 23 | 24 | // AppendCreate will append the object to the Create OSM object. 25 | func (c *Change) AppendCreate(o Object) { 26 | if c.Create == nil { 27 | c.Create = &OSM{} 28 | } 29 | 30 | c.Create.Append(o) 31 | } 32 | 33 | // AppendModify will append the object to the Modify OSM object. 34 | func (c *Change) AppendModify(o Object) { 35 | if c.Modify == nil { 36 | c.Modify = &OSM{} 37 | } 38 | 39 | c.Modify.Append(o) 40 | } 41 | 42 | // AppendDelete will append the object to the Delete OSM object. 43 | func (c *Change) AppendDelete(o Object) { 44 | if c.Delete == nil { 45 | c.Delete = &OSM{} 46 | } 47 | 48 | c.Delete.Append(o) 49 | } 50 | 51 | // HistoryDatasource converts the change object to a datasource accessible 52 | // by feature id. All the creates, modifies and deletes will be added 53 | // in that order. 54 | func (c *Change) HistoryDatasource() *HistoryDatasource { 55 | ds := &HistoryDatasource{} 56 | 57 | ds.add(c.Create, true) 58 | ds.add(c.Modify, true) 59 | ds.add(c.Delete, false) 60 | 61 | return ds 62 | } 63 | 64 | // MarshalXML implements the xml.Marshaller method to allow for the 65 | // correct wrapper/start element case and attr data. 66 | func (c Change) MarshalXML(e *xml.Encoder, start xml.StartElement) error { 67 | start.Name.Local = "osmChange" 68 | start.Attr = []xml.Attr{} 69 | 70 | if c.Version != "" { 71 | start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: "version"}, Value: c.Version}) 72 | } 73 | 74 | if c.Generator != "" { 75 | start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: "generator"}, Value: c.Generator}) 76 | } 77 | 78 | if c.Copyright != "" { 79 | start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: "copyright"}, Value: c.Copyright}) 80 | } 81 | 82 | if c.Attribution != "" { 83 | start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: "attribution"}, Value: c.Attribution}) 84 | } 85 | 86 | if c.License != "" { 87 | start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: "license"}, Value: c.License}) 88 | } 89 | 90 | if err := e.EncodeToken(start); err != nil { 91 | return err 92 | } 93 | 94 | if err := marshalInnerChange(e, "create", c.Create); err != nil { 95 | return err 96 | } 97 | 98 | if err := marshalInnerChange(e, "modify", c.Modify); err != nil { 99 | return err 100 | } 101 | 102 | if err := marshalInnerChange(e, "delete", c.Delete); err != nil { 103 | return err 104 | } 105 | 106 | return e.EncodeToken(start.End()) 107 | } 108 | 109 | func marshalInnerChange(e *xml.Encoder, name string, o *OSM) error { 110 | if o == nil { 111 | return nil 112 | } 113 | 114 | t := xml.StartElement{Name: xml.Name{Local: name}} 115 | if err := e.EncodeToken(t); err != nil { 116 | return err 117 | } 118 | 119 | if err := o.marshalInnerXML(e); err != nil { 120 | return err 121 | } 122 | 123 | return e.EncodeToken(t.End()) 124 | } 125 | -------------------------------------------------------------------------------- /changeset.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "encoding/xml" 5 | "time" 6 | ) 7 | 8 | // ChangesetID is the primary key for an osm changeset. 9 | type ChangesetID int64 10 | 11 | // ObjectID is a helper returning the object id for this changeset id. 12 | func (id ChangesetID) ObjectID() ObjectID { 13 | return ObjectID(changesetMask | (id << versionBits)) 14 | } 15 | 16 | // Changesets is a collection with some helper functions attached. 17 | type Changesets []*Changeset 18 | 19 | // A Changeset is a set of metadata around a set of osm changes. 20 | type Changeset struct { 21 | XMLName xmlNameJSONTypeCS `xml:"changeset" json:"type"` 22 | ID ChangesetID `xml:"id,attr" json:"id"` 23 | User string `xml:"user,attr" json:"user,omitempty"` 24 | UserID UserID `xml:"uid,attr" json:"uid,omitempty"` 25 | CreatedAt time.Time `xml:"created_at,attr" json:"created_at"` 26 | ClosedAt time.Time `xml:"closed_at,attr" json:"closed_at"` 27 | Open bool `xml:"open,attr" json:"open"` 28 | ChangesCount int `xml:"num_changes,attr,omitempty" json:"num_changes,omitempty"` 29 | MinLat float64 `xml:"min_lat,attr" json:"min_lat,omitempty"` 30 | MaxLat float64 `xml:"max_lat,attr" json:"max_lat,omitempty"` 31 | MinLon float64 `xml:"min_lon,attr" json:"min_lon,omitempty"` 32 | MaxLon float64 `xml:"max_lon,attr" json:"max_lon,omitempty"` 33 | CommentsCount int `xml:"comments_count,attr,omitempty" json:"comments_count,omitempty"` 34 | Tags Tags `xml:"tag" json:"tags,omitempty"` 35 | Discussion *ChangesetDiscussion `xml:"discussion,omitempty" json:"discussion,omitempty"` 36 | 37 | Change *Change `xml:"-" json:"change,omitempty"` 38 | } 39 | 40 | // ObjectID returns the object id of the changeset. 41 | func (c *Changeset) ObjectID() ObjectID { 42 | return c.ID.ObjectID() 43 | } 44 | 45 | // Bounds returns the bounds of the changeset as a bounds object. 46 | func (c *Changeset) Bounds() *Bounds { 47 | return &Bounds{ 48 | MinLat: c.MinLat, 49 | MaxLat: c.MaxLat, 50 | MinLon: c.MinLon, 51 | MaxLon: c.MaxLon, 52 | } 53 | } 54 | 55 | // Comment is a helper and returns the changeset comment from the tag. 56 | func (c *Changeset) Comment() string { 57 | return c.Tags.Find("comment") 58 | } 59 | 60 | // CreatedBy is a helper and returns the changeset created by from the tag. 61 | func (c *Changeset) CreatedBy() string { 62 | return c.Tags.Find("created_by") 63 | } 64 | 65 | // Locale is a helper and returns the changeset locale from the tag. 66 | func (c *Changeset) Locale() string { 67 | return c.Tags.Find("locale") 68 | } 69 | 70 | // Host is a helper and returns the changeset host from the tag. 71 | func (c *Changeset) Host() string { 72 | return c.Tags.Find("host") 73 | } 74 | 75 | // ImageryUsed is a helper and returns imagery used for the changeset from the tag. 76 | func (c *Changeset) ImageryUsed() string { 77 | return c.Tags.Find("imagery_used") 78 | } 79 | 80 | // Source is a helper and returns source for the changeset from the tag. 81 | func (c *Changeset) Source() string { 82 | return c.Tags.Find("source") 83 | } 84 | 85 | // Bot is a helper and returns true if the bot tag is a yes. 86 | func (c *Changeset) Bot() bool { 87 | // As of July 5, 2015: 300k yes, 123 no, 8 other 88 | return c.Tags.Find("bot") == "yes" 89 | } 90 | 91 | // IDs returns the ids of the changesets in the slice. 92 | func (cs Changesets) IDs() []ChangesetID { 93 | if len(cs) == 0 { 94 | return nil 95 | } 96 | 97 | r := make([]ChangesetID, 0, len(cs)) 98 | for _, c := range cs { 99 | r = append(r, c.ID) 100 | } 101 | 102 | return r 103 | } 104 | 105 | // ChangesetDiscussion is a conversation about a changeset. 106 | type ChangesetDiscussion struct { 107 | Comments []*ChangesetComment `xml:"comment" json:"comments"` 108 | } 109 | 110 | // ChangesetComment is a specific comment in a changeset discussion. 111 | type ChangesetComment struct { 112 | User string `xml:"user,attr" json:"user"` 113 | UserID UserID `xml:"uid,attr" json:"uid"` 114 | Timestamp time.Time `xml:"date,attr" json:"date"` 115 | Text string `xml:"text" json:"text"` 116 | } 117 | 118 | // MarshalXML implements the xml.Marshaller method to exclude this 119 | // whole element if the comments are empty. 120 | func (csd ChangesetDiscussion) MarshalXML(e *xml.Encoder, start xml.StartElement) error { 121 | if len(csd.Comments) == 0 { 122 | return nil 123 | } 124 | 125 | if err := e.EncodeToken(start); err != nil { 126 | return err 127 | } 128 | 129 | t := xml.StartElement{Name: xml.Name{Local: "comment"}} 130 | if err := e.EncodeElement(csd.Comments, t); err != nil { 131 | return err 132 | } 133 | 134 | return e.EncodeToken(start.End()) 135 | } 136 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: off 4 | patch: 5 | default: 6 | threshold: 50% 7 | 8 | precision: 2 9 | round: down 10 | range: "70...90" 11 | 12 | comment: false 13 | -------------------------------------------------------------------------------- /datasource.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | ) 7 | 8 | // A HistoryDatasourcer defines an interface to osm history data. 9 | type HistoryDatasourcer interface { 10 | NodeHistory(context.Context, NodeID) (Nodes, error) 11 | WayHistory(context.Context, WayID) (Ways, error) 12 | RelationHistory(context.Context, RelationID) (Relations, error) 13 | NotFound(error) bool 14 | } 15 | 16 | var errNotFound = errors.New("osm: feature not found") 17 | 18 | // A HistoryDatasource wraps maps to implement the HistoryDataSource interface. 19 | type HistoryDatasource struct { 20 | Nodes map[NodeID]Nodes 21 | Ways map[WayID]Ways 22 | Relations map[RelationID]Relations 23 | } 24 | 25 | var _ HistoryDatasourcer = &HistoryDatasource{} 26 | 27 | func (ds *HistoryDatasource) add(o *OSM, visible ...bool) { 28 | if o == nil { 29 | return 30 | } 31 | 32 | if len(o.Nodes) > 0 { 33 | if ds.Nodes == nil { 34 | ds.Nodes = make(map[NodeID]Nodes) 35 | } 36 | 37 | for _, n := range o.Nodes { 38 | if len(visible) == 1 { 39 | n.Visible = visible[0] 40 | } 41 | ds.Nodes[n.ID] = append(ds.Nodes[n.ID], n) 42 | } 43 | } 44 | 45 | if len(o.Ways) > 0 { 46 | if ds.Ways == nil { 47 | ds.Ways = make(map[WayID]Ways) 48 | } 49 | 50 | for _, w := range o.Ways { 51 | if len(visible) == 1 { 52 | w.Visible = visible[0] 53 | } 54 | ds.Ways[w.ID] = append(ds.Ways[w.ID], w) 55 | } 56 | } 57 | 58 | if len(o.Relations) > 0 { 59 | if ds.Relations == nil { 60 | ds.Relations = make(map[RelationID]Relations) 61 | } 62 | 63 | for _, r := range o.Relations { 64 | if len(visible) == 1 { 65 | r.Visible = visible[0] 66 | } 67 | ds.Relations[r.ID] = append(ds.Relations[r.ID], r) 68 | } 69 | } 70 | } 71 | 72 | // NodeHistory returns the history for the given id from the map. 73 | func (ds *HistoryDatasource) NodeHistory(ctx context.Context, id NodeID) (Nodes, error) { 74 | if ds.Nodes == nil { 75 | return nil, errNotFound 76 | } 77 | 78 | v := ds.Nodes[id] 79 | if v == nil { 80 | return nil, errNotFound 81 | } 82 | 83 | return v, nil 84 | } 85 | 86 | // WayHistory returns the history for the given id from the map. 87 | func (ds *HistoryDatasource) WayHistory(ctx context.Context, id WayID) (Ways, error) { 88 | if ds.Ways == nil { 89 | return nil, errNotFound 90 | } 91 | 92 | v := ds.Ways[id] 93 | if v == nil { 94 | return nil, errNotFound 95 | } 96 | 97 | return v, nil 98 | } 99 | 100 | // RelationHistory returns the history for the given id from the map. 101 | func (ds *HistoryDatasource) RelationHistory(ctx context.Context, id RelationID) (Relations, error) { 102 | if ds.Relations == nil { 103 | return nil, errNotFound 104 | } 105 | 106 | v := ds.Relations[id] 107 | if v == nil { 108 | return nil, errNotFound 109 | } 110 | 111 | return v, nil 112 | } 113 | 114 | // NotFound returns true if the error returned is a not found error. 115 | func (ds *HistoryDatasource) NotFound(err error) bool { 116 | return err == errNotFound 117 | } 118 | -------------------------------------------------------------------------------- /datasource_test.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | ) 7 | 8 | func TestHistoryDatasource(t *testing.T) { 9 | ctx := context.Background() 10 | 11 | t.Run("empty datasource", func(t *testing.T) { 12 | ds := &HistoryDatasource{} 13 | 14 | if _, err := ds.NodeHistory(ctx, 1); !ds.NotFound(err) { 15 | t.Errorf("should be not found error: %v", err) 16 | } 17 | 18 | if _, err := ds.WayHistory(ctx, 1); !ds.NotFound(err) { 19 | t.Errorf("should be not found error: %v", err) 20 | } 21 | 22 | if _, err := ds.RelationHistory(ctx, 1); !ds.NotFound(err) { 23 | t.Errorf("should be not found error: %v", err) 24 | } 25 | }) 26 | 27 | o := &OSM{ 28 | Nodes: Nodes{ 29 | {ID: 1, Version: 1}, 30 | {ID: 1, Version: 2}, 31 | }, 32 | Ways: Ways{ 33 | {ID: 1, Version: 1}, 34 | {ID: 1, Version: 2}, 35 | {ID: 1, Version: 3}, 36 | }, 37 | Relations: Relations{ 38 | {ID: 1, Version: 1}, 39 | {ID: 1, Version: 2}, 40 | {ID: 1, Version: 3}, 41 | {ID: 1, Version: 4}, 42 | }, 43 | } 44 | 45 | t.Run("non-empty datasource", func(t *testing.T) { 46 | ds := o.HistoryDatasource() 47 | 48 | ns, err := ds.NodeHistory(ctx, 1) 49 | if err != nil { 50 | t.Errorf("should not return error: %v", err) 51 | } 52 | 53 | if len(ns) != 2 { 54 | t.Errorf("incorrect nodes: %v", ns) 55 | } 56 | 57 | ws, err := ds.WayHistory(ctx, 1) 58 | if err != nil { 59 | t.Errorf("should not return error: %v", err) 60 | } 61 | 62 | if len(ws) != 3 { 63 | t.Errorf("incorrect ways: %v", ns) 64 | } 65 | 66 | rs, err := ds.RelationHistory(ctx, 1) 67 | if err != nil { 68 | t.Errorf("should not return error: %v", err) 69 | } 70 | 71 | if len(rs) != 4 { 72 | t.Errorf("incorrect relations: %v", ns) 73 | } 74 | }) 75 | 76 | t.Run("not found non-empty datasource", func(t *testing.T) { 77 | ds := o.HistoryDatasource() 78 | 79 | if _, err := ds.NodeHistory(ctx, 2); !ds.NotFound(err) { 80 | t.Errorf("should be not found error: %v", err) 81 | } 82 | 83 | if _, err := ds.WayHistory(ctx, 2); !ds.NotFound(err) { 84 | t.Errorf("should be not found error: %v", err) 85 | } 86 | 87 | if _, err := ds.RelationHistory(ctx, 2); !ds.NotFound(err) { 88 | t.Errorf("should be not found error: %v", err) 89 | } 90 | }) 91 | 92 | } 93 | -------------------------------------------------------------------------------- /diff.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import "encoding/xml" 4 | 5 | // Diff represents a difference of osm data with old and new data. 6 | type Diff struct { 7 | XMLName xml.Name `xml:"osm"` 8 | Actions Actions `xml:"action"` 9 | Changesets Changesets `xml:"changeset"` 10 | } 11 | 12 | // Actions is a set of diff actions. 13 | type Actions []Action 14 | 15 | // Action is an explicit create, modify or delete action with 16 | // old and new data if applicable. Different properties of this 17 | // struct will be populated depending on the action. 18 | // Create: da.OSM will contain the new element 19 | // Modify: da.Old and da.New will contain the old and new elements. 20 | // Delete: da.Old and da.New will contain the old and new elements. 21 | type Action struct { 22 | Type ActionType `xml:"type,attr"` 23 | *OSM `xml:",omitempty"` 24 | Old *OSM `xml:"old,omitempty"` 25 | New *OSM `xml:"new,omitempty"` 26 | } 27 | 28 | // UnmarshalXML converts xml into a diff action. 29 | func (a *Action) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { 30 | for _, attr := range start.Attr { 31 | if attr.Name.Local == "type" { 32 | a.Type = ActionType(attr.Value) 33 | break 34 | } 35 | } 36 | 37 | for { 38 | token, err := d.Token() 39 | if err != nil { 40 | break 41 | } 42 | 43 | start, ok := token.(xml.StartElement) 44 | if !ok { 45 | continue 46 | } 47 | 48 | switch start.Name.Local { 49 | case "old": 50 | a.Old = &OSM{} 51 | if err := d.DecodeElement(a.Old, &start); err != nil { 52 | return err 53 | } 54 | case "new": 55 | a.New = &OSM{} 56 | if err := d.DecodeElement(a.New, &start); err != nil { 57 | return err 58 | } 59 | case "node": 60 | n := &Node{} 61 | if err := d.DecodeElement(&n, &start); err != nil { 62 | return err 63 | } 64 | a.OSM = &OSM{Nodes: Nodes{n}} 65 | case "way": 66 | w := &Way{} 67 | if err := d.DecodeElement(&w, &start); err != nil { 68 | return err 69 | } 70 | a.OSM = &OSM{Ways: Ways{w}} 71 | case "relation": 72 | r := &Relation{} 73 | if err := d.DecodeElement(&r, &start); err != nil { 74 | return err 75 | } 76 | a.OSM = &OSM{Relations: Relations{r}} 77 | } 78 | } 79 | 80 | return nil 81 | } 82 | 83 | // MarshalXML converts a diff action to xml creating the proper structures. 84 | func (a Action) MarshalXML(e *xml.Encoder, start xml.StartElement) error { 85 | start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: "type"}, Value: string(a.Type)}) 86 | if err := e.EncodeToken(start); err != nil { 87 | return err 88 | } 89 | 90 | if a.OSM != nil { 91 | if err := a.OSM.marshalInnerElementsXML(e); err != nil { 92 | return err 93 | } 94 | } 95 | 96 | if a.Old != nil { 97 | if err := marshalInnerChange(e, "old", a.Old); err != nil { 98 | return err 99 | } 100 | } 101 | 102 | if a.New != nil { 103 | if err := marshalInnerChange(e, "new", a.New); err != nil { 104 | return err 105 | } 106 | } 107 | 108 | return e.EncodeToken(start.End()) 109 | } 110 | 111 | // ActionType is a strong type for the different diff actions. 112 | type ActionType string 113 | 114 | // The different types of diff actions. 115 | const ( 116 | ActionCreate ActionType = "create" 117 | ActionModify ActionType = "modify" 118 | ActionDelete ActionType = "delete" 119 | ) 120 | -------------------------------------------------------------------------------- /diff_test.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "encoding/xml" 5 | "os" 6 | "reflect" 7 | "testing" 8 | ) 9 | 10 | func TestDiff_MarshalXML(t *testing.T) { 11 | data := []byte(` 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | `) 24 | 25 | diff := &Diff{} 26 | err := xml.Unmarshal(data, &diff) 27 | if err != nil { 28 | t.Errorf("unmarshal error: %v", err) 29 | } 30 | 31 | if l := len(diff.Actions); l != 2 { 32 | t.Errorf("incorrect num of actions: %v", l) 33 | } 34 | 35 | marshalled, err := xml.MarshalIndent(diff, "", " ") 36 | if err != nil { 37 | t.Errorf("marshal error: %v", err) 38 | } 39 | 40 | if !reflect.DeepEqual(marshalled, data) { 41 | t.Errorf("incorrect marshal") 42 | t.Logf("%v", string(marshalled)) 43 | t.Logf("%v", string(data)) 44 | } 45 | 46 | // specifics 47 | diff = &Diff{} 48 | _, err = xml.Marshal(diff) 49 | if err != nil { 50 | t.Errorf("unable to marshal: %v", err) 51 | } 52 | 53 | // create 54 | diff.Actions = append(diff.Actions, Action{ 55 | Type: ActionCreate, 56 | OSM: &OSM{Nodes: Nodes{{ID: 1}}}, 57 | }) 58 | _, err = xml.Marshal(diff) 59 | if err != nil { 60 | t.Errorf("unable to marshal: %v", err) 61 | } 62 | 63 | // modify 64 | diff.Actions = append(diff.Actions, Action{ 65 | Type: ActionModify, 66 | Old: &OSM{Nodes: Nodes{{ID: 1}}}, 67 | New: &OSM{Nodes: Nodes{{ID: 1}}}, 68 | }) 69 | _, err = xml.Marshal(diff) 70 | if err != nil { 71 | t.Errorf("unable to marshal: %v", err) 72 | } 73 | } 74 | 75 | func TestDiff(t *testing.T) { 76 | data, err := os.ReadFile("testdata/annotated_diff.xml") 77 | if err != nil { 78 | t.Fatalf("unable to read file: %v", err) 79 | } 80 | 81 | diff := &Diff{} 82 | err = xml.Unmarshal(data, &diff) 83 | if err != nil { 84 | t.Errorf("unable to unmarshal: %v", err) 85 | } 86 | 87 | if l := len(diff.Actions); l != 1094 { 88 | t.Fatalf("incorrect number of actions, got %d", l) 89 | } 90 | 91 | // create way 92 | if at := diff.Actions[1075].Type; at != ActionCreate { 93 | t.Errorf("not a create action, %v", at) 94 | } 95 | 96 | way := diff.Actions[1075].Ways[0] 97 | 98 | if id := way.ID; id != 180669361 { 99 | t.Errorf("incorrect way id, got %v", id) 100 | } 101 | 102 | // modify relation 103 | if at := diff.Actions[1088].Type; at != ActionModify { 104 | t.Errorf("not a modify action, %v", at) 105 | } 106 | 107 | oldRelation := diff.Actions[1088].Old.Relations[0] 108 | newRelation := diff.Actions[1088].New.Relations[0] 109 | 110 | if oldRelation.ID != newRelation.ID { 111 | t.Errorf("modify diff is not correct") 112 | t.Logf("old: %v", oldRelation) 113 | t.Logf("new: %v", newRelation) 114 | } 115 | 116 | // delete node 117 | if at := diff.Actions[44].Type; at != ActionDelete { 118 | t.Fatalf("not a delete action, %v", at) 119 | } 120 | 121 | oldNode := diff.Actions[44].Old.Nodes[0] 122 | newNode := diff.Actions[44].New.Nodes[0] 123 | 124 | if oldNode.ID != newNode.ID { 125 | t.Errorf("delete diff is not correct") 126 | t.Logf("old: %v", oldNode) 127 | t.Logf("new: %v", newNode) 128 | } 129 | 130 | if newNode.Visible { 131 | t.Errorf("new node must not be visible") 132 | t.Logf("old: %v", oldNode) 133 | t.Logf("new: %v", newNode) 134 | } 135 | 136 | // should marshal the unmarshalled data 137 | _, err = xml.Marshal(diff) 138 | if err != nil { 139 | t.Errorf("unable to marshal: %v", err) 140 | } 141 | } 142 | 143 | func BenchmarkDiff_Marshal(b *testing.B) { 144 | data, err := os.ReadFile("testdata/annotated_diff.xml") 145 | if err != nil { 146 | b.Fatalf("unable to read file: %v", err) 147 | } 148 | 149 | diff := &Diff{} 150 | err = xml.Unmarshal(data, &diff) 151 | if err != nil { 152 | b.Fatalf("unmarshal error: %v", err) 153 | } 154 | 155 | b.ReportAllocs() 156 | b.ResetTimer() 157 | for i := 0; i < b.N; i++ { 158 | _, err := xml.Marshal(diff) 159 | if err != nil { 160 | b.Fatalf("marshal error: %v", err) 161 | } 162 | } 163 | } 164 | 165 | func BenchmarkDiff_Unmarshal(b *testing.B) { 166 | data, err := os.ReadFile("testdata/annotated_diff.xml") 167 | if err != nil { 168 | b.Fatalf("unable to read file: %v", err) 169 | } 170 | 171 | b.ReportAllocs() 172 | b.ResetTimer() 173 | for i := 0; i < b.N; i++ { 174 | diff := &Diff{} 175 | err := xml.Unmarshal(data, &diff) 176 | if err != nil { 177 | b.Fatalf("unmarshal error: %v", err) 178 | } 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/paulmach/osm 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2 7 | github.com/paulmach/orb v0.1.3 8 | github.com/paulmach/protoscan v0.2.1 9 | golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 10 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 11 | google.golang.org/protobuf v1.27.1 12 | ) 13 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2 h1:ISaMhBq2dagaoptFGUyywT5SzpysCbHofX3sCNw1djo= 2 | github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2/go.mod h1:2yDaWzisHKoQoxm+EU4YgKBaD7g1M0pxy7THWG44Lro= 3 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 4 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 5 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 6 | github.com/paulmach/orb v0.1.3 h1:Wa1nzU269Zv7V9paVEY1COWW8FCqv4PC/KJRbJSimpM= 7 | github.com/paulmach/orb v0.1.3/go.mod h1:VFlX/8C+IQ1p6FTRRKzKoOPJnvEtA5G0Veuqwbu//Vk= 8 | github.com/paulmach/protoscan v0.2.1 h1:rM0FpcTjUMvPUNk2BhPJrreDKetq43ChnL+x1sRg8O8= 9 | github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= 10 | golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w= 11 | golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 12 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 13 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 14 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 15 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 16 | google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= 17 | google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 18 | -------------------------------------------------------------------------------- /internal/mputil/join.go: -------------------------------------------------------------------------------- 1 | package mputil 2 | 3 | // Join will join a set of segments into a set of connected MultiSegments. 4 | func Join(segments []Segment) []MultiSegment { 5 | lists := []MultiSegment{} 6 | segments = compact(segments) 7 | 8 | // matches are removed from `segments` and put into the current 9 | // group, so when `segments` is empty we're done. 10 | for len(segments) != 0 { 11 | current := MultiSegment{segments[len(segments)-1]} 12 | segments = segments[:len(segments)-1] 13 | 14 | // if the current group is a ring, we're done. 15 | // else add in all the lines. 16 | for len(segments) != 0 && !current.First().Equal(current.Last()) { 17 | first := current.First() 18 | last := current.Last() 19 | 20 | foundAt := -1 21 | for i, segment := range segments { 22 | if last.Equal(segment.First()) { 23 | // nice fit at the end of current 24 | 25 | segment.Line = segment.Line[1:] 26 | current = append(current, segment) 27 | foundAt = i 28 | break 29 | } else if last.Equal(segment.Last()) { 30 | // reverse it and it'll fit at the end 31 | segment.Reverse() 32 | 33 | segment.Line = segment.Line[1:] 34 | current = append(current, segment) 35 | foundAt = i 36 | break 37 | } else if first.Equal(segment.Last()) { 38 | // nice fit at the start of current 39 | segment.Line = segment.Line[:len(segment.Line)-1] 40 | current = append(MultiSegment{segment}, current...) 41 | 42 | foundAt = i 43 | break 44 | } else if first.Equal(segment.First()) { 45 | // reverse it and it'll fit at the start 46 | segment.Reverse() 47 | 48 | segment.Line = segment.Line[:len(segment.Line)-1] 49 | current = append(MultiSegment{segment}, current...) 50 | 51 | foundAt = i 52 | break 53 | } 54 | } 55 | 56 | if foundAt == -1 { 57 | break // Invalid geometry (dangling way, unclosed ring) 58 | } 59 | 60 | // remove the found/matched segment from the list. 61 | if foundAt < len(segments)/2 { 62 | // first half, shift up 63 | for i := foundAt; i > 0; i-- { 64 | segments[i] = segments[i-1] 65 | } 66 | segments = segments[1:] 67 | } else { 68 | // second half, shift down 69 | for i := foundAt + 1; i < len(segments); i++ { 70 | segments[i-1] = segments[i] 71 | } 72 | segments = segments[:len(segments)-1] 73 | } 74 | } 75 | 76 | lists = append(lists, current) 77 | } 78 | 79 | return lists 80 | } 81 | 82 | func compact(ms MultiSegment) MultiSegment { 83 | at := 0 84 | for _, s := range ms { 85 | if len(s.Line) <= 1 { 86 | continue 87 | } 88 | 89 | ms[at] = s 90 | at++ 91 | } 92 | 93 | return ms[:at] 94 | } 95 | -------------------------------------------------------------------------------- /internal/mputil/mputil.go: -------------------------------------------------------------------------------- 1 | package mputil 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/paulmach/orb" 7 | "github.com/paulmach/osm" 8 | ) 9 | 10 | // Segment is a section of a multipolygon with some extra information 11 | // on the member it came from. 12 | type Segment struct { 13 | Index uint32 14 | Orientation orb.Orientation 15 | Reversed bool 16 | Line orb.LineString 17 | } 18 | 19 | // Reverse will reverse the line string of the segment. 20 | func (s *Segment) Reverse() { 21 | s.Reversed = !s.Reversed 22 | s.Line.Reverse() 23 | } 24 | 25 | // First returns the first point in the segment linestring. 26 | func (s Segment) First() orb.Point { 27 | return s.Line[0] 28 | } 29 | 30 | // Last returns the last point in the segment linestring. 31 | func (s Segment) Last() orb.Point { 32 | return s.Line[len(s.Line)-1] 33 | } 34 | 35 | // MultiSegment is an ordered set of segments that form a continuous 36 | // section of a multipolygon. 37 | type MultiSegment []Segment 38 | 39 | // First returns the first point in the list of linestrings. 40 | func (ms MultiSegment) First() orb.Point { 41 | return ms[0].Line[0] 42 | } 43 | 44 | // Last returns the last point in the list of linestrings. 45 | func (ms MultiSegment) Last() orb.Point { 46 | line := ms[len(ms)-1].Line 47 | return line[len(line)-1] 48 | } 49 | 50 | // LineString converts a multisegment into a geo linestring object. 51 | func (ms MultiSegment) LineString() orb.LineString { 52 | length := 0 53 | for _, s := range ms { 54 | length += len(s.Line) 55 | } 56 | 57 | line := make(orb.LineString, 0, length) 58 | for _, s := range ms { 59 | line = append(line, s.Line...) 60 | } 61 | 62 | return line 63 | } 64 | 65 | // Ring converts the multisegment to a ring of the given orientation. 66 | // It uses the orientation on the members if possible. 67 | func (ms MultiSegment) Ring(o orb.Orientation) orb.Ring { 68 | length := 0 69 | for _, s := range ms { 70 | length += len(s.Line) 71 | } 72 | 73 | ring := make(orb.Ring, 0, length) 74 | 75 | haveOrient := false 76 | reversed := false 77 | for _, s := range ms { 78 | if s.Orientation != 0 { 79 | haveOrient = true 80 | 81 | // if s.Orientation == o && s.Reversed { 82 | // reversed = true 83 | // } 84 | // if s.Orientation != 0 && !s.Reversed { 85 | // reversed = true 86 | // } 87 | 88 | if (s.Orientation == o) == s.Reversed { 89 | reversed = true 90 | } 91 | } 92 | 93 | ring = append(ring, s.Line...) 94 | } 95 | 96 | if (haveOrient && reversed) || (!haveOrient && ring.Orientation() != o) { 97 | ring.Reverse() 98 | } 99 | 100 | return ring 101 | } 102 | 103 | // Orientation computes the orientation of a multisegment like if it was ring. 104 | func (ms MultiSegment) Orientation() orb.Orientation { 105 | area := 0.0 106 | prev := ms.First() 107 | 108 | // implicitly move everything to near the origin to help with roundoff 109 | offset := prev 110 | for _, segment := range ms { 111 | for _, point := range segment.Line { 112 | area += (prev[0]-offset[0])*(point[1]-offset[1]) - 113 | (point[0]-offset[0])*(prev[1]-offset[1]) 114 | 115 | prev = point 116 | } 117 | } 118 | 119 | if area > 0 { 120 | return orb.CCW 121 | } 122 | 123 | return orb.CW 124 | } 125 | 126 | // Group will take the members and group them by inner our outer parts 127 | // of the relation. Will also build the way geometry. 128 | func Group( 129 | members osm.Members, 130 | ways map[osm.WayID]*osm.Way, 131 | at time.Time, 132 | ) (outer, inner []Segment, tainted bool) { 133 | for i, m := range members { 134 | if m.Type != osm.TypeWay { 135 | continue 136 | } 137 | 138 | w := ways[osm.WayID(m.Ref)] 139 | if w == nil { 140 | tainted = true 141 | continue // could be not found error, or something else. 142 | } 143 | 144 | line := w.LineStringAt(at) 145 | if len(line) != len(w.Nodes) { 146 | tainted = true 147 | } 148 | 149 | // zero length ways exist and don't make any sense when 150 | // building the multipolygon rings. 151 | if len(line) == 0 { 152 | continue 153 | } 154 | 155 | l := Segment{ 156 | Index: uint32(i), 157 | Orientation: m.Orientation, 158 | Reversed: false, 159 | Line: line, 160 | } 161 | 162 | if m.Role == "outer" { 163 | if l.Orientation == orb.CW { 164 | l.Reverse() 165 | } 166 | outer = append(outer, l) 167 | } else if m.Role == "inner" { 168 | if l.Orientation == orb.CCW { 169 | l.Reverse() 170 | } 171 | inner = append(inner, l) 172 | } 173 | } 174 | 175 | return outer, inner, tainted 176 | } 177 | -------------------------------------------------------------------------------- /json.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "encoding/json" 5 | "encoding/xml" 6 | ) 7 | 8 | // CustomJSONMarshaler can be set to have the code use a different 9 | // json marshaler than the default in the standard library. 10 | // One use case in enabling `github.com/json-iterator/go` 11 | // with something like this: 12 | // 13 | // import ( 14 | // jsoniter "github.com/json-iterator/go" 15 | // "github.com/paulmach/osm" 16 | // ) 17 | // 18 | // var c = jsoniter.Config{ 19 | // EscapeHTML: true, 20 | // SortMapKeys: false, 21 | // MarshalFloatWith6Digits: true, 22 | // }.Froze() 23 | // 24 | // osm.CustomJSONMarshaler = c 25 | // osm.CustomJSONUnmarshaler = c 26 | // 27 | // Note that any errors encountered during marshaling will be different. 28 | var CustomJSONMarshaler interface { 29 | Marshal(v interface{}) ([]byte, error) 30 | } 31 | 32 | // CustomJSONUnmarshaler can be set to have the code use a different 33 | // json unmarshaler than the default in the standard library. 34 | // One use case in enabling `github.com/json-iterator/go` 35 | // with something like this: 36 | // 37 | // import ( 38 | // jsoniter "github.com/json-iterator/go" 39 | // "github.com/paulmach/osm" 40 | // ) 41 | // 42 | // var c = jsoniter.Config{ 43 | // EscapeHTML: true, 44 | // SortMapKeys: false, 45 | // MarshalFloatWith6Digits: true, 46 | // }.Froze() 47 | // 48 | // osm.CustomJSONMarshaler = c 49 | // osm.CustomJSONUnmarshaler = c 50 | // 51 | // Note that any errors encountered during unmarshaling will be different. 52 | var CustomJSONUnmarshaler interface { 53 | Unmarshal(data []byte, v interface{}) error 54 | } 55 | 56 | func marshalJSON(v interface{}) ([]byte, error) { 57 | if CustomJSONMarshaler == nil { 58 | return json.Marshal(v) 59 | } 60 | 61 | return CustomJSONMarshaler.Marshal(v) 62 | } 63 | 64 | func unmarshalJSON(data []byte, v interface{}) error { 65 | if CustomJSONUnmarshaler == nil { 66 | return json.Unmarshal(data, v) 67 | } 68 | 69 | return CustomJSONUnmarshaler.Unmarshal(data, v) 70 | } 71 | 72 | type nocopyRawMessage []byte 73 | 74 | func (m *nocopyRawMessage) UnmarshalJSON(data []byte) error { 75 | *m = data 76 | return nil 77 | } 78 | 79 | // xmlNameJSONTypeNode is kind of a hack to encode the proper json 80 | // object type attribute for this struct type. 81 | type xmlNameJSONTypeNode xml.Name 82 | 83 | func (x xmlNameJSONTypeNode) MarshalJSON() ([]byte, error) { 84 | return []byte(`"node"`), nil 85 | } 86 | 87 | func (x xmlNameJSONTypeNode) UnmarshalJSON(data []byte) error { 88 | return nil 89 | } 90 | 91 | // xmlNameJSONTypeWay is kind of a hack to encode the proper json 92 | // object type attribute for this struct type. 93 | type xmlNameJSONTypeWay xml.Name 94 | 95 | func (x xmlNameJSONTypeWay) MarshalJSON() ([]byte, error) { 96 | return []byte(`"way"`), nil 97 | } 98 | 99 | func (x xmlNameJSONTypeWay) UnmarshalJSON(data []byte) error { 100 | return nil 101 | } 102 | 103 | // xmlNameJSONTypeRel is kind of a hack to encode the proper json 104 | // object type attribute for this struct type. 105 | type xmlNameJSONTypeRel xml.Name 106 | 107 | func (x xmlNameJSONTypeRel) MarshalJSON() ([]byte, error) { 108 | return []byte(`"relation"`), nil 109 | } 110 | 111 | func (x xmlNameJSONTypeRel) UnmarshalJSON(data []byte) error { 112 | return nil 113 | } 114 | 115 | // xmlNameJSONTypeCS is kind of a hack to encode the proper json 116 | // object type attribute for this struct type. 117 | type xmlNameJSONTypeCS xml.Name 118 | 119 | func (x xmlNameJSONTypeCS) MarshalJSON() ([]byte, error) { 120 | return []byte(`"changeset"`), nil 121 | } 122 | 123 | func (x xmlNameJSONTypeCS) UnmarshalJSON(data []byte) error { 124 | return nil 125 | } 126 | 127 | // xmlNameJSONTypeUser is kind of a hack to encode the proper json 128 | // object type attribute for this struct type. 129 | type xmlNameJSONTypeUser xml.Name 130 | 131 | func (x xmlNameJSONTypeUser) MarshalJSON() ([]byte, error) { 132 | return []byte(`"user"`), nil 133 | } 134 | 135 | func (x xmlNameJSONTypeUser) UnmarshalJSON(data []byte) error { 136 | return nil 137 | } 138 | 139 | // xmlNameJSONTypeNote is kind of a hack to encode the proper json 140 | // object type attribute for this struct type. 141 | type xmlNameJSONTypeNote xml.Name 142 | 143 | func (x xmlNameJSONTypeNote) MarshalJSON() ([]byte, error) { 144 | return []byte(`"note"`), nil 145 | } 146 | 147 | func (x xmlNameJSONTypeNote) UnmarshalJSON(data []byte) error { 148 | return nil 149 | } 150 | -------------------------------------------------------------------------------- /node.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "sort" 5 | "time" 6 | 7 | "github.com/paulmach/orb" 8 | ) 9 | 10 | // NodeID corresponds the primary key of a node. 11 | // The node id + version uniquely identify a node. 12 | type NodeID int64 13 | 14 | // ObjectID is a helper returning the object id for this node id. 15 | func (id NodeID) ObjectID(v int) ObjectID { 16 | return ObjectID(id.ElementID(v)) 17 | } 18 | 19 | // FeatureID is a helper returning the feature id for this node id. 20 | func (id NodeID) FeatureID() FeatureID { 21 | return FeatureID(nodeMask | (id << versionBits)) 22 | } 23 | 24 | // ElementID is a helper to convert the id to an element id. 25 | func (id NodeID) ElementID(v int) ElementID { 26 | return id.FeatureID().ElementID(v) 27 | } 28 | 29 | // Node is an osm point and allows for marshalling to/from osm xml. 30 | type Node struct { 31 | XMLName xmlNameJSONTypeNode `xml:"node" json:"type"` 32 | ID NodeID `xml:"id,attr" json:"id"` 33 | Lat float64 `xml:"lat,attr" json:"lat"` 34 | Lon float64 `xml:"lon,attr" json:"lon"` 35 | User string `xml:"user,attr" json:"user,omitempty"` 36 | UserID UserID `xml:"uid,attr" json:"uid,omitempty"` 37 | Visible bool `xml:"visible,attr" json:"visible"` 38 | Version int `xml:"version,attr" json:"version,omitempty"` 39 | ChangesetID ChangesetID `xml:"changeset,attr" json:"changeset,omitempty"` 40 | Timestamp time.Time `xml:"timestamp,attr" json:"timestamp"` 41 | Tags Tags `xml:"tag" json:"tags,omitempty"` 42 | 43 | // Committed, is the estimated time this object was committed 44 | // and made visible in the central OSM database. 45 | Committed *time.Time `xml:"committed,attr,omitempty" json:"committed,omitempty"` 46 | } 47 | 48 | // ObjectID returns the object id of the node. 49 | func (n *Node) ObjectID() ObjectID { 50 | return n.ID.ObjectID(n.Version) 51 | } 52 | 53 | // FeatureID returns the feature id of the node. 54 | func (n *Node) FeatureID() FeatureID { 55 | return n.ID.FeatureID() 56 | } 57 | 58 | // ElementID returns the element id of the node. 59 | func (n *Node) ElementID() ElementID { 60 | return n.ID.ElementID(n.Version) 61 | } 62 | 63 | // CommittedAt returns the best estimate on when this element 64 | // became was written/committed into the database. 65 | func (n *Node) CommittedAt() time.Time { 66 | if n.Committed != nil { 67 | return *n.Committed 68 | } 69 | 70 | return n.Timestamp 71 | } 72 | 73 | // TagMap returns the element tags as a key/value map. 74 | func (n *Node) TagMap() map[string]string { 75 | return n.Tags.Map() 76 | } 77 | 78 | // Point returns the orb.Point location for the node. 79 | // Will be (0, 0) for "deleted" nodes. 80 | func (n *Node) Point() orb.Point { 81 | return orb.Point{n.Lon, n.Lat} 82 | } 83 | 84 | // Nodes is a list of nodes with helper functions on top. 85 | type Nodes []*Node 86 | 87 | // IDs returns the ids for all the ways. 88 | func (ns Nodes) IDs() []NodeID { 89 | result := make([]NodeID, len(ns)) 90 | for i, n := range ns { 91 | result[i] = n.ID 92 | } 93 | 94 | return result 95 | } 96 | 97 | // FeatureIDs returns the feature ids for all the nodes. 98 | func (ns Nodes) FeatureIDs() FeatureIDs { 99 | r := make(FeatureIDs, len(ns)) 100 | for i, n := range ns { 101 | r[i] = n.FeatureID() 102 | } 103 | 104 | return r 105 | } 106 | 107 | // ElementIDs returns the element ids for all the nodes. 108 | func (ns Nodes) ElementIDs() ElementIDs { 109 | r := make(ElementIDs, len(ns)) 110 | for i, n := range ns { 111 | r[i] = n.ElementID() 112 | } 113 | 114 | return r 115 | } 116 | 117 | type nodesSort Nodes 118 | 119 | // SortByIDVersion will sort the set of nodes first by id and then version 120 | // in ascending order. 121 | func (ns Nodes) SortByIDVersion() { 122 | sort.Sort(nodesSort(ns)) 123 | } 124 | 125 | func (ns nodesSort) Len() int { return len(ns) } 126 | func (ns nodesSort) Swap(i, j int) { ns[i], ns[j] = ns[j], ns[i] } 127 | func (ns nodesSort) Less(i, j int) bool { 128 | if ns[i].ID == ns[j].ID { 129 | return ns[i].Version < ns[j].Version 130 | } 131 | 132 | return ns[i].ID < ns[j].ID 133 | } 134 | -------------------------------------------------------------------------------- /node_test.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "encoding/xml" 7 | "reflect" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func TestNode(t *testing.T) { 13 | data := []byte(``) 14 | 15 | n := Node{} 16 | err := xml.Unmarshal(data, &n) 17 | if err != nil { 18 | t.Fatalf("unmarshal error: %v", err) 19 | } 20 | 21 | if v := n.ID; v != 123 { 22 | t.Errorf("incorrect id, got %v", v) 23 | } 24 | 25 | if v := n.ChangesetID; v != 456 { 26 | t.Errorf("incorrect changeset, got %v", v) 27 | } 28 | 29 | if v := n.Timestamp; v != time.Date(2014, 4, 10, 0, 43, 05, 0, time.UTC) { 30 | t.Errorf("incorrect timestamp, got %v", v) 31 | } 32 | 33 | if v := n.Version; v != 1 { 34 | t.Errorf("incorrect version, got %v", v) 35 | } 36 | 37 | if v := n.Visible; !v { 38 | t.Errorf("incorrect visible, got %v", v) 39 | } 40 | 41 | if v := n.User; v != "user" { 42 | t.Errorf("incorrect user, got %v", v) 43 | } 44 | 45 | if v := n.UserID; v != 1357 { 46 | t.Errorf("incorrect user id, got %v", v) 47 | } 48 | 49 | if v := n.Lat; v != 50.7107023 { 50 | t.Errorf("incorrect lat, got %v", v) 51 | } 52 | 53 | if v := n.Lon; v != 6.0043943 { 54 | t.Errorf("incorrect lon, got %v", v) 55 | } 56 | } 57 | 58 | func TestNode_MarshalJSON(t *testing.T) { 59 | n := Node{ 60 | ID: 123, 61 | } 62 | 63 | data, err := json.Marshal(n) 64 | if err != nil { 65 | t.Fatalf("marshal error: %v", err) 66 | } 67 | 68 | if !bytes.Equal(data, []byte(`{"type":"node","id":123,"lat":0,"lon":0,"visible":false,"timestamp":"0001-01-01T00:00:00Z"}`)) { 69 | t.Errorf("incorrect json: %v", string(data)) 70 | } 71 | } 72 | 73 | func TestNode_MarshalXML(t *testing.T) { 74 | n := Node{ 75 | ID: 123, 76 | } 77 | 78 | data, err := xml.Marshal(n) 79 | if err != nil { 80 | t.Fatalf("xml marshal error: %v", err) 81 | } 82 | 83 | expected := `` 84 | if !bytes.Equal(data, []byte(expected)) { 85 | t.Errorf("incorrect marshal, got: %s", string(data)) 86 | } 87 | } 88 | 89 | func TestNodes_ids(t *testing.T) { 90 | ns := Nodes{ 91 | {ID: 1, Version: 3}, 92 | {ID: 2, Version: 4}, 93 | } 94 | 95 | eids := ElementIDs{NodeID(1).ElementID(3), NodeID(2).ElementID(4)} 96 | if ids := ns.ElementIDs(); !reflect.DeepEqual(ids, eids) { 97 | t.Errorf("incorrect element ids: %v", ids) 98 | } 99 | 100 | fids := FeatureIDs{NodeID(1).FeatureID(), NodeID(2).FeatureID()} 101 | if ids := ns.FeatureIDs(); !reflect.DeepEqual(ids, fids) { 102 | t.Errorf("incorrect feature ids: %v", ids) 103 | } 104 | 105 | nids := []NodeID{1, 2} 106 | if ids := ns.IDs(); !reflect.DeepEqual(ids, nids) { 107 | t.Errorf("incorrect node ids: %v", nids) 108 | } 109 | } 110 | 111 | func TestNodes_SortByIDVersion(t *testing.T) { 112 | ns := Nodes{ 113 | {ID: 7, Version: 3}, 114 | {ID: 2, Version: 4}, 115 | {ID: 5, Version: 2}, 116 | {ID: 5, Version: 3}, 117 | {ID: 5, Version: 4}, 118 | {ID: 3, Version: 4}, 119 | {ID: 4, Version: 4}, 120 | {ID: 9, Version: 4}, 121 | } 122 | 123 | ns.SortByIDVersion() 124 | 125 | eids := ElementIDs{ 126 | NodeID(2).ElementID(4), 127 | NodeID(3).ElementID(4), 128 | NodeID(4).ElementID(4), 129 | NodeID(5).ElementID(2), 130 | NodeID(5).ElementID(3), 131 | NodeID(5).ElementID(4), 132 | NodeID(7).ElementID(3), 133 | NodeID(9).ElementID(4), 134 | } 135 | 136 | if ids := ns.ElementIDs(); !reflect.DeepEqual(ids, eids) { 137 | t.Errorf("incorrect sort: %v", eids) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /note.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "encoding/xml" 5 | "time" 6 | ) 7 | 8 | // NoteID is the unique identifier for an osm note. 9 | type NoteID int64 10 | 11 | // ObjectID is a helper returning the object id for this note id. 12 | func (id NoteID) ObjectID() ObjectID { 13 | return ObjectID(noteMask | (id << versionBits)) 14 | } 15 | 16 | const dateLayout = "2006-01-02 15:04:05 MST" 17 | 18 | // Date is an object to decode the date format used in the osm notes xml api. 19 | // The format is '2006-01-02 15:04:05 MST'. 20 | type Date struct { 21 | time.Time 22 | } 23 | 24 | // UnmarshalXML is meant to decode the osm note date formation of 25 | // '2006-01-02 15:04:05 MST' into a time.Time object. 26 | func (d *Date) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) error { 27 | var s string 28 | err := dec.DecodeElement(&s, &start) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | d.Time, err = time.Parse(dateLayout, s) 34 | return err 35 | } 36 | 37 | // MarshalXML is meant to encode the time.Time into the osm note date formation 38 | // of '2006-01-02 15:04:05 MST'. 39 | func (d Date) MarshalXML(e *xml.Encoder, start xml.StartElement) error { 40 | return e.EncodeElement(d.Format(dateLayout), start) 41 | } 42 | 43 | // MarshalJSON will return null if the date is empty. 44 | func (d Date) MarshalJSON() ([]byte, error) { 45 | if d.IsZero() { 46 | return []byte(`null`), nil 47 | } 48 | return marshalJSON(d.Time) 49 | } 50 | 51 | // Notes is a collection of notes with some helpers attached. 52 | type Notes []*Note 53 | 54 | // Note is information for other mappers dropped at a map location. 55 | type Note struct { 56 | XMLName xmlNameJSONTypeNote `xml:"note" json:"type"` 57 | ID NoteID `xml:"id" json:"id"` 58 | Lat float64 `xml:"lat,attr" json:"lat"` 59 | Lon float64 `xml:"lon,attr" json:"lon"` 60 | URL string `xml:"url" json:"url,omitempty"` 61 | CommentURL string `xml:"comment_url" json:"comment_url,omitempty"` 62 | CloseURL string `xml:"close_url" json:"close_url,omitempty"` 63 | ReopenURL string `xml:"reopen_url" json:"reopen_url,omitempty"` 64 | DateCreated Date `xml:"date_created" json:"date_created"` 65 | DateClosed Date `xml:"date_closed" json:"date_closed,omitempty"` 66 | Status NoteStatus `xml:"status" json:"status,omitempty"` 67 | Comments []*NoteComment `xml:"comments>comment" json:"comments"` 68 | } 69 | 70 | // NoteComment is a comment on a note. 71 | type NoteComment struct { 72 | XMLName xml.Name `xml:"comment" json:"-"` 73 | Date Date `xml:"date" json:"date"` 74 | UserID UserID `xml:"uid" json:"uid,omitempty"` 75 | User string `xml:"user" json:"user,omitempty"` 76 | UserURL string `xml:"user_url" json:"user_url,omitempty"` 77 | Action NoteCommentAction `xml:"action" json:"action"` 78 | Text string `xml:"text" json:"text"` 79 | HTML string `xml:"html" json:"html"` 80 | } 81 | 82 | // ObjectID returns the object id of the note. 83 | func (n *Note) ObjectID() ObjectID { 84 | return n.ID.ObjectID() 85 | } 86 | 87 | // NoteCommentAction are actions that a note comment took. 88 | type NoteCommentAction string 89 | 90 | // The set of comment actions. 91 | var ( 92 | NoteCommentOpened NoteCommentAction = "opened" 93 | NoteCommentComment NoteCommentAction = "commented" 94 | NoteCommentClosed NoteCommentAction = "closed" 95 | ) 96 | 97 | // NoteStatus is the status of the note. 98 | type NoteStatus string 99 | 100 | // A note can be open or closed. 101 | var ( 102 | NoteOpen NoteStatus = "open" 103 | NoteClosed NoteStatus = "closed" 104 | ) 105 | -------------------------------------------------------------------------------- /note_test.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "encoding/xml" 7 | "reflect" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func TestNote_UnmarshalXML(t *testing.T) { 13 | rawXML := []byte(` 14 | 15 | 1302953 16 | note url 17 | comment url 18 | close url 19 | reopen url 20 | 2018-02-17 17:34:48 UTC 21 | closed 22 | 2018-02-17 22:16:03 UTC 23 | 24 | 25 | 2018-02-17 17:34:48 UTC 26 | 251221 27 | spiregrain 28 | user url 29 | opened 30 | comment text 31 | comment html 32 | 33 | 34 | 2018-02-17 22:16:03 UTC 35 | 251221 36 | spiregrain 37 | https://api.openstreetmap.org/user/spiregrain 38 | closed 39 | 40 |

41 | 42 | 43 | `) 44 | n := &Note{} 45 | 46 | err := xml.Unmarshal(rawXML, &n) 47 | if err != nil { 48 | t.Fatalf("unmarshal error: %v", err) 49 | } 50 | 51 | if v := n.ID; v != 1302953 { 52 | t.Errorf("incorrect value: %v", v) 53 | } 54 | 55 | if v := n.Lat; v != 51.5438971 { 56 | t.Errorf("incorrect value: %v", v) 57 | } 58 | 59 | if v := n.Lon; v != 0.0088488 { 60 | t.Errorf("incorrect value: %v", v) 61 | } 62 | 63 | if v := n.URL; v != "note url" { 64 | t.Errorf("incorrect value: %v", v) 65 | } 66 | 67 | if v := n.CommentURL; v != "comment url" { 68 | t.Errorf("incorrect value: %v", v) 69 | } 70 | 71 | if v := n.CloseURL; v != "close url" { 72 | t.Errorf("incorrect value: %v", v) 73 | } 74 | 75 | if v := n.ReopenURL; v != "reopen url" { 76 | t.Errorf("incorrect value: %v", v) 77 | } 78 | 79 | if v := n.DateCreated; !v.Equal(time.Date(2018, 2, 17, 17, 34, 48, 0, time.UTC)) { 80 | t.Errorf("incorrect value: %v", v) 81 | } 82 | 83 | if v := n.DateClosed; !v.Equal(time.Date(2018, 2, 17, 22, 16, 3, 0, time.UTC)) { 84 | t.Errorf("incorrect value: %v", v) 85 | } 86 | 87 | if v := n.Status; v != NoteClosed { 88 | t.Errorf("incorrect value: %v", v) 89 | } 90 | 91 | // comments 92 | if v := len(n.Comments); v != 2 { 93 | t.Errorf("incorrect value: %v", v) 94 | } 95 | 96 | if v := n.Comments[0].Date; !v.Equal(time.Date(2018, 2, 17, 17, 34, 48, 0, time.UTC)) { 97 | t.Errorf("incorrect value: %v", v) 98 | } 99 | 100 | if v := n.Comments[0].UserID; v != 251221 { 101 | t.Errorf("incorrect value: %v", v) 102 | } 103 | 104 | if v := n.Comments[0].User; v != "spiregrain" { 105 | t.Errorf("incorrect value: %v", v) 106 | } 107 | 108 | if v := n.Comments[0].UserURL; v != "user url" { 109 | t.Errorf("incorrect value: %v", v) 110 | } 111 | 112 | if v := n.Comments[0].Action; v != NoteCommentOpened { 113 | t.Errorf("incorrect value: %v", v) 114 | } 115 | 116 | if v := n.Comments[0].Text; v != "comment text" { 117 | t.Errorf("incorrect value: %v", v) 118 | } 119 | 120 | if v := n.Comments[0].HTML; v != "comment html" { 121 | t.Errorf("incorrect value: %v", v) 122 | } 123 | 124 | // should marshal correctly. 125 | data, err := xml.Marshal(n) 126 | if err != nil { 127 | t.Fatalf("xml marshal error: %v", err) 128 | } 129 | 130 | nn := &Note{} 131 | err = xml.Unmarshal(data, &nn) 132 | if err != nil { 133 | t.Fatalf("xml unmarshal error: %v", err) 134 | } 135 | 136 | if !reflect.DeepEqual(nn, n) { 137 | t.Errorf("incorrect marshal") 138 | t.Log(nn) 139 | t.Log(n) 140 | } 141 | } 142 | 143 | func TestNote_MarshalJSON(t *testing.T) { 144 | n := Note{ 145 | ID: 123, 146 | Lat: 10, 147 | Lon: 20, 148 | DateCreated: Date{time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)}, 149 | } 150 | 151 | data, err := json.Marshal(n) 152 | if err != nil { 153 | t.Fatalf("marshal error: %v", err) 154 | } 155 | 156 | if !bytes.Equal(data, []byte(`{"type":"note","id":123,"lat":10,"lon":20,"date_created":"2018-01-01T00:00:00Z","date_closed":null,"comments":null}`)) { 157 | t.Errorf("incorrect json: %v", string(data)) 158 | } 159 | } 160 | 161 | func TestNote_ObjectID(t *testing.T) { 162 | n := Note{ID: 123} 163 | id := n.ObjectID() 164 | 165 | if v := id.Type(); v != TypeNote { 166 | t.Errorf("incorrect type: %v", v) 167 | } 168 | 169 | if v := id.Ref(); v != 123 { 170 | t.Errorf("incorrect ref: %v", 123) 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /object.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | // ObjectID encodes the type and ref of an osm object, 10 | // e.g. nodes, ways, relations, changesets, notes and users. 11 | type ObjectID int64 12 | 13 | // Type returns the Type of the object. 14 | func (id ObjectID) Type() Type { 15 | switch id & typeMask { 16 | case nodeMask: 17 | return TypeNode 18 | case wayMask: 19 | return TypeWay 20 | case relationMask: 21 | return TypeRelation 22 | case changesetMask: 23 | return TypeChangeset 24 | case noteMask: 25 | return TypeNote 26 | case userMask: 27 | return TypeUser 28 | case boundsMask: 29 | return TypeBounds 30 | } 31 | 32 | panic("unknown type") 33 | } 34 | 35 | // Ref returns the ID reference for the object. Not unique without the type. 36 | func (id ObjectID) Ref() int64 { 37 | return int64((id & refMask) >> versionBits) 38 | } 39 | 40 | // Version returns the version of the object. 41 | // Will return 0 if the object doesn't have versions like users, notes and changesets. 42 | func (id ObjectID) Version() int { 43 | return int(id & (versionMask)) 44 | } 45 | 46 | // String returns "type/ref:version" for the object. 47 | func (id ObjectID) String() string { 48 | if id.Version() == 0 { 49 | return fmt.Sprintf("%s/%d:-", id.Type(), id.Ref()) 50 | } 51 | 52 | return fmt.Sprintf("%s/%d:%d", id.Type(), id.Ref(), id.Version()) 53 | } 54 | 55 | // ParseObjectID takes a string and tries to determine the object id from it. 56 | // The string must be formatted as "type/id:version", the same as the result of the String method. 57 | func ParseObjectID(s string) (ObjectID, error) { 58 | parts := strings.Split(s, "/") 59 | if len(parts) != 2 { 60 | return 0, fmt.Errorf("invalid element id: %v", s) 61 | } 62 | 63 | parts2 := strings.Split(parts[1], ":") 64 | if l := len(parts2); l == 0 || l > 2 { 65 | return 0, fmt.Errorf("invalid element id: %v", s) 66 | } 67 | 68 | var version int 69 | ref, err := strconv.ParseInt(parts2[0], 10, 64) 70 | if err != nil { 71 | return 0, fmt.Errorf("invalid element id: %v: %v", s, err) 72 | } 73 | 74 | if len(parts2) == 2 && parts2[1] != "-" { 75 | v, e := strconv.ParseInt(parts2[1], 10, 64) 76 | if e != nil { 77 | return 0, fmt.Errorf("invalid element id: %v: %v", s, err) 78 | } 79 | version = int(v) 80 | } 81 | 82 | oid, err := Type(parts[0]).objectID(ref, version) 83 | if err != nil { 84 | return 0, fmt.Errorf("invalid element id: %v: %v", s, err) 85 | } 86 | 87 | return oid, nil 88 | } 89 | 90 | // An Object represents a Node, Way, Relation, Changeset, Note or User only. 91 | type Object interface { 92 | ObjectID() ObjectID 93 | 94 | // private is so that **ID types don't implement this interface. 95 | private() 96 | } 97 | 98 | func (n *Node) private() {} 99 | func (w *Way) private() {} 100 | func (r *Relation) private() {} 101 | func (c *Changeset) private() {} 102 | func (n *Note) private() {} 103 | func (u *User) private() {} 104 | func (b *Bounds) private() {} 105 | 106 | // Objects is a set of objects with some helpers 107 | type Objects []Object 108 | 109 | // ObjectIDs returns a slice of the object ids of the osm objects. 110 | func (os Objects) ObjectIDs() ObjectIDs { 111 | if len(os) == 0 { 112 | return nil 113 | } 114 | 115 | ids := make(ObjectIDs, 0, len(os)) 116 | for _, o := range os { 117 | ids = append(ids, o.ObjectID()) 118 | } 119 | 120 | return ids 121 | } 122 | 123 | // ObjectIDs is a slice of ObjectIDs with some helpers on top. 124 | type ObjectIDs []ObjectID 125 | 126 | // A Scanner reads osm data from planet dump files. 127 | // It is based on the bufio.Scanner, common usage. 128 | // Scanners are not safe for parallel use. One should feed the 129 | // objects into their own channel and have workers read from that. 130 | // 131 | // s := scanner.New(r) 132 | // defer s.Close() 133 | // 134 | // for s.Next() { 135 | // o := s.Object() 136 | // // do something 137 | // } 138 | // 139 | // if s.Err() != nil { 140 | // // scanner did not complete fully 141 | // } 142 | type Scanner interface { 143 | Scan() bool 144 | Object() Object 145 | Err() error 146 | Close() error 147 | } 148 | -------------------------------------------------------------------------------- /object_test.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestParseObjectID(t *testing.T) { 9 | cases := []struct { 10 | name string 11 | string string 12 | id ObjectID 13 | }{ 14 | { 15 | name: "node", 16 | id: NodeID(0).ObjectID(1), 17 | }, 18 | { 19 | name: "zero version node", 20 | id: NodeID(3).ObjectID(0), 21 | }, 22 | { 23 | name: "way", 24 | id: WayID(10).ObjectID(2), 25 | }, 26 | { 27 | name: "relation", 28 | id: RelationID(100).ObjectID(3), 29 | }, 30 | { 31 | name: "changeset", 32 | id: ChangesetID(1000).ObjectID(), 33 | }, 34 | { 35 | name: "note", 36 | id: NoteID(10000).ObjectID(), 37 | }, 38 | { 39 | name: "user", 40 | id: UserID(5000).ObjectID(), 41 | }, 42 | { 43 | name: "node feature", 44 | string: "node/100", 45 | id: NodeID(100).ObjectID(0), 46 | }, 47 | { 48 | name: "bounds", 49 | string: "bounds/0", 50 | id: (&Bounds{}).ObjectID(), 51 | }, 52 | } 53 | 54 | for _, tc := range cases { 55 | t.Run(tc.name, func(t *testing.T) { 56 | var ( 57 | id ObjectID 58 | err error 59 | ) 60 | 61 | if tc.string == "" { 62 | id, err = ParseObjectID(tc.id.String()) 63 | if err != nil { 64 | t.Errorf("parse error: %v", err) 65 | return 66 | } 67 | } else { 68 | id, err = ParseObjectID(tc.string) 69 | if err != nil { 70 | t.Errorf("parse error: %v", err) 71 | return 72 | } 73 | } 74 | 75 | if id != tc.id { 76 | t.Errorf("incorrect id: %v != %v", id, tc.id) 77 | } 78 | }) 79 | } 80 | 81 | // errors 82 | if _, err := ParseObjectID("123"); err == nil { 83 | t.Errorf("should return error if only one part") 84 | } 85 | 86 | if _, err := ParseObjectID("node/1:1:1"); err == nil { 87 | t.Errorf("should return error if multiple :") 88 | } 89 | 90 | if _, err := ParseObjectID("node/abc:1"); err == nil { 91 | t.Errorf("should return error if id not a number") 92 | } 93 | 94 | if _, err := ParseObjectID("node/1:abc"); err == nil { 95 | t.Errorf("should return error if version not a number") 96 | } 97 | 98 | if _, err := ParseObjectID("lake/1:1"); err == nil { 99 | t.Errorf("should return error if not a valid type") 100 | } 101 | } 102 | 103 | func TestObjects_ObjectIDs(t *testing.T) { 104 | es := Objects{ 105 | &Node{ID: 1, Version: 5}, 106 | &Way{ID: 2, Version: 6}, 107 | &Relation{ID: 3, Version: 7}, 108 | &Node{ID: 4, Version: 8}, 109 | &User{ID: 5}, 110 | &Note{ID: 6}, 111 | } 112 | 113 | expected := ObjectIDs{ 114 | NodeID(1).ObjectID(5), 115 | WayID(2).ObjectID(6), 116 | RelationID(3).ObjectID(7), 117 | NodeID(4).ObjectID(8), 118 | UserID(5).ObjectID(), 119 | NoteID(6).ObjectID(), 120 | } 121 | 122 | if ids := es.ObjectIDs(); !reflect.DeepEqual(ids, expected) { 123 | t.Errorf("incorrect ids: %v", ids) 124 | } 125 | } 126 | 127 | func TestObjectID_implementations(t *testing.T) { 128 | type oid interface { 129 | ObjectID() ObjectID 130 | } 131 | 132 | var _ oid = ElementID(0) 133 | 134 | var _ oid = &Node{} 135 | var _ oid = &Way{} 136 | var _ oid = &Relation{} 137 | var _ oid = &Changeset{} 138 | var _ oid = &Note{} 139 | var _ oid = &User{} 140 | 141 | var _ oid = ChangesetID(0) 142 | var _ oid = NoteID(0) 143 | var _ oid = UserID(0) 144 | 145 | type oidv interface { 146 | ObjectID(v int) ObjectID 147 | } 148 | 149 | var _ oidv = FeatureID(0) 150 | var _ oidv = NodeID(0) 151 | var _ oidv = WayID(0) 152 | var _ oidv = RelationID(0) 153 | 154 | // These should not implement the ObjectID methods 155 | noImplement := []interface{}{ 156 | WayNode{}, 157 | Member{}, 158 | } 159 | 160 | for _, ni := range noImplement { 161 | if _, ok := ni.(oid); ok { 162 | t.Errorf("%T should not have ObjectID() method", ni) 163 | } 164 | 165 | if _, ok := ni.(oidv); ok { 166 | t.Errorf("%T should not have ObjectID(v int) method", ni) 167 | } 168 | } 169 | } 170 | 171 | func TestObject_implementations(t *testing.T) { 172 | var _ Object = &Node{} 173 | var _ Object = &Way{} 174 | var _ Object = &Relation{} 175 | var _ Object = &Changeset{} 176 | var _ Object = &Note{} 177 | var _ Object = &User{} 178 | 179 | // These should not implement the Object interface 180 | noImplement := []interface{}{ 181 | ObjectID(0), 182 | FeatureID(0), 183 | ElementID(0), 184 | WayNode{}, 185 | Member{}, 186 | 187 | NodeID(0), 188 | WayID(0), 189 | RelationID(0), 190 | ChangesetID(0), 191 | NoteID(0), 192 | UserID(0), 193 | 194 | Nodes{}, 195 | Ways{}, 196 | Relations{}, 197 | Changesets{}, 198 | Notes{}, 199 | Users{}, 200 | } 201 | 202 | for _, ni := range noImplement { 203 | if _, ok := ni.(Object); ok { 204 | t.Errorf("%T should not be an object", ni) 205 | } 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /osmapi/README.md: -------------------------------------------------------------------------------- 1 | osm/osmapi [![Godoc Reference](https://godoc.org/github.com/paulmach/osm/osmapi?status.svg)](https://godoc.org/github.com/paulmach/osm/osmapi) 2 | ========== 3 | 4 | Package osmapi provides an interface to the [OSM v0.6 API](https://wiki.openstreetmap.org/wiki/API_v0.6). 5 | 6 | Usage: 7 | 8 | ```go 9 | node, err := osmapi.Node(ctx, 1010) 10 | ``` 11 | 12 | This call issues a request to [api.openstreetmap.org/api/0.6/node/1010](https://api.openstreetmap.org/api/0.6/node/1010) 13 | and returns a parsed `osm.Node` object with all the methods attached. 14 | 15 | ## List of functions 16 | 17 | ```go 18 | func Map(context.Context, bounds *osm.Bounds) (*osm.OSM, error) 19 | 20 | func Node(context.Context, osm.NodeID) (*osm.Node, error) 21 | func Nodes(context.Context, []osm.NodeID) (osm.Nodes, error) 22 | func NodeVersion(context.Context, osm.NodeID, v int) (*osm.Node, error) 23 | func NodeHistory(context.Context, osm.NodeID) (osm.Nodes, error) 24 | 25 | func NodeWays(context.Context, osm.NodeID) (osm.Ways, error) 26 | func NodeRelations(context.Context, osm.NodeID) (osm.Relations, error) 27 | 28 | func Way(context.Context, osm.WayID) (*osm.Way, error) 29 | func Ways(context.Context, []osm.WayID) (osm.Ways, error) 30 | func WayFull(context.Context, osm.WayID) (*osm.OSM, error) 31 | func WayVersion(context.Context, osm.WayID, v int) (*osm.Way, error) 32 | func WayHistory(context.Context, osm.WayID) (osm.Ways, error) 33 | 34 | func WayRelations(context.Context, osm.WayID) (osm.Relations, error) 35 | 36 | func Relation(context.Context, osm.RelationID) (*osm.Relation, error) 37 | func Relations(context.Context, []osm.RelationID) (osm.Relations, error) 38 | func RelationFull(context.Context, osm.RelationID) (*osm.OSM, error) 39 | func RelationVersion(context.Context, osm.RelationID, v int) (*osm.Relation, error) 40 | func RelationHistory(context.Context, osm.RelationID) (osm.Relations, error) 41 | 42 | func RelationRelations(context.Context, osm.RelationID) (osm.Relations, error) 43 | 44 | func Changeset(context.Context, osm.ChangesetID) (*osm.Changeset, error) 45 | func ChangesetWithDiscussion(context.Context, osm.ChangesetID) (*osm.Changeset, error) 46 | func ChangesetDownload(context.Context, osm.ChangesetID) (*osm.Change, error) 47 | 48 | func Note(ctx context.Context, id osm.NoteID) (*osm.Note, error) { 49 | func Notes(ctx context.Context, bounds *osm.Bounds, opts ...NotesOption) (osm.Notes, error) 50 | func NotesSearch(ctx context.Context, query string, opts ...NotesOption) (osm.Notes, error) 51 | 52 | func User(ctx context.Context, id osm.UserID) (*osm.User, error) 53 | ``` 54 | 55 | See the [godoc reference](https://godoc.org/github.com/paulmach/osm/osmapi) 56 | for more details. 57 | 58 | ## Rate limiting 59 | 60 | This package can make sure of [`x/time/rate.Limiter`](https://godoc.org/golang.org/x/time/rate#Limiter) 61 | to throttle requests to the official api. Example usage: 62 | 63 | ```go 64 | // 10 qps 65 | osmapi.DefaultDatasource.Limiter = rate.NewLimiter(10, 1) 66 | ``` 67 | -------------------------------------------------------------------------------- /osmapi/changeset.go: -------------------------------------------------------------------------------- 1 | package osmapi 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/paulmach/osm" 8 | ) 9 | 10 | // Changeset returns a given changeset from the osm rest api. 11 | // Delegates to the DefaultDatasource and uses its http.Client to make the request. 12 | func Changeset(ctx context.Context, id osm.ChangesetID) (*osm.Changeset, error) { 13 | return DefaultDatasource.Changeset(ctx, id) 14 | } 15 | 16 | // Changeset returns a given changeset from the osm rest api. 17 | func (ds *Datasource) Changeset(ctx context.Context, id osm.ChangesetID) (*osm.Changeset, error) { 18 | url := fmt.Sprintf("%s/changeset/%d", ds.baseURL(), id) 19 | return ds.getChangeset(ctx, url) 20 | } 21 | 22 | // ChangesetWithDiscussion returns a changeset and its discussion from the osm rest api. 23 | // Delegates to the DefaultDatasource and uses its http.Client to make the request. 24 | func ChangesetWithDiscussion(ctx context.Context, id osm.ChangesetID) (*osm.Changeset, error) { 25 | return DefaultDatasource.ChangesetWithDiscussion(ctx, id) 26 | } 27 | 28 | // ChangesetWithDiscussion returns a changeset and its discussion from the osm rest api. 29 | func (ds *Datasource) ChangesetWithDiscussion(ctx context.Context, id osm.ChangesetID) (*osm.Changeset, error) { 30 | url := fmt.Sprintf("%s/changeset/%d?include_discussion=true", ds.baseURL(), id) 31 | return ds.getChangeset(ctx, url) 32 | } 33 | 34 | func (ds *Datasource) getChangeset(ctx context.Context, url string) (*osm.Changeset, error) { 35 | css := &osm.OSM{} 36 | if err := ds.getFromAPI(ctx, url, &css); err != nil { 37 | return nil, err 38 | } 39 | 40 | if l := len(css.Changesets); l != 1 { 41 | return nil, fmt.Errorf("wrong number of changesets, expected 1, got %v", l) 42 | } 43 | 44 | return css.Changesets[0], nil 45 | } 46 | 47 | // ChangesetDownload returns the full osmchange for the changeset using the osm rest api. 48 | // Delegates to the DefaultDatasource and uses its http.Client to make the request. 49 | func ChangesetDownload(ctx context.Context, id osm.ChangesetID) (*osm.Change, error) { 50 | return DefaultDatasource.ChangesetDownload(ctx, id) 51 | } 52 | 53 | // ChangesetDownload returns the full osmchange for the changeset using the osm rest api. 54 | func (ds *Datasource) ChangesetDownload(ctx context.Context, id osm.ChangesetID) (*osm.Change, error) { 55 | url := fmt.Sprintf("%s/changeset/%d/download", ds.baseURL(), id) 56 | 57 | change := &osm.Change{} 58 | if err := ds.getFromAPI(ctx, url, &change); err != nil { 59 | return nil, err 60 | } 61 | 62 | return change, nil 63 | } 64 | -------------------------------------------------------------------------------- /osmapi/changeset_test.go: -------------------------------------------------------------------------------- 1 | package osmapi 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | func TestChangeset_urls(t *testing.T) { 12 | ctx := context.Background() 13 | 14 | url := "" 15 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 16 | url = r.URL.String() 17 | w.Write([]byte(``)) 18 | })) 19 | defer ts.Close() 20 | 21 | DefaultDatasource.BaseURL = ts.URL 22 | defer func() { 23 | DefaultDatasource.BaseURL = BaseURL 24 | }() 25 | 26 | t.Run("changeset", func(t *testing.T) { 27 | Changeset(ctx, 1) 28 | if !strings.Contains(url, "changeset/1") { 29 | t.Errorf("incorrect path: %v", url) 30 | } 31 | }) 32 | 33 | t.Run("changeset with discussion", func(t *testing.T) { 34 | ChangesetWithDiscussion(ctx, 1) 35 | if !strings.Contains(url, "changeset/1?include_discussion=true") { 36 | t.Errorf("incorrect path: %v", url) 37 | } 38 | }) 39 | 40 | t.Run("changeset download", func(t *testing.T) { 41 | ChangesetDownload(ctx, 1) 42 | if !strings.Contains(url, "changeset/1/download") { 43 | t.Errorf("incorrect path: %v", url) 44 | } 45 | }) 46 | } 47 | -------------------------------------------------------------------------------- /osmapi/datasource.go: -------------------------------------------------------------------------------- 1 | // Package osmapi provides an interface to the OSM v0.6 API. 2 | package osmapi 3 | 4 | import ( 5 | "context" 6 | "encoding/xml" 7 | "fmt" 8 | "net/http" 9 | "time" 10 | 11 | "github.com/paulmach/osm" 12 | ) 13 | 14 | // BaseURL defines the api host. This can be change to hit 15 | // a dev server, for example, http://api06.dev.openstreetmap.org/api/0.6 16 | const BaseURL = "http://api.openstreetmap.org/api/0.6" 17 | 18 | // A RateLimiter is something that can wait until its next allowed request. 19 | // This interface is met by `golang.org/x/time/rate.Limiter` and is meant 20 | // to be used with it. For example: 21 | // // 10 qps 22 | // osmapi.DefaultDatasource.Limiter = rate.NewLimiter(10, 1) 23 | type RateLimiter interface { 24 | Wait(context.Context) error 25 | } 26 | 27 | // Datasource defines context about the http client to use to make requests. 28 | type Datasource struct { 29 | // If Limiter is non-nil. The datasource will wait/block until the request 30 | // is allowed by the rate limiter. To be a good citizen, it is recommended 31 | // to use this when making may concurrent requests against the prod osm api. 32 | // See the RateLimiter docs for more information. 33 | Limiter RateLimiter 34 | 35 | BaseURL string 36 | Client *http.Client 37 | } 38 | 39 | // DefaultDatasource is the Datasource used by package level convenience functions. 40 | var DefaultDatasource = &Datasource{ 41 | BaseURL: BaseURL, 42 | Client: &http.Client{ 43 | Timeout: 6 * time.Minute, // looks like the api server has a 5 min timeout. 44 | }, 45 | } 46 | 47 | var _ osm.HistoryDatasourcer = &Datasource{} 48 | 49 | // NewDatasource creates a Datasource using the given client. 50 | func NewDatasource(client *http.Client) *Datasource { 51 | return &Datasource{ 52 | Client: client, 53 | } 54 | } 55 | 56 | func (ds *Datasource) getFromAPI(ctx context.Context, url string, item interface{}) error { 57 | client := ds.Client 58 | if client == nil { 59 | client = DefaultDatasource.Client 60 | } 61 | 62 | if client == nil { 63 | client = http.DefaultClient 64 | } 65 | 66 | if ds.Limiter != nil { 67 | err := ds.Limiter.Wait(ctx) 68 | if err != nil { 69 | return err 70 | } 71 | } 72 | 73 | req, err := http.NewRequest("GET", url, nil) 74 | if err != nil { 75 | return err 76 | } 77 | 78 | resp, err := client.Do(req.WithContext(ctx)) 79 | if err != nil { 80 | return err 81 | } 82 | defer resp.Body.Close() 83 | 84 | if resp.StatusCode == http.StatusNotFound { 85 | return &NotFoundError{URL: url} 86 | } 87 | 88 | if resp.StatusCode == http.StatusForbidden { 89 | return &ForbiddenError{URL: url} 90 | } 91 | 92 | if resp.StatusCode == http.StatusGone { 93 | return &GoneError{URL: url} 94 | } 95 | 96 | if resp.StatusCode == http.StatusRequestURITooLong { 97 | return &RequestURITooLongError{URL: url} 98 | } 99 | 100 | if resp.StatusCode != http.StatusOK { 101 | return &UnexpectedStatusCodeError{ 102 | Code: resp.StatusCode, 103 | URL: url, 104 | } 105 | } 106 | 107 | return xml.NewDecoder(resp.Body).Decode(item) 108 | } 109 | 110 | func (ds *Datasource) baseURL() string { 111 | if ds.BaseURL != "" { 112 | return ds.BaseURL 113 | } 114 | 115 | return BaseURL 116 | } 117 | 118 | // NotFound error will return true if the result is not found. 119 | func (ds *Datasource) NotFound(err error) bool { 120 | if err == nil { 121 | return false 122 | } 123 | 124 | _, ok := err.(*NotFoundError) 125 | return ok 126 | } 127 | 128 | // NotFoundError means 404 from the api. 129 | type NotFoundError struct { 130 | URL string 131 | } 132 | 133 | // Error returns an error message with the url causing the problem. 134 | func (e *NotFoundError) Error() string { 135 | return fmt.Sprintf("osmapi: not found at %s", e.URL) 136 | } 137 | 138 | // ForbiddenError means 403 from the api. 139 | // Returned whenever the version of the element is not available (due to redaction). 140 | type ForbiddenError struct { 141 | URL string 142 | } 143 | 144 | // Error returns an error message with the url causing the problem. 145 | func (e *ForbiddenError) Error() string { 146 | return fmt.Sprintf("osmapi: forbidden at %s", e.URL) 147 | } 148 | 149 | // GoneError is returned for deleted elements that get 410 from the api. 150 | type GoneError struct { 151 | URL string 152 | } 153 | 154 | // Error returns an error message with the url causing the problem. 155 | func (e *GoneError) Error() string { 156 | return fmt.Sprintf("osmapi: gone at %s", e.URL) 157 | } 158 | 159 | // RequestURITooLongError is returned when requesting too many ids in 160 | // a multi id request, ie. Nodes, Ways, Relations functions. 161 | type RequestURITooLongError struct { 162 | URL string 163 | } 164 | 165 | // Error returns an error message with the url causing the problem. 166 | func (e *RequestURITooLongError) Error() string { 167 | return fmt.Sprintf("osmapi: uri too long at %s", e.URL) 168 | } 169 | 170 | // UnexpectedStatusCodeError is return for a non 200 or 404 status code. 171 | type UnexpectedStatusCodeError struct { 172 | Code int 173 | URL string 174 | } 175 | 176 | // Error returns an error message with some information. 177 | func (e *UnexpectedStatusCodeError) Error() string { 178 | return fmt.Sprintf("osmapi: unexpected status code of %d for url %s", e.Code, e.URL) 179 | } 180 | -------------------------------------------------------------------------------- /osmapi/datasource_test.go: -------------------------------------------------------------------------------- 1 | package osmapi 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | ) 7 | 8 | func TestDatasourceNotFound(t *testing.T) { 9 | ds := NewDatasource(nil) 10 | 11 | if ds.NotFound(nil) { 12 | t.Errorf("should be false for nil") 13 | } 14 | 15 | if ds.NotFound(errors.New("foo")) { 16 | t.Errorf("should be false for random error") 17 | } 18 | 19 | if ds.NotFound(&GoneError{}) { 20 | t.Errorf("should be false for gone error") 21 | } 22 | 23 | if !ds.NotFound(&NotFoundError{}) { 24 | t.Errorf("should be true for not found error") 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /osmapi/map.go: -------------------------------------------------------------------------------- 1 | package osmapi 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/paulmach/osm" 8 | ) 9 | 10 | // Map returns the latest elements in the given bounding box. 11 | // Delegates to the DefaultDatasource and uses its http.Client to make the request. 12 | func Map(ctx context.Context, bounds *osm.Bounds, opts ...FeatureOption) (*osm.OSM, error) { 13 | return DefaultDatasource.Map(ctx, bounds, opts...) 14 | } 15 | 16 | // Map returns the latest elements in the given bounding box. 17 | func (ds *Datasource) Map(ctx context.Context, bounds *osm.Bounds, opts ...FeatureOption) (*osm.OSM, error) { 18 | params, err := featureOptions(opts) 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | url := fmt.Sprintf("%s/map?bbox=%f,%f,%f,%f&%s", ds.baseURL(), 24 | bounds.MinLon, bounds.MinLat, 25 | bounds.MaxLon, bounds.MaxLat, 26 | params) 27 | o := &osm.OSM{} 28 | if err := ds.getFromAPI(ctx, url, &o); err != nil { 29 | return nil, err 30 | } 31 | 32 | return o, nil 33 | } 34 | -------------------------------------------------------------------------------- /osmapi/map_test.go: -------------------------------------------------------------------------------- 1 | package osmapi 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/paulmach/osm" 12 | ) 13 | 14 | func TestMap_urls(t *testing.T) { 15 | ctx := context.Background() 16 | 17 | url := "" 18 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 19 | url = r.URL.String() 20 | w.Write([]byte(``)) 21 | })) 22 | defer ts.Close() 23 | 24 | DefaultDatasource.BaseURL = ts.URL 25 | defer func() { 26 | DefaultDatasource.BaseURL = BaseURL 27 | }() 28 | 29 | t.Run("map", func(t *testing.T) { 30 | bound := &osm.Bounds{ 31 | MinLon: 1, MinLat: 2, 32 | MaxLon: 3, MaxLat: 4, 33 | } 34 | 35 | Map(ctx, bound) 36 | if !strings.Contains(url, "map?bbox=1.000000,2.000000,3.000000,4.000000") { 37 | t.Errorf("incorrect path: %v", url) 38 | } 39 | 40 | Map(ctx, bound, At(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC))) 41 | if !strings.Contains(url, "map?bbox=1.000000,2.000000,3.000000,4.000000&at=2016-01-01T00:00:00Z") { 42 | t.Errorf("incorrect path: %v", url) 43 | } 44 | }) 45 | } 46 | -------------------------------------------------------------------------------- /osmapi/node_test.go: -------------------------------------------------------------------------------- 1 | package osmapi 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/paulmach/osm" 12 | ) 13 | 14 | func TestNode_urls(t *testing.T) { 15 | ctx := context.Background() 16 | 17 | url := "" 18 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 19 | url = r.URL.String() 20 | w.Write([]byte(``)) 21 | })) 22 | defer ts.Close() 23 | 24 | DefaultDatasource.BaseURL = ts.URL 25 | defer func() { 26 | DefaultDatasource.BaseURL = BaseURL 27 | }() 28 | 29 | t.Run("node", func(t *testing.T) { 30 | Node(ctx, 1) 31 | if !strings.Contains(url, "node/1") { 32 | t.Errorf("incorrect path: %v", url) 33 | } 34 | 35 | Node(ctx, 1, At(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC))) 36 | if !strings.Contains(url, "node/1?at=2016-01-01T00:00:00Z") { 37 | t.Errorf("incorrect path: %v", url) 38 | } 39 | }) 40 | 41 | t.Run("nodes", func(t *testing.T) { 42 | Nodes(ctx, []osm.NodeID{1, 2}) 43 | if !strings.Contains(url, "nodes?nodes=1,2") { 44 | t.Errorf("incorrect path: %v", url) 45 | } 46 | 47 | Nodes(ctx, []osm.NodeID{1, 2}, At(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC))) 48 | if !strings.Contains(url, "nodes?nodes=1,2&at=2016-01-01T00:00:00Z") { 49 | t.Errorf("incorrect path: %v", url) 50 | } 51 | }) 52 | 53 | t.Run("node version", func(t *testing.T) { 54 | NodeVersion(ctx, 1, 2) 55 | if !strings.Contains(url, "node/1/2") { 56 | t.Errorf("incorrect path: %v", url) 57 | } 58 | }) 59 | 60 | t.Run("node history", func(t *testing.T) { 61 | NodeHistory(ctx, 1) 62 | if !strings.Contains(url, "node/1/history") { 63 | t.Errorf("incorrect path: %v", url) 64 | } 65 | }) 66 | 67 | t.Run("node ways", func(t *testing.T) { 68 | NodeWays(ctx, 1) 69 | if !strings.Contains(url, "node/1/ways") { 70 | t.Errorf("incorrect path: %v", url) 71 | } 72 | 73 | NodeWays(ctx, 1, At(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC))) 74 | if !strings.Contains(url, "node/1/ways?at=2016-01-01T00:00:00Z") { 75 | t.Errorf("incorrect path: %v", url) 76 | } 77 | }) 78 | 79 | t.Run("node relations", func(t *testing.T) { 80 | NodeRelations(ctx, 1) 81 | if !strings.Contains(url, "node/1/relations") { 82 | t.Errorf("incorrect path: %v", url) 83 | } 84 | 85 | NodeRelations(ctx, 1, At(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC))) 86 | if !strings.Contains(url, "node/1/relations?at=2016-01-01T00:00:00Z") { 87 | t.Errorf("incorrect path: %v", url) 88 | } 89 | }) 90 | } 91 | -------------------------------------------------------------------------------- /osmapi/note.go: -------------------------------------------------------------------------------- 1 | package osmapi 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/url" 7 | "strings" 8 | 9 | "github.com/paulmach/osm" 10 | ) 11 | 12 | // Note returns the note from the osm rest api. 13 | // Delegates to the DefaultDatasource and uses its http.Client to make the request. 14 | func Note(ctx context.Context, id osm.NoteID) (*osm.Note, error) { 15 | return DefaultDatasource.Note(ctx, id) 16 | } 17 | 18 | // Note returns the note from the osm rest api. 19 | func (ds *Datasource) Note(ctx context.Context, id osm.NoteID) (*osm.Note, error) { 20 | url := fmt.Sprintf("%s/notes/%d", ds.baseURL(), id) 21 | 22 | o := &osm.OSM{} 23 | if err := ds.getFromAPI(ctx, url, &o); err != nil { 24 | return nil, err 25 | } 26 | 27 | if l := len(o.Notes); l != 1 { 28 | return nil, fmt.Errorf("wrong number of notes, expected 1, got %v", l) 29 | } 30 | 31 | return o.Notes[0], nil 32 | } 33 | 34 | // Notes returns the notes in a bounding box. Can provide options to limit the results 35 | // or change what it means to be "closed". See the options or osm api v0.6 docs for details. 36 | // Delegates to the DefaultDatasource and uses its http.Client to make the request. 37 | func Notes(ctx context.Context, bounds *osm.Bounds, opts ...NotesOption) (osm.Notes, error) { 38 | return DefaultDatasource.Notes(ctx, bounds, opts...) 39 | } 40 | 41 | var _ NotesOption = Limit(1) 42 | var _ NotesOption = MaxDaysClosed(1) 43 | 44 | // Notes returns the notes in a bounding box. Can provide options to limit the results 45 | // or change what it means to be "closed". See the options or osm api v0.6 docs for details. 46 | func (ds *Datasource) Notes(ctx context.Context, bounds *osm.Bounds, opts ...NotesOption) (osm.Notes, error) { 47 | params := make([]string, 0, 1+len(opts)) 48 | params = append(params, fmt.Sprintf("bbox=%f,%f,%f,%f", 49 | bounds.MinLon, bounds.MinLat, 50 | bounds.MaxLon, bounds.MaxLat)) 51 | 52 | var err error 53 | for _, o := range opts { 54 | params, err = o.applyNotes(params) 55 | if err != nil { 56 | return nil, err 57 | } 58 | } 59 | 60 | url := fmt.Sprintf("%s/notes?%s", ds.baseURL(), strings.Join(params, "&")) 61 | 62 | o := &osm.OSM{} 63 | if err := ds.getFromAPI(ctx, url, &o); err != nil { 64 | return nil, err 65 | } 66 | 67 | return o.Notes, nil 68 | } 69 | 70 | // NotesSearch returns the notes in a bounding box whose text matches the query. 71 | // Can provide options to limit the results or change what it means to be "closed". 72 | // See the options or osm api v0.6 docs for details. 73 | // Delegates to the DefaultDatasource and uses its http.Client to make the request. 74 | func NotesSearch(ctx context.Context, query string, opts ...NotesOption) (osm.Notes, error) { 75 | return DefaultDatasource.NotesSearch(ctx, query, opts...) 76 | } 77 | 78 | // NotesSearch returns the notes whose text matches the query. 79 | // Can provide options to limit the results or change what it means to be "closed". 80 | // See the options or osm api v0.6 docs for details. 81 | func (ds *Datasource) NotesSearch(ctx context.Context, query string, opts ...NotesOption) (osm.Notes, error) { 82 | params := make([]string, 0, 1+len(opts)) 83 | params = append(params, fmt.Sprintf("q=%s", url.QueryEscape(query))) 84 | 85 | var err error 86 | for _, o := range opts { 87 | params, err = o.applyNotes(params) 88 | if err != nil { 89 | return nil, err 90 | } 91 | } 92 | 93 | url := fmt.Sprintf("%s/notes/search?%s", ds.baseURL(), strings.Join(params, "&")) 94 | 95 | o := &osm.OSM{} 96 | if err := ds.getFromAPI(ctx, url, &o); err != nil { 97 | return nil, err 98 | } 99 | 100 | return o.Notes, nil 101 | } 102 | -------------------------------------------------------------------------------- /osmapi/note_test.go: -------------------------------------------------------------------------------- 1 | package osmapi 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/paulmach/osm" 11 | ) 12 | 13 | func TestNote_urls(t *testing.T) { 14 | ctx := context.Background() 15 | 16 | url := "" 17 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 18 | url = r.URL.String() 19 | w.Write([]byte(``)) 20 | })) 21 | defer ts.Close() 22 | 23 | DefaultDatasource.BaseURL = ts.URL 24 | defer func() { 25 | DefaultDatasource.BaseURL = BaseURL 26 | }() 27 | 28 | t.Run("note", func(t *testing.T) { 29 | Note(ctx, 1) 30 | if !strings.Contains(url, "notes/1") { 31 | t.Errorf("incorrect path: %v", url) 32 | } 33 | }) 34 | 35 | t.Run("notes", func(t *testing.T) { 36 | bound := &osm.Bounds{ 37 | MinLon: 1, MinLat: 2, 38 | MaxLon: 3, MaxLat: 4, 39 | } 40 | 41 | Notes(ctx, bound) 42 | if !strings.Contains(url, "notes?bbox=1.000000,2.000000,3.000000,4.000000") { 43 | t.Errorf("incorrect path: %v", url) 44 | } 45 | 46 | Notes(ctx, bound, Limit(1), MaxDaysClosed(4)) 47 | if !strings.Contains(url, "notes?bbox=1.000000,2.000000,3.000000,4.000000&limit=1&closed=4") { 48 | t.Errorf("incorrect path: %v", url) 49 | } 50 | }) 51 | 52 | t.Run("nodes search", func(t *testing.T) { 53 | NotesSearch(ctx, "asdf") 54 | if !strings.Contains(url, "notes/search?q=asdf") { 55 | t.Errorf("incorrect path: %v", url) 56 | } 57 | 58 | NotesSearch(ctx, "asdf", Limit(1), MaxDaysClosed(4)) 59 | if !strings.Contains(url, "notes/search?q=asdf&limit=1&closed=4") { 60 | t.Errorf("incorrect path: %v", url) 61 | } 62 | }) 63 | } 64 | -------------------------------------------------------------------------------- /osmapi/options.go: -------------------------------------------------------------------------------- 1 | package osmapi 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | // FeatureOption can be used when fetching a feature or a set of different features. 11 | type FeatureOption interface { 12 | applyFeature([]string) ([]string, error) 13 | } 14 | 15 | // At adds an `at=2006-01-02T15:04:05Z` parameter to the request. 16 | // The osm.fyi supports requesting features and maps as they were at the given time. 17 | func At(t time.Time) FeatureOption { 18 | return &at{t} 19 | } 20 | 21 | type at struct{ t time.Time } 22 | 23 | func (o *at) applyFeature(p []string) ([]string, error) { 24 | return append(p, "at="+o.t.UTC().Format("2006-01-02T15:04:05Z")), nil 25 | } 26 | 27 | // NotesOption defines a valid option for the osmapi.Notes by bounding box api. 28 | type NotesOption interface { 29 | applyNotes([]string) ([]string, error) 30 | } 31 | 32 | // Limit indicates the number of results to return valid values [1,10000]. 33 | // Default is 100. 34 | func Limit(num int) NotesOption { 35 | return &limit{num} 36 | } 37 | 38 | // MaxDaysClosed specifies the number of days a note needs to be closed to 39 | // no longer be returned. 0 will return only open notes, -1 will return all notes. 40 | // Default is 7. 41 | func MaxDaysClosed(num int) NotesOption { 42 | return &maxDaysClosed{num} 43 | } 44 | 45 | type limit struct{ n int } 46 | 47 | func (o *limit) applyNotes(p []string) ([]string, error) { 48 | if o.n < 1 || 10000 < o.n { 49 | return nil, errors.New("osmapi: limit must be between 1 and 10000") 50 | } 51 | return append(p, fmt.Sprintf("limit=%d", o.n)), nil 52 | } 53 | 54 | type maxDaysClosed struct{ n int } 55 | 56 | func (o *maxDaysClosed) applyNotes(p []string) ([]string, error) { 57 | return append(p, fmt.Sprintf("closed=%d", o.n)), nil 58 | } 59 | 60 | func featureOptions(opts []FeatureOption) (string, error) { 61 | if len(opts) == 0 { 62 | return "", nil 63 | } 64 | 65 | params := make([]string, 0, len(opts)) 66 | 67 | var err error 68 | for _, o := range opts { 69 | params, err = o.applyFeature(params) 70 | if err != nil { 71 | return "", err 72 | } 73 | } 74 | 75 | return strings.Join(params, "&"), nil 76 | } 77 | -------------------------------------------------------------------------------- /osmapi/relation_test.go: -------------------------------------------------------------------------------- 1 | package osmapi 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/paulmach/osm" 12 | ) 13 | 14 | func TestRelation_urls(t *testing.T) { 15 | ctx := context.Background() 16 | 17 | url := "" 18 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 19 | url = r.URL.String() 20 | w.Write([]byte(``)) 21 | })) 22 | defer ts.Close() 23 | 24 | DefaultDatasource.BaseURL = ts.URL 25 | defer func() { 26 | DefaultDatasource.BaseURL = BaseURL 27 | }() 28 | 29 | t.Run("relation", func(t *testing.T) { 30 | Relation(ctx, 1) 31 | if !strings.Contains(url, "relation/1") { 32 | t.Errorf("incorrect path: %v", url) 33 | } 34 | 35 | Relation(ctx, 1, At(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC))) 36 | if !strings.Contains(url, "relation/1?at=2016-01-01T00:00:00Z") { 37 | t.Errorf("incorrect path: %v", url) 38 | } 39 | }) 40 | 41 | t.Run("relations", func(t *testing.T) { 42 | Relations(ctx, []osm.RelationID{1, 2}) 43 | if !strings.Contains(url, "relations?relations=1,2") { 44 | t.Errorf("incorrect path: %v", url) 45 | } 46 | 47 | Relations(ctx, []osm.RelationID{1, 2}, At(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC))) 48 | if !strings.Contains(url, "relations?relations=1,2&at=2016-01-01T00:00:00Z") { 49 | t.Errorf("incorrect path: %v", url) 50 | } 51 | }) 52 | 53 | t.Run("relation version", func(t *testing.T) { 54 | RelationVersion(ctx, 1, 2) 55 | if !strings.Contains(url, "relation/1/2") { 56 | t.Errorf("incorrect path: %v", url) 57 | } 58 | }) 59 | 60 | t.Run("relation history", func(t *testing.T) { 61 | RelationHistory(ctx, 1) 62 | if !strings.Contains(url, "relation/1/history") { 63 | t.Errorf("incorrect path: %v", url) 64 | } 65 | }) 66 | 67 | t.Run("relation relations", func(t *testing.T) { 68 | RelationRelations(ctx, 1) 69 | if !strings.Contains(url, "relation/1/relations") { 70 | t.Errorf("incorrect path: %v", url) 71 | } 72 | 73 | RelationRelations(ctx, 1, At(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC))) 74 | if !strings.Contains(url, "relation/1/relations?at=2016-01-01T00:00:00Z") { 75 | t.Errorf("incorrect path: %v", url) 76 | } 77 | }) 78 | 79 | t.Run("relation full", func(t *testing.T) { 80 | RelationFull(ctx, 1) 81 | if !strings.Contains(url, "relation/1/full") { 82 | t.Errorf("incorrect path: %v", url) 83 | } 84 | 85 | RelationFull(ctx, 1, At(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC))) 86 | if !strings.Contains(url, "relation/1/full?at=2016-01-01T00:00:00Z") { 87 | t.Errorf("incorrect path: %v", url) 88 | } 89 | }) 90 | } 91 | -------------------------------------------------------------------------------- /osmapi/user.go: -------------------------------------------------------------------------------- 1 | package osmapi 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/paulmach/osm" 8 | ) 9 | 10 | // User returns the user from the osm rest api. 11 | // Delegates to the DefaultDatasource and uses its http.Client to make the request. 12 | func User(ctx context.Context, id osm.UserID) (*osm.User, error) { 13 | return DefaultDatasource.User(ctx, id) 14 | } 15 | 16 | // User returns the user from the osm rest api. 17 | func (ds *Datasource) User(ctx context.Context, id osm.UserID) (*osm.User, error) { 18 | url := fmt.Sprintf("%s/user/%d", ds.baseURL(), id) 19 | 20 | o := &osm.OSM{} 21 | if err := ds.getFromAPI(ctx, url, &o); err != nil { 22 | return nil, err 23 | } 24 | 25 | if l := len(o.Users); l != 1 { 26 | return nil, fmt.Errorf("wrong number of users, expected 1, got %v", l) 27 | } 28 | 29 | return o.Users[0], nil 30 | } 31 | -------------------------------------------------------------------------------- /osmapi/user_test.go: -------------------------------------------------------------------------------- 1 | package osmapi 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | func TestUser_urls(t *testing.T) { 12 | ctx := context.Background() 13 | 14 | url := "" 15 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 16 | url = r.URL.String() 17 | w.Write([]byte(``)) 18 | })) 19 | defer ts.Close() 20 | 21 | DefaultDatasource.BaseURL = ts.URL 22 | defer func() { 23 | DefaultDatasource.BaseURL = BaseURL 24 | }() 25 | 26 | t.Run("user", func(t *testing.T) { 27 | User(ctx, 1) 28 | if !strings.Contains(url, "user/1") { 29 | t.Errorf("incorrect path: %v", url) 30 | } 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /osmapi/way_test.go: -------------------------------------------------------------------------------- 1 | package osmapi 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/paulmach/osm" 12 | ) 13 | 14 | func TestWay_urls(t *testing.T) { 15 | ctx := context.Background() 16 | 17 | url := "" 18 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 19 | url = r.URL.String() 20 | w.Write([]byte(``)) 21 | })) 22 | defer ts.Close() 23 | 24 | DefaultDatasource.BaseURL = ts.URL 25 | defer func() { 26 | DefaultDatasource.BaseURL = BaseURL 27 | }() 28 | 29 | t.Run("way", func(t *testing.T) { 30 | Way(ctx, 1) 31 | if !strings.Contains(url, "way/1") { 32 | t.Errorf("incorrect path: %v", url) 33 | } 34 | 35 | Way(ctx, 1, At(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC))) 36 | if !strings.Contains(url, "way/1?at=2016-01-01T00:00:00Z") { 37 | t.Errorf("incorrect path: %v", url) 38 | } 39 | }) 40 | 41 | t.Run("ways", func(t *testing.T) { 42 | Ways(ctx, []osm.WayID{1, 2}) 43 | if !strings.Contains(url, "ways?ways=1,2") { 44 | t.Errorf("incorrect path: %v", url) 45 | } 46 | 47 | Ways(ctx, []osm.WayID{1, 2}, At(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC))) 48 | if !strings.Contains(url, "ways?ways=1,2&at=2016-01-01T00:00:00Z") { 49 | t.Errorf("incorrect path: %v", url) 50 | } 51 | }) 52 | 53 | t.Run("way version", func(t *testing.T) { 54 | WayVersion(ctx, 1, 2) 55 | if !strings.Contains(url, "way/1/2") { 56 | t.Errorf("incorrect path: %v", url) 57 | } 58 | }) 59 | 60 | t.Run("way history", func(t *testing.T) { 61 | WayHistory(ctx, 1) 62 | if !strings.Contains(url, "way/1/history") { 63 | t.Errorf("incorrect path: %v", url) 64 | } 65 | }) 66 | 67 | t.Run("way relations", func(t *testing.T) { 68 | WayRelations(ctx, 1) 69 | if !strings.Contains(url, "way/1/relations") { 70 | t.Errorf("incorrect path: %v", url) 71 | } 72 | 73 | WayRelations(ctx, 1, At(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC))) 74 | if !strings.Contains(url, "way/1/relations?at=2016-01-01T00:00:00Z") { 75 | t.Errorf("incorrect path: %v", url) 76 | } 77 | }) 78 | 79 | t.Run("way full", func(t *testing.T) { 80 | WayFull(ctx, 1) 81 | if !strings.Contains(url, "way/1/full") { 82 | t.Errorf("incorrect path: %v", url) 83 | } 84 | 85 | WayFull(ctx, 1, At(time.Date(2016, 1, 1, 0, 0, 0, 0, time.UTC))) 86 | if !strings.Contains(url, "way/1/full?at=2016-01-01T00:00:00Z") { 87 | t.Errorf("incorrect path: %v", url) 88 | } 89 | }) 90 | } 91 | -------------------------------------------------------------------------------- /osmgeojson/README.md: -------------------------------------------------------------------------------- 1 | osm/osmgeojson [![Godoc Reference](https://godoc.org/github.com/paulmach/osm/osmgeojson?status.svg)](https://godoc.org/github.com/paulmach/osm/osmgeojson) 2 | ============== 3 | 4 | Package `osmgeojson` converts OSM data to GeoJSON. It is a **full** port of the 5 | nodejs library [osmtogeojson](https://github.com/tyrasd/osmtogeojson) and sports 6 | the same features and tests (plus more): 7 | 8 | * real OSM polygon detection 9 | * OSM multipolygon support, e.g. buildings with holes become proper multipolygons 10 | * supports annotated geometries 11 | * well tested 12 | 13 | ### Usage 14 | 15 | ```go 16 | delta := 0.0001 17 | 18 | lon, lat := -83.5997038, 41.5923682 19 | bounds := &osm.Bounds{ 20 | MinLat: lat - delta, MaxLat: lat + delta, 21 | MinLon: lon - delta, MaxLon: lon + delta, 22 | } 23 | 24 | o, _ := osmapi.Map(ctx, bounds) // fetch data from the osm api. 25 | 26 | // run the conversion 27 | fc, err := osmgeojson.Convert(o, opts) 28 | 29 | // marshal the json 30 | gj, _ := json.MarshalIndent(fc, "", " ") 31 | fmt.Println(string(gj)) 32 | ``` 33 | 34 | ### Options 35 | 36 | The package provides several options to control what is included in the feature properties. 37 | If possible, excluding some of the extra properties can greatly improve the performance. 38 | All of the options **default to false**, i.e. everything will be included. 39 | 40 | * `NoID(yes bool)` 41 | 42 | Controls whether to set the feature.ID to "type/id" e.g. "node/475373687". For some use cases 43 | this may be of limited use since the feature.Properies "type" and "id" are also set. 44 | 45 | * `NoMeta(yes bool)` 46 | 47 | Controls whether to populate the "meta" property which is a sub-map with the 48 | following values from the osm element: "timestamp", "version", "changeset", "user", "uid". 49 | 50 | * `NoRelationMembership(yes bool)` 51 | 52 | Controls whether to include a list of the relations the osm element is a member of. 53 | This info is set as the "relation" property which is an array of objects with the 54 | following values from the relation: "id", "role", "tags". 55 | 56 | * `IncludeInvalidPolygons(yes bool)` 57 | 58 | By default, inner rings of 'multipolygon' without a matching outer ring will be ignored. 59 | However, in some use cases the outer ring can be implied as the viewport bound and the inner rings 60 | can then be rendered correctly. Polygons with a nil first ring will be need to be updated such 61 | that the first ring is the viewport bound. This options will also include rings that do not 62 | have matching endpoints. Usually this means one or more of the outer ways are missing. 63 | 64 | 65 | ### Benchmarks 66 | 67 | These benchmarks are meant to show the performance impact of the different options. 68 | They were run on a 2012 MacBook Air with a 2 ghz processor and 8 gigs of ram. 69 | 70 | ``` 71 | BenchmarkConvert-4 10000 2520891 ns/op 935697 B/op 11299 allocs/op 72 | BenchmarkConvertAnnotated-4 10000 2196433 ns/op 853544 B/op 11239 allocs/op 73 | BenchmarkConvert_NoID-4 10000 2310816 ns/op 913915 B/op 9687 allocs/op 74 | BenchmarkConvert_NoMeta-4 10000 2026031 ns/op 716953 B/op 7546 allocs/op 75 | BenchmarkConvert_NoRelationMembership-4 10000 2397634 ns/op 912454 B/op 10716 allocs/op 76 | BenchmarkConvert_NoIDsMetaMembership-4 20000 1718224 ns/op 671984 B/op 5353 allocs/op 77 | ``` 78 | 79 | #### Similar libraries in other languages: 80 | 81 | * [osmtogeojson](https://github.com/tyrasd/osmtogeojson) - Node 82 | -------------------------------------------------------------------------------- /osmgeojson/benchmarks_test.go: -------------------------------------------------------------------------------- 1 | package osmgeojson 2 | 3 | import ( 4 | "encoding/xml" 5 | "os" 6 | "testing" 7 | 8 | "github.com/paulmach/osm" 9 | ) 10 | 11 | func BenchmarkConvert(b *testing.B) { 12 | o := parseFile(b, "testdata/benchmark.osm") 13 | 14 | b.ReportAllocs() 15 | b.ResetTimer() 16 | for n := 0; n < b.N; n++ { 17 | _, err := Convert(o) 18 | if err != nil { 19 | b.Fatalf("convert error: %v", err) 20 | } 21 | } 22 | } 23 | 24 | func BenchmarkConvertAnnotated(b *testing.B) { 25 | o := parseFile(b, "testdata/benchmark.osm") 26 | annotate(o) 27 | 28 | b.ReportAllocs() 29 | b.ResetTimer() 30 | for n := 0; n < b.N; n++ { 31 | _, err := Convert(o) 32 | if err != nil { 33 | b.Fatalf("convert error: %v", err) 34 | } 35 | } 36 | } 37 | 38 | func BenchmarkConvert_NoID(b *testing.B) { 39 | o := parseFile(b, "testdata/benchmark.osm") 40 | 41 | b.ReportAllocs() 42 | b.ResetTimer() 43 | for n := 0; n < b.N; n++ { 44 | _, err := Convert(o, NoID(true)) 45 | if err != nil { 46 | b.Fatalf("convert error: %v", err) 47 | } 48 | } 49 | } 50 | 51 | func BenchmarkConvert_NoMeta(b *testing.B) { 52 | o := parseFile(b, "testdata/benchmark.osm") 53 | 54 | b.ReportAllocs() 55 | b.ResetTimer() 56 | for n := 0; n < b.N; n++ { 57 | _, err := Convert(o, NoMeta(true)) 58 | if err != nil { 59 | b.Fatalf("convert error: %v", err) 60 | } 61 | } 62 | } 63 | 64 | func BenchmarkConvert_NoRelationMembership(b *testing.B) { 65 | o := parseFile(b, "testdata/benchmark.osm") 66 | 67 | b.ReportAllocs() 68 | b.ResetTimer() 69 | for n := 0; n < b.N; n++ { 70 | _, err := Convert(o, NoRelationMembership(true)) 71 | if err != nil { 72 | b.Fatalf("convert error: %v", err) 73 | } 74 | } 75 | } 76 | 77 | func BenchmarkConvert_NoIDsMetaMembership(b *testing.B) { 78 | o := parseFile(b, "testdata/benchmark.osm") 79 | 80 | b.ReportAllocs() 81 | b.ResetTimer() 82 | for n := 0; n < b.N; n++ { 83 | _, err := Convert(o, NoID(true), NoMeta(true), NoRelationMembership(true)) 84 | if err != nil { 85 | b.Fatalf("convert error: %v", err) 86 | } 87 | } 88 | } 89 | 90 | func parseFile(t testing.TB, filename string) *osm.OSM { 91 | data, err := os.ReadFile(filename) 92 | if err != nil { 93 | t.Fatalf("could not read file: %v", err) 94 | } 95 | 96 | o := &osm.OSM{} 97 | err = xml.Unmarshal(data, &o) 98 | if err != nil { 99 | t.Fatalf("failed to unmarshal %s: %v", filename, err) 100 | } 101 | 102 | return o 103 | } 104 | 105 | func annotate(o *osm.OSM) { 106 | nodes := make(map[osm.NodeID]*osm.Node) 107 | for _, n := range o.Nodes { 108 | nodes[n.ID] = n 109 | } 110 | 111 | for _, w := range o.Ways { 112 | for i, wn := range w.Nodes { 113 | n := nodes[wn.ID] 114 | if n == nil { 115 | continue 116 | } 117 | 118 | w.Nodes[i].Lat = n.Lat 119 | w.Nodes[i].Lon = n.Lon 120 | w.Nodes[i].Version = n.Version 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /osmgeojson/options.go: -------------------------------------------------------------------------------- 1 | package osmgeojson 2 | 3 | // An Option is a setting for creating the geojson. 4 | type Option func(*context) error 5 | 6 | // NoID will omit setting the geojson feature.ID 7 | func NoID(yes bool) Option { 8 | return func(ctx *context) error { 9 | ctx.noID = yes 10 | return nil 11 | } 12 | } 13 | 14 | // NoMeta will omit the meta (timestamp, user, changeset, etc) info 15 | // from the output geojson feature properties. 16 | func NoMeta(yes bool) Option { 17 | return func(ctx *context) error { 18 | ctx.noMeta = yes 19 | return nil 20 | } 21 | } 22 | 23 | // NoRelationMembership will omit the list of relations 24 | // an element is a member of from the output geojson features. 25 | func NoRelationMembership(yes bool) Option { 26 | return func(ctx *context) error { 27 | ctx.noRelationMembership = yes 28 | return nil 29 | } 30 | } 31 | 32 | // IncludeInvalidPolygons will return a polygon with nil outer/first ring 33 | // if the outer ringer is not found in the data. It may also return 34 | // rings whose endpoints do not match and are probably missing sections. 35 | func IncludeInvalidPolygons(yes bool) Option { 36 | return func(ctx *context) error { 37 | ctx.includeInvalidPolygons = yes 38 | return nil 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /osmgeojson/options_test.go: -------------------------------------------------------------------------------- 1 | package osmgeojson 2 | 3 | import ( 4 | "encoding/xml" 5 | "testing" 6 | 7 | "github.com/paulmach/orb/geojson" 8 | "github.com/paulmach/osm" 9 | ) 10 | 11 | var nodeXML = ` 12 | 13 | 14 | 15 | 16 | 24 | ` 25 | 26 | var wayXML = ` 27 | 28 | 29 | 30 | 31 | 39 | 40 | 41 | 42 | 43 | 44 | ` 45 | 46 | var relationXML = ` 47 | 48 | 49 | 50 | 51 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | ` 72 | 73 | func TestOptionNoID(t *testing.T) { 74 | test := func(t *testing.T, xml string) { 75 | feature := convertXML(t, xml).Features[0] 76 | if v := feature.ID; v == nil { 77 | t.Errorf("id should be set: %v", v) 78 | } 79 | 80 | feature = convertXML(t, xml, NoID(true)).Features[0] 81 | if v := feature.ID; v != nil { 82 | t.Errorf("id should be nil: %v", v) 83 | } 84 | } 85 | 86 | t.Run("node", func(t *testing.T) { 87 | test(t, nodeXML) 88 | }) 89 | 90 | t.Run("way", func(t *testing.T) { 91 | test(t, wayXML) 92 | }) 93 | 94 | t.Run("relation", func(t *testing.T) { 95 | test(t, relationXML) 96 | }) 97 | } 98 | 99 | func TestOptionNoMeta(t *testing.T) { 100 | test := func(t *testing.T, xml string) { 101 | feature := convertXML(t, xml).Features[0] 102 | if v := feature.Properties["meta"]; v == nil { 103 | t.Errorf("meta should be set: %v", v) 104 | } 105 | 106 | feature = convertXML(t, xml, NoMeta(true)).Features[0] 107 | if v := feature.Properties["meta"]; v != nil { 108 | t.Errorf("meta should be nil: %v", v) 109 | } 110 | } 111 | 112 | t.Run("node", func(t *testing.T) { 113 | test(t, nodeXML) 114 | }) 115 | 116 | t.Run("way", func(t *testing.T) { 117 | test(t, wayXML) 118 | }) 119 | 120 | t.Run("relation", func(t *testing.T) { 121 | test(t, relationXML) 122 | }) 123 | } 124 | 125 | func TestOptionNoRelationMembership(t *testing.T) { 126 | test := func(t *testing.T, xml string) { 127 | feature := convertXML(t, xml).Features[0] 128 | if v := feature.Properties["relations"]; v == nil { 129 | t.Errorf("relations should be set: %v", v) 130 | } 131 | 132 | feature = convertXML(t, xml, NoRelationMembership(true)).Features[0] 133 | if v := feature.Properties["relations"]; v != nil { 134 | t.Errorf("relations should be nil: %v", v) 135 | } 136 | } 137 | 138 | t.Run("node", func(t *testing.T) { 139 | test(t, nodeXML) 140 | }) 141 | 142 | t.Run("way", func(t *testing.T) { 143 | test(t, wayXML) 144 | }) 145 | 146 | t.Run("relation", func(t *testing.T) { 147 | test(t, relationXML) 148 | }) 149 | } 150 | 151 | func convertXML(t *testing.T, data string, opts ...Option) *geojson.FeatureCollection { 152 | o := &osm.OSM{} 153 | err := xml.Unmarshal([]byte(data), &o) 154 | if err != nil { 155 | t.Fatalf("failed to unmarshal xml: %v", err) 156 | } 157 | 158 | fc, err := Convert(o, opts...) 159 | if err != nil { 160 | t.Fatalf("failed to convert: %v", err) 161 | } 162 | 163 | return fc 164 | } 165 | -------------------------------------------------------------------------------- /osmpbf/README.md: -------------------------------------------------------------------------------- 1 | # osm/osmpbf [![Go Reference](https://pkg.go.dev/badge/github.com/paulmach/osm.svg)](https://pkg.go.dev/github.com/paulmach/osm/osmpbf) 2 | 3 | Package osmpbf provides a scanner for decoding large [OSM PBF](https://wiki.openstreetmap.org/wiki/PBF_Format) files. 4 | They are typically found at [planet.osm.org](https://planet.openstreetmap.org/) or [Geofabrik Download](https://download.geofabrik.de/). 5 | 6 | ## Example: 7 | 8 | ```go 9 | file, err := os.Open("./delaware-latest.osm.pbf") 10 | if err != nil { 11 | panic(err) 12 | } 13 | defer file.Close() 14 | 15 | // The third parameter is the number of parallel decoders to use. 16 | scanner := osmpbf.New(context.Background(), file, runtime.GOMAXPROCS(-1)) 17 | defer scanner.Close() 18 | 19 | for scanner.Scan() { 20 | switch o := scanner.Object().(type) { 21 | case *osm.Node: 22 | 23 | case *osm.Way: 24 | 25 | case *osm.Relation: 26 | 27 | } 28 | } 29 | 30 | if err := scanner.Err(); err != nil { 31 | panic(err) 32 | } 33 | ``` 34 | 35 | **Note:** Scanners are **not** safe for parallel use. One should feed the 36 | objects into a channel and have workers read from that. 37 | 38 | ## Skipping Types 39 | 40 | Sometimes only ways or relations are needed. In this case reading and creating 41 | those objects can be skipped completely. After creating the Scanner set the appropriate 42 | attributes to true. 43 | 44 | ``` 45 | type Scanner struct { 46 | // Skip element types that are not needed. The data is skipped 47 | // at the encoded protobuf level, but each block still 48 | // needs to be decompressed. 49 | SkipNodes bool 50 | SkipWays bool 51 | SkipRelations bool 52 | 53 | // contains filtered or unexported fields 54 | } 55 | ``` 56 | 57 | ## Filtering Elements 58 | 59 | The above skips all elements of a type. To filter based on the element's tags or 60 | other values, use the filter functions. These filter functions are called in parallel 61 | and not in a predefined order. This can be a performant way to filter for elements 62 | with a certain set of tags. 63 | 64 | ``` 65 | type Scanner struct { 66 | // If the Filter function is false, the element well be skipped 67 | // at the decoding level. The functions should be fast, they block the 68 | // decoder, there are `procs` number of concurrent decoders. 69 | // Elements can be stored if the function returns true. Memory is 70 | // reused if the filter returns false. 71 | FilterNode func(*osm.Node) bool 72 | FilterWay func(*osm.Way) bool 73 | FilterRelation func(*osm.Relation) bool 74 | 75 | // contains filtered or unexported fields 76 | } 77 | ``` 78 | 79 | ## OSM PBF files with node locations on ways 80 | 81 | This package supports reading OSM PBF files where the ways have been annotated with the coordinates of each node. Such files can be generated using [osmium](https://osmcode.org/osmium-tool), with the [add-locations-to-ways](https://docs.osmcode.org/osmium/latest/osmium-add-locations-to-ways.html) subcommand. This feature makes it possible to work with the ways and their geometries without having to keep all node locations in some index (which takes work and memory resources). 82 | 83 | Coordinates are stored in the `Lat` and `Lon` fields of each `WayNode`. There is no need to specify an explicit option; when the node locations are present on the ways, they are loaded automatically. For more info about the OSM PBF format extension, see [the original blog post](https://blog.jochentopf.com/2016-04-20-node-locations-on-ways.html). 84 | 85 | ## Using cgo/czlib for decompression 86 | 87 | OSM PBF files are a set of blocks that are zlib compressed. When using the pure golang 88 | implementation this can account for about 1/3 of the read time. When cgo is enabled 89 | the package will used [czlib](https://github.com/DataDog/czlib). 90 | 91 | ``` 92 | $ CGO_ENABLED=0 go test -bench . > disabled.txt 93 | $ CGO_ENABLED=1 go test -bench . > enabled.txt 94 | $ benchcmp disabled.txt enabled.txt 95 | benchmark old ns/op new ns/op delta 96 | BenchmarkLondon-12 312294630 229927205 -26.37% 97 | BenchmarkLondon_nodes-12 246562457 160021768 -35.10% 98 | BenchmarkLondon_ways-12 216803544 134747327 -37.85% 99 | BenchmarkLondon_relations-12 158722633 80560144 -49.24% 100 | 101 | benchmark old allocs new allocs delta 102 | BenchmarkLondon-12 2469128 2416804 -2.12% 103 | BenchmarkLondon_nodes-12 1056166 1003850 -4.95% 104 | BenchmarkLondon_ways-12 1845032 1792716 -2.84% 105 | BenchmarkLondon_relations-12 509090 456772 -10.28% 106 | 107 | benchmark old bytes new bytes delta 108 | BenchmarkLondon-12 963734544 954877896 -0.92% 109 | BenchmarkLondon_nodes-12 658337435 649482060 -1.35% 110 | BenchmarkLondon_ways-12 441674734 432819378 -2.00% 111 | BenchmarkLondon_relations-12 187941609 179086389 -4.71% 112 | ``` 113 | -------------------------------------------------------------------------------- /osmpbf/internal/osmpbf/README.md: -------------------------------------------------------------------------------- 1 | *.proto files were downloaded from https://github.com/scrosby/OSM-binary/tree/master/src and changed in following ways: 2 | 3 | * To eliminate continuous conversions from `[]byte` to `string`, this 4 | 5 | ```protobuf 6 | message StringTable { 7 | repeated bytes s = 1; 8 | } 9 | ``` 10 | 11 | was changed to 12 | 13 | ```protobuf 14 | message StringTable { 15 | repeated string s = 1; 16 | } 17 | ``` 18 | 19 | This changes is expected to be fully compatible with all PBF files. 20 | -------------------------------------------------------------------------------- /osmpbf/internal/osmpbf/fileformat.proto: -------------------------------------------------------------------------------- 1 | /** Copyright (c) 2010 Scott A. Crosby. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | 9 | */ 10 | 11 | syntax = "proto2"; 12 | 13 | option java_package = "crosby.binary"; 14 | package osmpbf; 15 | 16 | //protoc --java_out=../.. fileformat.proto 17 | option go_package = "github.com/paulmach/osmpbf/internal/osmpbf"; 18 | 19 | // 20 | // STORAGE LAYER: Storing primitives. 21 | // 22 | 23 | message Blob { 24 | optional bytes raw = 1; // No compression 25 | optional int32 raw_size = 2; // When compressed, the uncompressed size 26 | 27 | // Possible compressed versions of the data. 28 | optional bytes zlib_data = 3; 29 | 30 | // PROPOSED feature for LZMA compressed data. SUPPORT IS NOT REQUIRED. 31 | optional bytes lzma_data = 4; 32 | 33 | // Formerly used for bzip2 compressed data. Depreciated in 2010. 34 | optional bytes OBSOLETE_bzip2_data = 5 [deprecated=true]; // Don't reuse this tag number. 35 | } 36 | 37 | /* A file contains an sequence of fileblock headers, each prefixed by 38 | their length in network byte order, followed by a data block 39 | containing the actual data. types staring with a "_" are reserved. 40 | */ 41 | 42 | message BlobHeader { 43 | required string type = 1; 44 | optional bytes indexdata = 2; 45 | required int32 datasize = 3; 46 | } 47 | 48 | 49 | -------------------------------------------------------------------------------- /osmpbf/internal/osmpbf/generate.go: -------------------------------------------------------------------------------- 1 | package osmpbf 2 | 3 | //go:generate protoc --proto_path=. --go_opt=module=github.com/paulmach/osmpbf/internal/osmpbf --go_out=. fileformat.proto osmformat.proto 4 | -------------------------------------------------------------------------------- /osmpbf/scanner.go: -------------------------------------------------------------------------------- 1 | package osmpbf 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "sync/atomic" 7 | 8 | "github.com/paulmach/osm" 9 | ) 10 | 11 | var _ osm.Scanner = &Scanner{} 12 | 13 | // Scanner provides a convenient interface reading a stream of osm data 14 | // from a file or url. Successive calls to the Scan method will step through the data. 15 | // 16 | // Scanning stops unrecoverably at EOF, the first I/O error, the first xml error or 17 | // the context being cancelled. When a scan stops, the reader may have advanced 18 | // arbitrarily far past the last token. 19 | // 20 | // The Scanner API is based on bufio.Scanner 21 | // https://golang.org/pkg/bufio/#Scanner 22 | type Scanner struct { 23 | // Skip element types that are not needed. The data is skipped 24 | // at the encoded protobuf level, but each block still needs to be decompressed. 25 | SkipNodes bool 26 | SkipWays bool 27 | SkipRelations bool 28 | 29 | // If the Filter function is false, the element well be skipped 30 | // at the decoding level. The functions should be fast, they block the 31 | // decoder, there are `procs` number of concurrent decoders. 32 | // Elements can be stored if the function returns true. Memory is 33 | // reused if the filter returns false. 34 | FilterNode func(*osm.Node) bool 35 | FilterWay func(*osm.Way) bool 36 | FilterRelation func(*osm.Relation) bool 37 | 38 | ctx context.Context 39 | closed bool 40 | 41 | decoder *decoder 42 | started bool 43 | procs int 44 | next osm.Object 45 | err error 46 | } 47 | 48 | // New returns a new Scanner to read from r. 49 | // procs indicates amount of paralellism, when reading blocks 50 | // which will off load the unzipping/decoding to multiple cpus. 51 | func New(ctx context.Context, r io.Reader, procs int) *Scanner { 52 | if ctx == nil { 53 | ctx = context.Background() 54 | } 55 | 56 | s := &Scanner{ 57 | ctx: ctx, 58 | procs: procs, 59 | } 60 | s.decoder = newDecoder(ctx, s, r) 61 | return s 62 | } 63 | 64 | // FullyScannedBytes returns the number of bytes that have been read 65 | // and fully scanned. OSM protobuf files contain data blocks with 66 | // 8000 nodes each. The returned value contains the bytes for the blocks 67 | // that have been fully scanned. 68 | // 69 | // A user can use this number of seek forward in a file 70 | // and begin reading mid-data. Note that while elements are usually sorted 71 | // by Type, ID, Version in OSM protobuf files, versions of given element may 72 | // span blocks. 73 | func (s *Scanner) FullyScannedBytes() int64 { 74 | return atomic.LoadInt64(&s.decoder.cOffset) 75 | } 76 | 77 | // PreviousFullyScannedBytes returns the previous value of FullyScannedBytes. 78 | // This is interesting because it's not totally clear if a feature spans a block. 79 | // For example, if one quits after finding the first relation, upon restarting there 80 | // is no way of knowing if the first relation is complete, so skip it. But if this relation 81 | // is the first relation in the file we'll skip a full relation. 82 | func (s *Scanner) PreviousFullyScannedBytes() int64 { 83 | return atomic.LoadInt64(&s.decoder.pOffset) 84 | } 85 | 86 | // Close cleans up all the reading goroutines, it does not 87 | // close the underlying reader. 88 | func (s *Scanner) Close() error { 89 | s.closed = true 90 | return s.decoder.Close() 91 | } 92 | 93 | // Header returns the pbf file header with interesting information 94 | // about how it was created. 95 | func (s *Scanner) Header() (*Header, error) { 96 | if !s.started { 97 | s.started = true 98 | // the header gets read before Start returns 99 | s.err = s.decoder.Start(s.procs) 100 | } 101 | 102 | return s.decoder.header, s.err 103 | } 104 | 105 | // Scan advances the Scanner to the next element, which will then be available 106 | // through the Element method. It returns false when the scan stops, either 107 | // by reaching the end of the input, an io error, an xml error or the context 108 | // being cancelled. After Scan returns false, the Err method will return any 109 | // error that occurred during scanning, except that if it was io.EOF, Err will 110 | // return nil. 111 | func (s *Scanner) Scan() bool { 112 | if !s.started { 113 | s.started = true 114 | s.err = s.decoder.Start(s.procs) 115 | } 116 | 117 | if s.err != nil || s.closed || s.ctx.Err() != nil { 118 | return false 119 | } 120 | 121 | s.next, s.err = s.decoder.Next() 122 | return s.err == nil 123 | } 124 | 125 | // Object returns the most recent token generated by a call to Scan 126 | // as a new osm.Object. Currently osm.pbf files only contain nodes, ways and 127 | // relations. This method returns an object so match the osm.Scanner interface 128 | // and allows this Scanner to share an interface with osmxml.Scanner. 129 | func (s *Scanner) Object() osm.Object { 130 | return s.next 131 | } 132 | 133 | // Err returns the first non-EOF error that was encountered by the Scanner. 134 | func (s *Scanner) Err() error { 135 | if s.err == io.EOF { 136 | return nil 137 | } 138 | 139 | if s.err != nil { 140 | return s.err 141 | } 142 | 143 | if s.closed { 144 | return osm.ErrScannerClosed 145 | } 146 | 147 | return s.ctx.Err() 148 | } 149 | -------------------------------------------------------------------------------- /osmpbf/zlib_cgo.go: -------------------------------------------------------------------------------- 1 | // +build cgo 2 | 3 | package osmpbf 4 | 5 | import ( 6 | "bytes" 7 | "io" 8 | 9 | "github.com/datadog/czlib" 10 | ) 11 | 12 | func zlibReader(data []byte) (io.ReadCloser, error) { 13 | return czlib.NewReader(bytes.NewReader(data)) 14 | } 15 | -------------------------------------------------------------------------------- /osmpbf/zlib_go.go: -------------------------------------------------------------------------------- 1 | // +build !cgo 2 | 3 | package osmpbf 4 | 5 | import ( 6 | "bytes" 7 | "compress/zlib" 8 | "io" 9 | ) 10 | 11 | func zlibReader(data []byte) (io.ReadCloser, error) { 12 | return zlib.NewReader(bytes.NewReader(data)) 13 | } 14 | -------------------------------------------------------------------------------- /osmtest/scanner.go: -------------------------------------------------------------------------------- 1 | package osmtest 2 | 3 | import "github.com/paulmach/osm" 4 | 5 | // Scanner implements the osm.Scanner interface with 6 | // just a list of objects. 7 | type Scanner struct { 8 | // ScanError can be used to trigger an error. 9 | // If non-nil, Next() will return false and Err() will 10 | // return this error. 11 | ScanError error 12 | 13 | offset int 14 | objects osm.Objects 15 | } 16 | 17 | var _ osm.Scanner = &Scanner{} 18 | 19 | // NewScanner creates a new test scanner useful for test stubbing. 20 | func NewScanner(objects osm.Objects) *Scanner { 21 | return &Scanner{ 22 | offset: -1, 23 | objects: objects, 24 | } 25 | } 26 | 27 | // Scan progresses the scanner to the next object. 28 | func (s *Scanner) Scan() bool { 29 | if s.ScanError != nil { 30 | return false 31 | } 32 | 33 | s.offset++ 34 | return s.offset < len(s.objects) 35 | } 36 | 37 | // Object returns the current object. 38 | func (s *Scanner) Object() osm.Object { 39 | return s.objects[s.offset] 40 | } 41 | 42 | // Err returns the scanner.ScanError. 43 | func (s *Scanner) Err() error { 44 | return s.ScanError 45 | } 46 | 47 | // Close is a stub for this test scanner. 48 | func (s *Scanner) Close() error { 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /osmtest/scanner_test.go: -------------------------------------------------------------------------------- 1 | package osmtest 2 | 3 | import ( 4 | "errors" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/paulmach/osm" 9 | ) 10 | 11 | func TestScanner(t *testing.T) { 12 | objs := osm.Objects{ 13 | &osm.Node{ID: 1, Version: 4}, 14 | &osm.Way{ID: 2, Version: 5}, 15 | &osm.Relation{ID: 3, Version: 6}, 16 | } 17 | 18 | scanner := NewScanner(objs) 19 | defer scanner.Close() 20 | 21 | expected := osm.ObjectIDs{ 22 | osm.NodeID(1).ObjectID(4), 23 | osm.WayID(2).ObjectID(5), 24 | osm.RelationID(3).ObjectID(6), 25 | } 26 | 27 | ids := osm.ObjectIDs{} 28 | for scanner.Scan() { 29 | ids = append(ids, scanner.Object().ObjectID()) 30 | } 31 | 32 | if !reflect.DeepEqual(ids, expected) { 33 | t.Errorf("incorrect ids: %v", ids) 34 | } 35 | } 36 | 37 | func TestScanner_error(t *testing.T) { 38 | objs := osm.Objects{ 39 | &osm.Node{ID: 1, Version: 4}, 40 | &osm.Way{ID: 2, Version: 5}, 41 | &osm.Relation{ID: 3, Version: 6}, 42 | } 43 | 44 | scanner := NewScanner(objs) 45 | defer scanner.Close() 46 | 47 | if scanner.Err() != nil { 48 | t.Errorf("error should not be set initially") 49 | } 50 | 51 | if !scanner.Scan() { 52 | t.Errorf("should be true initially") 53 | } 54 | 55 | scanner.ScanError = errors.New("some error") 56 | 57 | if scanner.Scan() { 58 | t.Errorf("should be false after error") 59 | } 60 | 61 | if scanner.Err() == nil { 62 | t.Errorf("should return error if there is one") 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /osmxml/example_test.go: -------------------------------------------------------------------------------- 1 | package osmxml_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/paulmach/osm" 9 | "github.com/paulmach/osm/osmxml" 10 | ) 11 | 12 | func ExampleScanner() { 13 | scanner := osmxml.New(context.Background(), os.Stdin) 14 | for scanner.Scan() { 15 | fmt.Println(scanner.Object().(*osm.Changeset)) 16 | } 17 | 18 | if err := scanner.Err(); err != nil { 19 | fmt.Fprintln(os.Stderr, "reading standard input:", err) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /osmxml/scanner.go: -------------------------------------------------------------------------------- 1 | package osmxml 2 | 3 | import ( 4 | "context" 5 | "encoding/xml" 6 | "io" 7 | "strings" 8 | 9 | "github.com/paulmach/osm" 10 | ) 11 | 12 | var _ osm.Scanner = &Scanner{} 13 | 14 | // Scanner provides a convenient interface reading a stream of osm data 15 | // from a file or url. Successive calls to the Scan method will step through the data. 16 | // 17 | // Scanning stops unrecoverably at EOF, the first I/O error, the first xml error or 18 | // the context being cancelled. When a scan stops, the reader may have advanced 19 | // arbitrarily far past the last token. 20 | // 21 | // The Scanner API is based on bufio.Scanner 22 | // https://golang.org/pkg/bufio/#Scanner 23 | type Scanner struct { 24 | ctx context.Context 25 | done context.CancelFunc 26 | closed bool 27 | 28 | decoder *xml.Decoder 29 | next osm.Object 30 | err error 31 | } 32 | 33 | // New returns a new Scanner to read from r. 34 | func New(ctx context.Context, r io.Reader) *Scanner { 35 | if ctx == nil { 36 | ctx = context.Background() 37 | } 38 | 39 | s := &Scanner{ 40 | decoder: xml.NewDecoder(r), 41 | } 42 | 43 | s.ctx, s.done = context.WithCancel(ctx) 44 | return s 45 | } 46 | 47 | // Close causes all future calls to Scan to return false. 48 | // Does not close the underlying reader. 49 | func (s *Scanner) Close() error { 50 | s.closed = true 51 | s.done() 52 | 53 | return nil 54 | } 55 | 56 | // Scan advances the Scanner to the next element, which will then be available 57 | // through the Object method. It returns false when the scan stops, either 58 | // by reaching the end of the input, an io error, an xml error or the context 59 | // being cancelled. After Scan returns false, the Err method will return any 60 | // error that occurred during scanning, except if it was io.EOF, Err will 61 | // return nil. 62 | func (s *Scanner) Scan() bool { 63 | if s.err != nil { 64 | return false 65 | } 66 | 67 | Loop: 68 | for { 69 | if s.ctx.Err() != nil { 70 | return false 71 | } 72 | 73 | t, err := s.decoder.Token() 74 | if err != nil { 75 | s.err = err 76 | return false 77 | } 78 | 79 | se, ok := t.(xml.StartElement) 80 | if !ok { 81 | continue 82 | } 83 | 84 | s.next = nil 85 | switch strings.ToLower(se.Name.Local) { 86 | case "bounds": 87 | bounds := &osm.Bounds{} 88 | err = s.decoder.DecodeElement(&bounds, &se) 89 | s.next = bounds 90 | case "node": 91 | node := &osm.Node{} 92 | err = s.decoder.DecodeElement(&node, &se) 93 | s.next = node 94 | case "way": 95 | way := &osm.Way{} 96 | err = s.decoder.DecodeElement(&way, &se) 97 | s.next = way 98 | case "relation": 99 | relation := &osm.Relation{} 100 | err = s.decoder.DecodeElement(&relation, &se) 101 | s.next = relation 102 | case "changeset": 103 | cs := &osm.Changeset{} 104 | err = s.decoder.DecodeElement(&cs, &se) 105 | s.next = cs 106 | case "note": 107 | n := &osm.Note{} 108 | err = s.decoder.DecodeElement(&n, &se) 109 | s.next = n 110 | case "user": 111 | u := &osm.User{} 112 | err = s.decoder.DecodeElement(&u, &se) 113 | s.next = u 114 | default: 115 | continue Loop 116 | } 117 | 118 | if err != nil { 119 | s.err = err 120 | return false 121 | } 122 | 123 | return true 124 | } 125 | } 126 | 127 | // Object returns the most recent token generated by a call to Scan 128 | // as a new osm.Object. This interface is implemented by: 129 | // *osm.Bounds 130 | // *osm.Node 131 | // *osm.Way 132 | // *osm.Relation 133 | // *osm.Changeset 134 | // *osm.Note 135 | // *osm.User 136 | func (s *Scanner) Object() osm.Object { 137 | return s.next 138 | } 139 | 140 | // Err returns the first non-EOF error that was encountered by the Scanner. 141 | func (s *Scanner) Err() error { 142 | if s.err == io.EOF { 143 | return nil 144 | } 145 | 146 | if s.err != nil { 147 | return s.err 148 | } 149 | 150 | if s.closed { 151 | return osm.ErrScannerClosed 152 | } 153 | 154 | return s.ctx.Err() 155 | } 156 | -------------------------------------------------------------------------------- /polygon_test.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestWay_Polygon(t *testing.T) { 9 | w := &Way{} 10 | w.Nodes = WayNodes{ 11 | {ID: 1}, {ID: 2}, 12 | {ID: 3}, {ID: 1}, 13 | } 14 | 15 | w2 := &Way{Nodes: w.Nodes[:3]} 16 | if w2.Polygon() { 17 | t.Errorf("should be over 3 nodes to be polygon") 18 | } 19 | 20 | w.Nodes[3].ID = 10 21 | if w2.Polygon() { 22 | t.Errorf("first and last node must have same id") 23 | } 24 | 25 | c := polyConditions[1].Values 26 | if !reflect.DeepEqual(c, []string{"elevator", "escape", "rest_area", "services"}) { 27 | t.Errorf("values not sorted") 28 | } 29 | 30 | cases := []struct { 31 | name string 32 | tags []Tag 33 | value bool 34 | }{ 35 | { 36 | name: "area no overrides", 37 | tags: []Tag{ 38 | {Key: "area", Value: "no"}, 39 | {Key: "building", Value: "yes"}, 40 | }, 41 | value: false, 42 | }, 43 | { 44 | name: "non-empty area and not no", 45 | tags: []Tag{ 46 | {Key: "area", Value: "maybe"}, 47 | {Key: "building", Value: "no"}, 48 | }, 49 | value: true, 50 | }, 51 | { 52 | name: "at least one condition met", 53 | tags: []Tag{ 54 | {Key: "building", Value: "no"}, 55 | {Key: "boundary", Value: "yes"}, 56 | }, 57 | value: true, 58 | }, 59 | { 60 | name: "match within whitelist", 61 | tags: []Tag{ 62 | {Key: "railway", Value: "station"}, 63 | }, 64 | value: true, 65 | }, 66 | { 67 | name: "not match if not within whitelist", 68 | tags: []Tag{ 69 | {Key: "railway", Value: "line"}, 70 | }, 71 | value: false, 72 | }, 73 | { 74 | name: "not match within blacklist", 75 | tags: []Tag{ 76 | {Key: "man_made", Value: "cutline"}, 77 | }, 78 | value: false, 79 | }, 80 | { 81 | name: "match if not within blacklist", 82 | tags: []Tag{ 83 | {Key: "man_made", Value: "thing"}, 84 | }, 85 | value: true, 86 | }, 87 | { 88 | name: "indoor anything is a polygon", 89 | tags: []Tag{ 90 | {Key: "indoor", Value: "anything"}, 91 | }, 92 | value: true, 93 | }, 94 | } 95 | 96 | for _, tc := range cases { 97 | t.Run(tc.name, func(t *testing.T) { 98 | w := &Way{ 99 | Nodes: WayNodes{ 100 | {ID: 1}, {ID: 2}, 101 | {ID: 3}, {ID: 1}, 102 | }, 103 | Tags: Tags(tc.tags), 104 | } 105 | 106 | if v := w.Polygon(); v != tc.value { 107 | t.Errorf("not correctly detected, %v != %v", v, tc.value) 108 | } 109 | }) 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /replication/README.md: -------------------------------------------------------------------------------- 1 | # osm/replication [![Godoc Reference](https://pkg.go.dev/badge/github.com/paulmach/osm)](https://pkg.go.dev/github.com/paulmach/osm/replication) 2 | 3 | Package `replication` handles fetching the Minute, Hour, Day and Changeset replication 4 | and the associated state value from [Planet OSM](http://planet.osm.org). 5 | 6 | For example, to fetch the current Minute replication state: 7 | 8 | ```go 9 | num, fullState, err := replication.CurrentMinuteState(ctx) 10 | ``` 11 | 12 | This is the data in [http://planet.osm.org/replication/minute/state.txt](http://planet.osm.org/replication/minute/state.txt) 13 | updated every minute. 14 | 15 | Once you know the change number you want, fetch the change using: 16 | 17 | ```go 18 | change, err := replication.Minute(ctx, num) 19 | ``` 20 | 21 | ## Finding sequences numbers by timestamp 22 | 23 | It's also possible to find the sequence number by timestamp. 24 | These calls make multiple requests for state to find the one matching the given timestamp. 25 | 26 | ```go 27 | MinuteStateAt(ctx context.Context, timestamp time.Time) (MinuteSeqNum, *State, error) 28 | HourStateAt(ctx context.Context, timestamp time.Time) (HourSeqNum, *State, error) 29 | DayStateAt(ctx context.Context, timestamp time.Time) (DaySeqNum, *State, error) 30 | ChangesetStateAt(ctx context.Context, timestamp time.Time) (ChangesetSeqNum, *State, error) 31 | ``` 32 | -------------------------------------------------------------------------------- /replication/changesets_test.go: -------------------------------------------------------------------------------- 1 | package replication 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "context" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestDecodeChangesetState(t *testing.T) { 12 | data := []byte(`--- 13 | last_run: 2016-07-02 22:46:01.422137422 Z 14 | sequence: 1912325 15 | `) 16 | 17 | state, err := decodeChangesetState(data) 18 | if v := ChangesetSeqNum(state.SeqNum); v != 1912325 { 19 | t.Errorf("incorrect sequence number, got %v", v) 20 | } 21 | 22 | if !state.Timestamp.Equal(time.Date(2016, 7, 2, 22, 46, 1, 422137422, time.UTC)) { 23 | t.Errorf("incorrect time, got %v", state.Timestamp) 24 | } 25 | 26 | if err != nil { 27 | t.Errorf("got error: %v", err) 28 | } 29 | } 30 | 31 | func TestChangesetDecoder(t *testing.T) { 32 | ctx := context.Background() 33 | 34 | buf := bytes.NewBuffer(nil) 35 | w := gzip.NewWriter(buf) 36 | w.Write([]byte(` 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | `)) 52 | w.Close() 53 | 54 | changesets, err := changesetDecoder(ctx, buf) 55 | if err != nil { 56 | t.Fatalf("decode error: %v", err) 57 | } 58 | 59 | if len(changesets) != 2 { 60 | t.Errorf("incorrect number of changes: %d", len(changesets)) 61 | } 62 | } 63 | 64 | func TestBaseChangesetURL(t *testing.T) { 65 | url := DefaultDatasource.baseChangesetURL(123456789) 66 | if url != "https://planet.osm.org/replication/changesets/123/456/789" { 67 | t.Errorf("incorrect url, got %v", url) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /replication/datasource.go: -------------------------------------------------------------------------------- 1 | package replication 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "time" 7 | ) 8 | 9 | // BaseURL defines the default planet server to hit. 10 | const BaseURL = "https://planet.osm.org" 11 | 12 | // Datasource defines context around replication data requests. 13 | type Datasource struct { 14 | BaseURL string // will use package level BaseURL if empty 15 | Client *http.Client 16 | } 17 | 18 | // DefaultDatasource is the Datasource used by the package level convenience functions. 19 | var DefaultDatasource = &Datasource{ 20 | Client: &http.Client{ 21 | Timeout: 30 * time.Minute, 22 | }, 23 | } 24 | 25 | // NewDatasource creates a Datasource using the given client. 26 | func NewDatasource(client *http.Client) *Datasource { 27 | return &Datasource{ 28 | Client: client, 29 | } 30 | } 31 | 32 | func (ds Datasource) baseURL() string { 33 | if ds.BaseURL != "" { 34 | return ds.BaseURL 35 | } 36 | 37 | return BaseURL 38 | } 39 | 40 | func (ds Datasource) client() *http.Client { 41 | if ds.Client != nil { 42 | return ds.Client 43 | } 44 | 45 | if DefaultDatasource.Client != nil { 46 | return DefaultDatasource.Client 47 | } 48 | 49 | return http.DefaultClient 50 | } 51 | 52 | // UnexpectedStatusCodeError is return for a non 200 or 404 status code. 53 | type UnexpectedStatusCodeError struct { 54 | Code int 55 | URL string 56 | } 57 | 58 | // Error returns an error message with some information. 59 | func (e *UnexpectedStatusCodeError) Error() string { 60 | return fmt.Sprintf("replication: unexpected status code of %d for url %s", e.Code, e.URL) 61 | } 62 | 63 | // NotFound will return try if the error from one of the methods was due 64 | // to the file not found on the remote host. 65 | func NotFound(err error) bool { 66 | if err == nil { 67 | return false 68 | } 69 | 70 | if e, ok := err.(*UnexpectedStatusCodeError); ok { 71 | return e.Code == http.StatusNotFound 72 | } 73 | 74 | return false 75 | } 76 | 77 | // timeFormats contains the set of different formats we've see the time in. 78 | var timeFormats = []string{ 79 | "2006-01-02 15:04:05.999999999 Z", 80 | "2006-01-02 15:04:05.999999999 +00:00", 81 | "2006-01-02T15\\:04\\:05Z", 82 | } 83 | 84 | func decodeTime(s string) (time.Time, error) { 85 | var ( 86 | t time.Time 87 | err error 88 | ) 89 | for _, format := range timeFormats { 90 | t, err = time.Parse(format, s) 91 | if err == nil { 92 | return t, nil 93 | } 94 | } 95 | 96 | return t, err 97 | } 98 | -------------------------------------------------------------------------------- /replication/interval_test.go: -------------------------------------------------------------------------------- 1 | package replication 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestDecodeIntervalState(t *testing.T) { 9 | data := []byte(`#Sat Jul 16 06:28:03 UTC 2016 10 | txnMaxQueried=836441250 11 | sequenceNumber=2010594 12 | timestamp=2016-07-16T06\:28\:02Z 13 | txnReadyList= 14 | txnMax=836441259 15 | txnActiveList=836441203 16 | `) 17 | 18 | state, err := decodeIntervalState(data) 19 | if v := MinuteSeqNum(state.SeqNum); v != 2010594 { 20 | t.Errorf("incorrect id, got %v", v) 21 | } 22 | 23 | if !state.Timestamp.Equal(time.Date(2016, 7, 16, 6, 28, 2, 0, time.UTC)) { 24 | t.Errorf("incorrect time, got %v", state.Timestamp) 25 | } 26 | 27 | if v := state.TxnMax; v != 836441259 { 28 | t.Errorf("incorrect txnMax, got %v", v) 29 | } 30 | 31 | if v := state.TxnMaxQueried; v != 836441250 { 32 | t.Errorf("incorrect txnMaxQueried, got %v", v) 33 | } 34 | 35 | if err != nil { 36 | t.Errorf("got error: %v", err) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /replication/live_test.go: -------------------------------------------------------------------------------- 1 | package replication 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func liveOnly(t testing.TB) { 10 | if os.Getenv("LIVE_TEST") != "true" { 11 | t.Skipf("skipping live test, set LIVE_TEST=true to enable") 12 | } 13 | } 14 | 15 | func TestCurrentState(t *testing.T) { 16 | liveOnly(t) 17 | ctx := context.Background() 18 | 19 | _, _, err := CurrentMinuteState(ctx) 20 | if err != nil { 21 | t.Fatalf("request error: %v", err) 22 | } 23 | 24 | _, _, err = CurrentHourState(ctx) 25 | if err != nil { 26 | t.Fatalf("request error: %v", err) 27 | } 28 | 29 | _, _, err = CurrentDayState(ctx) 30 | if err != nil { 31 | t.Fatalf("request error: %v", err) 32 | } 33 | } 34 | 35 | func TestDownloadChanges(t *testing.T) { 36 | liveOnly(t) 37 | ctx := context.Background() 38 | 39 | _, err := Minute(ctx, 10) 40 | if err != nil { 41 | t.Fatalf("request error: %v", err) 42 | } 43 | 44 | _, err = Hour(ctx, 10) 45 | if err != nil { 46 | t.Fatalf("request error: %v", err) 47 | } 48 | 49 | _, err = Day(ctx, 1) 50 | if err != nil { 51 | t.Fatalf("request error: %v", err) 52 | } 53 | } 54 | 55 | func TestCurrentChangesetState(t *testing.T) { 56 | liveOnly(t) 57 | 58 | ctx := context.Background() 59 | _, _, err := CurrentChangesetState(ctx) 60 | if err != nil { 61 | t.Fatalf("request error: %v", err) 62 | } 63 | } 64 | 65 | func TestChangesets(t *testing.T) { 66 | liveOnly(t) 67 | 68 | ctx := context.Background() 69 | sets, err := Changesets(ctx, 100) 70 | if err != nil { 71 | t.Fatalf("request error: %v", err) 72 | } 73 | 74 | if l := len(sets); l != 12 { 75 | t.Errorf("incorrect number of changesets: %v", l) 76 | } 77 | } 78 | 79 | func TestChangesetState(t *testing.T) { 80 | liveOnly(t) 81 | 82 | ctx := context.Background() 83 | state, err := ChangesetState(ctx, 5001990) 84 | if err != nil { 85 | t.Fatalf("request error: %v", err) 86 | } 87 | 88 | if state.SeqNum != 5001990 { 89 | t.Errorf("incorrect state: %+v", state) 90 | } 91 | 92 | // current state 93 | n, state, err := CurrentChangesetState(ctx) 94 | if err != nil { 95 | t.Fatalf("request error: %v", err) 96 | } 97 | 98 | changes, err := Changesets(ctx, n) 99 | if err != nil { 100 | t.Fatalf("request error: %v", err) 101 | } 102 | 103 | for _, c := range changes { 104 | if c.CreatedAt.After(state.Timestamp) { 105 | t.Errorf("data is after the state file?") 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /tag.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "encoding/json" 5 | "sort" 6 | ) 7 | 8 | // UninterestingTags are boring tags. If an element only has 9 | // these tags it does not usually need to be displayed. 10 | // For example, if a node with just these tags is part of a way, it 11 | // probably does not need its own icon along the way. 12 | var UninterestingTags = map[string]bool{ 13 | "source": true, 14 | "source_ref": true, 15 | "source:ref": true, 16 | "history": true, 17 | "attribution": true, 18 | "created_by": true, 19 | "tiger:county": true, 20 | "tiger:tlid": true, 21 | "tiger:upload_uuid": true, 22 | } 23 | 24 | // Tag is a key+value item attached to osm nodes, ways and relations. 25 | type Tag struct { 26 | Key string `xml:"k,attr"` 27 | Value string `xml:"v,attr"` 28 | } 29 | 30 | // Tags is a collection of Tag objects with some helper functions. 31 | type Tags []Tag 32 | 33 | // Find will return the value for the key. 34 | // Will return an empty string if not found. 35 | func (ts Tags) Find(k string) string { 36 | for _, t := range ts { 37 | if t.Key == k { 38 | return t.Value 39 | } 40 | } 41 | 42 | return "" 43 | } 44 | 45 | // FindTag will return the Tag for the given key. 46 | // Can be used to determine if a key exists, even with an empty value. 47 | // Returns nil if not found. 48 | func (ts Tags) FindTag(k string) *Tag { 49 | for _, t := range ts { 50 | if t.Key == k { 51 | return &t 52 | } 53 | } 54 | 55 | return nil 56 | } 57 | 58 | // HasTag will return the true if a tag exists for the given key. 59 | func (ts Tags) HasTag(k string) bool { 60 | for _, t := range ts { 61 | if t.Key == k { 62 | return true 63 | } 64 | } 65 | 66 | return false 67 | } 68 | 69 | // Map returns the tags as a key/value map. 70 | func (ts Tags) Map() map[string]string { 71 | result := make(map[string]string, len(ts)) 72 | for _, t := range ts { 73 | result[t.Key] = t.Value 74 | } 75 | 76 | return result 77 | } 78 | 79 | // AnyInteresting will return true if there is at last one interesting tag. 80 | func (ts Tags) AnyInteresting() bool { 81 | for _, t := range ts { 82 | if !UninterestingTags[t.Key] { 83 | return true 84 | } 85 | } 86 | 87 | return false 88 | } 89 | 90 | // MarshalJSON allows the tags to be marshalled as a key/value object, 91 | // as defined by the overpass osmjson. 92 | func (ts Tags) MarshalJSON() ([]byte, error) { 93 | return marshalJSON(ts.Map()) 94 | } 95 | 96 | // UnmarshalJSON allows the tags to be unmarshalled from a key/value object, 97 | // as defined by the overpass osmjson. 98 | func (ts *Tags) UnmarshalJSON(data []byte) error { 99 | o := make(map[string]string) 100 | err := json.Unmarshal(data, &o) 101 | if err != nil { 102 | return err 103 | } 104 | 105 | tags := make(Tags, 0, len(o)) 106 | 107 | for k, v := range o { 108 | tags = append(tags, Tag{Key: k, Value: v}) 109 | } 110 | 111 | *ts = tags 112 | return nil 113 | } 114 | 115 | type tagsSort Tags 116 | 117 | // SortByKeyValue will do an inplace sort of the tags. 118 | func (ts Tags) SortByKeyValue() { 119 | sort.Sort(tagsSort(ts)) 120 | } 121 | func (ts tagsSort) Len() int { return len(ts) } 122 | func (ts tagsSort) Swap(i, j int) { ts[i], ts[j] = ts[j], ts[i] } 123 | func (ts tagsSort) Less(i, j int) bool { 124 | if ts[i].Key == ts[j].Key { 125 | return ts[i].Value < ts[j].Value 126 | } 127 | 128 | return ts[i].Key < ts[j].Key 129 | } 130 | -------------------------------------------------------------------------------- /tag_test.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "bytes" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | func TestTags_FindTag(t *testing.T) { 10 | cases := []struct { 11 | name string 12 | tags Tags 13 | key string 14 | tag *Tag 15 | }{ 16 | { 17 | name: "find tag", 18 | tags: Tags{ 19 | {Key: "area", Value: "true"}, 20 | {Key: "building", Value: "yes"}, 21 | }, 22 | key: "building", 23 | tag: &Tag{Key: "building", Value: "yes"}, 24 | }, 25 | { 26 | name: "not found", 27 | tags: Tags{ 28 | {Key: "building", Value: "yes"}, 29 | }, 30 | key: "not found", 31 | tag: nil, 32 | }, 33 | { 34 | name: "empty value", 35 | tags: Tags{ 36 | {Key: "present", Value: ""}, 37 | }, 38 | key: "present", 39 | tag: &Tag{Key: "present", Value: ""}, 40 | }, 41 | } 42 | 43 | for _, tc := range cases { 44 | t.Run(tc.name, func(t *testing.T) { 45 | v := tc.tags.FindTag(tc.key) 46 | if !reflect.DeepEqual(v, tc.tag) { 47 | t.Errorf("incorrect find tag: %v != %v", v, tc.tag) 48 | } 49 | }) 50 | } 51 | } 52 | 53 | func TestTags_HasTag(t *testing.T) { 54 | cases := []struct { 55 | name string 56 | tags Tags 57 | key string 58 | has bool 59 | }{ 60 | { 61 | name: "has tag", 62 | tags: Tags{ 63 | {Key: "area", Value: "true"}, 64 | {Key: "building", Value: "yes"}, 65 | }, 66 | key: "building", 67 | has: true, 68 | }, 69 | { 70 | name: "not found", 71 | tags: Tags{ 72 | {Key: "building", Value: "yes"}, 73 | }, 74 | key: "not found", 75 | has: false, 76 | }, 77 | { 78 | name: "empty value", 79 | tags: Tags{ 80 | {Key: "present", Value: ""}, 81 | }, 82 | key: "present", 83 | has: true, 84 | }, 85 | } 86 | 87 | for _, tc := range cases { 88 | t.Run(tc.name, func(t *testing.T) { 89 | v := tc.tags.HasTag(tc.key) 90 | if v != tc.has { 91 | t.Errorf("incorrect has tag: %v != %v", v, tc.has) 92 | } 93 | }) 94 | } 95 | } 96 | 97 | func TestTags_AnyInteresting(t *testing.T) { 98 | cases := []struct { 99 | name string 100 | tags Tags 101 | interesting bool 102 | }{ 103 | { 104 | name: "has interesting", 105 | tags: Tags{ 106 | {Key: "building", Value: "yes"}, 107 | }, 108 | interesting: true, 109 | }, 110 | { 111 | name: "no tags", 112 | tags: Tags{}, 113 | interesting: false, 114 | }, 115 | { 116 | name: "no interesting tags", 117 | tags: Tags{ 118 | {Key: "source", Value: "whatever"}, 119 | {Key: "history", Value: "lots"}, 120 | }, 121 | interesting: false, 122 | }, 123 | } 124 | 125 | for _, tc := range cases { 126 | t.Run(tc.name, func(t *testing.T) { 127 | v := tc.tags.AnyInteresting() 128 | if v != tc.interesting { 129 | t.Errorf("incorrect interesting: %v != %v", v, tc.interesting) 130 | } 131 | }) 132 | } 133 | } 134 | 135 | func TestTags_MarshalJSON(t *testing.T) { 136 | data, err := Tags{}.MarshalJSON() 137 | if err != nil { 138 | t.Errorf("marshal error: %v", err) 139 | } 140 | 141 | if !bytes.Equal(data, []byte(`{}`)) { 142 | t.Errorf("incorrect data, got: %v", string(data)) 143 | } 144 | 145 | t2 := Tags{ 146 | Tag{Key: "highway 🏤 ", Value: "crossing"}, 147 | Tag{Key: "source", Value: "Bind 🏤 "}, 148 | } 149 | 150 | data, err = t2.MarshalJSON() 151 | if err != nil { 152 | t.Errorf("marshal error: %v", err) 153 | } 154 | if !bytes.Equal(data, []byte(`{"highway 🏤 ":"crossing","source":"Bind 🏤 "}`)) { 155 | t.Errorf("incorrect data, got: %v", string(data)) 156 | } 157 | } 158 | 159 | func TestTags_UnmarshalJSON(t *testing.T) { 160 | tags := Tags{} 161 | data := []byte(`{"highway 🏤 ":"crossing","source":"Bind 🏤 "}`) 162 | 163 | err := tags.UnmarshalJSON(data) 164 | if err != nil { 165 | t.Errorf("unmarshal error: %v", err) 166 | } 167 | 168 | tags.SortByKeyValue() 169 | t2 := Tags{ 170 | Tag{Key: "highway 🏤 ", Value: "crossing"}, 171 | Tag{Key: "source", Value: "Bind 🏤 "}, 172 | } 173 | 174 | if !reflect.DeepEqual(tags, t2) { 175 | t.Errorf("incorrect tags: %v", tags) 176 | } 177 | } 178 | 179 | func TestTags_SortByKeyValue(t *testing.T) { 180 | tags := Tags{ 181 | Tag{Key: "highway", Value: "crossing"}, 182 | Tag{Key: "source", Value: "Bind"}, 183 | } 184 | 185 | tags.SortByKeyValue() 186 | if v := tags[0].Key; v != "highway" { 187 | t.Errorf("incorrect sort got %v", v) 188 | } 189 | 190 | if v := tags[1].Key; v != "source" { 191 | t.Errorf("incorrect sort got %v", v) 192 | } 193 | 194 | tags = Tags{ 195 | Tag{Key: "source", Value: "Bind"}, 196 | Tag{Key: "highway", Value: "crossing"}, 197 | } 198 | 199 | tags.SortByKeyValue() 200 | if v := tags[0].Key; v != "highway" { 201 | t.Errorf("incorrect sort got %v", v) 202 | } 203 | 204 | if v := tags[1].Key; v != "source" { 205 | t.Errorf("incorrect sort got %v", v) 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /testdata/andorra-latest.osm.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paulmach/osm/db9d8ebc69cde0241f4989a74ea94ca9e7066e12/testdata/andorra-latest.osm.bz2 -------------------------------------------------------------------------------- /testdata/compare_scanners.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "compress/bzip2" 5 | "context" 6 | "log" 7 | "math" 8 | "os" 9 | "reflect" 10 | 11 | "github.com/paulmach/osm" 12 | "github.com/paulmach/osm/osmpbf" 13 | "github.com/paulmach/osm/osmxml" 14 | ) 15 | 16 | func main() { 17 | // these can be downloaded at http://download.geofabrik.de/north-america.html 18 | pbffile := "delaware-latest.osm.pbf" 19 | bz2file := "delaware-latest.osm.bz2" 20 | 21 | o1 := readPbf(pbffile) 22 | o2 := readBz2(bz2file) 23 | 24 | log.Printf("Are they the same? %v", reflect.DeepEqual(o1, o2)) 25 | 26 | log.Printf("nodes: %v %v", len(o1.Nodes), len(o2.Nodes)) 27 | if len(o1.Nodes) == len(o2.Nodes) { 28 | for i := range o1.Nodes { 29 | if !reflect.DeepEqual(o1.Nodes[i], o2.Nodes[i]) { 30 | log.Printf("unequal nodes") 31 | log.Printf("%v", o1.Nodes[i]) 32 | log.Printf("%v", o2.Nodes[i]) 33 | } 34 | } 35 | } 36 | 37 | log.Printf("ways: %v %v", len(o1.Ways), len(o2.Ways)) 38 | if len(o1.Ways) == len(o2.Ways) { 39 | for i := range o1.Ways { 40 | if !reflect.DeepEqual(o1.Ways[i], o2.Ways[i]) { 41 | log.Printf("unequal ways") 42 | log.Printf("%v", o1.Ways[i]) 43 | log.Printf("%v", o2.Ways[i]) 44 | } 45 | } 46 | } 47 | 48 | log.Printf("relations: %v %v", len(o1.Relations), len(o2.Relations)) 49 | if len(o1.Relations) == len(o2.Relations) { 50 | for i := range o1.Relations { 51 | if !reflect.DeepEqual(o1.Relations[i], o2.Relations[i]) { 52 | log.Printf("unequal relations") 53 | log.Printf("%v", o1.Relations[i]) 54 | log.Printf("%v", o2.Relations[i]) 55 | } 56 | } 57 | } 58 | } 59 | 60 | func readPbf(filename string) *osm.OSM { 61 | log.Printf("Reading pbf file %v", filename) 62 | f, err := os.Open(filename) 63 | if err != nil { 64 | panic(err) 65 | } 66 | defer f.Close() 67 | 68 | scanner := osmpbf.New(context.Background(), f, 1) 69 | return scanner2osm(scanner) 70 | } 71 | 72 | func readBz2(filename string) *osm.OSM { 73 | log.Printf("Reading bz2 file %v", filename) 74 | f, err := os.Open(filename) 75 | if err != nil { 76 | panic(err) 77 | } 78 | defer f.Close() 79 | 80 | r := bzip2.NewReader(f) 81 | scanner := osmxml.New(context.Background(), r) 82 | return scanner2osm(scanner) 83 | } 84 | 85 | func scanner2osm(scanner osm.Scanner) *osm.OSM { 86 | o := &osm.OSM{} 87 | for scanner.Scan() { 88 | e := scanner.Element() 89 | 90 | if e.Node != nil { 91 | e.Node.Lat = math.Floor(e.Node.Lat*1e7+0.5) / 1e7 92 | e.Node.Lon = math.Floor(e.Node.Lon*1e7+0.5) / 1e7 93 | e.Node.Visible = true 94 | e.Node.Tags.SortByKeyValue() 95 | o.Nodes = append(o.Nodes, e.Node) 96 | } 97 | 98 | if e.Way != nil { 99 | e.Way.Visible = true 100 | e.Way.Tags.SortByKeyValue() 101 | o.Ways = append(o.Ways, e.Way) 102 | } 103 | 104 | if e.Relation != nil { 105 | e.Relation.Visible = true 106 | e.Relation.Tags.SortByKeyValue() 107 | o.Relations = append(o.Relations, e.Relation) 108 | } 109 | } 110 | 111 | if err := scanner.Err(); err != nil { 112 | panic(err) 113 | } 114 | 115 | o.Nodes.SortByIDVersion() 116 | o.Ways.SortByIDVersion() 117 | o.Relations.SortByIDVersion() 118 | 119 | return o 120 | } 121 | -------------------------------------------------------------------------------- /testdata/delaware-latest.osm.pbf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paulmach/osm/db9d8ebc69cde0241f4989a74ea94ca9e7066e12/testdata/delaware-latest.osm.pbf -------------------------------------------------------------------------------- /testdata/relation-updates.osm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /testdata/way-updates.osm: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /update.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "time" 7 | ) 8 | 9 | // CommitInfoStart is the start time when we know committed at information. 10 | // Any update.Timestamp >= this date is a committed at time. Anything before 11 | // this date is the element timestamp. 12 | var CommitInfoStart = time.Date(2012, 9, 12, 9, 30, 3, 0, time.UTC) 13 | 14 | // An Update is a change to children of a way or relation. 15 | // The child type, id, ref and/or role are the same as the child 16 | // at the given index. Lon/Lat are only updated for nodes. 17 | type Update struct { 18 | Index int `xml:"index,attr" json:"index"` 19 | Version int `xml:"version,attr" json:"version"` 20 | 21 | // Timestamp is the committed at time if time > CommitInfoStart or the 22 | // element timestamp if before that date. 23 | Timestamp time.Time `xml:"timestamp,attr" json:"timestamp"` 24 | 25 | ChangesetID ChangesetID `xml:"changeset,attr,omitempty" json:"changeset,omitempty"` 26 | Lat float64 `xml:"lat,attr,omitempty" json:"lat,omitempty"` 27 | Lon float64 `xml:"lon,attr,omitempty" json:"lon,omitempty"` 28 | Reverse bool `xml:"reverse,attr,omitempty" json:"reverse,omitempty"` 29 | } 30 | 31 | // Updates are collections of updates. 32 | type Updates []Update 33 | 34 | // UpTo will return the subset of updates taking place upto and on 35 | // the given time. 36 | func (us Updates) UpTo(t time.Time) Updates { 37 | var result Updates 38 | 39 | for _, u := range us { 40 | if u.Timestamp.After(t) { 41 | continue 42 | } 43 | 44 | result = append(result, u) 45 | } 46 | 47 | return result 48 | } 49 | 50 | // UpdateIndexOutOfRangeError is return when applying an update to an object 51 | // and the update index is out of range. 52 | type UpdateIndexOutOfRangeError struct { 53 | Index int 54 | } 55 | 56 | var _ error = &UpdateIndexOutOfRangeError{} 57 | 58 | // Error returns a string representation of the error. 59 | func (e *UpdateIndexOutOfRangeError) Error() string { 60 | return fmt.Sprintf("osm: index %d is out of range", e.Index) 61 | } 62 | 63 | type updatesSortTS Updates 64 | 65 | // SortByTimestamp will sort the updates by timestamp in ascending order. 66 | func (us Updates) SortByTimestamp() { sort.Sort(updatesSortTS(us)) } 67 | func (us updatesSortTS) Len() int { return len(us) } 68 | func (us updatesSortTS) Swap(i, j int) { us[i], us[j] = us[j], us[i] } 69 | func (us updatesSortTS) Less(i, j int) bool { 70 | return us[i].Timestamp.Before(us[j].Timestamp) 71 | } 72 | 73 | type updatesSortIndex Updates 74 | 75 | // SortByIndex will sort the updates by index in ascending order. 76 | func (us Updates) SortByIndex() { sort.Sort(updatesSortIndex(us)) } 77 | func (us updatesSortIndex) Len() int { return len(us) } 78 | func (us updatesSortIndex) Swap(i, j int) { us[i], us[j] = us[j], us[i] } 79 | func (us updatesSortIndex) Less(i, j int) bool { 80 | if us[i].Index != us[j].Index { 81 | return us[i].Index < us[j].Index 82 | } 83 | 84 | return us[i].Timestamp.Before(us[j].Timestamp) 85 | } 86 | -------------------------------------------------------------------------------- /update_test.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestUpdates_UpTo(t *testing.T) { 9 | us := Updates{ 10 | {Index: 1, Timestamp: time.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC)}, 11 | {Index: 2, Timestamp: time.Date(2014, 1, 1, 0, 0, 0, 0, time.UTC)}, 12 | {Index: 3, Timestamp: time.Date(2013, 1, 1, 0, 0, 0, 0, time.UTC)}, 13 | } 14 | 15 | if v := len(us.UpTo(time.Date(2011, 1, 1, 0, 0, 0, 0, time.UTC))); v != 0 { 16 | t.Errorf("incorrect number of updates, got %v", v) 17 | } 18 | 19 | u := us.UpTo(time.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC)) 20 | if v := len(u); v != 1 { 21 | t.Errorf("incorrect number of updates, got %v", v) 22 | } 23 | 24 | if v := u[0].Index; v != 1 { 25 | t.Errorf("incorrect value, got index: %v", v) 26 | } 27 | 28 | u = us.UpTo(time.Date(2013, 1, 1, 0, 0, 0, 0, time.UTC)) 29 | if v := len(u); v != 2 { 30 | t.Errorf("incorrect number of updates, got %v", v) 31 | } 32 | 33 | if v := u[0].Index; v != 1 { 34 | t.Errorf("incorrect value, got index: %v", v) 35 | } 36 | 37 | if v := u[1].Index; v != 3 { 38 | t.Errorf("incorrect value, got index: %v", v) 39 | } 40 | 41 | if v := len(us.UpTo(time.Date(2013, 2, 1, 0, 0, 0, 0, time.UTC))); v != 2 { 42 | t.Errorf("incorrect number of updates, got %v", v) 43 | } 44 | 45 | if v := len(us.UpTo(time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC))); v != 3 { 46 | t.Errorf("incorrect number of updates, got %v", v) 47 | } 48 | } 49 | 50 | func TestUpdates_SortByIndex(t *testing.T) { 51 | us := Updates{ 52 | {Index: 1, Version: 0, Timestamp: time.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC)}, 53 | {Index: 1, Version: 2, Timestamp: time.Date(2014, 1, 1, 0, 0, 0, 0, time.UTC)}, 54 | {Index: 1, Version: 1, Timestamp: time.Date(2013, 1, 1, 0, 0, 0, 0, time.UTC)}, 55 | } 56 | 57 | us.SortByIndex() 58 | 59 | for i, u := range us { 60 | if u.Version != i { 61 | t.Fatalf("incorrect sort: %v", us) 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /user.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // UserID is the primary key for a user. 8 | // This is unique as the display name may change. 9 | type UserID int64 10 | 11 | // ObjectID is a helper returning the object id for this user id. 12 | func (id UserID) ObjectID() ObjectID { 13 | return ObjectID(userMask | (id << versionBits)) 14 | } 15 | 16 | // Users is a collection of users with some helpers attached. 17 | type Users []*User 18 | 19 | // A User is a registered OSM user. 20 | type User struct { 21 | XMLName xmlNameJSONTypeUser `xml:"user" json:"type"` 22 | ID UserID `xml:"id,attr" json:"id"` 23 | Name string `xml:"display_name,attr" json:"name"` 24 | Description string `xml:"description" json:"description,omitempty"` 25 | Img struct { 26 | Href string `xml:"href,attr" json:"href"` 27 | } `xml:"img" json:"img"` 28 | Changesets struct { 29 | Count int `xml:"count,attr" json:"count"` 30 | } `xml:"changesets" json:"changesets"` 31 | Traces struct { 32 | Count int `xml:"count,attr" json:"count"` 33 | } `xml:"traces" json:"traces"` 34 | Home struct { 35 | Lat float64 `xml:"lat,attr" json:"lat"` 36 | Lon float64 `xml:"lon,attr" json:"lon"` 37 | Zoom int `xml:"zoom,attr" json:"zoom"` 38 | } `xml:"home" json:"home"` 39 | Languages []string `xml:"languages>lang" json:"languages"` 40 | Blocks struct { 41 | Received struct { 42 | Count int `xml:"count,attr" json:"count"` 43 | Active int `xml:"active,attr" json:"active"` 44 | } `xml:"received" json:"received"` 45 | } `xml:"blocks" json:"blocks"` 46 | Messages struct { 47 | Received struct { 48 | Count int `xml:"count,attr" json:"count"` 49 | Unread int `xml:"unread,attr" json:"unread"` 50 | } `xml:"received" json:"received"` 51 | Sent struct { 52 | Count int `xml:"count,attr" json:"count"` 53 | } `xml:"sent" json:"sent"` 54 | } `xml:"messages" json:"messages"` 55 | CreatedAt time.Time `xml:"account_created,attr" json:"created_at"` 56 | } 57 | 58 | // ObjectID returns the object id of the user. 59 | func (u *User) ObjectID() ObjectID { 60 | return u.ID.ObjectID() 61 | } 62 | -------------------------------------------------------------------------------- /user_test.go: -------------------------------------------------------------------------------- 1 | package osm 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "encoding/xml" 7 | "reflect" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func TestUser_UnmarshalXML(t *testing.T) { 13 | rawXML := []byte(` 14 | 15 | mapper 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | en-UK 25 | en 26 | 27 | 28 | 29 | 30 | 31 | `) 32 | u := &User{} 33 | 34 | err := xml.Unmarshal(rawXML, &u) 35 | if err != nil { 36 | t.Fatalf("unmarshal error: %v", err) 37 | } 38 | 39 | if v := u.ID; v != 91499 { 40 | t.Errorf("incorrect value: %v", v) 41 | } 42 | 43 | if v := u.Name; v != "pm" { 44 | t.Errorf("incorrect value: %v", v) 45 | } 46 | 47 | if v := u.Description; v != "mapper" { 48 | t.Errorf("incorrect value: %v", v) 49 | } 50 | 51 | if v := u.Img.Href; v != "image url" { 52 | t.Errorf("incorrect value: %v", v) 53 | } 54 | 55 | if v := u.Changesets.Count; v != 2638 { 56 | t.Errorf("incorrect value: %v", v) 57 | } 58 | 59 | if v := u.Traces.Count; v != 1 { 60 | t.Errorf("incorrect value: %v", v) 61 | } 62 | 63 | // home 64 | if v := u.Home.Lat; v != 37.793 { 65 | t.Errorf("incorrect value: %v", v) 66 | } 67 | 68 | if v := u.Home.Lon; v != -122.2712 { 69 | t.Errorf("incorrect value: %v", v) 70 | } 71 | 72 | if v := u.Home.Zoom; v != 3 { 73 | t.Errorf("incorrect value: %v", v) 74 | } 75 | 76 | if v := u.Languages; !reflect.DeepEqual(v, []string{"en-UK", "en"}) { 77 | t.Errorf("incorrect value: %v", v) 78 | } 79 | 80 | // blocks 81 | if v := u.Blocks.Received.Count; v != 5 { 82 | t.Errorf("incorrect value: %v", v) 83 | } 84 | 85 | if v := u.Blocks.Received.Active; v != 6 { 86 | t.Errorf("incorrect value: %v", v) 87 | } 88 | 89 | // messages 90 | if v := u.Messages.Received.Count; v != 15 { 91 | t.Errorf("incorrect value: %v", v) 92 | } 93 | 94 | if v := u.Messages.Received.Unread; v != 3 { 95 | t.Errorf("incorrect value: %v", v) 96 | } 97 | 98 | if v := u.Messages.Sent.Count; v != 7 { 99 | t.Errorf("incorrect value: %v", v) 100 | } 101 | 102 | // created 103 | if v := u.CreatedAt; !v.Equal(time.Date(2009, 1, 13, 19, 49, 59, 0, time.UTC)) { 104 | t.Errorf("incorrect value: %v", v) 105 | } 106 | 107 | // should marshal correctly. 108 | data, err := xml.Marshal(u) 109 | if err != nil { 110 | t.Fatalf("xml marshal error: %v", err) 111 | } 112 | 113 | nu := &User{} 114 | err = xml.Unmarshal(data, &nu) 115 | if err != nil { 116 | t.Fatalf("xml unmarshal error: %v", err) 117 | } 118 | 119 | if !reflect.DeepEqual(nu, u) { 120 | t.Errorf("incorrect marshal") 121 | t.Log(nu) 122 | t.Log(u) 123 | } 124 | } 125 | 126 | func TestUser_ObjectID(t *testing.T) { 127 | u := User{ID: 123} 128 | id := u.ObjectID() 129 | 130 | if v := id.Type(); v != TypeUser { 131 | t.Errorf("incorrect type: %v", v) 132 | } 133 | 134 | if v := id.Ref(); v != 123 { 135 | t.Errorf("incorrect ref: %v", 123) 136 | } 137 | } 138 | 139 | func TestUser_MarshalJSON(t *testing.T) { 140 | u := User{ 141 | ID: 123, 142 | Name: "user", 143 | } 144 | 145 | data, err := json.Marshal(u) 146 | if err != nil { 147 | t.Fatalf("marshal error: %v", err) 148 | } 149 | 150 | if !bytes.Equal(data, []byte(`{"type":"user","id":123,"name":"user","img":{"href":""},"changesets":{"count":0},"traces":{"count":0},"home":{"lat":0,"lon":0,"zoom":0},"languages":null,"blocks":{"received":{"count":0,"active":0}},"messages":{"received":{"count":0,"unread":0},"sent":{"count":0}},"created_at":"0001-01-01T00:00:00Z"}`)) { 151 | t.Errorf("incorrect json: %v", string(data)) 152 | } 153 | } 154 | --------------------------------------------------------------------------------