├── web
├── style.css
├── app.js
└── index.html
├── kernel.go
├── LICENSE.md
├── stitch
└── main.go
├── renderer.go
├── server
└── main.go
├── layer.go
├── tile.go
├── README.md
└── loader
└── main.go
/web/style.css:
--------------------------------------------------------------------------------
1 | #map {
2 | position: absolute;
3 | left: 0;
4 | right: 0;
5 | top: 0;
6 | bottom: 0;
7 | }
8 |
--------------------------------------------------------------------------------
/kernel.go:
--------------------------------------------------------------------------------
1 | package density
2 |
3 | import "math"
4 |
5 | type KernelItem struct {
6 | Dx, Dy int
7 | Weight float64
8 | }
9 |
10 | type Kernel []KernelItem
11 |
12 | func NewKernel(n int) Kernel {
13 | var result Kernel
14 | for dy := -n; dy <= n; dy++ {
15 | for dx := -n; dx <= n; dx++ {
16 | d := math.Sqrt(float64(dx*dx + dy*dy))
17 | w := math.Max(0, 1-d/float64(n))
18 | w = math.Pow(w, 2)
19 | result = append(result, KernelItem{dx, dy, w})
20 | }
21 | }
22 | return result
23 | }
24 |
--------------------------------------------------------------------------------
/web/app.js:
--------------------------------------------------------------------------------
1 | var initialLat = 40.7831;
2 | var initialLng = -73.9712;
3 |
4 | var map = L.map('map').setView([initialLat, initialLng], 16);
5 |
6 | L.tileLayer('http://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}.png', {
7 | attribution: '© OpenStreetMap © CartoDB',
8 | subdomains: 'abcd',
9 | // maxZoom: 19
10 | }).addTo(map);
11 |
12 | L.tileLayer('http://localhost:5000/{z}/{x}/{y}.png', {
13 | // maxZoom: 19,
14 | // opacity: 0.8
15 | }).addTo(map);
16 |
--------------------------------------------------------------------------------
/web/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Density Map
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
1 | Copyright (C) 2019 Michael Fogleman
2 |
3 | Permission is hereby granted, free of charge, to any person obtaining a copy
4 | of this software and associated documentation files (the "Software"), to deal
5 | in the Software without restriction, including without limitation the rights
6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7 | copies of the Software, and to permit persons to whom the Software is
8 | furnished to do so, subject to the following conditions:
9 |
10 | The above copyright notice and this permission notice shall be included in all
11 | copies or substantial portions of the Software.
12 |
13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19 | SOFTWARE.
20 |
21 |
--------------------------------------------------------------------------------
/stitch/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "image"
5 | "image/draw"
6 | "image/png"
7 | "os"
8 |
9 | "github.com/fogleman/density"
10 | )
11 |
12 | const (
13 | lat = 0
14 | lng = 0
15 | zoom = 3
16 | w = 2048
17 | h = 2048
18 | )
19 |
20 | var urls = []string{
21 | "http://a.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}.png",
22 | "http://localhost:5000/{z}/{x}/{y}.png",
23 | }
24 |
25 | func SavePNG(path string, im image.Image) error {
26 | file, err := os.Create(path)
27 | if err != nil {
28 | return err
29 | }
30 | defer file.Close()
31 | return png.Encode(file, im)
32 | }
33 |
34 | func MergeLayers(layers []*image.NRGBA) *image.NRGBA {
35 | result := image.NewNRGBA(layers[0].Bounds())
36 | for _, layer := range layers {
37 | draw.Draw(result, result.Bounds(), layer, image.ZP, draw.Over)
38 | }
39 | return result
40 | }
41 |
42 | func main() {
43 | layers := make([]*image.NRGBA, len(urls))
44 | for i := range layers {
45 | im, err := density.Stitch(urls[i], lat, lng, zoom, w, h)
46 | if err != nil {
47 | panic(err)
48 | }
49 | layers[i] = im
50 | }
51 | im := MergeLayers(layers)
52 | SavePNG("out.png", im)
53 | }
54 |
--------------------------------------------------------------------------------
/renderer.go:
--------------------------------------------------------------------------------
1 | package density
2 |
3 | import (
4 | "fmt"
5 | "image"
6 | "log"
7 | "math"
8 | "time"
9 |
10 | "github.com/gocql/gocql"
11 | )
12 |
13 | type Renderer struct {
14 | Cluster *gocql.ClusterConfig
15 | Query string
16 | BaseZoom int
17 | }
18 |
19 | func NewRenderer(host, keyspace, table string, baseZoom int) *Renderer {
20 | cluster := gocql.NewCluster(host)
21 | cluster.Keyspace = keyspace
22 | query := "SELECT lat, lng FROM %s WHERE zoom = ? AND x = ? AND y = ?;"
23 | query = fmt.Sprintf(query, table)
24 | return &Renderer{cluster, query, baseZoom}
25 | }
26 |
27 | func (r *Renderer) Render(zoom, x, y int) (image.Image, bool) {
28 | start := time.Now()
29 | session, _ := r.Cluster.CreateSession()
30 | defer session.Close()
31 | tile := r.loadTile(session, zoom, x, y)
32 | kernel := NewKernel(2)
33 | scale := 32 / math.Pow(4, float64(r.BaseZoom-zoom))
34 | im, ok := tile.Render(kernel, scale)
35 | elapsed := time.Now().Sub(start).Seconds()
36 | if ok {
37 | fmt.Printf("RENDER (%d %d %d) %8d pts %.3fs\n",
38 | zoom, x, y, tile.Points, elapsed)
39 | }
40 | return im, ok
41 | }
42 |
43 | func (r *Renderer) loadPoints(session *gocql.Session, x, y int, tile *Tile) {
44 | iter := session.Query(r.Query, r.BaseZoom, x, y).Iter()
45 | var lat, lng float64
46 | for iter.Scan(&lat, &lng) {
47 | tile.Add(lat, lng)
48 | }
49 | if err := iter.Close(); err != nil {
50 | log.Fatal(err)
51 | }
52 | }
53 |
54 | func (r *Renderer) loadTile(session *gocql.Session, zoom, x, y int) *Tile {
55 | tile := NewTile(zoom, x, y)
56 | if zoom < 3 {
57 | return tile
58 | }
59 | p := 1 // padding
60 | var x0, y0, x1, y1 int
61 | if zoom > r.BaseZoom {
62 | d := int(math.Pow(2, float64(zoom-r.BaseZoom)))
63 | x0, y0 = x/d-p, y/d-p
64 | x1, y1 = x/d+p, y/d+p
65 | } else if zoom < r.BaseZoom {
66 | d := int(math.Pow(2, float64(r.BaseZoom-zoom)))
67 | x0, y0 = x*d-p, y*d-p
68 | x1, y1 = (x+1)*d-1+p, (y+1)*d-1+p
69 | } else {
70 | x0, y0 = x-p, y-p
71 | x1, y1 = x+p, y+p
72 | }
73 | for tx := x0; tx <= x1; tx++ {
74 | for ty := y0; ty <= y1; ty++ {
75 | r.loadPoints(session, tx, ty, tile)
76 | }
77 | }
78 | return tile
79 | }
80 |
--------------------------------------------------------------------------------
/server/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "flag"
5 | "fmt"
6 | "image/png"
7 | "log"
8 | "net/http"
9 | "os"
10 | "path"
11 | "strconv"
12 |
13 | "github.com/fogleman/density"
14 | "github.com/gorilla/mux"
15 | )
16 |
17 | const CqlHost = "127.0.0.1"
18 |
19 | var Port int
20 | var CacheDirectory string
21 | var Keyspace string
22 | var Table string
23 | var BaseZoom int
24 |
25 | func init() {
26 | flag.IntVar(&Port, "port", 5000, "server port")
27 | flag.StringVar(&CacheDirectory, "cache", "cache", "cache directory")
28 | flag.StringVar(&Keyspace, "keyspace", "density", "keyspace name")
29 | flag.StringVar(&Table, "table", "points", "table name")
30 | flag.IntVar(&BaseZoom, "zoom", 18, "tile zoom")
31 | }
32 |
33 | func cachePath(zoom, x, y int) string {
34 | return fmt.Sprintf("%s/%d/%d/%d.png", CacheDirectory, zoom, x, y)
35 | }
36 |
37 | func pathExists(p string) bool {
38 | _, err := os.Stat(p)
39 | return err == nil
40 | }
41 |
42 | func parseInt(x string) int {
43 | value, _ := strconv.ParseInt(x, 0, 0)
44 | return int(value)
45 | }
46 |
47 | func Handler(w http.ResponseWriter, r *http.Request) {
48 | vars := mux.Vars(r)
49 | zoom := parseInt(vars["zoom"])
50 | x := parseInt(vars["x"])
51 | y := parseInt(vars["y"])
52 | p := cachePath(zoom, x, y)
53 | if !pathExists(p) {
54 | // nothing in cache, render the tile
55 | renderer := density.NewRenderer(CqlHost, Keyspace, Table, BaseZoom)
56 | im, ok := renderer.Render(zoom, x, y)
57 | if ok {
58 | // save tile in cache
59 | d, _ := path.Split(p)
60 | os.MkdirAll(d, 0777)
61 | f, err := os.Create(p)
62 | if err != nil {
63 | // unable to cache, just send the png
64 | w.Header().Set("Content-Type", "image/png")
65 | w.Header().Set("Access-Control-Allow-Origin", "*")
66 | png.Encode(w, im)
67 | return
68 | }
69 | png.Encode(f, im)
70 | f.Close()
71 | } else {
72 | // blank tile
73 | http.NotFound(w, r)
74 | return
75 | }
76 | } else {
77 | fmt.Printf("CACHED (%d %d %d)\n", zoom, x, y)
78 | }
79 | // serve cached tile
80 | w.Header().Set("Content-Type", "image/png")
81 | w.Header().Set("Access-Control-Allow-Origin", "*")
82 | http.ServeFile(w, r, p)
83 | }
84 |
85 | func main() {
86 | flag.Parse()
87 | router := mux.NewRouter()
88 | router.HandleFunc("/{zoom:\\d+}/{x:\\d+}/{y:\\d+}.png", Handler)
89 | addr := fmt.Sprintf(":%d", Port)
90 | log.Fatal(http.ListenAndServe(addr, router))
91 | }
92 |
--------------------------------------------------------------------------------
/layer.go:
--------------------------------------------------------------------------------
1 | package density
2 |
3 | import (
4 | "fmt"
5 | "image"
6 | "image/draw"
7 | "math"
8 | "net/http"
9 | "strconv"
10 | "strings"
11 | )
12 |
13 | func Stitch(urlTemplate string, lat, lng float64, zoom, w, h int) (*image.NRGBA, error) {
14 | layer := NewTileLayer(urlTemplate)
15 | return layer.GetTiles(lat, lng, zoom, w, h)
16 | }
17 |
18 | func GetImage(url string) (image.Image, error) {
19 | fmt.Println(url)
20 | response, err := http.Get(url)
21 | if err != nil {
22 | return nil, err
23 | }
24 | defer response.Body.Close()
25 | im, _, err := image.Decode(response.Body)
26 | return im, err
27 | }
28 |
29 | type TileLayer struct {
30 | URLTemplate string
31 | }
32 |
33 | func NewTileLayer(urlTemplate string) *TileLayer {
34 | return &TileLayer{urlTemplate}
35 | }
36 |
37 | func (layer *TileLayer) GetTile(z, x, y int) (image.Image, error) {
38 | url := layer.URLTemplate
39 | url = strings.Replace(url, "{z}", strconv.Itoa(z), -1)
40 | url = strings.Replace(url, "{x}", strconv.Itoa(x), -1)
41 | url = strings.Replace(url, "{y}", strconv.Itoa(y), -1)
42 | return GetImage(url)
43 | }
44 |
45 | func (layer *TileLayer) GetTiles(lat, lng float64, zoom, w, h int) (*image.NRGBA, error) {
46 | im := image.NewNRGBA(image.Rect(0, 0, w, h))
47 | cx, cy := TileFloatXY(zoom, lat, lng)
48 | x0 := cx - float64(w)/2/TileSize
49 | y0 := cy - float64(h)/2/TileSize
50 | x1 := cx + float64(w)/2/TileSize
51 | y1 := cy + float64(h)/2/TileSize
52 | x0i := int(math.Floor(x0))
53 | y0i := int(math.Floor(y0))
54 | x1i := int(math.Floor(x1))
55 | y1i := int(math.Floor(y1))
56 | ch := make(chan error)
57 | sem := make(chan int, 16)
58 | for x := x0i; x <= x1i; x++ {
59 | for y := y0i; y <= y1i; y++ {
60 | px := int(float64(w) * (float64(x) - x0) / (x1 - x0))
61 | py := int(float64(h) * (float64(y) - y0) / (y1 - y0))
62 | go layer.getTilesWorker(im, zoom, x, y, px, py, ch, sem)
63 | }
64 | }
65 | for x := x0i; x <= x1i; x++ {
66 | for y := y0i; y <= y1i; y++ {
67 | if err := <-ch; err != nil {
68 | return nil, err
69 | }
70 | }
71 | }
72 | return im, nil
73 | }
74 |
75 | func (layer *TileLayer) getTilesWorker(im *image.NRGBA, zoom, x, y, px, py int, ch chan error, sem chan int) {
76 | sem <- 1
77 | t, err := layer.GetTile(zoom, x, y)
78 | <-sem
79 | if err != nil {
80 | ch <- nil
81 | return
82 | }
83 | draw.Draw(im, image.Rect(px, py, px+TileSize, py+TileSize), t, image.ZP, draw.Src)
84 | ch <- nil
85 | }
86 |
--------------------------------------------------------------------------------
/tile.go:
--------------------------------------------------------------------------------
1 | package density
2 |
3 | import (
4 | "image"
5 | "image/color"
6 | "math"
7 |
8 | "github.com/lucasb-eyer/go-colorful"
9 | )
10 |
11 | const TileSize = 256
12 |
13 | func TileXY(zoom int, lat, lng float64) (x, y int) {
14 | fx, fy := TileFloatXY(zoom, lat, lng)
15 | x = int(math.Floor(fx))
16 | y = int(math.Floor(fy))
17 | return
18 | }
19 |
20 | func TileFloatXY(zoom int, lat, lng float64) (x, y float64) {
21 | lat_rad := lat * math.Pi / 180
22 | n := math.Pow(2, float64(zoom))
23 | x = (lng + 180) / 360 * n
24 | y = (1 - math.Log(math.Tan(lat_rad)+(1/math.Cos(lat_rad)))/math.Pi) / 2 * n
25 | return
26 | }
27 |
28 | func TileLatLng(zoom, x, y int) (lat, lng float64) {
29 | n := math.Pow(2, float64(zoom))
30 | lng = float64(x)/n*360 - 180
31 | lat = math.Atan(math.Sinh(math.Pi*(1-2*float64(y)/n))) * 180 / math.Pi
32 | return
33 | }
34 |
35 | type IntPoint struct {
36 | X, Y int
37 | }
38 |
39 | type Tile struct {
40 | Zoom, X, Y int
41 | Grid map[IntPoint]float64
42 | Points int
43 | }
44 |
45 | func NewTile(zoom, x, y int) *Tile {
46 | grid := make(map[IntPoint]float64)
47 | return &Tile{zoom, x, y, grid, 0}
48 | }
49 |
50 | func (tile *Tile) Add(lat, lng float64) {
51 | u, v := TileFloatXY(tile.Zoom, lat, lng)
52 | u -= float64(tile.X)
53 | v -= float64(tile.Y)
54 | v = 1 - v
55 | u *= TileSize
56 | v *= TileSize
57 | x := int(math.Floor(u))
58 | y := int(math.Floor(v))
59 | u = u - math.Floor(u)
60 | v = v - math.Floor(v)
61 | tile.Grid[IntPoint{x + 0, y + 0}] += (1 - u) * (1 - v)
62 | tile.Grid[IntPoint{x + 0, y + 1}] += (1 - u) * v
63 | tile.Grid[IntPoint{x + 1, y + 0}] += u * (1 - v)
64 | tile.Grid[IntPoint{x + 1, y + 1}] += u * v
65 | tile.Points++
66 | }
67 |
68 | func (tile *Tile) Render(kernel Kernel, scale float64) (image.Image, bool) {
69 | im := image.NewNRGBA(image.Rect(0, 0, TileSize, TileSize))
70 | ok := false
71 | for y := 0; y < TileSize; y++ {
72 | for x := 0; x < TileSize; x++ {
73 | var t, tw float64
74 | for _, k := range kernel {
75 | nx := x + k.Dx
76 | ny := y + k.Dy
77 | t += tile.Grid[IntPoint{nx, ny}] * k.Weight
78 | tw += k.Weight
79 | }
80 | if t == 0 {
81 | continue
82 | }
83 | t *= scale
84 | t /= tw
85 | t = t / (t + 1)
86 | a := uint8(255 * math.Pow(t, 0.5))
87 | c := colorful.Hsv(215.0, 1-t*t, 1)
88 | r, g, b := c.RGB255()
89 | im.SetNRGBA(x, TileSize-1-y, color.NRGBA{r, g, b, a})
90 | ok = true
91 | }
92 | }
93 | return im, ok
94 | }
95 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # High-density Point Maps
2 |
3 | Render millions of points on a map.
4 |
5 | 
6 |
7 | ### Demo Site
8 |
9 | This demo shows 77 million taxi pickups in NYC - from January to June 2015.
10 |
11 | https://www.michaelfogleman.com/static/density/
12 |
13 | ### Dependencies
14 |
15 | - Go
16 | - Cassandra
17 |
18 | ### Download
19 |
20 | go get github.com/fogleman/density
21 |
22 | ### Loading Data
23 |
24 | Cassandra is used to store large amounts of data. Data is clustered by `(zoom, x, y)` so
25 | that all of the points inside of a tile can quickly be fetched for rendering, even
26 | faster than PostGIS with an index.
27 |
28 | First, create a new keyspace and table to house the data.
29 |
30 | create keyspace density
31 | with replication = {
32 | 'class': 'SimpleStrategy',
33 | 'replication_factor': 1
34 | }
35 | and durable_writes = false;
36 |
37 | create table points (
38 | zoom int,
39 | x int,
40 | y int,
41 | lat double,
42 | lng double,
43 | primary key ((zoom, x, y), lat, lng)
44 | );
45 |
46 | Next, load points into the database from a CSV file using the loader script.
47 |
48 | go run loader/main.go < input.csv
49 |
50 | Several command line options are available:
51 |
52 | | Flag | Default | Description |
53 | | --- | --- | --- |
54 | | -keyspace | density | Cassandra keyspace to load into |
55 | | -table | points | Cassandra table to load into |
56 | | -lat | 0 | CSV column index of latitude values |
57 | | -lng | 1 | CSV column index of longitude values |
58 | | -zoom | 18 | Zoom level to use for binning points |
59 |
60 | Just run the loader whenever you need to insert more data.
61 |
62 | ### Serving Tiles
63 |
64 | Once the data is loaded into Cassandra, the tile server can be run for rendering tiles on the fly.
65 |
66 | go run server/main.go
67 |
68 | | Flag | Default | Description |
69 | | --- | --- | --- |
70 | | -keyspace | density | Cassandra keyspace to load from |
71 | | -table | points | Cassandra table to load from |
72 | | -zoom | 18 | Zoom level that was used for binning points |
73 | | -port | 5000 | Tile server port number |
74 | | -cache | cache | Directory for caching tile images |
75 |
76 | ### Serving Maps
77 |
78 | A simple Leaflet map is provided to display a base map with the point tiles on top in a separate layer.
79 |
80 | cd web
81 | python -m SimpleHTTPServer
82 |
83 | Then visit [http://localhost:8000/](http://localhost:8000/) in your browser!
84 |
85 | ### TODO
86 |
87 | - tile rendering options
88 | - multiple layers
89 |
--------------------------------------------------------------------------------
/loader/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/csv"
5 | "flag"
6 | "fmt"
7 | "log"
8 | "os"
9 | "strconv"
10 | "time"
11 |
12 | "github.com/fogleman/density"
13 | "github.com/gocql/gocql"
14 | )
15 |
16 | /*
17 | create keyspace density
18 | with replication = {
19 | 'class': 'SimpleStrategy',
20 | 'replication_factor': 1
21 | }
22 | and durable_writes = false;
23 |
24 | create table points (
25 | zoom int,
26 | x int,
27 | y int,
28 | lat double,
29 | lng double,
30 | primary key ((zoom, x, y), lat, lng)
31 | );
32 | */
33 |
34 | const CqlHost = "127.0.0.1"
35 | const Workers = 64
36 |
37 | var Keyspace string
38 | var Table string
39 | var LatIndex int
40 | var LngIndex int
41 | var Zoom int
42 | var Query string
43 |
44 | func init() {
45 | flag.StringVar(&Keyspace, "keyspace", "density", "keyspace name")
46 | flag.StringVar(&Table, "table", "points", "table name")
47 | flag.IntVar(&LatIndex, "lat", 0, "column index for latitude")
48 | flag.IntVar(&LngIndex, "lng", 1, "column index for longitude")
49 | flag.IntVar(&Zoom, "zoom", 18, "tile zoom")
50 | }
51 |
52 | type Point struct {
53 | Lat, Lng float64
54 | }
55 |
56 | func insert(session *gocql.Session, lat, lng float64) {
57 | x, y := density.TileXY(Zoom, lat, lng)
58 | if x < 0 || y < 0 {
59 | return // bad data point
60 | }
61 | if err := session.Query(Query, Zoom, x, y, lat, lng).Exec(); err != nil {
62 | log.Fatal(err)
63 | }
64 | }
65 |
66 | func worker(session *gocql.Session, points <-chan Point) {
67 | for point := range points {
68 | insert(session, point.Lat, point.Lng)
69 | }
70 | }
71 |
72 | func main() {
73 | flag.Parse()
74 |
75 | Query = "INSERT INTO %s (zoom, x, y, lat, lng) VALUES (?, ?, ?, ?, ?);"
76 | Query = fmt.Sprintf(Query, Table)
77 |
78 | cluster := gocql.NewCluster(CqlHost)
79 | cluster.Keyspace = Keyspace
80 | cluster.Timeout = 10 * time.Second
81 | session, _ := cluster.CreateSession()
82 | defer session.Close()
83 |
84 | points := make(chan Point, 1024)
85 | for i := 0; i < Workers; i++ {
86 | go worker(session, points)
87 | }
88 |
89 | reader := csv.NewReader(os.Stdin)
90 | for {
91 | record, err := reader.Read()
92 | if err != nil {
93 | break
94 | }
95 | lat, _ := strconv.ParseFloat(record[LatIndex], 64)
96 | lng, _ := strconv.ParseFloat(record[LngIndex], 64)
97 | if lat == 0 || lng == 0 {
98 | continue
99 | }
100 | if lat < -90 || lat > 90 {
101 | continue
102 | }
103 | if lng < -180 || lng > 180 {
104 | continue
105 | }
106 | points <- Point{lat, lng}
107 | }
108 | close(points)
109 | }
110 |
--------------------------------------------------------------------------------