├── .gitignore ├── LICENSE ├── README.md ├── docs ├── 01-setup.md ├── 02-elasticsearch.md ├── 03-elasticsearch-geo.md ├── README.md └── images │ ├── deployment.png │ ├── kibana-dev-tools.png │ └── open-sky-viewer.png ├── lab ├── .dockerignore ├── .env ├── .gitignore ├── README.md ├── airports │ ├── README.md │ ├── airports.bulk │ ├── airports.csv │ ├── airports.geo.json │ ├── airports.ldjson │ ├── generate_bulk.sh │ └── screenshot.png ├── dashboard-saved-object.ndjson ├── docker-compose-opensky.yml ├── docker-compose.yml ├── elastic-config.js ├── elastic-config.js.cloud.sample ├── opensky-loader │ ├── .dockerignore │ ├── Dockerfile │ ├── config.js │ ├── download │ │ ├── README.md │ │ ├── data_header.csv │ │ ├── download.sh │ │ └── flight_tracking.geojson.gz │ ├── index.js │ ├── package-lock.json │ └── package.json ├── opensky-viewer │ ├── .dockerignore │ ├── .gitignore │ ├── Dockerfile │ ├── config.js │ ├── index.html │ ├── index.js │ ├── package-lock.json │ └── package.json └── vector-tile-viewer │ ├── .eleventy.js │ ├── .env │ ├── .gitignore │ ├── README.md │ ├── _data │ └── meta.js │ ├── _includes │ ├── base.njk │ ├── map-docs.njk │ ├── map.njk │ └── title.njk │ ├── package.json │ ├── pages │ ├── 1-basemap │ │ └── 1-1-basemap.html │ ├── 2-documents │ │ ├── 2-1-documents.html │ │ ├── 2-2-documents.html │ │ ├── 2-3-documents.html │ │ ├── 2-4-documents.html │ │ └── 2-5-documents.html │ ├── 3-hexagons │ │ ├── 3-1-hexagons.html │ │ ├── 3-2-hexagons.html │ │ └── 3-3-hexagons.html │ └── 4-aggs │ │ ├── 4-1-geotile.html │ │ └── 4-2-heatmap.html │ ├── static │ ├── banner.png │ ├── css │ │ └── map-and-info.css │ ├── favicon.png │ ├── js │ │ └── highlight.min.js │ ├── style-ad.json │ └── style.json │ └── yarn.lock └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | lab/esdata/* 3 | lab/opensky-loader/config/*.js 4 | lab/opensky-loader/node_modules 5 | lab/opensky-loader/download/*flight_tracking*.csv 6 | lab/opensky-loader/download/*flight_tracking*.geojson 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Let’s go geospatial with Kibana and Elasticsearch 2 | 3 | ## About 4 | 5 | On this workhshop we will get introduced to Kibana and the Elastic Stack geospatial capabilities, starting with the Kibana Maps for easy exploration of our cluster geodata, and then continuing with the powerful Elastic query API to perform search and aggregating operations. 6 | 7 | ## Facilitator 8 | 9 | * Name: [Jorge Sanz](https://links.jorgesanz.net) 10 | * Twitter: [@xurxosanz](https://twitter.com/xurxosanz) 11 | * Mastodon: [@xurxosanz@mastodon.social](https://mastodon.social/@xurxosanz) 12 | * Github: [@jsanz](https://github.com/jsanz) 13 | 14 | Bio: 15 | 16 | > Jorge is a Cartographer from Valencia, Spain, with more than 15 years of experience in the geospatial industry and a strong focus in both Open Source and Data. He joined [Elastic](https://www.elastic.co/) and the [Kibana Maps](https://www.elastic.co/products/maps) team in September 2019. 17 | 18 | ## Agenda 19 | 20 | - Lab Set up and Data import 21 | - Quick overview of Elastic Stack 22 | - Kibana overview: Dashboards, Lens, Maps, and Canvas 23 | - Elasticsearch (geo) queries 24 | - Webmapping 1o1 25 | -------------------------------------------------------------------------------- /docs/01-setup.md: -------------------------------------------------------------------------------- 1 | # Laboratory 2 | 3 | ## Elastic Stack set up 4 | 5 | There are two recommended ways to set up the Elastic Stack for this laboratory: 6 | 7 | * Create a trial account with the [Elastic Cloud][1]. This is the easiest and **recommended** procedure. 8 | * Use [Docker](https://docs.docker.com/get-started/overview/) and [Composer](https://docs.docker.com/compose/) to start the stack locally 9 | 10 | ### Set up: Elastic Stack with [Elastic Cloud][1] 11 | 12 | This is the easiest way to start this lab, simply create a new account at the [Elastic Cloud][1] and start a deployment. You can leave the default settings. Once you have the cluster up and running you need to save three important settings: 13 | 14 | * Cloud ID 15 | * Username and password will be available to download as CSV: **do it**. 16 | 17 | From the deployment page you can also get your endpoint URLs for Kibana and Elastic. 18 | 19 | ![](images/deployment.png) 20 | 21 | You have a full guide on how to set up the trial on the [Elastic Cloud Getting Started][2] page. 22 | 23 | ### Set up: Docker Compose 24 | 25 | 26 | | ⚠ Important ⚠ | 27 | | :-- | 28 | | On linux systems, even running on Docker, Elasticsearch needs a memory parameter in the host system to be tuned since the default is usually too low. You can run the following command if you want it just once `sysctl -w vm.max_map_count=262144` or add `vm.max_map_count=262144` to `/etc/sysctl.conf` to make it permanent. More details in the [Virtual memory](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html) section of the documentation. 29 | | 30 | 31 | On the `/lab` folder you have a `docker-compose.yaml` file with the definition of all the services for this lab. It refers to a number of variables stored in the `.env` file. You may want to adapt this file but by default it should be fine. 32 | 33 | | ⚠ Warning ⚠ | 34 | | :-- | 35 | | You should always set up a strong password for `elastic` and `kibana_system` users if your environment is exposed to the internet. Do not leave the defaults if your cluster is accessible to others! | 36 | 37 | First time you run it will take some minutes since it needs to download all the images, so maybe you'll want to run `docker compose pull` and `docker compose build` from a location with good bandwidth **before** the workshop to ensure you have all the docker images installed. 38 | 39 | To start the Elastic Stack services you can run `docker compose up -d`. This command will start two nodes of Elasticsearch, a Kibana instance, and the OpenSky loader and viewer applications. 40 | 41 | Once running you can check their status with `docker compose ps` and `docker compose logs -f`. 42 | 43 | If everything goes as expected you can visit kibana from `http://localhost:5601`. 44 | 45 | ## Getting [Open Sky][3] data into Elastic 46 | 47 | Apart from having access to a Elastic Stack (both Elasticsearch and Kibana), you need data to explore them. Kibana has three well-know datasets that include geospatial information and are ready to load from the home page. For this lab we are going to load flights data in real time from the [Open Sky Network][3] using a simple nodejs script at the `opensky-loader` folder. Depending on how you access the stack the environment needs to be adapted minimally. The script can also be run in two different ways, if you have a Node development environment then you can run it locally, but if you don't have it you can use Docker Compose also to run this script. 48 | 49 | The file `lab/elastic-config.js` is configured with some default settings, a couple of them can be overridden by environment variables. 50 | 51 | * Elasticsearch client configuration (host and password can be set up with environment variables). 52 | * Prefix for the name of indexes that will store the flights positions. 53 | * How often you want to retrieve the data from OpenSky API (every 60 seconds is more than fine) 54 | * OpenSky optional credentials pulled from environment variables (read from the `.env` file if running from Docker). 55 | 56 | You only need to adapt the Elasticsearch configuration, that will differs depending if you are running in Elastic Cloud, Docker Compose, or Local. 57 | 58 | ### Running the OpenSky loader and viewer applications 59 | 60 | #### Running the Elastic stack with Docker Compose 61 | 62 | Nothing to do, both applications started when you run `docker compose up -d`. 63 | 64 | #### Using Docker Compose and Elastic Cloud 65 | 66 | If you are using Elastic Cloud and docker compose you need to set up in the `.env` file the following variables: 67 | 68 | * `ELASTIC_HOST` is the Elasticsearch endpoint. You can get this URL from Elastic cloud management UI. 69 | * `ELASTIC_PASSWORD` is the password for the `elastic` user and you should have this stored when you created your cluster. 70 | 71 | Now you can run just the OpenSky loader and viewer using this command: 72 | 73 | ``` 74 | docker compose -f docker-compose-opensky.yml up 75 | ``` 76 | 77 | This will start the two services defined in the [alternate compose YAML file](../lab/docker-compose-opensky.yml) pointing to your cloud instance. 78 | 79 | #### Running locally 80 | 81 | | 🗣 Note 🗣 | 82 | | :-- | 83 | | To run the script locally you need to have `node` and `npm` tools installed in your computer.| 84 | 85 | If you prefer to run the applications directly on your computer you just need on separate terminals to go to the `opensky-loader` and `opensky-viewer` folders and run: 86 | 87 | ```sh 88 | $ export ELASTIC_HOST="https://your-elasticsearch-endpoint:9643" 89 | $ export ELASTIC_PASSWORD="your_elastic_user_password" 90 | $ export OPENSKY_USER="your_opensky_user" 91 | $ export OPENSKY_PASSWORD="your_opensky_password" 92 | $ npm install 93 | $ npm start 94 | ``` 95 | 96 | 97 | ### Confirmation 98 | 99 | The easiest way to check if your data is being ingested is going to the Kibana DevTools application and run this query 100 | 101 | ``` 102 | GET /flight_tracking*/_count 103 | ``` 104 | 105 | ``` 106 | { 107 | "count" : 737482, 108 | "_shards" : { 109 | "total" : 1, 110 | "successful" : 1, 111 | "skipped" : 0, 112 | "failed" : 0 113 | } 114 | } 115 | ``` 116 | ![](./images/kibana-dev-tools.png) 117 | 118 | Also, your OpenSky viewer application should render data from the `flight_tracking*` index pattern. 119 | 120 | ![](./images/open-sky-viewer.png) 121 | 122 | [1]: https://www.elastic.co/cloud/elasticsearch-service/signup 123 | [2]: https://www.elastic.co/guide/en/cloud/current/ec-getting-started.html 124 | [3]: https://opensky-network.org/ 125 | -------------------------------------------------------------------------------- /docs/02-elasticsearch.md: -------------------------------------------------------------------------------- 1 | ## Elasticsearch queries 2 | 3 | Elasticsearch is entirely managed via REST API endpoints. All management, ingesting, querying, aggregating, etc. is done through REST endpoints. Queries in particular need a rich query language that allow to express all kind of requirements. This is just a glimpse of some interesting queries to give you an idea but you should navigate the [documentation][apis] for further details. 4 | 5 | There's plenty of resources to learn more about Elasticsearch, you may want to start from: 6 | 7 | * [DZone article][dzone] covering non geospatial queries 8 | * Official Elasticsearch [webinar][webinar] 9 | * [Documentation][docs] 10 | 11 | 12 | ### Index creation 13 | 14 | Create an index with a given mapping that contains a [`geo_point`][geo_point] type: 15 | 16 | ``` 17 | PUT workshop_test 18 | { 19 | "settings": { 20 | "number_of_replicas": 1, 21 | "number_of_shards": 1 22 | }, 23 | "mappings":{ 24 | "properties": { 25 | "location": { 26 | "type": "geo_point" 27 | }, 28 | "category": { 29 | "type": "keyword" 30 | }, 31 | "title": { 32 | "type": "text" 33 | } 34 | } 35 | } 36 | } 37 | ``` 38 | 39 | ### Inserting points 40 | 41 | Inserting documents in Elasticsearch means making a `POST` request with your document fields in a simple JSON format, but for geospatial data there are a number of different ways to specify the coordinates. 42 | 43 | As a string: latitude, longitude 44 | 45 | ``` 46 | POST workshop_test/_doc/1 47 | { 48 | "location": "41.12,-71.34", 49 | "category": "place name", 50 | "title": "Null Island" 51 | } 52 | ``` 53 | 54 | As a [geohash][geohash]: 55 | 56 | ``` 57 | POST workshop_test/_doc/2 58 | { 59 | "location": "drm3btev3e86", 60 | "category": "place name 2", 61 | "title": "Somewhere" 62 | } 63 | ``` 64 | 65 | As an array: longitude, latitude 66 | 67 | ``` 68 | POST workshop_test/_doc/3 69 | { 70 | "location": [ -71.34, 41.12 ] , 71 | "category": "place name 3", 72 | "title": "Somewhere 3" 73 | } 74 | ``` 75 | 76 | As an object: 77 | 78 | ``` 79 | POST workshop_test/_doc/4 80 | { 81 | "location": { 82 | "lat": 41.12, 83 | "lon": -71.34 84 | } , 85 | "category": "place name 4", 86 | "title": "Somewhere 4" 87 | } 88 | ``` 89 | 90 | ### Bulk insertion: 91 | 92 | Let's define a new index with a mapping: 93 | 94 | ``` 95 | PUT airports 96 | { 97 | "mappings": { 98 | "properties": { 99 | "coords": { 100 | "type": "geo_point" 101 | }, 102 | "abbrev": { 103 | "type": "keyword" 104 | }, 105 | "name": { 106 | "type": "text" 107 | }, 108 | "type": { 109 | "type": "keyword" 110 | } 111 | } 112 | } 113 | } 114 | ``` 115 | 116 | We can insert more than one document in a single `_bulk` request: 117 | 118 | ``` 119 | PUT _bulk 120 | { "index" : { "_index" : "airports", "_id" : "1" } } 121 | {"coords":[75.9570722,30.8503599],"name":"Sahnewal","abbrev":"LUH","type":"small"} 122 | { "index" : { "_index" : "airports", "_id" : "2" } } 123 | {"coords":[75.9330598,17.6254152],"name":"Solapur","abbrev":"SSE","type":"mid"} 124 | { "index" : { "_index" : "airports", "_id" : "3" } } 125 | {"coords":[85.323597,23.3177246],"name":"Birsa Munda","abbrev":"IXR","type":"mid"} 126 | { "index" : { "_index" : "airports", "_id" : "4" } } 127 | {"coords":[48.7471065,31.3431586],"name":"Ahwaz","abbrev":"AWZ","type":"mid"} 128 | { "index" : { "_index" : "airports", "_id" : "5" } } 129 | {"coords":[78.2172187,26.2854877],"name":"Gwalior","abbrev":"GWL","type":"mid and military"} 130 | { "index" : { "_index" : "airports", "_id" : "6" } } 131 | {"coords":[42.9710963,14.7552534],"name":"Hodeidah Int'l","abbrev":"HOD","type":"mid"} 132 | { "index" : { "_index" : "airports", "_id" : "7" } } 133 | {"coords":[75.8092915,22.7277492],"name":"Devi Ahilyabai Holkar Int'l","abbrev":"IDR","type":"mid"} 134 | { "index" : { "_index" : "airports", "_id" : "8" } } 135 | {"coords":[73.8105675,19.9660206],"name":"Gandhinagar","abbrev":"ISK","type":"mid"} 136 | { "index" : { "_index" : "airports", "_id" : "9" } } 137 | {"coords":[76.8017261,30.6707249],"name":"Chandigarh Int'l","abbrev":"IXC","type":"major and military"} 138 | { "index" : { "_index" : "airports", "_id" : "10" } } 139 | {"coords":[75.3958433,19.867297],"name":"Aurangabad","abbrev":"IXU","type":"mid"} 140 | ``` 141 | 142 | You can find a complete dataset with airports from all over the world in the `/lab/airports/airports.bulk` file if you want to populate your index with almost 900 points. 143 | 144 | ### Querying 145 | 146 | [Filter][bool] by value, get only a number of columns and order the results 147 | 148 | ``` 149 | GET flight_tracking*/_search 150 | { 151 | "size": 5, 152 | "_source": ["timePosition", "callsign", "location", "velocity"], 153 | "query":{ 154 | "bool": { 155 | "filter": { 156 | "term": { 157 | "originCountry": "China" 158 | } 159 | } 160 | } 161 | }, 162 | "sort": [ 163 | { 164 | "timePosition": "desc" 165 | } 166 | ] 167 | } 168 | ``` 169 | 170 | Just get the number of results using `_count` instead of `_search` using a [`bool`][bool] query with a filter. 171 | 172 | ``` 173 | GET flight_tracking*/_count 174 | { 175 | "query":{ 176 | "bool": { 177 | "filter": { 178 | "term": { 179 | "originCountry": "China" 180 | } 181 | } 182 | } 183 | } 184 | } 185 | ``` 186 | 187 | A more complex [`query_string`][query_string] query using wildcards and operators 188 | 189 | ``` 190 | GET flight_tracking*/_search 191 | { 192 | "query": { 193 | "query_string": { 194 | "query" : "RYR* OR ACA*", 195 | "default_field" : "callsign" 196 | } 197 | } 198 | } 199 | ``` 200 | 201 | 202 | Combining queries with filters using the [`bool` compounded query][bool]. 203 | 204 | ``` 205 | GET flight_tracking*/_search 206 | { 207 | "_source": [ "callsign", "timePosition", "onGround" ], 208 | "query": { 209 | "bool": { 210 | "must": [ 211 | { 212 | "query_string": { 213 | "query": "RYR*", 214 | "default_field": "callsign" 215 | } 216 | } 217 | ], 218 | "filter": [ 219 | { "term": { "onGround": "true" } }, 220 | { "range": { "timePosition": { "gte": "now-1d/h" } } } 221 | ] 222 | } 223 | } 224 | } 225 | ``` 226 | 227 | 228 | ### Aggregations 229 | 230 | Get some aggregations (metrics and histogram buckets) for positions that are not on the ground, for the last 30 minutes, and with positive altitudes. 231 | 232 | ``` 233 | GET flight_tracking*/_search 234 | { 235 | "size": 0, 236 | "query": { 237 | "bool": { 238 | "filter": [ 239 | { "term": { "onGround": "false" } }, 240 | { "range": { "timePosition": { "gte": "now-30m/m" } } }, 241 | { "range": { "geoAltitude": { "gte": 0 } } } 242 | ] 243 | } 244 | }, 245 | "aggs": { 246 | "avg_speed": { "avg": { "field": "velocity" } }, 247 | "geoAltitude_stats": { "stats": { "field": "geoAltitude" } }, 248 | "altitude_percentiles": { 249 | "percentiles": { 250 | "field": "geoAltitude", 251 | "percents": [ 0, 5, 10, 25, 50, 75, 90, 95, 100 ] 252 | } 253 | }, 254 | "positions_over_time": { 255 | "date_histogram": { 256 | "field": "timePosition", 257 | "fixed_interval": "10m" 258 | } 259 | }, 260 | "speed_histogram": { 261 | "histogram": { 262 | "field": "velocity", 263 | "interval": 50 264 | } 265 | } 266 | } 267 | } 268 | ``` 269 | 270 | See that by default aggregations are returned along with the search results but usually you want one or another, thus this query asks for no individual documents (`"size": 0`). 271 | 272 | [apis]: https://www.elastic.co/guide/en/elasticsearch/reference/current/rest-apis.html 273 | [webinar]: https://www.elastic.co/webinars/getting-started-elasticsearch 274 | [docs]: https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html 275 | [dzone]: https://dzone.com/articles/23-useful-elasticsearch-example-queries 276 | 277 | 278 | [users]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-put-user.html 279 | [geo_point]: https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-point.html 280 | [query_string]: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html 281 | [bool]: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-bool-query.html 282 | [geohash]: https://en.wikipedia.org/wiki/Geohash 283 | -------------------------------------------------------------------------------- /docs/03-elasticsearch-geo.md: -------------------------------------------------------------------------------- 1 | # Elasticsearch Geospatial Queries 2 | 3 | ## Search 4 | 5 | Find documents in your index using geospatial conditions. 6 | 7 | ### Point and radius query 8 | 9 | With the [`geo_distance`][geo_distance] query type get the positions near Barajas airport: 10 | 11 | ``` 12 | GET flight_tracking*/_search 13 | { 14 | "query":{ 15 | "geo_distance": { 16 | "distance": "5km", 17 | "location":{ 18 | "lat": 40.469674, 19 | "lon": -3.559828 20 | } } } } 21 | ``` 22 | 23 | **TIP**: you can use the website to get coordinates and bounding boxes. 24 | 25 | ### Bounding box query 26 | 27 | Get the locations in the approximate [bounding box][bbox] of the JFK airport: 28 | 29 | ``` 30 | GET flight_tracking*/_search 31 | { 32 | "query": { 33 | "bool": { 34 | "must": [ { "match_all": {} } ], 35 | "filter": { 36 | "geo_bounding_box": { 37 | "location": { 38 | "top_left": { "lat": 40.666, "lon": -73.824 }, 39 | "bottom_right": { "lat": 40.620, "lon": -73.744 } 40 | } } } } } } 41 | ``` 42 | 43 | ### Shape query 44 | 45 | Let's find how many positions go over a [polygon][poly] that covers the city of Wuhan: 46 | 47 | ``` 48 | GET flight_tracking*/_count 49 | { 50 | "query": { 51 | "bool": { 52 | "must": [ 53 | { 54 | "match_all": {} 55 | } 56 | ], 57 | "filter": { 58 | "geo_shape": { 59 | "location": { 60 | "shape": """POLYGON(( 61 | 114.52 30.35, 62 | 114.19 30.38, 63 | 114.05 30.50, 64 | 114.05 30.61, 65 | 114.22 30.77, 66 | 114.54 30.81, 67 | 114.65 30.69, 68 | 114.69 30.53, 69 | 114.52 30.35)) 70 | """ 71 | } } } } } } 72 | ``` 73 | 74 | 75 | 76 | **TIP**: You can get quickly a polygon representation using [this tool][bbox_tool] and getting the `GeoJSON` output. 77 | 78 | 79 | ## Metric aggregations 80 | 81 | ### By bounding box 82 | 83 | Let's find the bounding box of all positions where `countryOrigin` is Monaco using the [`geo_bounds`][geo_bounds] aggregation: 84 | 85 | ``` 86 | GET flight_tracking*/_search 87 | { 88 | "size": 0, 89 | "query": { 90 | "match": { 91 | "originCountry": "Italy" 92 | } 93 | }, 94 | "aggs": { 95 | "viewport": { 96 | "geo_bounds": { 97 | "field": "location", 98 | "wrap_longitude": true 99 | } 100 | } 101 | } 102 | } 103 | ``` 104 | 105 | ### Centroid 106 | 107 | Get the [centroids][centroids] of the top 5 Ryanair flights with more positions: 108 | 109 | ``` 110 | GET flight_tracking*/_search 111 | { 112 | "size": 0, 113 | "query": { 114 | "query_string": { "default_field": "callsign", "query": "RYR*" } 115 | }, 116 | "aggs": { 117 | "centroids_by_callsign":{ 118 | "terms": { "field": "callsign.keyword", "size": 5 }, 119 | "aggs": { 120 | "cetroid": { 121 | "geo_centroid": { "field": "location" } 122 | } } } } } 123 | ``` 124 | 125 | **TIP**: yes, aggregations can be nested!! 126 | 127 | 128 | 129 | ### Geoline aggregation 130 | 131 | This aggregation takes a group of points and returns the line that connects them given a sorting field. You usually want this aggregation to be combined with a filter or a terms aggregation to retrieve lines that connect the locations of a particular asset or grouped by an identifier like an airplane `callsign` field. 132 | 133 | In this example we filter the last 15 minutes data for the airplane `JST574`, and the request the [line] aggregation representation using the `timePosition` field. 134 | 135 | **IMPORTANT**: You need to adapt the `callsign` and the date filter values to your own data, using Discover or Maps. 136 | 137 | ``` 138 | GET flight_tracking_*/_search 139 | { 140 | "size": 0, 141 | "query": { 142 | "bool": { 143 | "filter": [ 144 | { 145 | "range": { "timePosition": { "gte": "now-15m" }} 146 | }, 147 | { 148 | "match_phrase": { "callsign": "JST574" } 149 | } 150 | ] 151 | } 152 | }, 153 | "aggs": { 154 | "line": { 155 | "geo_line": { 156 | "point": {"field": "location"}, 157 | "sort": {"field": "timePosition"} 158 | } 159 | } 160 | } 161 | } 162 | ``` 163 | 164 | 165 | 166 | ## Bucket aggregations 167 | 168 | Group your query results using geospatial aggregations. 169 | 170 | ### Buffers 171 | 172 | Group positions around CDG airport in [rings][rings] (also known as **buffers** in the geospatial world) of 10, 20, and 30 kilometers and return results using an object instead of an array: 173 | 174 | ``` 175 | GET flight_tracking*/_search 176 | { 177 | "size": 0, 178 | "query": { "match_all": {} }, 179 | "aggs": { 180 | "rings_around_cdg": { 181 | "geo_distance": { 182 | "field": "location", 183 | "origin": [ 2.561, 49.01 ], 184 | "unit": "km", 185 | "keyed": true, 186 | "ranges": [ 187 | { "to": 10, "key": "<10km" }, 188 | { "from": 10, "to": 20, "key": "10-20km" }, 189 | { "from": 20, "to": 30, "key": "20-30km" } 190 | ] 191 | } } } } 192 | ``` 193 | 194 | ### Tile grid 195 | 196 | In the geospatial industry there is a common way to bucket the Earth using the square grid many online maps use. This schema uses a `Z/X/Y` notation that Elasticsearch can use to return your buckets. 197 | 198 | Let's find the zoom level 6 buckets for positions in mainland France. 199 | 200 | ``` 201 | GET flight_tracking*/_search 202 | { 203 | "size": 0, 204 | "query": { "match_all": {} }, 205 | "aggregations": { 206 | "europe": { 207 | "filter": { 208 | "geo_shape": { 209 | "location": { 210 | "shape": { 211 | "type": "Polygon", 212 | "coordinates": [[ 213 | [ 3.315, 42.207 ],[ -2.332, 43.607 ], 214 | [ -5.166, 48.439 ],[ -1.98, 49.749 ], 215 | [ 1.975, 51.244 ],[ 8.478, 49.077 ], 216 | [ 6.567, 46.765 ],[ 7.973, 43.384 ], 217 | [ 3.315, 42.207 ] 218 | ]] 219 | } 220 | } 221 | } 222 | }, 223 | "aggregations": { 224 | "zoom6": { 225 | "geotile_grid": { 226 | "field": "location", 227 | "precision": 6 228 | } } } } } } 229 | ``` 230 | 231 | **IMPORTANT**: Be careful with the `precision` parameter, a high value can potentially return **millions** of buckets, so you should only ask for high-precision results in a very small bounding box, or for small datasets. 232 | 233 | **TIP**: You can get quickly a polygon representation using [this tool][bbox_tool] and getting the `GeoJSON` output. 234 | 235 | 236 | ### Hex grid 237 | 238 | You can perform a similar query to the previous but instead of getting back buckets in the `Z/X/Y` schema, you get [hexagons] with the Uber's [h3] cell identifier. Same note about the `precision` parameter applies to this aggregation. 239 | 240 | **TIP**: You may find [this viewer](https://wolf-h3-viewer.glitch.me/) useful to render the location of a given h3 cell id. 241 | 242 | ``` 243 | GET flight_tracking*/_search 244 | { 245 | "size": 0, 246 | "query": { "match_all": {} }, 247 | "aggregations": { 248 | "europe": { 249 | "filter": { 250 | "geo_shape": { 251 | "location": { 252 | "shape": { 253 | "type": "Polygon", 254 | "coordinates": [[ 255 | [ 3.315, 42.207 ],[ -2.332, 43.607 ], 256 | [ -5.166, 48.439 ],[ -1.98, 49.749 ], 257 | [ 1.975, 51.244 ],[ 8.478, 49.077 ], 258 | [ 6.567, 46.765 ],[ 7.973, 43.384 ], 259 | [ 3.315, 42.207 ] 260 | ]] 261 | } 262 | } 263 | } 264 | }, 265 | "aggregations": { 266 | "h3_z3": { 267 | "geohex_grid": { 268 | "field": "location", 269 | "precision": 3 270 | } } } } } } 271 | ``` 272 | 273 | 274 | 275 | [geo_distance]: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-geo-distance-query.html 276 | [bbox]: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-geo-bounding-box-query.html 277 | [poly]: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-geo-shape-query.html 278 | [bbox_tool]: https://boundingbox.klokantech.com/ 279 | [geo_bounds]: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-geobounds-aggregation.html 280 | [centroids]: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-geocentroid-aggregation.html 281 | [line]: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-geo-line.html 282 | [rings]: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-geodistance-aggregation.html 283 | [hexagons]: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-geohexgrid-aggregation.html 284 | 285 | [h3]: https://h3geo.org -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Elastic Lab 2 | 3 | * [Lab Set up](01-setup.md) 4 | * [Elasticsearch Queries](02-elasticsearch.md) 5 | * [Elasticsearch Spatial Queries](03-elasticsearch-geo.md) 6 | 7 | 8 | -------------------------------------------------------------------------------- /docs/images/deployment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsanz/elastic-workshop/248063c5b57a21b157583e2df18724a4346b2037/docs/images/deployment.png -------------------------------------------------------------------------------- /docs/images/kibana-dev-tools.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsanz/elastic-workshop/248063c5b57a21b157583e2df18724a4346b2037/docs/images/kibana-dev-tools.png -------------------------------------------------------------------------------- /docs/images/open-sky-viewer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsanz/elastic-workshop/248063c5b57a21b157583e2df18724a4346b2037/docs/images/open-sky-viewer.png -------------------------------------------------------------------------------- /lab/.dockerignore: -------------------------------------------------------------------------------- 1 | certs 2 | node_modules 3 | .vscode 4 | -------------------------------------------------------------------------------- /lab/.env: -------------------------------------------------------------------------------- 1 | # Password for the 'elastic' user (at least 6 characters) 2 | # IMPORTANT: should be changed if your stack is not run locally!! 3 | ELASTIC_PASSWORD=changeme 4 | 5 | # Password for the 'kibana_system' user (at least 6 characters) 6 | # IMPORTANT: should be changed if your stack is not run locally!! 7 | KIBANA_PASSWORD=changeme 8 | 9 | # Version of Elastic products 10 | STACK_VERSION=8.3.3 11 | 12 | # Set the cluster name 13 | CLUSTER_NAME=docker-cluster 14 | 15 | # Set to 'basic' or 'trial' to automatically start the 30-day trial 16 | #LICENSE=basic 17 | LICENSE=trial 18 | 19 | # Port to expose Elasticsearch HTTP API to the host 20 | ES_PORT=9200 21 | 22 | # Port to expose Kibana to the host 23 | KIBANA_PORT=5601 24 | 25 | # Increase or decrease based on the available host memory (in bytes) 26 | MEM_LIMIT=1073741824 27 | 28 | # Project namespace (defaults to the current folder name if not set) 29 | #COMPOSE_PROJECT_NAME=myproject 30 | 31 | # Port to expose the OpenSky viewer sample app 32 | OPENSKY_VIEWER_PORT=80 33 | 34 | # OpenSky optnional creedentials 35 | OPENSKY_USER= 36 | OPENSKY_PASSWORD= 37 | 38 | # To be used if Elasticsearch is not running locally 39 | # ELASTIC_HOST="https://your-elasticsearch-host:9643" 40 | -------------------------------------------------------------------------------- /lab/.gitignore: -------------------------------------------------------------------------------- 1 | certs 2 | -------------------------------------------------------------------------------- /lab/README.md: -------------------------------------------------------------------------------- 1 | # Elastic Workshop laboratory 2 | 3 | Please follow the [setup instructions](../docs/01-setup.md) to generate your Elastic Stack environment. -------------------------------------------------------------------------------- /lab/airports/README.md: -------------------------------------------------------------------------------- 1 | # Airports dataset 2 | 3 | The [Natural Earth Airports dataset](https://www.naturalearthdata.com/downloads/10m-cultural-vectors/airports/) is provided here in different formats for your convenience. 4 | 5 | * The easiest path is to upload `airports.geo.json` using Elastic Maps GeoJSON upload. 6 | * If you want more control on the index mappings, or experiment with ingest pipelines, go to Kibana Machine Learning file uploader and select `airports.csv` and `airports.ldjson` 7 | * To upload using Kibana Dev Tools: `airports.bulk` 8 | 9 | ## Using dev tools: 10 | 11 | Create the index: 12 | 13 | ``` 14 | PUT airports 15 | { 16 | "settings": { 17 | "number_of_replicas": 0, 18 | "number_of_shards": 1 19 | }, 20 | "mappings": { 21 | "properties": { 22 | "abbrev": { 23 | "type": "keyword" 24 | }, 25 | "coords": { 26 | "type": "geo_point" 27 | }, 28 | "name": { 29 | "type": "text" 30 | }, 31 | "type": { 32 | "type": "keyword" 33 | } 34 | } 35 | } 36 | } 37 | ``` 38 | 39 | Send the contents of the `airports.bulk` file using the `POST _bulk` query. 40 | 41 | ![](screenshot.png) 42 | -------------------------------------------------------------------------------- /lab/airports/generate_bulk.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | count=1 4 | input="airports.ldjson" 5 | 6 | while IFS= read -r line 7 | do 8 | echo "{ \"index\" : { \"_index\" : \"workshop_airports\", \"_id\" : \"${count}\" } }" 9 | echo "$line" 10 | (( count++ )) 11 | done < "$input" -------------------------------------------------------------------------------- /lab/airports/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsanz/elastic-workshop/248063c5b57a21b157583e2df18724a4346b2037/lab/airports/screenshot.png -------------------------------------------------------------------------------- /lab/dashboard-saved-object.ndjson: -------------------------------------------------------------------------------- 1 | {"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"timePosition","title":"flight_tracking_*","typeMeta":"{}"},"coreMigrationVersion":"8.2.0","id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","migrationVersion":{"index-pattern":"8.0.0"},"references":[],"type":"index-pattern","updated_at":"2022-05-16T10:41:20.783Z","version":"Wzc5MCw1XQ=="} 2 | {"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{}","title":"airports","typeMeta":"{}"},"coreMigrationVersion":"8.2.0","id":"53e86190-d4ff-11ec-bc4f-b5a4ce689d85","migrationVersion":{"index-pattern":"8.0.0"},"references":[],"type":"index-pattern","updated_at":"2022-05-16T10:02:43.118Z","version":"Wzc2MSw1XQ=="} 3 | {"attributes":{"description":"","layerListJSON":"[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true,\"lightModeDefault\":\"road_map_desaturated\"},\"id\":\"ca93bfa6-a84b-4def-99a2-f745819bfc85\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{\"type\":\"TILE\"},\"includeInFitToBounds\":true,\"type\":\"EMS_VECTOR_TILE\"},{\"sourceDescriptor\":{\"geoField\":\"location\",\"requestType\":\"hex\",\"resolution\":\"MOST_FINE\",\"id\":\"fc0d1f39-fb0a-4b58-8302-aa139cfa6ae8\",\"type\":\"ES_GEO_GRID\",\"applyGlobalQuery\":true,\"applyGlobalTime\":true,\"applyForceRefresh\":true,\"metrics\":[{\"type\":\"count\"}],\"indexPatternRefName\":\"layer_1_source_index_pattern\"},\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"color\":\"Blue to Red\",\"colorCategory\":\"palette_0\",\"field\":{\"name\":\"doc_count\",\"origin\":\"source\"},\"fieldMetaOptions\":{\"isEnabled\":true,\"sigma\":3},\"type\":\"ORDINAL\",\"useCustomColorRamp\":false}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFF\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":0}},\"iconSize\":{\"type\":\"DYNAMIC\",\"options\":{\"minSize\":7,\"maxSize\":24,\"field\":{\"name\":\"doc_count\",\"origin\":\"source\"},\"fieldMetaOptions\":{\"isEnabled\":true,\"sigma\":3}}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"id\":\"55e0a639-4d99-4206-bb70-eb8e2f98f377\",\"label\":\"Positions grid\",\"minZoom\":0,\"maxZoom\":7,\"alpha\":0.75,\"visible\":true,\"includeInFitToBounds\":true,\"type\":\"MVT_VECTOR\",\"joins\":[]},{\"sourceDescriptor\":{\"geoField\":\"location\",\"scalingType\":\"MVT\",\"id\":\"81f93291-54f2-4da1-a868-2b0a71b5b5d2\",\"type\":\"ES_SEARCH\",\"applyGlobalQuery\":true,\"applyGlobalTime\":true,\"applyForceRefresh\":true,\"filterByMapBounds\":true,\"tooltipProperties\":[],\"sortField\":\"\",\"sortOrder\":\"desc\",\"topHitsSplitField\":\"\",\"topHitsSize\":1,\"indexPatternRefName\":\"layer_2_source_index_pattern\"},\"id\":\"fd0a6c1e-29a8-4a8c-89d6-86d1a24b9c44\",\"label\":\"Positions\",\"minZoom\":7,\"maxZoom\":24,\"alpha\":0.32,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"color\":\"Blues\",\"colorCategory\":\"palette_30\",\"field\":{\"name\":\"originCountry\",\"origin\":\"source\"},\"fieldMetaOptions\":{\"isEnabled\":true,\"sigma\":3},\"type\":\"CATEGORICAL\",\"useCustomColorPalette\":false}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#c83868\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":0}},\"iconSize\":{\"type\":\"DYNAMIC\",\"options\":{\"minSize\":1,\"maxSize\":15,\"field\":{\"label\":\"geoAltitude\",\"name\":\"geoAltitude\",\"origin\":\"source\",\"type\":\"number\",\"supportsAutoDomain\":true},\"fieldMetaOptions\":{\"isEnabled\":true,\"sigma\":3}}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"includeInFitToBounds\":true,\"type\":\"MVT_VECTOR\",\"joins\":[]},{\"sourceDescriptor\":{\"geoField\":\"location\",\"scalingType\":\"TOP_HITS\",\"sortField\":\"timePosition\",\"sortOrder\":\"desc\",\"tooltipProperties\":[\"callsign.keyword\",\"originCountry\",\"geoAltitude\",\"velocity\"],\"topHitsSplitField\":\"callsign.keyword\",\"topHitsSize\":1,\"id\":\"fb277ae2-290a-43cc-b064-78c71d35f0df\",\"type\":\"ES_SEARCH\",\"applyGlobalQuery\":true,\"applyGlobalTime\":true,\"applyForceRefresh\":true,\"filterByMapBounds\":true,\"indexPatternRefName\":\"layer_3_source_index_pattern\"},\"id\":\"a2542bca-acee-47d6-a5b1-3cffffd21e9a\",\"label\":\"Last position\",\"minZoom\":7,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"airport\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#6092C0\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#41937c\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"DYNAMIC\",\"options\":{\"minSize\":2,\"maxSize\":16,\"field\":{\"label\":\"geoAltitude\",\"name\":\"geoAltitude\",\"origin\":\"source\",\"type\":\"number\",\"supportsAutoDomain\":true},\"fieldMetaOptions\":{\"isEnabled\":true,\"sigma\":3}}},\"iconOrientation\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"label\":\"heading\",\"name\":\"heading\",\"origin\":\"source\",\"type\":\"number\",\"supportsAutoDomain\":true},\"fieldMetaOptions\":{\"isEnabled\":true,\"sigma\":3}}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"icon\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"includeInFitToBounds\":true,\"type\":\"GEOJSON_VECTOR\",\"joins\":[]},{\"id\":\"55c3ccc5-03b2-4ebe-ae35-5fbeabee19e7\",\"sourceDescriptor\":{\"geoField\":\"coords\",\"id\":\"5a3144c1-4ebe-4040-b9db-0b11ce3ea87c\",\"label\":\"airports\",\"scalingType\":\"MVT\",\"tooltipProperties\":[\"name\",\"abbrev\",\"type\"],\"type\":\"ES_SEARCH\",\"indexPatternRefName\":\"layer_4_source_index_pattern\"},\"type\":\"MVT_VECTOR\",\"visible\":false,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"color\":\"Blues\",\"colorCategory\":\"palette_0\",\"field\":{\"name\":\"type\",\"origin\":\"source\"},\"fieldMetaOptions\":{\"isEnabled\":true,\"sigma\":3},\"type\":\"CATEGORICAL\",\"useCustomColorPalette\":true,\"customColorPalette\":[{\"stop\":null,\"color\":\"#e3dfdf\"},{\"stop\":\"major\",\"color\":\"#54B399\"},{\"stop\":\"major and military\",\"color\":\"#74baa7\"},{\"stop\":\"mid\",\"color\":\"#B9A888\"},{\"stop\":\"mid and military\",\"color\":\"#bab3a6\"},{\"stop\":\"military major\",\"color\":\"#6092C0\"},{\"stop\":\"military mid\",\"color\":\"#84a4c2\"},{\"stop\":\"small\",\"color\":\"#D36086\"},{\"stop\":\"spaceport\",\"color\":\"#E7664C\"}]}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"label\":\"Airports\"}]","mapStateJSON":"{\"zoom\":7.03,\"center\":{\"lon\":2.90447,\"lat\":51.69168},\"timeFilters\":{\"from\":\"now-15m\",\"to\":\"now\"},\"refreshConfig\":{\"isPaused\":false,\"interval\":0},\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filters\":[],\"settings\":{\"autoFitToDataBounds\":false,\"backgroundColor\":\"#ffffff\",\"customIcons\":[],\"disableInteractive\":false,\"disableTooltipControl\":false,\"hideToolbarOverlay\":false,\"hideLayerControl\":false,\"hideViewControl\":false,\"initialLocation\":\"LAST_SAVED_LOCATION\",\"fixedLocation\":{\"lat\":0,\"lon\":0,\"zoom\":2},\"browserLocation\":{\"zoom\":2},\"maxZoom\":24,\"minZoom\":0,\"showScaleControl\":false,\"showSpatialFilters\":true,\"showTimesliderToggleButton\":true,\"spatialFiltersAlpa\":0.3,\"spatialFiltersFillColor\":\"#DA8B45\",\"spatialFiltersLineColor\":\"#DA8B45\"}}","title":"Open Sky","uiStateJSON":"{\"isLayerTOCOpen\":true,\"openTOCDetails\":[]}"},"coreMigrationVersion":"8.2.0","id":"cc6a24a0-d4ff-11ec-bc4f-b5a4ce689d85","migrationVersion":{"map":"8.1.0"},"references":[{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"layer_1_source_index_pattern","type":"index-pattern"},{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"layer_2_source_index_pattern","type":"index-pattern"},{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"layer_3_source_index_pattern","type":"index-pattern"},{"id":"53e86190-d4ff-11ec-bc4f-b5a4ce689d85","name":"layer_4_source_index_pattern","type":"index-pattern"}],"type":"map","updated_at":"2022-05-16T12:20:29.660Z","version":"Wzc5Niw1XQ=="} 4 | {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"8.2.0\",\"type\":\"map\",\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":36,\"i\":\"a43e2915-1e1e-4e61-8735-f304bfefb35e\"},\"panelIndex\":\"a43e2915-1e1e-4e61-8735-f304bfefb35e\",\"embeddableConfig\":{\"mapCenter\":{\"lat\":19.15487,\"lon\":-7.44617,\"zoom\":1.79},\"mapBuffer\":{\"minLon\":-180,\"minLat\":-66.51326,\"maxLon\":180,\"maxLat\":85.05113},\"isLayerTOCOpen\":true,\"openTOCDetails\":[],\"hiddenLayers\":[],\"filterByMapExtent\":false,\"enhancements\":{}},\"panelRefName\":\"panel_a43e2915-1e1e-4e61-8735-f304bfefb35e\"},{\"version\":\"8.2.0\",\"type\":\"lens\",\"gridData\":{\"x\":24,\"y\":0,\"w\":6,\"h\":6,\"i\":\"ab7e7c62-c79f-408b-9a29-08e1c1ebb8b2\"},\"panelIndex\":\"ab7e7c62-c79f-408b-9a29-08e1c1ebb8b2\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsMetric\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"b95825b0-d504-11ec-bc4f-b5a4ce689d85\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"b95825b0-d504-11ec-bc4f-b5a4ce689d85\",\"name\":\"indexpattern-datasource-layer-ff589e5b-4284-437a-a5af-eaf0fe9016f0\"}],\"state\":{\"visualization\":{\"layerId\":\"ff589e5b-4284-437a-a5af-eaf0fe9016f0\",\"accessor\":\"2f8d74a2-8617-412c-a95b-0c4d8618ce9b\",\"layerType\":\"data\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"ff589e5b-4284-437a-a5af-eaf0fe9016f0\":{\"columns\":{\"2f8d74a2-8617-412c-a95b-0c4d8618ce9b\":{\"label\":\"Positions\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"___records___\",\"params\":{\"emptyAsNull\":true},\"customLabel\":true}},\"columnOrder\":[\"2f8d74a2-8617-412c-a95b-0c4d8618ce9b\"],\"incompleteColumns\":{}}}}}}},\"enhancements\":{},\"hidePanelTitles\":false},\"title\":\"Count\"},{\"version\":\"8.2.0\",\"type\":\"lens\",\"gridData\":{\"x\":30,\"y\":0,\"w\":18,\"h\":15,\"i\":\"24c8d122-d96c-4ea2-9b82-ffa792c4e0d9\"},\"panelIndex\":\"24c8d122-d96c-4ea2-9b82-ffa792c4e0d9\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"b95825b0-d504-11ec-bc4f-b5a4ce689d85\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"b95825b0-d504-11ec-bc4f-b5a4ce689d85\",\"name\":\"indexpattern-datasource-layer-ebd928d3-2870-4094-9aed-9fb558610450\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"bar_stacked\",\"layers\":[{\"layerId\":\"ebd928d3-2870-4094-9aed-9fb558610450\",\"accessors\":[\"53850181-7eaf-46f7-bed1-f0474c881ddd\"],\"position\":\"top\",\"seriesType\":\"bar_stacked\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"9452a9c9-08c5-4d90-a5a7-c43f0e0d6cc1\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"ebd928d3-2870-4094-9aed-9fb558610450\":{\"columns\":{\"9452a9c9-08c5-4d90-a5a7-c43f0e0d6cc1\":{\"label\":\"timePosition\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"timePosition\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"53850181-7eaf-46f7-bed1-f0474c881ddd\":{\"label\":\"Count of records\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"___records___\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"9452a9c9-08c5-4d90-a5a7-c43f0e0d6cc1\",\"53850181-7eaf-46f7-bed1-f0474c881ddd\"],\"incompleteColumns\":{}}}}}}},\"enhancements\":{},\"hidePanelTitles\":false},\"title\":\"Positions\"},{\"version\":\"8.2.0\",\"type\":\"lens\",\"gridData\":{\"x\":24,\"y\":6,\"w\":6,\"h\":8,\"i\":\"96bd5564-a095-4fc7-b8dd-065326286cf9\"},\"panelIndex\":\"96bd5564-a095-4fc7-b8dd-065326286cf9\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsMetric\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"b95825b0-d504-11ec-bc4f-b5a4ce689d85\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"b95825b0-d504-11ec-bc4f-b5a4ce689d85\",\"name\":\"indexpattern-datasource-layer-ff589e5b-4284-437a-a5af-eaf0fe9016f0\"}],\"state\":{\"visualization\":{\"layerId\":\"ff589e5b-4284-437a-a5af-eaf0fe9016f0\",\"accessor\":\"2f8d74a2-8617-412c-a95b-0c4d8618ce9b\",\"layerType\":\"data\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"ff589e5b-4284-437a-a5af-eaf0fe9016f0\":{\"columns\":{\"2f8d74a2-8617-412c-a95b-0c4d8618ce9b\":{\"label\":\"meters\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"baroAltitude\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true,\"format\":{\"id\":\"number\",\"params\":{\"decimals\":0}}},\"customLabel\":true}},\"columnOrder\":[\"2f8d74a2-8617-412c-a95b-0c4d8618ce9b\"],\"incompleteColumns\":{}}}}}}},\"enhancements\":{},\"hidePanelTitles\":false},\"title\":\"Average barometric altitude\"},{\"version\":\"8.2.0\",\"type\":\"lens\",\"gridData\":{\"x\":24,\"y\":14,\"w\":6,\"h\":8,\"i\":\"ec502401-08f2-4329-8098-adeef854f9b7\"},\"panelIndex\":\"ec502401-08f2-4329-8098-adeef854f9b7\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsMetric\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"b95825b0-d504-11ec-bc4f-b5a4ce689d85\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"b95825b0-d504-11ec-bc4f-b5a4ce689d85\",\"name\":\"indexpattern-datasource-layer-ff589e5b-4284-437a-a5af-eaf0fe9016f0\"}],\"state\":{\"visualization\":{\"layerId\":\"ff589e5b-4284-437a-a5af-eaf0fe9016f0\",\"accessor\":\"2f8d74a2-8617-412c-a95b-0c4d8618ce9b\",\"layerType\":\"data\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"ff589e5b-4284-437a-a5af-eaf0fe9016f0\":{\"columns\":{\"2f8d74a2-8617-412c-a95b-0c4d8618ce9bX0\":{\"label\":\"Part of m/s\",\"dataType\":\"number\",\"operationType\":\"average\",\"sourceField\":\"velocity\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":false},\"customLabel\":true},\"2f8d74a2-8617-412c-a95b-0c4d8618ce9bX1\":{\"label\":\"Part of m/s\",\"dataType\":\"number\",\"operationType\":\"math\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"tinymathAst\":{\"type\":\"function\",\"name\":\"multiply\",\"args\":[\"2f8d74a2-8617-412c-a95b-0c4d8618ce9bX0\",3.6],\"location\":{\"min\":0,\"max\":31},\"text\":\"multiply(average(velocity),3.6)\"}},\"references\":[\"2f8d74a2-8617-412c-a95b-0c4d8618ce9bX0\"],\"customLabel\":true},\"2f8d74a2-8617-412c-a95b-0c4d8618ce9b\":{\"label\":\"km/h\",\"dataType\":\"number\",\"operationType\":\"formula\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"formula\":\"multiply(average(velocity),3.6)\",\"isFormulaBroken\":false,\"format\":{\"id\":\"number\",\"params\":{\"decimals\":0}}},\"references\":[\"2f8d74a2-8617-412c-a95b-0c4d8618ce9bX1\"],\"customLabel\":true}},\"columnOrder\":[\"2f8d74a2-8617-412c-a95b-0c4d8618ce9b\",\"2f8d74a2-8617-412c-a95b-0c4d8618ce9bX0\",\"2f8d74a2-8617-412c-a95b-0c4d8618ce9bX1\"],\"incompleteColumns\":{}}}}}}},\"enhancements\":{},\"hidePanelTitles\":false},\"title\":\"Average velocity\"},{\"version\":\"8.2.0\",\"type\":\"lens\",\"gridData\":{\"x\":30,\"y\":15,\"w\":18,\"h\":21,\"i\":\"a2e9e854-f30e-49ec-8cdd-f2472b7f2947\"},\"panelIndex\":\"a2e9e854-f30e-49ec-8cdd-f2472b7f2947\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsPie\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"b95825b0-d504-11ec-bc4f-b5a4ce689d85\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"b95825b0-d504-11ec-bc4f-b5a4ce689d85\",\"name\":\"indexpattern-datasource-layer-4ae1e7da-5673-4e6a-aa87-b0827cc0dba8\"}],\"state\":{\"visualization\":{\"shape\":\"treemap\",\"layers\":[{\"layerId\":\"4ae1e7da-5673-4e6a-aa87-b0827cc0dba8\",\"groups\":[\"2b327f2e-f088-4a19-a71e-b77272d9390f\"],\"metric\":\"c1462641-477d-4323-b74c-f9e4ed200205\",\"numberDisplay\":\"percent\",\"categoryDisplay\":\"default\",\"legendDisplay\":\"default\",\"nestedLegend\":false,\"layerType\":\"data\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"4ae1e7da-5673-4e6a-aa87-b0827cc0dba8\":{\"columns\":{\"2b327f2e-f088-4a19-a71e-b77272d9390f\":{\"label\":\"Top 30 values of originCountry\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"originCountry\",\"isBucketed\":true,\"params\":{\"size\":30,\"orderBy\":{\"type\":\"column\",\"columnId\":\"c1462641-477d-4323-b74c-f9e4ed200205\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"}}},\"c1462641-477d-4323-b74c-f9e4ed200205\":{\"label\":\"Count of records\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"___records___\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"2b327f2e-f088-4a19-a71e-b77272d9390f\",\"c1462641-477d-4323-b74c-f9e4ed200205\"],\"incompleteColumns\":{}}}}}}},\"enhancements\":{},\"hidePanelTitles\":false},\"title\":\"Origin Country\"},{\"version\":\"8.2.0\",\"type\":\"lens\",\"gridData\":{\"x\":24,\"y\":22,\"w\":6,\"h\":14,\"i\":\"26b76c3f-d714-4e2b-b93b-e458fde0f4be\"},\"panelIndex\":\"26b76c3f-d714-4e2b-b93b-e458fde0f4be\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsPie\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"b95825b0-d504-11ec-bc4f-b5a4ce689d85\",\"name\":\"indexpattern-datasource-current-indexpattern\"},{\"type\":\"index-pattern\",\"id\":\"b95825b0-d504-11ec-bc4f-b5a4ce689d85\",\"name\":\"indexpattern-datasource-layer-387ab670-f4d0-491e-b5cf-714a2ea9cdd2\"}],\"state\":{\"visualization\":{\"shape\":\"donut\",\"layers\":[{\"layerId\":\"387ab670-f4d0-491e-b5cf-714a2ea9cdd2\",\"groups\":[\"c929d968-8aa2-4cfb-8a52-52b21c3e6122\"],\"metric\":\"a5f77e90-2d83-476e-88b6-16c2f5c1ff2d\",\"numberDisplay\":\"percent\",\"categoryDisplay\":\"default\",\"legendDisplay\":\"default\",\"nestedLegend\":false,\"layerType\":\"data\",\"legendPosition\":\"top\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"387ab670-f4d0-491e-b5cf-714a2ea9cdd2\":{\"columns\":{\"c929d968-8aa2-4cfb-8a52-52b21c3e6122\":{\"label\":\"Top 5 values of onGround\",\"dataType\":\"boolean\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"onGround\",\"isBucketed\":true,\"params\":{\"size\":5,\"orderBy\":{\"type\":\"column\",\"columnId\":\"a5f77e90-2d83-476e-88b6-16c2f5c1ff2d\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"}}},\"a5f77e90-2d83-476e-88b6-16c2f5c1ff2d\":{\"label\":\"Count of records\",\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"___records___\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"c929d968-8aa2-4cfb-8a52-52b21c3e6122\",\"a5f77e90-2d83-476e-88b6-16c2f5c1ff2d\"],\"incompleteColumns\":{}}}}}}},\"hidePanelTitles\":false,\"enhancements\":{}},\"title\":\"On Ground?\"}]","timeRestore":false,"title":"OpenSky","version":1},"coreMigrationVersion":"8.2.0","id":"a4b89e90-d52d-11ec-bbd6-a3ae01a1af6e","migrationVersion":{"dashboard":"8.2.0"},"references":[{"id":"cc6a24a0-d4ff-11ec-bc4f-b5a4ce689d85","name":"a43e2915-1e1e-4e61-8735-f304bfefb35e:panel_a43e2915-1e1e-4e61-8735-f304bfefb35e","type":"map"},{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"ab7e7c62-c79f-408b-9a29-08e1c1ebb8b2:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"ab7e7c62-c79f-408b-9a29-08e1c1ebb8b2:indexpattern-datasource-layer-ff589e5b-4284-437a-a5af-eaf0fe9016f0","type":"index-pattern"},{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"24c8d122-d96c-4ea2-9b82-ffa792c4e0d9:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"24c8d122-d96c-4ea2-9b82-ffa792c4e0d9:indexpattern-datasource-layer-ebd928d3-2870-4094-9aed-9fb558610450","type":"index-pattern"},{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"96bd5564-a095-4fc7-b8dd-065326286cf9:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"96bd5564-a095-4fc7-b8dd-065326286cf9:indexpattern-datasource-layer-ff589e5b-4284-437a-a5af-eaf0fe9016f0","type":"index-pattern"},{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"ec502401-08f2-4329-8098-adeef854f9b7:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"ec502401-08f2-4329-8098-adeef854f9b7:indexpattern-datasource-layer-ff589e5b-4284-437a-a5af-eaf0fe9016f0","type":"index-pattern"},{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"a2e9e854-f30e-49ec-8cdd-f2472b7f2947:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"a2e9e854-f30e-49ec-8cdd-f2472b7f2947:indexpattern-datasource-layer-4ae1e7da-5673-4e6a-aa87-b0827cc0dba8","type":"index-pattern"},{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"26b76c3f-d714-4e2b-b93b-e458fde0f4be:indexpattern-datasource-current-indexpattern","type":"index-pattern"},{"id":"b95825b0-d504-11ec-bc4f-b5a4ce689d85","name":"26b76c3f-d714-4e2b-b93b-e458fde0f4be:indexpattern-datasource-layer-387ab670-f4d0-491e-b5cf-714a2ea9cdd2","type":"index-pattern"}],"type":"dashboard","updated_at":"2022-05-16T15:38:38.042Z","version":"WzE0ODgsNV0="} 5 | {"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":4,"missingRefCount":0,"missingReferences":[]} -------------------------------------------------------------------------------- /lab/docker-compose-opensky.yml: -------------------------------------------------------------------------------- 1 | #Minimal Docker Compose cluster for ElasticSearch and Kibana 2 | # Put this file any folder creating "logs" and "esdata" folders 3 | # to persist the cluster indices and kibana state 4 | 5 | version: "3.7" 6 | 7 | services: 8 | opensky-loader: 9 | init: true 10 | build: ./opensky-loader/ 11 | container_name: opensky-loader 12 | volumes: 13 | - ./certs/ca/ca.crt:/etc/ssl/opensky/ca/ca.crt:ro 14 | - ./opensky-loader/index.js:/usr/src/app/index.js 15 | - ./elastic-config.js:/usr/src/app/config.js 16 | environment: 17 | - ELASTIC_HOST=${ELASTIC_HOST} 18 | - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} 19 | - OPENSKY_USER=${OPENSKY_USER} 20 | - OPENSKY_PASSWORD=${OPENSKY_PASSWORD} 21 | 22 | opensky-viewer: 23 | init: true 24 | build: ./opensky-viewer/ 25 | container_name: opensky-viewer 26 | environment: 27 | - ELASTIC_HOST=${ELASTIC_HOST} 28 | - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} 29 | - PORT=3000 30 | ports: 31 | - $OPENSKY_VIEWER_PORT:3000 32 | volumes: 33 | - ./certs/ca/ca.crt:/etc/ssl/opensky/ca/ca.crt:ro 34 | - ./opensky-viewer/index.js:/usr/src/app/index.js 35 | - ./opensky-viewer/index.html:/usr/src/app/index.html 36 | - ./elastic-config.js:/usr/src/app/config.js 37 | 38 | -------------------------------------------------------------------------------- /lab/docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Minimal Docker Compose cluster for ElasticSearch and Kibana 2 | # Put this file any folder creating "logs" and "esdata" folders 3 | # to persist the cluster indices and kibana state 4 | # 5 | # More details at: https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-compose-file 6 | 7 | version: "3.7" 8 | 9 | services: 10 | setup: 11 | image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} 12 | volumes: 13 | - ./certs:/usr/share/elasticsearch/config/certs 14 | user: "0" 15 | command: > 16 | bash -c ' 17 | if [ x${ELASTIC_PASSWORD} == x ]; then 18 | echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; 19 | exit 1; 20 | elif [ x${KIBANA_PASSWORD} == x ]; then 21 | echo "Set the KIBANA_PASSWORD environment variable in the .env file"; 22 | exit 1; 23 | fi; 24 | if [ ! -f certs/ca.zip ]; then 25 | echo "Creating CA"; 26 | bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; 27 | unzip config/certs/ca.zip -d config/certs; 28 | fi; 29 | if [ ! -f certs/certs.zip ]; then 30 | echo "Creating certs"; 31 | echo -ne \ 32 | "instances:\n"\ 33 | " - name: es01\n"\ 34 | " dns:\n"\ 35 | " - es01\n"\ 36 | " - localhost\n"\ 37 | " ip:\n"\ 38 | " - 127.0.0.1\n"\ 39 | " - name: es02\n"\ 40 | " dns:\n"\ 41 | " - es02\n"\ 42 | " - localhost\n"\ 43 | " ip:\n"\ 44 | " - 127.0.0.1\n"\ 45 | > config/certs/instances.yml; 46 | bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; 47 | unzip config/certs/certs.zip -d config/certs; 48 | fi; 49 | echo "Setting file permissions" 50 | chown -R root:root config/certs; 51 | find . -type d -exec chmod 750 \{\} \;; 52 | find . -type f -exec chmod 640 \{\} \;; 53 | echo "Waiting for Elasticsearch availability"; 54 | until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; 55 | echo "Setting kibana_system password"; 56 | until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; 57 | echo "All done!"; 58 | ' 59 | healthcheck: 60 | test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"] 61 | interval: 1s 62 | timeout: 5s 63 | retries: 120 64 | 65 | es01: 66 | depends_on: 67 | setup: 68 | condition: service_healthy 69 | image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} 70 | volumes: 71 | - ./certs:/usr/share/elasticsearch/config/certs 72 | - esdata01:/usr/share/elasticsearch/data 73 | ports: 74 | - ${ES_PORT}:9200 75 | environment: 76 | - node.name=es01 77 | - cluster.name=${CLUSTER_NAME} 78 | - cluster.initial_master_nodes=es01,es02 79 | - discovery.seed_hosts=es02 80 | - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} 81 | - bootstrap.memory_lock=true 82 | - xpack.security.enabled=true 83 | - xpack.security.http.ssl.enabled=true 84 | - xpack.security.http.ssl.key=certs/es01/es01.key 85 | - xpack.security.http.ssl.certificate=certs/es01/es01.crt 86 | - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt 87 | - xpack.security.http.ssl.verification_mode=certificate 88 | - xpack.security.transport.ssl.enabled=true 89 | - xpack.security.transport.ssl.key=certs/es01/es01.key 90 | - xpack.security.transport.ssl.certificate=certs/es01/es01.crt 91 | - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt 92 | - xpack.security.transport.ssl.verification_mode=certificate 93 | - xpack.license.self_generated.type=${LICENSE} 94 | mem_limit: ${MEM_LIMIT} 95 | ulimits: 96 | memlock: 97 | soft: -1 98 | hard: -1 99 | healthcheck: 100 | test: 101 | [ 102 | "CMD-SHELL", 103 | "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", 104 | ] 105 | interval: 10s 106 | timeout: 10s 107 | retries: 120 108 | deploy: 109 | restart_policy: 110 | condition: on-failure 111 | delay: 5s 112 | max_attempts: 3 113 | window: 60s 114 | 115 | es02: 116 | depends_on: 117 | - es01 118 | image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} 119 | volumes: 120 | - ./certs:/usr/share/elasticsearch/config/certs 121 | - esdata02:/usr/share/elasticsearch/data 122 | environment: 123 | - node.name=es02 124 | - cluster.name=${CLUSTER_NAME} 125 | - cluster.initial_master_nodes=es01,es02 126 | - discovery.seed_hosts=es01 127 | - bootstrap.memory_lock=true 128 | - xpack.security.enabled=true 129 | - xpack.security.http.ssl.enabled=true 130 | - xpack.security.http.ssl.key=certs/es02/es02.key 131 | - xpack.security.http.ssl.certificate=certs/es02/es02.crt 132 | - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt 133 | - xpack.security.http.ssl.verification_mode=certificate 134 | - xpack.security.transport.ssl.enabled=true 135 | - xpack.security.transport.ssl.key=certs/es02/es02.key 136 | - xpack.security.transport.ssl.certificate=certs/es02/es02.crt 137 | - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt 138 | - xpack.security.transport.ssl.verification_mode=certificate 139 | - xpack.license.self_generated.type=${LICENSE} 140 | mem_limit: ${MEM_LIMIT} 141 | ulimits: 142 | memlock: 143 | soft: -1 144 | hard: -1 145 | healthcheck: 146 | test: 147 | [ 148 | "CMD-SHELL", 149 | "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", 150 | ] 151 | interval: 10s 152 | timeout: 10s 153 | retries: 120 154 | deploy: 155 | restart_policy: 156 | condition: on-failure 157 | delay: 5s 158 | max_attempts: 3 159 | window: 60s 160 | 161 | kibana: 162 | depends_on: 163 | es01: 164 | condition: service_healthy 165 | es02: 166 | condition: service_healthy 167 | image: docker.elastic.co/kibana/kibana:${STACK_VERSION} 168 | volumes: 169 | - ./certs:/usr/share/kibana/config/certs 170 | - kibanadata:/usr/share/kibana/data 171 | ports: 172 | - ${KIBANA_PORT}:5601 173 | environment: 174 | - SERVERNAME=kibana 175 | - ELASTICSEARCH_HOSTS=https://es01:9200 176 | - ELASTICSEARCH_USERNAME=kibana_system 177 | - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} 178 | - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt 179 | mem_limit: ${MEM_LIMIT} 180 | healthcheck: 181 | test: 182 | [ 183 | "CMD-SHELL", 184 | "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'", 185 | ] 186 | interval: 10s 187 | timeout: 10s 188 | retries: 120 189 | 190 | opensky-loader: 191 | init: true 192 | build: ./opensky-loader/ 193 | container_name: opensky-loader 194 | volumes: 195 | - ./certs/ca/ca.crt:/etc/ssl/opensky/ca/ca.crt:ro 196 | - ./opensky-loader/index.js:/usr/src/app/index.js 197 | - ./elastic-config.js:/usr/src/app/config.js 198 | environment: 199 | - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} 200 | - OPENSKY_USER=${OPENSKY_USER} 201 | - OPENSKY_PASSWORD=${OPENSKY_PASSWORD} 202 | depends_on: 203 | es01: 204 | condition: service_healthy 205 | es02: 206 | condition: service_healthy 207 | 208 | opensky-viewer: 209 | init: true 210 | build: ./opensky-viewer/ 211 | container_name: opensky-viewer 212 | environment: 213 | - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} 214 | - PORT=3000 215 | ports: 216 | - $OPENSKY_VIEWER_PORT:3000 217 | volumes: 218 | - ./certs/ca/ca.crt:/etc/ssl/opensky/ca/ca.crt:ro 219 | - ./opensky-viewer/index.js:/usr/src/app/index.js 220 | - ./opensky-viewer/index.html:/usr/src/app/index.html 221 | - ./elastic-config.js:/usr/src/app/config.js 222 | depends_on: 223 | es01: 224 | condition: service_healthy 225 | es02: 226 | condition: service_healthy 227 | 228 | volumes: 229 | esdata01: 230 | driver: local 231 | esdata02: 232 | driver: local 233 | kibanadata: 234 | driver: local 235 | 236 | -------------------------------------------------------------------------------- /lab/elastic-config.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | 3 | const default_node = 'https://es01:9200'; 4 | 5 | const node = process.env.ELASTIC_HOST || default_node; 6 | const password = process.env.ELASTIC_PASSWORD || 'changeme'; 7 | 8 | const opensky_user = process.env.OPENSKY_USER; 9 | const opensky_password = process.env.OPENSKY_PASSWORD; 10 | 11 | const tls = (node === default_node) ? 12 | { 13 | ca: fs.readFileSync('/etc/ssl/opensky/ca/ca.crt'), 14 | rejectUnauthorized: true 15 | } : null 16 | 17 | module.exports = { 18 | es_config: { 19 | node, 20 | auth: { 21 | username: 'elastic', 22 | password 23 | }, 24 | tls 25 | }, 26 | index_name: 'flight_tracking', 27 | sleep_seconds: 90, 28 | opensky: { 29 | user: opensky_user, 30 | password: opensky_password, 31 | url: 'https://opensky-network.org/api/states/all' 32 | } 33 | }; 34 | 35 | -------------------------------------------------------------------------------- /lab/elastic-config.js.cloud.sample: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | es_config: { 3 | cloud:{ 4 | id: 'your-cloud-id' 5 | }, 6 | auth: { 7 | username: 'your-elastic-user', 8 | password: 'your-elastic-password' 9 | } 10 | }, 11 | /* optional pipeline for ingesting processing 12 | pipeline_name: 'my_pipeline' 13 | */ 14 | index_name: 'flight_tracking', 15 | sleep_seconds: 60 16 | }; -------------------------------------------------------------------------------- /lab/opensky-loader/.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | download 3 | -------------------------------------------------------------------------------- /lab/opensky-loader/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:current-alpine 2 | 3 | WORKDIR /usr/src/app 4 | 5 | COPY . . 6 | 7 | RUN npm install 8 | 9 | CMD npm start -------------------------------------------------------------------------------- /lab/opensky-loader/config.js: -------------------------------------------------------------------------------- 1 | ../elastic-config.js -------------------------------------------------------------------------------- /lab/opensky-loader/download/README.md: -------------------------------------------------------------------------------- 1 | # OpenSky Data Downloader 2 | 3 | Use the script `download.sh` along with [`curl`][c], [`jq`][j], and [`ogr2ogr`][o] to download and generate CSV and GeoJSON files. The script takes a single parameter with the number or requests to the API you want to do, using `5` as a default value. 4 | 5 | The script makes a request to OpenSky API and appends to a CSV file the contents, waits for 60 seconds and repeat. The CSV will be named using a date scheme so you can run the process several times without loosing any observations. 6 | 7 | Once the file is generate it will use `ogr2ogr` to convert the CSV into a GeoJSON file you can upload to Elasticsearct using Elastic Maps [GeoJSON Upload][u]. 8 | 9 | Sample execution: 10 | 11 | ``` 12 | $ bash download.sh 2 13 | Iterating 2 times 14 | Writing to flight_tracking_2020-02-05_18_42.csv 15 | [1] Downloading... (mié feb 5 18:42:03 CET 2020) 16 | Done, sleeping 60 seconds 17 | [2] Downloading... (mié feb 5 18:43:12 CET 2020) 18 | Finished downloading 19 | Generating the GeoJSON file: flight_tracking_2020-02-05_18_42.geojson 20 | ``` 21 | 22 | ## Upload the CSV 23 | 24 | If you want to have all the fields correctly mapped you can use the [Kibana File Upload](https://www.elastic.co/guide/en/kibana/current/connect-to-elasticsearch.html#upload-data-kibana) to process the `CSV` file instead of the `GeoJSON` output. 25 | 26 | Once the file is initially processed, use these mappings and ingest pipeline definition in the **Advanced** tab 27 | 28 | ### Mappings 29 | 30 | ```json 31 | { 32 | "properties": { 33 | "@timestamp": { 34 | "type": "date" 35 | }, 36 | "_": { 37 | "type": "keyword" 38 | }, 39 | "baroAltitude": { 40 | "type": "double" 41 | }, 42 | "callsign": { 43 | "type": "keyword" 44 | }, 45 | "country": { 46 | "type": "keyword" 47 | }, 48 | "geoAltitude": { 49 | "type": "double" 50 | }, 51 | "heading": { 52 | "type": "double" 53 | }, 54 | "icao24": { 55 | "type": "keyword" 56 | }, 57 | "lastContact": { 58 | "type": "date", 59 | "format": "epoch_second" 60 | }, 61 | "timePosition": { 62 | "type": "date", 63 | "format": "epoch_second" 64 | }, 65 | "latitude": { 66 | "type": "float" 67 | }, 68 | "longitude": { 69 | "type": "float" 70 | }, 71 | "onGround": { 72 | "type": "boolean" 73 | }, 74 | "positionSource": { 75 | "type": "keyword" 76 | }, 77 | "spi": { 78 | "type": "boolean" 79 | }, 80 | "transponderCode": { 81 | "type": "keyword" 82 | }, 83 | "velocity": { 84 | "type": "double" 85 | }, 86 | "verticalRate": { 87 | "type": "double" 88 | }, 89 | "location": { 90 | "type": "geo_point" 91 | } 92 | } 93 | } 94 | ``` 95 | 96 | ### Ingest pipeline 97 | 98 | ```json 99 | { 100 | "description": "Ingest pipeline created by text structure finder", 101 | "processors": [ 102 | { 103 | "csv": { 104 | "field": "message", 105 | "target_fields": [ 106 | "icao24", 107 | "callsign", 108 | "country", 109 | "timePosition", 110 | "lastContact", 111 | "longitude", 112 | "latitude", 113 | "baroAltitude", 114 | "onGround", 115 | "velocity", 116 | "heading", 117 | "verticalRate", 118 | "_", 119 | "geoAltitude", 120 | "transponderCode", 121 | "spi", 122 | "positionSource" 123 | ], 124 | "ignore_missing": false 125 | } 126 | }, 127 | { 128 | "date": { 129 | "field": "timePosition", 130 | "formats": [ 131 | "UNIX" 132 | ] 133 | } 134 | }, 135 | { 136 | "convert": { 137 | "field": "baroAltitude", 138 | "type": "double", 139 | "ignore_missing": true 140 | } 141 | }, 142 | { 143 | "convert": { 144 | "field": "geoAltitude", 145 | "type": "double", 146 | "ignore_missing": true 147 | } 148 | }, 149 | { 150 | "convert": { 151 | "field": "heading", 152 | "type": "double", 153 | "ignore_missing": true 154 | } 155 | }, 156 | { 157 | "date": { 158 | "field": "lastContact", 159 | "formats": [ 160 | "UNIX" 161 | ] 162 | } 163 | }, 164 | { 165 | "convert": { 166 | "field": "latitude", 167 | "type": "double", 168 | "ignore_missing": true 169 | } 170 | }, 171 | { 172 | "convert": { 173 | "field": "longitude", 174 | "type": "double", 175 | "ignore_missing": true 176 | } 177 | }, 178 | { 179 | "convert": { 180 | "field": "onGround", 181 | "type": "boolean", 182 | "ignore_missing": true 183 | } 184 | }, 185 | { 186 | "convert": { 187 | "field": "spi", 188 | "type": "boolean", 189 | "ignore_missing": true 190 | } 191 | }, 192 | { 193 | "convert": { 194 | "field": "velocity", 195 | "type": "double", 196 | "ignore_missing": true 197 | } 198 | }, 199 | { 200 | "convert": { 201 | "field": "verticalRate", 202 | "type": "double", 203 | "ignore_missing": true 204 | } 205 | }, 206 | { 207 | "set": { 208 | "field": "location", 209 | "value": "{{latitude}},{{longitude}}" 210 | } 211 | }, 212 | { 213 | "remove": { 214 | "field": "message" 215 | } 216 | }, 217 | { 218 | "remove": { 219 | "field": "_" 220 | } 221 | }, 222 | { 223 | "remove": { 224 | "field": "latitude" 225 | } 226 | }, 227 | { 228 | "remove": { 229 | "field": "longitude" 230 | } 231 | } 232 | ] 233 | } 234 | ``` 235 | 236 | 237 | 238 | [c]: https://curl.haxx.se/ 239 | [o]: https://gdal.org/programs/ogr2ogr.html 240 | [j]: https://stedolan.github.io/jq/ 241 | [u]: https://www.elastic.co/guide/en/kibana/current/geojson-upload.html -------------------------------------------------------------------------------- /lab/opensky-loader/download/data_header.csv: -------------------------------------------------------------------------------- 1 | icao24,callsign,country,timePosition,lastContact,longitude,latitude,baroAltitude,onGround,velocity,heading,verticalRate,_,geoAltitude,transponderCode,spi,positionSource 2 | -------------------------------------------------------------------------------- /lab/opensky-loader/download/download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | filename_base=flight_tracking_$(date "+%Y-%m-%d_%H_%M") 4 | 5 | csv_file="${filename_base}.csv" 6 | geojson_file="${filename_base}.geojson" 7 | 8 | if [ -z "$1" ] 9 | then 10 | echo "No iterations provided, we'll do 5" 11 | iterations=5 12 | else 13 | iterations=$1 14 | echo "Iterating ${iterations} times" 15 | fi 16 | 17 | echo "Writing to ${csv_file}" 18 | cat data_header.csv > ${csv_file} 19 | for i in $(seq 1 ${iterations}); do 20 | echo "[${i}] Downloading... ($(date))"; 21 | curl -sL "https://opensky-network.org/api/states/all" \ 22 | | jq -c ".states[]" \ 23 | | sed -e 's/^\[//g' -e 's/\]$//g' \ 24 | >> ${csv_file} 25 | if [ $i -ne $iterations ]; then 26 | echo "Done, sleeping 60 seconds" 27 | sleep 60 28 | else 29 | echo "Finished downloading" 30 | fi 31 | done 32 | 33 | echo "Generating the GeoJSON file: ${geojson_file}" 34 | ogr2ogr -f GeoJSON ${geojson_file} \ 35 | -oo "X_POSSIBLE_NAMES=Lon*" \ 36 | -oo "Y_POSSIBLE_NAMES=Lat*" \ 37 | -oo KEEP_GEOM_COLUMNS=NO \ 38 | ${csv_file} 39 | -------------------------------------------------------------------------------- /lab/opensky-loader/download/flight_tracking.geojson.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsanz/elastic-workshop/248063c5b57a21b157583e2df18724a4346b2037/lab/opensky-loader/download/flight_tracking.geojson.gz -------------------------------------------------------------------------------- /lab/opensky-loader/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { Client } = require('@elastic/elasticsearch'); 4 | const axios = require('axios').default; 5 | const config = require('./config'); 6 | 7 | const CONFIG = config.es_config || { node: 'http://localhost:9200' } 8 | const INTERVAL_SECS = config.sleep_seconds || 60; 9 | const ES_INDEX_NAME = config.index_name || 'flight_tracking'; 10 | 11 | const client = new Client(CONFIG); 12 | 13 | function checkConnection() { 14 | return new Promise(async (resolve) => { 15 | let isConnected = false; 16 | while (!isConnected) { 17 | try { 18 | sleep(1000); 19 | console.debug("Checking connection to ElasticSearch..."); 20 | await client.cluster.health({}); 21 | console.debug("Successfully connected to ElasticSearch"); 22 | isConnected = true; 23 | } catch (e) { 24 | console.log(e) 25 | } 26 | } 27 | resolve(true); 28 | }); 29 | } 30 | 31 | async function loadFlights() { 32 | console.log('---- OPENSKY LOADER CONFIG ----'); 33 | console.log(config); 34 | console.log('---- OPENSKY LOADER CONFIG ----'); 35 | 36 | await checkConnection(); 37 | 38 | 39 | // Infinite loop to load the resources 40 | while (true) { 41 | const today = (new Date()).toISOString().split('T')[0]; 42 | const index_name = ES_INDEX_NAME + '_' + today; 43 | try { 44 | 45 | await checkConnection(); 46 | // Create the index if necessary 47 | const doesIndexExistResp = await client.indices.exists({ index: index_name, }); 48 | 49 | if (!doesIndexExistResp) { 50 | createIndex(index_name); 51 | } 52 | 53 | await indexFlights(index_name, await getFlights()); 54 | } catch (error) { 55 | console.log(error) 56 | } 57 | await sleep(INTERVAL_SECS * 1000); 58 | } 59 | } 60 | 61 | function sleep(ms) { 62 | console.log(`Sleeping for ${ms / 1000} seconds`); 63 | return new Promise(resolve => setTimeout(resolve, ms)); 64 | } 65 | 66 | async function createIndex(index_name) { 67 | console.log(`Creating index ${index_name}`); 68 | await client.indices.create({ 69 | index: index_name, 70 | body: { 71 | settings: { index: { number_of_shards: 1, auto_expand_replicas: '0-1' } }, 72 | mappings: { 73 | properties: { 74 | icao24: { type: 'keyword' }, 75 | callsign: { 76 | "type": 'text', 77 | "fields": { 78 | "keyword": { 79 | "type": "keyword", 80 | "ignore_above": 256 81 | } 82 | } 83 | }, 84 | originCountry: { type: 'keyword' }, 85 | location: { type: 'geo_point' }, 86 | timePosition: { type: 'date' }, 87 | lastContact: { type: 'date' }, 88 | '@timestamp': { type: 'date' }, 89 | baroAltitude: { type: 'float' }, 90 | onGround: { type: 'boolean' }, 91 | velocity: { type: 'float' }, 92 | verticalRate: { type: 'float' }, 93 | 'heading': { type: 'float' }, 94 | geoAltitude: { type: 'float' }, 95 | transponderCode: { type: 'keyword' }, 96 | spi: { type: 'boolean' }, 97 | positionSource: { type: 'integer' } 98 | } 99 | }, 100 | }, 101 | }); 102 | } 103 | 104 | async function indexFlights(index_name, flights) { 105 | console.log(`Indexing ${flights.length} flights into ${index_name}`); 106 | 107 | const bulk = []; 108 | flights.forEach(async (flight) => { 109 | bulk.push({ index: { _index: index_name } }); 110 | bulk.push(flight); 111 | }); 112 | 113 | const bulkObj = { body: bulk }; 114 | if (config.pipeline_name) { 115 | console.log('Using pipeline ', config.pipeline_name); 116 | bulkObj['pipeline'] = config.pipeline_name; 117 | } 118 | const resp = await client.bulk(bulkObj); 119 | 120 | if (resp.errors) { 121 | console.log(`Failed to load some flights`); 122 | } 123 | } 124 | 125 | async function getFlights() { 126 | console.log(`Fetching flights @ ${(new Date()).toISOString()}`); 127 | 128 | const options = { 129 | headers: { 'X-Requested-With': 'XMLHttpRequest' }, 130 | auth: config.opensky.user == undefined 131 | ? {} 132 | : { 133 | username: config.opensky.user, 134 | password: config.opensky.password 135 | } 136 | }; 137 | 138 | const response = await axios.get(config.opensky.url, options); 139 | 140 | console.log(`Fetched ${response.data.states.length} flights`); 141 | 142 | return response.data.states.map(convertStateToFlight); 143 | } 144 | 145 | function convertStateToFlight(state) { 146 | // state is an array, see 'All State Vectors' @ https://opensky-network.org/apidoc/rest.html for details 147 | const flight = { 148 | ['@timestamp']: Date.now(), 149 | onGround: state[8], 150 | spi: state[15], // Whether flight status indicates special purpose indicator 151 | }; 152 | if (state[0]) { 153 | flight.icao24 = state[0]; 154 | } 155 | if (state[1]) { 156 | flight.callsign = state[1] ? state[1].trim() : undefined; 157 | } 158 | if (state[2]) { 159 | flight.originCountry = state[2]; 160 | } 161 | if (state[3]) { 162 | // provided in seconds, convert to milliseconds so ES will convert to Date 163 | flight.timePosition = state[3] * 1000; 164 | } 165 | if (state[4]) { 166 | // provided in seconds, convert to milliseconds so ES will convert to Date 167 | flight.lastContact = state[4] * 1000; 168 | } 169 | if (state[5] && state[6]) { 170 | flight.location = { lat: state[6], lon: state[5] }; 171 | } 172 | if (state[7]) { 173 | flight.baroAltitude = state[7]; 174 | } 175 | if (state[9]) { 176 | flight.velocity = state[9]; 177 | } 178 | if (state[10]) { 179 | // decimal degrees clockwise from north (north=0°) 180 | flight.heading = state[10]; 181 | } 182 | if (state[11]) { 183 | // Vertical rate in m/s. A positive value indicates that the airplane is climbing, a negative value indicates that it descends 184 | flight.verticalRate = state[11]; 185 | } 186 | if (state[13]) { 187 | // Geometric altitude in meters 188 | flight.geoAltitude = state[13]; 189 | } 190 | if (state[14]) { 191 | flight.transponderCode = state[14]; 192 | } 193 | if (state[16]) { 194 | flight.positionSource = state[16]; 195 | } 196 | 197 | return flight; 198 | } 199 | 200 | loadFlights().catch(console.log); 201 | -------------------------------------------------------------------------------- /lab/opensky-loader/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "opensky-loader", 3 | "version": "1.0.0", 4 | "lockfileVersion": 2, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "opensky-loader", 9 | "version": "1.0.0", 10 | "license": "ISC", 11 | "dependencies": { 12 | "@elastic/elasticsearch": "^8.2.0-patch.1", 13 | "axios": "^0.27.2" 14 | } 15 | }, 16 | "node_modules/@elastic/elasticsearch": { 17 | "version": "8.2.0-patch.1", 18 | "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-8.2.0-patch.1.tgz", 19 | "integrity": "sha512-ROx9WLPoRrJ/NvT3NEz/4ZhzvGRmB1dSvhkNyuQ+LoHSR2kh4CFqFmxoFtJmWN1uqbw8WvQ0AmMCyLbDI04IVA==", 20 | "dependencies": { 21 | "@elastic/transport": "^8.2.0", 22 | "tslib": "^2.4.0" 23 | }, 24 | "engines": { 25 | "node": ">=14" 26 | } 27 | }, 28 | "node_modules/@elastic/transport": { 29 | "version": "8.2.0", 30 | "resolved": "https://registry.npmjs.org/@elastic/transport/-/transport-8.2.0.tgz", 31 | "integrity": "sha512-H/HmefMNQfLiBSVTmNExu2lYs5EzwipUnQB53WLr17RCTDaQX0oOLHcWpDsbKQSRhDAMPPzp5YZsZMJxuxPh7A==", 32 | "dependencies": { 33 | "debug": "^4.3.4", 34 | "hpagent": "^1.0.0", 35 | "ms": "^2.1.3", 36 | "secure-json-parse": "^2.4.0", 37 | "tslib": "^2.4.0", 38 | "undici": "^5.1.1" 39 | }, 40 | "engines": { 41 | "node": ">=14" 42 | } 43 | }, 44 | "node_modules/asynckit": { 45 | "version": "0.4.0", 46 | "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", 47 | "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" 48 | }, 49 | "node_modules/axios": { 50 | "version": "0.27.2", 51 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", 52 | "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", 53 | "dependencies": { 54 | "follow-redirects": "^1.14.9", 55 | "form-data": "^4.0.0" 56 | } 57 | }, 58 | "node_modules/combined-stream": { 59 | "version": "1.0.8", 60 | "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", 61 | "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", 62 | "dependencies": { 63 | "delayed-stream": "~1.0.0" 64 | }, 65 | "engines": { 66 | "node": ">= 0.8" 67 | } 68 | }, 69 | "node_modules/debug": { 70 | "version": "4.3.4", 71 | "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", 72 | "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", 73 | "dependencies": { 74 | "ms": "2.1.2" 75 | }, 76 | "engines": { 77 | "node": ">=6.0" 78 | }, 79 | "peerDependenciesMeta": { 80 | "supports-color": { 81 | "optional": true 82 | } 83 | } 84 | }, 85 | "node_modules/debug/node_modules/ms": { 86 | "version": "2.1.2", 87 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", 88 | "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" 89 | }, 90 | "node_modules/delayed-stream": { 91 | "version": "1.0.0", 92 | "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", 93 | "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", 94 | "engines": { 95 | "node": ">=0.4.0" 96 | } 97 | }, 98 | "node_modules/follow-redirects": { 99 | "version": "1.15.1", 100 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.1.tgz", 101 | "integrity": "sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==", 102 | "funding": [ 103 | { 104 | "type": "individual", 105 | "url": "https://github.com/sponsors/RubenVerborgh" 106 | } 107 | ], 108 | "engines": { 109 | "node": ">=4.0" 110 | }, 111 | "peerDependenciesMeta": { 112 | "debug": { 113 | "optional": true 114 | } 115 | } 116 | }, 117 | "node_modules/form-data": { 118 | "version": "4.0.0", 119 | "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", 120 | "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", 121 | "dependencies": { 122 | "asynckit": "^0.4.0", 123 | "combined-stream": "^1.0.8", 124 | "mime-types": "^2.1.12" 125 | }, 126 | "engines": { 127 | "node": ">= 6" 128 | } 129 | }, 130 | "node_modules/hpagent": { 131 | "version": "1.0.0", 132 | "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.0.0.tgz", 133 | "integrity": "sha512-SCleE2Uc1bM752ymxg8QXYGW0TWtAV4ZW3TqH1aOnyi6T6YW2xadCcclm5qeVjvMvfQ2RKNtZxO7uVb9CTPt1A==", 134 | "engines": { 135 | "node": ">=14" 136 | } 137 | }, 138 | "node_modules/mime-db": { 139 | "version": "1.52.0", 140 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", 141 | "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", 142 | "engines": { 143 | "node": ">= 0.6" 144 | } 145 | }, 146 | "node_modules/mime-types": { 147 | "version": "2.1.35", 148 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", 149 | "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", 150 | "dependencies": { 151 | "mime-db": "1.52.0" 152 | }, 153 | "engines": { 154 | "node": ">= 0.6" 155 | } 156 | }, 157 | "node_modules/ms": { 158 | "version": "2.1.3", 159 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", 160 | "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" 161 | }, 162 | "node_modules/secure-json-parse": { 163 | "version": "2.4.0", 164 | "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.4.0.tgz", 165 | "integrity": "sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg==" 166 | }, 167 | "node_modules/tslib": { 168 | "version": "2.4.0", 169 | "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", 170 | "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" 171 | }, 172 | "node_modules/undici": { 173 | "version": "5.9.1", 174 | "resolved": "https://registry.npmjs.org/undici/-/undici-5.9.1.tgz", 175 | "integrity": "sha512-6fB3a+SNnWEm4CJbgo0/CWR8RGcOCQP68SF4X0mxtYTq2VNN8T88NYrWVBAeSX+zb7bny2dx2iYhP3XHi00omg==", 176 | "engines": { 177 | "node": ">=12.18" 178 | } 179 | } 180 | }, 181 | "dependencies": { 182 | "@elastic/elasticsearch": { 183 | "version": "8.2.0-patch.1", 184 | "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-8.2.0-patch.1.tgz", 185 | "integrity": "sha512-ROx9WLPoRrJ/NvT3NEz/4ZhzvGRmB1dSvhkNyuQ+LoHSR2kh4CFqFmxoFtJmWN1uqbw8WvQ0AmMCyLbDI04IVA==", 186 | "requires": { 187 | "@elastic/transport": "^8.2.0", 188 | "tslib": "^2.4.0" 189 | } 190 | }, 191 | "@elastic/transport": { 192 | "version": "8.2.0", 193 | "resolved": "https://registry.npmjs.org/@elastic/transport/-/transport-8.2.0.tgz", 194 | "integrity": "sha512-H/HmefMNQfLiBSVTmNExu2lYs5EzwipUnQB53WLr17RCTDaQX0oOLHcWpDsbKQSRhDAMPPzp5YZsZMJxuxPh7A==", 195 | "requires": { 196 | "debug": "^4.3.4", 197 | "hpagent": "^1.0.0", 198 | "ms": "^2.1.3", 199 | "secure-json-parse": "^2.4.0", 200 | "tslib": "^2.4.0", 201 | "undici": "^5.1.1" 202 | } 203 | }, 204 | "asynckit": { 205 | "version": "0.4.0", 206 | "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", 207 | "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" 208 | }, 209 | "axios": { 210 | "version": "0.27.2", 211 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", 212 | "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", 213 | "requires": { 214 | "follow-redirects": "^1.14.9", 215 | "form-data": "^4.0.0" 216 | } 217 | }, 218 | "combined-stream": { 219 | "version": "1.0.8", 220 | "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", 221 | "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", 222 | "requires": { 223 | "delayed-stream": "~1.0.0" 224 | } 225 | }, 226 | "debug": { 227 | "version": "4.3.4", 228 | "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", 229 | "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", 230 | "requires": { 231 | "ms": "2.1.2" 232 | }, 233 | "dependencies": { 234 | "ms": { 235 | "version": "2.1.2", 236 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", 237 | "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" 238 | } 239 | } 240 | }, 241 | "delayed-stream": { 242 | "version": "1.0.0", 243 | "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", 244 | "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" 245 | }, 246 | "follow-redirects": { 247 | "version": "1.15.1", 248 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.1.tgz", 249 | "integrity": "sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==" 250 | }, 251 | "form-data": { 252 | "version": "4.0.0", 253 | "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", 254 | "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", 255 | "requires": { 256 | "asynckit": "^0.4.0", 257 | "combined-stream": "^1.0.8", 258 | "mime-types": "^2.1.12" 259 | } 260 | }, 261 | "hpagent": { 262 | "version": "1.0.0", 263 | "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.0.0.tgz", 264 | "integrity": "sha512-SCleE2Uc1bM752ymxg8QXYGW0TWtAV4ZW3TqH1aOnyi6T6YW2xadCcclm5qeVjvMvfQ2RKNtZxO7uVb9CTPt1A==" 265 | }, 266 | "mime-db": { 267 | "version": "1.52.0", 268 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", 269 | "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" 270 | }, 271 | "mime-types": { 272 | "version": "2.1.35", 273 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", 274 | "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", 275 | "requires": { 276 | "mime-db": "1.52.0" 277 | } 278 | }, 279 | "ms": { 280 | "version": "2.1.3", 281 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", 282 | "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" 283 | }, 284 | "secure-json-parse": { 285 | "version": "2.4.0", 286 | "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.4.0.tgz", 287 | "integrity": "sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg==" 288 | }, 289 | "tslib": { 290 | "version": "2.4.0", 291 | "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", 292 | "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" 293 | }, 294 | "undici": { 295 | "version": "5.9.1", 296 | "resolved": "https://registry.npmjs.org/undici/-/undici-5.9.1.tgz", 297 | "integrity": "sha512-6fB3a+SNnWEm4CJbgo0/CWR8RGcOCQP68SF4X0mxtYTq2VNN8T88NYrWVBAeSX+zb7bny2dx2iYhP3XHi00omg==" 298 | } 299 | } 300 | } 301 | -------------------------------------------------------------------------------- /lab/opensky-loader/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "opensky-loader", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "description": "Loading OpenSky data into Elasticsearch", 6 | "repository": { 7 | "type": "git", 8 | "url": "git+https://github.com/jsanz/elastic-workshop.git" 9 | }, 10 | "keywords": [ 11 | "elastic", 12 | "kibana", 13 | "geo", 14 | "maps" 15 | ], 16 | "author": "Jorge Sanz ", 17 | "license": "ISC", 18 | "dependencies": { 19 | "@elastic/elasticsearch": "^8.2.0-patch.1", 20 | "axios": "^0.27.2" 21 | }, 22 | "scripts": { 23 | "start": "node ./index.js" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /lab/opensky-viewer/.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /lab/opensky-viewer/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /lab/opensky-viewer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:current-alpine 2 | 3 | EXPOSE 3000 4 | 5 | WORKDIR /usr/src/app 6 | 7 | COPY . . 8 | 9 | RUN npm install 10 | 11 | CMD npm start 12 | -------------------------------------------------------------------------------- /lab/opensky-viewer/config.js: -------------------------------------------------------------------------------- 1 | ../elastic-config.js -------------------------------------------------------------------------------- /lab/opensky-viewer/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 44 | 45 | 46 |
47 | Add an Elasticsearch layer 48 |
49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 |
57 |
58 |
59 | 60 |
61 | 62 | 63 | 64 | 65 | 66 |

Features:

67 | 68 |
69 | 70 |
71 |
72 | 240 | 241 | -------------------------------------------------------------------------------- /lab/opensky-viewer/index.js: -------------------------------------------------------------------------------- 1 | const http = require('http'); 2 | const url = require('url'); 3 | const querystring = require('querystring'); 4 | const { Client } = require('@elastic/elasticsearch'); 5 | const fs = require('fs'); 6 | const config = require('./config'); 7 | 8 | const CONFIG = config.es_config || { node: process.env.ES_URL || 'http://elastic:changeme@localhost:9200/' } 9 | const client = new Client(CONFIG); 10 | 11 | 12 | const port = process.env.PORT || 8080; 13 | const server = http.createServer(async function (request, response) { 14 | 15 | // Set CORS headers 16 | response.setHeader('Access-Control-Allow-Origin', '*'); 17 | response.setHeader('Access-Control-Request-Method', '*'); 18 | response.setHeader('Access-Control-Allow-Methods', 'OPTIONS, GET'); 19 | response.setHeader('Access-Control-Allow-Headers', '*'); 20 | 21 | if (request.url.startsWith('/tile')) { 22 | const urlParse = url.parse(request.url); 23 | const params = querystring.decode(urlParse.query); 24 | 25 | console.log(`Tile request: ${JSON.stringify(params)}`); 26 | 27 | // Precision level for aggregation cells. Accepts 0-8. Larger numbers result in smaller aggregation cells. If 0, results don’t include the aggs layer. 28 | let gridPrecision = 0; 29 | if (params.renderMethod === 'grid') { 30 | gridPrecision = 8; 31 | } else if (params.renderMethod === 'hex') { 32 | gridPrecision = 5; 33 | } 34 | 35 | const body = { 36 | exact_bounds: true, 37 | extent: 4096, 38 | grid_agg: params.renderMethod === 'grid' ? 'geotile' : 'geohex', 39 | grid_precision: gridPrecision, 40 | grid_type: 'grid', 41 | size: params.renderMethod === 'hits' ? 10000 : 0,// only populate the hits layer when necessary 42 | track_total_hits: false, 43 | query: params.searchQuery ? { //use Lucene query_string syntax 44 | "query_string": { 45 | "query": params.searchQuery, 46 | "analyze_wildcard": true 47 | } 48 | } : { 49 | "match_all": {} 50 | } 51 | } 52 | 53 | try { 54 | const tile = await client.searchMvt({ 55 | index: params.index, 56 | field: params.geometry, 57 | zoom: parseInt(params.z), 58 | x: parseInt(params.x), 59 | y: parseInt(params.y), 60 | ...body, 61 | }, { meta: true }); 62 | 63 | // set response header 64 | response.writeHead(tile.statusCode, { 65 | 'content-disposition': 'inline', 66 | 'content-length': 'content-length' in tile.headers ? tile.headers['content-length'] : `0`, 67 | 'Content-Type': 'content-type' in tile.headers ? tile.headers['content-type'] : 'application/x-protobuf', 68 | 'Cache-Control': `public, max-age=0`, 69 | 'Last-Modified': `${new Date().toUTCString()}`, 70 | }); 71 | 72 | // set response content 73 | response.write(tile.body); 74 | response.end(); 75 | } catch (e) { 76 | console.error(e); 77 | response.writeHead('statusCode' in e ? e.statusCode : 500); 78 | response.write(e?.meta?.body ? JSON.stringify(e?.meta?.body) : ''); 79 | response.end(); 80 | } 81 | } else if (request.url === '/'){ 82 | response.writeHead(200, { 'Content-Type': 'text/html' }); 83 | response.write(fs.readFileSync('./index.html')); 84 | response.end(''); 85 | } else { 86 | response.writeHead(404); 87 | response.write('Page does not exist') 88 | response.end(); 89 | } 90 | }); 91 | 92 | server.listen(port); 93 | console.log(`Tile server running on port ${port}`); 94 | -------------------------------------------------------------------------------- /lab/opensky-viewer/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mvt_sample", 3 | "version": "0.0.1", 4 | "dependencies": { 5 | "@elastic/elasticsearch": "^8.2.0", 6 | "request": "^2.88.2" 7 | }, 8 | "scripts": { 9 | "start": "node index.js" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/.eleventy.js: -------------------------------------------------------------------------------- 1 | const { EleventyHtmlBasePlugin } = require("@11ty/eleventy"); 2 | 3 | require('dotenv').config(); 4 | 5 | env = Object.keys(process.env). 6 | filter((k) => k.startsWith('ELASTIC')). 7 | reduce((cur, key) => { return Object.assign(cur, { [key]: process.env[key] })}, {}); 8 | 9 | module.exports = function(eleventyConfig) { 10 | eleventyConfig.addPlugin(EleventyHtmlBasePlugin); 11 | 12 | eleventyConfig.addGlobalData('env', env); 13 | 14 | eleventyConfig.addPassthroughCopy({ 15 | "./node_modules/maplibre-gl/dist/maplibre-gl-dev.js": "assets/js/maplibre-gl-dev.js", 16 | "./node_modules/maplibre-gl/dist/maplibre-gl-dev.js.map": "assets/js/maplibre-gl-dev.js.map", 17 | "./node_modules/maplibre-gl/dist/maplibre-gl.css": "assets/css/maplibre-gl.css", 18 | "./node_modules/pmtiles/dist/index.js": "assets/js/pmtiles.js", 19 | "./node_modules/@picocss/pico/css/pico.css": "assets/css/pico.css", 20 | "static/js/highlight.min.js": "assets/js/highlight.min.js", 21 | "node_modules/highlight.js/styles/nord.css": "assets/css/nord.css", 22 | "static/*": "assets/", 23 | "static/css/*": "assets/css/" 24 | }); 25 | 26 | // Sort with `Array.sort` 27 | eleventyConfig.addCollection("maps", function(collectionApi) { 28 | return collectionApi 29 | .getAll() 30 | .filter(function(item){ 31 | return "ordering" in item.data; 32 | }) 33 | .sort(function(a, b) { 34 | return a.data.ordering - b.data.ordering; 35 | }); 36 | }); 37 | 38 | return { 39 | pathPrefix: "/jsanz-bucket/vector-tile-viewer/" 40 | } 41 | }; 42 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/.env: -------------------------------------------------------------------------------- 1 | ELASTIC_HOST="https://jorge-sanz.es.eastus2.azure.elastic-cloud.com" 2 | ELASTIC_APIKEY="aWM1c1JZd0I5STRESWxLSzdSN1o6bmoxVmxtV01SbXV5eE0tZXN6a1lxdw==" -------------------------------------------------------------------------------- /lab/vector-tile-viewer/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | 3 | dist 4 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: title.njk 3 | title: Elasticsearch and Webmapping 4 | description: > 5 | Two hours workshop on how different ways 6 | to render Elasticsearch data on a web map application using 7 | Maplibre. 8 | permalink: / 9 | --- 10 | 11 | ## About 12 | 13 | > This is part of the larger [Elastic Workshop](https://github.com/jsanz/elastic-workshop/) repository. 14 | 15 | In this two hours session, first, there are details on how to set up an Elasticsearch cluster to accept search and aggregation requests using Vector Tiles as the output format. 16 | 17 | With a cluster ready, we can explore two main types of data rendering: individual documents and grid aggregations using different spatial indices. 18 | 19 | ## Slides 20 | 21 | 22 | 23 | ## Data preparation 24 | 25 | > Full details on how to install these datasets and others can be found [here](https://gist.github.com/jsanz/235570f46634269ee354c831f87caf65) 26 | 27 | On an Elasticsearch 8.x cluster add this setting to the `elasticsearch.yml` file to allow restoring snapshots from a [Read-only URL repository](https://www.elastic.co/guide/en/elasticsearch/reference/master/snapshots-read-only-repository.html). 28 | 29 | ```yaml 30 | repositories.url.allowed_urls: 31 | - "https://storage.googleapis.com/jsanz-bucket/*" 32 | ``` 33 | 34 | To accept requests for vector tiles from a browser we also need to enable CORS in Elasticsearch so, again in the `elasticsearch.yml`: 35 | 36 | ```yaml 37 | http.cors: 38 | enabled : true 39 | allow-origin: "*" 40 | allow-methods: OPTIONS, HEAD, GET, POST 41 | allow-headers: "X-Requested-With, Content-Type, Content-Length, Authorization, Accept, User-Agent, X-Elastic-Client-Meta, Cache-Control" 42 | ``` 43 | 44 | > ⚠ Be sure to read the [networking documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html) carefully if you are working with a production cluster to understand the implications of allowing CORS requests 45 | 46 | 47 | Restart the cluster to activate this URL and then you can create a couple of snapshots repositories and restore some indices with [NYC 311](https://data.cityofnewyork.us/Social-Services/311-Service-Requests-from-2010-to-Present/erm2-nwe9) data and the [Geonames database](http://www.geonames.org/). 48 | 49 |
50 | Loading datasets 🔽 51 | 52 | ```text 53 | # ==== NYC 311 ==== 54 | # Add the NYC 311 snapshots repository 55 | PUT /_snapshot/nyc311 56 | { 57 | "type": "url", 58 | "settings": { 59 | "url": "https://storage.googleapis.com/jsanz-bucket/nyc311_repo/" 60 | } 61 | } 62 | 63 | # Check two snapshots are available 64 | GET _snapshot/nyc311/* 65 | 66 | # Restore 311 data (async) 67 | POST /_snapshot/nyc311/snapshot_1/_restore 68 | 69 | # Restore NYC boroughs data (async) 70 | POST /_snapshot/nyc311/snapshot_2/_restore 71 | 72 | 73 | # ==== Geonames ==== 74 | 75 | # Add the Geonames snapshots repository 76 | PUT /_snapshot/geonames 77 | { 78 | "type": "url", 79 | "settings": { 80 | "url": "https://storage.googleapis.com/jsanz-bucket/v8/geospatial_demos/" 81 | } 82 | } 83 | 84 | # Check the geonames snapshot is available available 85 | GET _snapshot/geonames/geonames 86 | 87 | # Restore Geonames data (async) 88 | POST /_snapshot/geonames/geonames/_restore 89 | 90 | 91 | # ==== OSM Andorra ==== 92 | 93 | # Add the OSM snapshots repository 94 | PUT /_snapshot/osm 95 | { 96 | "type": "url", 97 | "settings": { 98 | "url": "https://storage.googleapis.com/jsanz-bucket/v8/osm/" 99 | } 100 | } 101 | 102 | # Check the osm_andorra snapshot is available available 103 | GET _snapshot/osm/osm_andorra 104 | 105 | # Restore osm_andorra data (async) 106 | POST /_snapshot/osm/osm_andorra/_restore 107 | 108 | # Expose this index with two filtered aliases 109 | POST _aliases 110 | { 111 | "actions": [ 112 | { 113 | "add": { 114 | "index": "osm_andorra", 115 | "alias": "osm_highways_andorra", 116 | "filter": { 117 | "bool": { 118 | "filter": [ 119 | { 120 | "bool": { 121 | "minimum_should_match": 1, 122 | "should": [ 123 | { 124 | "exists": { 125 | "field": "highway" 126 | } 127 | } 128 | ] 129 | } 130 | }, 131 | { 132 | "bool": { 133 | "minimum_should_match": 1, 134 | "should": [ 135 | { 136 | "term": { 137 | "osm_type": { 138 | "value": "way" 139 | } 140 | } 141 | } 142 | ] 143 | } 144 | } 145 | ] 146 | } 147 | } 148 | } 149 | }, 150 | { 151 | "add": { 152 | "index": "osm_andorra", 153 | "alias": "osm_buildings_andorra", 154 | "filter": { 155 | "bool": { 156 | "filter": [ 157 | { 158 | "bool": { 159 | "minimum_should_match": 1, 160 | "should": [ 161 | { 162 | "exists": { 163 | "field": "building" 164 | } 165 | } 166 | ] 167 | } 168 | }, 169 | { 170 | "bool": { 171 | "minimum_should_match": 1, 172 | "should": [ 173 | { 174 | "term": { 175 | "osm_type": { 176 | "value": "area" 177 | } 178 | } 179 | } 180 | ] 181 | } 182 | } 183 | ] 184 | } 185 | } 186 | } 187 | } 188 | ] 189 | } 190 | ``` 191 |
192 | 193 | You need to wait for the data to be downloaded and restored. Check the indices in your cluster with: 194 | 195 | ```text 196 | GET _cat/indices?v&h=index,docs.count&s=index 197 | ``` 198 | 199 | Create the corresponding data views for Kibana from the Stack Management interface or with the following Console commands for the [Create Data View API](https://www.elastic.co/guide/en/kibana/master/data-views-api-create.html) 200 | 201 |
202 | Creating Kibana Data Views 🔽 203 | 204 | ```text 205 | POST kbn://api/data_views/data_view 206 | { 207 | "data_view": { 208 | "title": "311", 209 | "name": "NYC 311 calls", 210 | "timeFieldName": "Created Date" 211 | } 212 | } 213 | 214 | POST kbn://api/data_views/data_view 215 | { 216 | "data_view": { 217 | "title": "nyc_boroughs", 218 | "name": "NYC Boroughs" 219 | } 220 | } 221 | 222 | POST kbn://api/data_views/data_view 223 | { 224 | "data_view": { 225 | "title": "NYC", 226 | "name": "NYC 311 calls", 227 | "timeFieldName": "Created Date" 228 | } 229 | } 230 | 231 | 232 | POST kbn://api/data_views/data_view 233 | { 234 | "data_view": { 235 | "title": "osm_andorra", 236 | "name": "OpenStreetMap Andorra", 237 | "timeFieldName": "timestamp" 238 | } 239 | } 240 | ``` 241 |
242 | 243 | ## Setting up Elasticsearch API key 244 | 245 | Create a `workshop` API key that can view the indices we just created and that will expire in five days. 246 | 247 | 248 |
249 | Create an API key 🔽 250 | 251 | ```text 252 | POST /_security/api_key 253 | { 254 | "name": "workshop-api-key", 255 | "expiration": "5d", 256 | "role_descriptors": { 257 | "workshop": { 258 | "index": [ 259 | { 260 | "names": [ 261 | "geonames", 262 | "311", 263 | "nyc_boroughs", 264 | "osm_*" 265 | ], 266 | "privileges": [ 267 | "read", 268 | "view_index_metadata" 269 | ], 270 | "field_security": { 271 | "grant": [ 272 | "*" 273 | ] 274 | } 275 | } 276 | ] 277 | } 278 | } 279 | } 280 | ``` 281 |
282 | 283 | Write down the result as you'll need those fields on your requests. 284 | 285 | ```json 286 | { 287 | "id": "YqAxoIgBclR1XK5t5Ixm", 288 | "name": "workshop-api-key", 289 | "expiration": 1688906804330, 290 | "api_key": "your-api-key-here", 291 | "encoded": "your-encoded-name-and-api-key-here" 292 | } 293 | ``` 294 | 295 | 296 | You can test your API key from curl: 297 | 298 | ```bash 299 | $ ELASTIC_HOST="https://your-cluster-url" 300 | $ ELASTIC_APIKEY="your-encoded-name-and-api-key-here" 301 | $ curl -H "Authorization: ApiKey ${ELASTIC_APIKEY}" \ 302 | "${ELASTIC_HOST}/geonames/_count?pretty=true" 303 | ``` 304 | 305 | This should return: 306 | 307 | ```json 308 | { 309 | "count" : 11968314, 310 | "_shards" : { 311 | "total" : 1, 312 | "successful" : 1, 313 | "skipped" : 0, 314 | "failed" : 0 315 | } 316 | } 317 | ``` 318 | 319 | ## Starting the viewer 320 | 321 | If you are familiar with the NodeJS stack then you can download the dependencies (`yarn install` or `npm install`) and start a development server (`yarn start` or `npm start`) so you can edit the code in the source files and the page will reload automatically. 322 | 323 | * `_includes/map.njk` and ` _includes/map-docs.njk` contain the common code to initialize the different map pages 324 | * `pages/X.html` contains the HTML markup and the JavaScript code to run that page. 325 | 326 | The target `yarn build` or `npm build` will generate the output files in the `dist` folder that can be uploaded to any static webserver (Github Pages, Vercel, Netlify, and so on). 327 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/_data/meta.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | siteName: "Web mapping with Elasticsearch", 3 | siteDescription: "A workshop on how to use Elasticsearch in web mapping projects", 4 | authorName: "Jorge Sanz ", 5 | } -------------------------------------------------------------------------------- /lab/vector-tile-viewer/_includes/base.njk: -------------------------------------------------------------------------------- 1 | --- 2 | title: Elasticsearch Webmapping 3 | --- 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | {{ title or meta.siteName | trim }} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | {% for style in css -%} 20 | 21 | {% endfor -%} 22 | 23 | {% for script in js -%} 24 | 25 | {% endfor -%} 26 | 27 | 28 | {{ content | safe }} 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/_includes/map-docs.njk: -------------------------------------------------------------------------------- 1 | --- 2 | layout: map.njk 3 | --- 4 | 5 | 6 | 62 | 63 | {{ content | safe }} 64 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/_includes/map.njk: -------------------------------------------------------------------------------- 1 | --- 2 | layout: base.njk 3 | css: 4 | - maplibre-gl.css 5 | - map-and-info.css 6 | js: 7 | - maplibre-gl-dev.js 8 | - pmtiles.js 9 | --- 10 | 11 | 23 | 24 | {{ content | safe }} 25 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/_includes/title.njk: -------------------------------------------------------------------------------- 1 | --- 2 | layout: base.njk 3 | css: 4 | - pico.css 5 | - nord.css 6 | js: 7 | - highlight.min.js 8 | --- 9 | 10 | 11 | 12 |
13 | 14 |
15 |

{{ title }}

16 |

{{ description }}

17 |
18 | 19 |
20 | 21 |
22 | 23 |
24 |

🗺 Maps! 🗺

25 | 26 |
27 | {%- for post in collections.maps %} 28 |
{{ post.data.title }}
29 |
{{post.data.description}}
30 | {%- endfor %} 31 |
32 |
33 | 34 |
35 | {{content | safe }} 36 |
37 | 38 | 39 |
40 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "elasticsearch-webmapping", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "license": "MIT", 6 | "scripts": { 7 | "clean": "rimraf dist", 8 | "start": "eleventy --config=.eleventy.js --output=dist --serve", 9 | "build": "eleventy --config=.eleventy.js --output=dist", 10 | "prepreview": "yarn clean && yarn build", 11 | "preview": "http-server dist", 12 | "publish": "gsutil -m rsync -d -r dist gs://jsanz-bucket/vector-tile-viewer/" 13 | }, 14 | "devDependencies": { 15 | "@11ty/eleventy": "^2.0.1", 16 | "@picocss/pico": "^1.5.10", 17 | "dotenv": "^16.1.4", 18 | "highlight.js": "^11.8.0", 19 | "http-server": "^14.1.1", 20 | "maplibre-gl": "^3.0.0", 21 | "rimraf": "^5.0.1" 22 | }, 23 | "dependencies": { 24 | "pmtiles": "^2.7.2" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/pages/1-basemap/1-1-basemap.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: map.njk 3 | title: "[1-1] Just the basemap" 4 | description: > 5 | Just rendereing the basemap from PMTiles vector tiles and 6 | Elastic OSM Bright desaturated schema 7 | permalink: 1-1-basemap.html 8 | ordering: 10 9 | --- 10 | 11 |
12 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/pages/2-documents/2-1-documents.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: map-docs.njk 3 | title: "[2-1] Documents: render everything between two dates" 4 | description: > 5 | Learn how to render Elasticsearch data with a simple query 6 | into a Maplibre layer. 7 | permalink: 2-1-documents.html 8 | ordering: 21 9 | --- 10 | 11 |
12 |
13 |

311 noise claims from 2019 Q1

14 |

Count:

15 |
16 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/pages/2-documents/2-2-documents.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: map-docs.njk 3 | title: "[2-2] Documents: terms query and thematic mapping" 4 | description: > 5 | Extend your basic layer with a color palette depending 6 | on the value of a document field. 7 | permalink: 2-2-documents.html 8 | ordering: 22 9 | --- 10 | 11 |
12 |
13 |

311 noise claims from 2019 Q1

14 |

Count:

15 |
16 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/pages/2-documents/2-3-documents.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: map-docs.njk 3 | title: "[2-3] Documents: add a popup" 4 | description: > 5 | Include more fields on the vector tiles retrieved to allow showing 6 | a pop up with details of each 311 call. 7 | permalink: 2-3-documents.html 8 | ordering: 23 9 | --- 10 | 11 |
12 |
13 |

311 noise claims from 2019 Q1

14 |

Click on a point to get details

15 |

Count:

16 |
17 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/pages/2-documents/2-4-documents.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: map-docs.njk 3 | title: "[2-4] Documents: search" 4 | description: > 5 | Filter data on the map by including a text string from an input form 6 | that will search for all fields in our index. 7 | permalink: 2-4-documents.html 8 | ordering: 24 9 | --- 10 | 11 |
12 |
13 |

311 noise claims from 2019 Q1

14 |

Click on a point to get details

15 |

Count:

16 |
17 | 21 | 22 |
23 |
24 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/pages/2-documents/2-5-documents.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: map-docs.njk 3 | title: "[2-5] Documents: geometry types" 4 | description: > 5 | Render some OpenStreetMap data from different geometry types 6 | permalink: 2-5-documents.html 7 | ordering: 25 8 | --- 9 | 10 |
11 |
12 |

Andoraa roads and buildings

13 |

OpenStreetMap Data

14 |
15 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/pages/3-hexagons/3-1-hexagons.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: map-docs.njk 3 | title: "[3-1] Hexagons" 4 | description: > 5 | Aggregate 311 calls using the H3 hexagon spatial index. 6 | permalink: 3-1-hexagons.html 7 | ordering: 31 8 | --- 9 | 10 |
11 |
12 |

311 noise claims
between 2015 and 2020

13 |

Hover an hex to get the count

14 |

Count:

15 |
16 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/pages/3-hexagons/3-2-hexagons.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: map-docs.njk 3 | title: "[3-2] Hexagons: adaptative legend" 4 | description: > 5 | Change the legend of the hexagons aggregated Maplibre layer depending on the zoom level. 6 | permalink: 3-2-hexagons.html 7 | ordering: 32 8 | --- 9 | 10 |
11 |
12 |

311 noise claims
between 2015 and 2020

13 |

Hover an hex to get the count

14 |

Count:

15 |
16 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/pages/3-hexagons/3-3-hexagons.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: map-docs.njk 3 | title: "[3-3] Hexagons: additional metric" 4 | description: > 5 | Include a new custom metric in the data aggregation. 6 | permalink: 3-3-hexagons.html 7 | ordering: 33 8 | --- 9 | 10 |
11 |
12 |

311 calls between 2015 and 2020
by complaint types

13 |

Hover an hex to get the types count

14 |

Count:

15 |
16 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/pages/4-aggs/4-1-geotile.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: map-docs.njk 3 | title: "[4-1] Geotile: square tiles" 4 | description: > 5 | Aggregate using square tiles and extend the Elasticsearch 6 | query to exclude some outlier data. 7 | permalink: 4-1-geotile.html 8 | ordering: 41 9 | --- 10 | 11 |
12 |
13 |

311 noise claims
between 2015 and 2020

14 |

Hover on a square to get the count

15 |

Count:

16 |
17 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/pages/4-aggs/4-2-heatmap.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: map-docs.njk 3 | title: "[4-2] Geotile: heatmap" 4 | description: > 5 | Use a heatmap styling to render geotile aggregated data. 6 | permalink: 4-2-heatmap.html 7 | ordering: 42 8 | --- 9 | 10 |
11 |
12 |

311 noise claims
between 2015 and 2020

13 |
14 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/static/banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsanz/elastic-workshop/248063c5b57a21b157583e2df18724a4346b2037/lab/vector-tile-viewer/static/banner.png -------------------------------------------------------------------------------- /lab/vector-tile-viewer/static/css/map-and-info.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | padding: 0; 4 | font-family: Helvetica Neue, Arial, Helvetica, sans-serif; 5 | } 6 | #map { 7 | position: absolute; 8 | top: 0; 9 | bottom: 0; 10 | width: 100%; 11 | } 12 | #info { 13 | display: table; 14 | position: relative; 15 | margin: 10px auto 0 10px; 16 | word-wrap: anywhere; 17 | padding: 10px; 18 | border: none; 19 | border-radius: 3px; 20 | font-size: 12px; 21 | text-align: center; 22 | color: #222; 23 | background: #fff; 24 | border: 2px solid darkslateblue; 25 | } 26 | #info h3, 27 | #info p { 28 | margin: 0 0 5px 0; 29 | } 30 | -------------------------------------------------------------------------------- /lab/vector-tile-viewer/static/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsanz/elastic-workshop/248063c5b57a21b157583e2df18724a4346b2037/lab/vector-tile-viewer/static/favicon.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "elastic-workshop", 3 | "version": "1.0.0", 4 | "description": "Workshop about ElasticSearch and Kibana geospatial capabilities", 5 | "repository": { 6 | "type": "git", 7 | "url": "git+https://github.com/jsanz/elastic-workshop.git" 8 | }, 9 | "keywords": ["elastic", "kibana", "geo", "maps"], 10 | "author": "Jorge Sanz ", 11 | "license": "ISC", 12 | "bugs": { 13 | "url": "https://github.com/jsanz/elastic-workshop/issues" 14 | }, 15 | "homepage": "https://github.com/jsanz/elastic-workshop" 16 | } 17 | --------------------------------------------------------------------------------