├── .gitignore ├── Dockerfile ├── FastImporter.postman_collection.json ├── LICENSE ├── README.md ├── config.py ├── data ├── cities.json ├── state_data.csv ├── states.geojson ├── states.json └── us-state-capitals.csv ├── db.py ├── main.py ├── models.py ├── requirements.txt ├── routers └── imports.py └── utilities.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | .vscode -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9 2 | 3 | WORKDIR /app 4 | 5 | COPY ./requirements.txt /app/requirements.txt 6 | 7 | RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt 8 | 9 | COPY . / 10 | 11 | CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] -------------------------------------------------------------------------------- /FastImporter.postman_collection.json: -------------------------------------------------------------------------------- 1 | { 2 | "info": { 3 | "_postman_id": "f72e5cc8-d27e-464e-af55-fa18ae7d67c9", 4 | "name": "FastImporter", 5 | "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" 6 | }, 7 | "item": [ 8 | { 9 | "name": "Arcgis Service", 10 | "request": { 11 | "method": "POST", 12 | "header": [], 13 | "body": { 14 | "mode": "raw", 15 | "raw": "{\n \"url\": \"https://services5.arcgis.com/bPacKTm9cauMXVfn/ArcGIS/rest/services/TN_State_Parks_Points/FeatureServer/0\",\n \"database\": \"data\"\n}", 16 | "options": { 17 | "raw": { 18 | "language": "json" 19 | } 20 | } 21 | }, 22 | "url": { 23 | "raw": "http://127.0.0.1:8000/api/v1/import/arcgis_service/", 24 | "protocol": "http", 25 | "host": [ 26 | "127", 27 | "0", 28 | "0", 29 | "1" 30 | ], 31 | "port": "8000", 32 | "path": [ 33 | "api", 34 | "v1", 35 | "import", 36 | "arcgis_service", 37 | "" 38 | ] 39 | } 40 | }, 41 | "response": [] 42 | }, 43 | { 44 | "name": "Import Geographic Data From Geographic File", 45 | "request": { 46 | "method": "POST", 47 | "header": [], 48 | "body": { 49 | "mode": "formdata", 50 | "formdata": [ 51 | { 52 | "key": "database", 53 | "value": "data", 54 | "type": "default" 55 | }, 56 | { 57 | "key": "files", 58 | "type": "file", 59 | "src": "/home/michael/Documents/geo_data/states.geojson" 60 | } 61 | ], 62 | "options": { 63 | "raw": { 64 | "language": "json" 65 | } 66 | } 67 | }, 68 | "url": { 69 | "raw": "http://127.0.0.1:8000/api/v1/import/geographic_data_from_geographic_file/", 70 | "protocol": "http", 71 | "host": [ 72 | "127", 73 | "0", 74 | "0", 75 | "1" 76 | ], 77 | "port": "8000", 78 | "path": [ 79 | "api", 80 | "v1", 81 | "import", 82 | "geographic_data_from_geographic_file", 83 | "" 84 | ] 85 | } 86 | }, 87 | "response": [] 88 | }, 89 | { 90 | "name": "Import Geographic Data From CSV", 91 | "request": { 92 | "method": "POST", 93 | "header": [], 94 | "body": { 95 | "mode": "formdata", 96 | "formdata": [ 97 | { 98 | "key": "database", 99 | "value": "data", 100 | "type": "default" 101 | }, 102 | { 103 | "key": "files", 104 | "type": "file", 105 | "src": "/home/michael/Documents/apps/python/FastImporter/data/state_data.csv" 106 | }, 107 | { 108 | "key": "map", 109 | "value": "states", 110 | "type": "default" 111 | }, 112 | { 113 | "key": "map_column", 114 | "value": "state_abbr", 115 | "type": "default" 116 | }, 117 | { 118 | "key": "map_columns", 119 | "value": "[\"state_abbr\"]", 120 | "type": "default" 121 | }, 122 | { 123 | "key": "table_column", 124 | "value": "state_abbr", 125 | "type": "default" 126 | }, 127 | { 128 | "key": "table_columns", 129 | "value": "[\"state_abbr\",\"Number of Rest Stops\"]", 130 | "type": "default" 131 | } 132 | ], 133 | "options": { 134 | "raw": { 135 | "language": "json" 136 | } 137 | } 138 | }, 139 | "url": { 140 | "raw": "http://127.0.0.1:8000/api/v1/import/geographic_data_from_csv/", 141 | "protocol": "http", 142 | "host": [ 143 | "127", 144 | "0", 145 | "0", 146 | "1" 147 | ], 148 | "port": "8000", 149 | "path": [ 150 | "api", 151 | "v1", 152 | "import", 153 | "geographic_data_from_csv", 154 | "" 155 | ] 156 | } 157 | }, 158 | "response": [] 159 | }, 160 | { 161 | "name": "Import Point Data From CSV", 162 | "request": { 163 | "method": "POST", 164 | "header": [], 165 | "body": { 166 | "mode": "formdata", 167 | "formdata": [ 168 | { 169 | "key": "database", 170 | "value": "data", 171 | "type": "default" 172 | }, 173 | { 174 | "key": "files", 175 | "type": "file", 176 | "src": "/home/michael/Documents/apps/python/FastImporter/data/us-state-capitals.csv" 177 | }, 178 | { 179 | "key": "longitude", 180 | "value": "longitude", 181 | "type": "default" 182 | }, 183 | { 184 | "key": "latitude", 185 | "value": "latitude", 186 | "type": "default" 187 | }, 188 | { 189 | "key": "table_columns", 190 | "value": "[\"name\",\"description\",\"latitude\",\"longitude\"]", 191 | "type": "default" 192 | } 193 | ], 194 | "options": { 195 | "raw": { 196 | "language": "json" 197 | } 198 | } 199 | }, 200 | "url": { 201 | "raw": "http://127.0.0.1:8000/api/v1/import/point_data_from_csv/", 202 | "protocol": "http", 203 | "host": [ 204 | "127", 205 | "0", 206 | "0", 207 | "1" 208 | ], 209 | "port": "8000", 210 | "path": [ 211 | "api", 212 | "v1", 213 | "import", 214 | "point_data_from_csv", 215 | "" 216 | ] 217 | } 218 | }, 219 | "response": [] 220 | }, 221 | { 222 | "name": "Import Geographic Data From JSON File", 223 | "request": { 224 | "method": "POST", 225 | "header": [], 226 | "body": { 227 | "mode": "formdata", 228 | "formdata": [ 229 | { 230 | "key": "database", 231 | "value": "data", 232 | "type": "default" 233 | }, 234 | { 235 | "key": "files", 236 | "type": "file", 237 | "src": "/home/michael/Documents/apps/python/FastImporter/data/states.json" 238 | }, 239 | { 240 | "key": "map_column", 241 | "value": "state_abbr", 242 | "type": "default" 243 | }, 244 | { 245 | "key": "table_column", 246 | "value": "code", 247 | "type": "default" 248 | }, 249 | { 250 | "key": "table_columns", 251 | "value": "[\"state\",\"slug\",\"code\",\"nickname\"]", 252 | "type": "default" 253 | }, 254 | { 255 | "key": "map", 256 | "value": "states", 257 | "type": "default" 258 | }, 259 | { 260 | "key": "map_columns", 261 | "value": "[\"state_abbr\"]", 262 | "type": "default" 263 | } 264 | ], 265 | "options": { 266 | "raw": { 267 | "language": "json" 268 | } 269 | } 270 | }, 271 | "url": { 272 | "raw": "http://127.0.0.1:8000/api/v1/import/geographic_data_from_json_file/", 273 | "protocol": "http", 274 | "host": [ 275 | "127", 276 | "0", 277 | "0", 278 | "1" 279 | ], 280 | "port": "8000", 281 | "path": [ 282 | "api", 283 | "v1", 284 | "import", 285 | "geographic_data_from_json_file", 286 | "" 287 | ] 288 | } 289 | }, 290 | "response": [] 291 | }, 292 | { 293 | "name": "Import Point Data From JSON File", 294 | "request": { 295 | "method": "POST", 296 | "header": [], 297 | "body": { 298 | "mode": "formdata", 299 | "formdata": [ 300 | { 301 | "key": "database", 302 | "value": "data", 303 | "type": "default" 304 | }, 305 | { 306 | "key": "files", 307 | "type": "file", 308 | "src": "/home/michael/Documents/apps/python/FastImporter/data/cities.json" 309 | }, 310 | { 311 | "key": "longitude", 312 | "value": "longitude", 313 | "type": "default" 314 | }, 315 | { 316 | "key": "latitude", 317 | "value": "latitude", 318 | "type": "default" 319 | }, 320 | { 321 | "key": "table_columns", 322 | "value": "[\"id\",\"name\",\"latitude\",\"longitude\",\"state_id\",\"state_code\",\"state_name\",\"country_id\",\"country_code\",\"country_name\",\"wikiDataId\"]", 323 | "type": "default" 324 | } 325 | ], 326 | "options": { 327 | "raw": { 328 | "language": "json" 329 | } 330 | } 331 | }, 332 | "url": { 333 | "raw": "http://127.0.0.1:8000/api/v1/import/point_data_from_json_file/", 334 | "protocol": "http", 335 | "host": [ 336 | "127", 337 | "0", 338 | "0", 339 | "1" 340 | ], 341 | "port": "8000", 342 | "path": [ 343 | "api", 344 | "v1", 345 | "import", 346 | "point_data_from_json_file", 347 | "" 348 | ] 349 | } 350 | }, 351 | "response": [] 352 | }, 353 | { 354 | "name": "Import Geographic Data From JSON URL", 355 | "request": { 356 | "method": "POST", 357 | "header": [], 358 | "body": { 359 | "mode": "raw", 360 | "raw": "{\n \"database\": \"data\",\n \"map_column\": \"state_abbr\",\n \"table_column\": \"code\",\n \"table_columns\": [\n \"state\",\n \"slug\",\n \"code\",\n \"nickname\"\n ],\n \"map\": \"states\",\n \"map_columns\": [\n \"state_abbr\"\n ],\n \"url\": \"https://raw.githubusercontent.com/CivilServiceUSA/us-states/master/data/states.json\"\n}", 361 | "options": { 362 | "raw": { 363 | "language": "json" 364 | } 365 | } 366 | }, 367 | "url": { 368 | "raw": "http://127.0.0.1:8000/api/v1/import/geographic_data_from_json_url/", 369 | "protocol": "http", 370 | "host": [ 371 | "127", 372 | "0", 373 | "0", 374 | "1" 375 | ], 376 | "port": "8000", 377 | "path": [ 378 | "api", 379 | "v1", 380 | "import", 381 | "geographic_data_from_json_url", 382 | "" 383 | ] 384 | } 385 | }, 386 | "response": [] 387 | }, 388 | { 389 | "name": "Import Point Data From JSON URL", 390 | "request": { 391 | "method": "POST", 392 | "header": [], 393 | "body": { 394 | "mode": "raw", 395 | "raw": "{\n \"database\": \"data\",\n \"longitude\": \"longitude\",\n \"latitude\": \"latitude\",\n \"table_columns\": [\n \"id\",\n \"name\",\n \"latitude\",\n \"longitude\",\n \"state_id\",\n \"state_code\",\n \"state_name\",\n \"country_id\",\n \"country_code\",\n \"country_name\",\n \"wikiDataId\"\n ],\n \"url\": \"https://raw.githubusercontent.com/dr5hn/countries-states-cities-database/master/states.json\"\n}", 396 | "options": { 397 | "raw": { 398 | "language": "json" 399 | } 400 | } 401 | }, 402 | "url": { 403 | "raw": "http://127.0.0.1:8000/api/v1/import/point_data_from_json_url/", 404 | "protocol": "http", 405 | "host": [ 406 | "127", 407 | "0", 408 | "0", 409 | "1" 410 | ], 411 | "port": "8000", 412 | "path": [ 413 | "api", 414 | "v1", 415 | "import", 416 | "point_data_from_json_url", 417 | "" 418 | ] 419 | } 420 | }, 421 | "response": [] 422 | }, 423 | { 424 | "name": "Import Geojson From URL", 425 | "request": { 426 | "method": "POST", 427 | "header": [], 428 | "body": { 429 | "mode": "raw", 430 | "raw": "{\n \"database\": \"data\",\n \"url\": \"https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/significant_month.geojson\"\n}", 431 | "options": { 432 | "raw": { 433 | "language": "json" 434 | } 435 | } 436 | }, 437 | "url": { 438 | "raw": "http://127.0.0.1:8000/api/v1/import/geojson_from_url/", 439 | "protocol": "http", 440 | "host": [ 441 | "127", 442 | "0", 443 | "0", 444 | "1" 445 | ], 446 | "port": "8000", 447 | "path": [ 448 | "api", 449 | "v1", 450 | "import", 451 | "geojson_from_url", 452 | "" 453 | ] 454 | } 455 | }, 456 | "response": [] 457 | } 458 | ] 459 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Michael Keller 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FastImporter 2 | 3 | FastImporter is a [PostGIS](https://github.com/postgis/postgis) geospatial api to enable loading data into a PostGIS database from a multitude of locations such as files, and url endpoints. FastImporter is written in [Python](https://www.python.org/) using the [FastAPI](https://fastapi.tiangolo.com/) web framework. 4 | 5 | --- 6 | 7 | **Source Code**: https://github.com/mkeller3/FastImporter 8 | 9 | --- 10 | 11 | ## Requirements 12 | 13 | FastImporter requires PostGIS >= 2.4.0 and ogr2ogr. 14 | 15 | ## Configuration 16 | 17 | In order for the api to work you will need to edit the `config.py` file with your database connections. 18 | 19 | Example 20 | ```python 21 | DATABASES = { 22 | "data": { 23 | "host": "localhost", 24 | "database": "data", 25 | "username": "postgres", 26 | "password": "postgres", 27 | "port": 5432, 28 | } 29 | } 30 | ``` 31 | 32 | ## Usage 33 | 34 | ### Running Locally 35 | 36 | To run the app locally `uvicorn main:app --reload` 37 | 38 | ### Production 39 | Build Dockerfile into a docker image to deploy to the cloud. 40 | 41 | ## API 42 | 43 | | Method | URL | Description | 44 | | ------ | --- | ----------- | 45 | | `GET` | `/api/v1/import/status/{process_id}` | [Import Status](#Import-Status) | 46 | | `POST` | `/api/v1/import/arcgis_service` | [ArcGIS Service](#ArcGIS-Service) | 47 | | `POST` | `/api/v1/import/geographic_data_from_geographic_file` | [Geographic Data From Geographic File](#Geographic-Data-From-Geographic-File) | 48 | | `POST` | `/api/v1/import/geographic_data_from_csv` | [Geographic Data From CSV](#Geographic-Data-From-CSV) | 49 | | `POST` | `/api/v1/import/point_data_from_csv` | [Point Data From CSV](#Point-Data-From-CSV) | 50 | | `POST` | `/api/v1/import/geographic_data_from_json_file` | [Geographic Data From Json File](#Geographic-Data-From-Json-File) | 51 | | `POST` | `/api/v1/import/point_data_from_json_file` | [Point Data From Json File ](#Point-Data-From-Json-File) | 52 | | `POST` | `/api/v1/import/geographic_data_from_json_url` | [Geographic Data From Json URL](#Geographic-Data-From-Json-Url) | 53 | | `POST` | `/api/v1/import/point_data_from_json_url` | [Point Data From Json URL](#Point-Data-From-Json-Url) | 54 | | `POST` | `/api/v1/import/geojson_from_url` | [Geojson From URL](#Geojson-From-Url) | 55 | 56 | 57 | ## Endpoint Description's 58 | 59 | ## Import Status 60 | Any time an import is submitted it given a process_id to have the import run in the background using [FastAPI's Background Tasks](https://fastapi.tiangolo.com/tutorial/background-tasks/). To check the 61 | status of an import, you can call this endpoint with the process_id. 62 | 63 | ## Example Call 64 | ```shell 65 | /api/v1/import/status/472e29dc-91a8-41d3-b05f-cee34006e3f7 66 | ``` 67 | 68 | ## Example Output - Still Running 69 | ```json 70 | { 71 | "status": "PENDING" 72 | } 73 | ``` 74 | 75 | ## Example Output - Complete 76 | ```json 77 | { 78 | "status": "SUCCESS", 79 | "new_table_id": "shnxppipxrppsdkozuroilkubktfodibtqorhucjvxlcdrqyhh", 80 | "completion_time": "2022-07-06T19:33:17.950059", 81 | "run_time_in_seconds": 1.78599 82 | } 83 | ``` 84 | 85 | ## Example Output - Error 86 | ```json 87 | { 88 | "status": "FAILURE", 89 | "error": "ERROR HERE", 90 | "completion_time": "2022-07-08T13:39:47.961389", 91 | "run_time_in_seconds": 0.040892 92 | } 93 | ``` 94 | 95 | ## ArcGIS Service 96 | 97 | ### Description 98 | Import data from any `FeatureServer` or `MapServer` that allows for geojson as an output. 99 | 100 | Example: Download a point dataset of Tennesse State Parks. 101 | 102 | ### Example Input 103 | ```json 104 | { 105 | "url": "https://services5.arcgis.com/bPacKTm9cauMXVfn/ArcGIS/rest/services/TN_State_Parks_Points/FeatureServer/0", 106 | "database": "data" 107 | } 108 | ``` 109 | 110 | ### Example Output 111 | ```json 112 | { 113 | "process_id": "c8d7b8d8-3e82-4f93-b441-55a5f51c4171", 114 | "url": "http://127.0.0.1:8000/api/v1/import/status/c8d7b8d8-3e82-4f93-b441-55a5f51c4171" 115 | } 116 | ``` 117 | 118 | ## Geographic Data From Geographic File 119 | 120 | ### Description 121 | Import geographic data from a file/files. 122 | 123 | Example: Import geojson from [file](/data/states.geojson). 124 | 125 | ### Example Input 126 | ```json 127 | { 128 | "database": "data", 129 | "files": "FILES IN MULTI PART FORM" 130 | } 131 | ``` 132 | 133 | ### Example Output 134 | ```json 135 | { 136 | "process_id": "c8d7b8d8-3e82-4f93-b441-55a5f51c4171", 137 | "url": "http://127.0.0.1:8000/api/v1/import/status/c8d7b8d8-3e82-4f93-b441-55a5f51c4171" 138 | } 139 | ``` 140 | 141 | ## Geographic Data From CSV 142 | 143 | ### Description 144 | Import a csv [file](/data/state_data.csv) and join to a map already within the database based off a column. 145 | 146 | Example: Uploading a csv with two columns `state_abbr` and `Number of Rest Stops` 147 | and joining to the `states` map based off of the `state_abbr` column. 148 | 149 | ### Example Input 150 | ```json 151 | { 152 | "database": "data", 153 | "map": "states", 154 | "map_column": "state_abbr", 155 | "map_columns": ["state_abbr"], 156 | "table_column": "state_abbr", 157 | "table_columns": ["state_abbr","Number of Rest Stops"], 158 | "files": "FILES IN MULTI PART FORM" 159 | } 160 | ``` 161 | 162 | ### Example Output 163 | ```json 164 | { 165 | "process_id": "c8d7b8d8-3e82-4f93-b441-55a5f51c4171", 166 | "url": "http://127.0.0.1:8000/api/v1/import/status/c8d7b8d8-3e82-4f93-b441-55a5f51c4171" 167 | } 168 | ``` 169 | 170 | ## Point Data From CSV 171 | 172 | ### Description 173 | Import a csv file with latitude and longitude columns into database. 174 | 175 | Example: A csv [file](/data/us-states-capitals.csv) with latitude and longitude columns for US Capitals. 176 | 177 | ### Example Input 178 | ```json 179 | { 180 | "database": "data", 181 | "longitude": "longitude", 182 | "latitude": "latitude", 183 | "table_columns": ["name","description","latitude","longitude"], 184 | "files": "FILES IN MULTI PART FORM" 185 | } 186 | ``` 187 | 188 | ### Example Output 189 | ```json 190 | { 191 | "process_id": "c8d7b8d8-3e82-4f93-b441-55a5f51c4171", 192 | "url": "http://127.0.0.1:8000/api/v1/import/status/c8d7b8d8-3e82-4f93-b441-55a5f51c4171" 193 | } 194 | ``` 195 | 196 | ## Geographic Data From Json File 197 | 198 | ### Description 199 | Import json from a file and join to a map already within the database based off a column. 200 | 201 | Example: Import state date from a json [file](/data/states.json). 202 | 203 | ### Example Input 204 | ```json 205 | { 206 | "database": "data", 207 | "map": "states", 208 | "map_column": "state_abbr", 209 | "map_columns": ["state_abbr"], 210 | "table_column": "code", 211 | "table_columns": ["state","slug","code","nickname"], 212 | "files": "FILES IN MULTI PART FORM" 213 | } 214 | ``` 215 | 216 | ### Example Output 217 | ```json 218 | { 219 | "process_id": "c8d7b8d8-3e82-4f93-b441-55a5f51c4171", 220 | "url": "http://127.0.0.1:8000/api/v1/import/status/c8d7b8d8-3e82-4f93-b441-55a5f51c4171" 221 | } 222 | ``` 223 | 224 | ## Point Data From Json File 225 | 226 | ### Description 227 | Import point data from a Json file with latitude and longitude columns. 228 | 229 | Example: A json [file](/data/cities.json) that contains cities for the entire world. 230 | 231 | ### Example Input 232 | ```json 233 | { 234 | "database": "data", 235 | "longitude": "longitude", 236 | "latitude": "latitude", 237 | "table_columns": ["id","name","latitude","longitude","state_id","state_code","state_name","country_id","country_code","country_name","wikiDataId"], 238 | "files": "FILES IN MULTI PART FORM" 239 | } 240 | ``` 241 | 242 | ### Example Output 243 | ```json 244 | { 245 | "process_id": "c8d7b8d8-3e82-4f93-b441-55a5f51c4171", 246 | "url": "http://127.0.0.1:8000/api/v1/import/status/c8d7b8d8-3e82-4f93-b441-55a5f51c4171" 247 | } 248 | ``` 249 | 250 | ## Geographic Data From Json URL 251 | 252 | ### Description 253 | Import json from a url and join to a map already within the database based off a column. 254 | 255 | Example: Import state information from a gitlab url 256 | 257 | ### Example Input 258 | ```json 259 | { 260 | "database": "data", 261 | "map_column": "state_abbr", 262 | "table_column": "code", 263 | "table_columns": [ 264 | "state", 265 | "slug", 266 | "code", 267 | "nickname" 268 | ], 269 | "map": "states", 270 | "map_columns": [ 271 | "state_abbr" 272 | ], 273 | "url": "https://raw.githubusercontent.com/CivilServiceUSA/us-states/master/data/states.json" 274 | } 275 | ``` 276 | 277 | ### Example Output 278 | ```json 279 | { 280 | "process_id": "c8d7b8d8-3e82-4f93-b441-55a5f51c4171", 281 | "url": "http://127.0.0.1:8000/api/v1/import/status/c8d7b8d8-3e82-4f93-b441-55a5f51c4171" 282 | } 283 | ``` 284 | 285 | ## Point Data From Json URL 286 | 287 | ### Description 288 | Import json data from a url with latitude and longitude columns into database. 289 | 290 | Example: Import state centroids from a gitlab url 291 | 292 | ### Example Input 293 | ```json 294 | { 295 | "url": "https://raw.githubusercontent.com/dr5hn/countries-states-cities-database/master/states.json", 296 | "database": "data", 297 | "longitude": "longitude", 298 | "latitude": "latitude", 299 | "table_columns": ["id","name","latitude","longitude","state_code","country_id","country_code","country_name","type"], 300 | "files": "FILES IN MULTI PART FORM" 301 | } 302 | ``` 303 | 304 | ### Example Output 305 | ```json 306 | { 307 | "process_id": "c8d7b8d8-3e82-4f93-b441-55a5f51c4171", 308 | "url": "http://127.0.0.1:8000/api/v1/import/status/c8d7b8d8-3e82-4f93-b441-55a5f51c4171" 309 | } 310 | ``` 311 | 312 | ## Geojson From URL 313 | 314 | ### Description 315 | Import geojson from any url. 316 | 317 | Example: Input large earthquakes for the past month 318 | 319 | ### Example Input 320 | ```json 321 | { 322 | "database": "data", 323 | "url": "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/significant_month.geojson" 324 | } 325 | ``` 326 | 327 | ### Example Output 328 | ```json 329 | { 330 | "process_id": "c8d7b8d8-3e82-4f93-b441-55a5f51c4171", 331 | "url": "http://127.0.0.1:8000/api/v1/import/status/c8d7b8d8-3e82-4f93-b441-55a5f51c4171" 332 | } 333 | ``` -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | """FastImporter App - Configuration File""" 2 | DATABASES = { 3 | "data": { 4 | "host": "localhost", 5 | "database": "data", 6 | "username": "postgres", 7 | "password": "postgres", 8 | "port": 5432, 9 | } 10 | } -------------------------------------------------------------------------------- /data/cities.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": 52, 4 | "name": "Ashkāsham", 5 | "state_id": 3901, 6 | "state_code": "BDS", 7 | "state_name": "Badakhshan", 8 | "country_id": 1, 9 | "country_code": "AF", 10 | "country_name": "Afghanistan", 11 | "latitude": "36.68333000", 12 | "longitude": "71.53333000", 13 | "wikiDataId": "Q4805192" 14 | }, 15 | { 16 | "id": 68, 17 | "name": "Fayzabad", 18 | "state_id": 3901, 19 | "state_code": "BDS", 20 | "state_name": "Badakhshan", 21 | "country_id": 1, 22 | "country_code": "AF", 23 | "country_name": "Afghanistan", 24 | "latitude": "37.11664000", 25 | "longitude": "70.58002000", 26 | "wikiDataId": "Q156558" 27 | }, 28 | { 29 | "id": 78, 30 | "name": "Jurm", 31 | "state_id": 3901, 32 | "state_code": "BDS", 33 | "state_name": "Badakhshan", 34 | "country_id": 1, 35 | "country_code": "AF", 36 | "country_name": "Afghanistan", 37 | "latitude": "36.86477000", 38 | "longitude": "70.83421000", 39 | "wikiDataId": "Q10308323" 40 | } 41 | ] -------------------------------------------------------------------------------- /data/state_data.csv: -------------------------------------------------------------------------------- 1 | State Abbr,Number of Rest Stops 2 | IL,56 3 | IN,84 4 | WI,213 5 | -------------------------------------------------------------------------------- /data/states.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "state": "Alabama", 4 | "slug": "alabama", 5 | "code": "AL", 6 | "nickname": "Yellowhammer State", 7 | "website": "http://www.alabama.gov", 8 | "admission_date": "1819-12-14", 9 | "admission_number": 22, 10 | "capital_city": "Montgomery", 11 | "capital_url": "http://www.montgomeryal.gov", 12 | "population": 4833722, 13 | "population_rank": 23, 14 | "constitution_url": "http://alisondb.legislature.state.al.us/alison/default.aspx", 15 | "state_flag_url": "https://cdn.civil.services/us-states/flags/alabama-large.png", 16 | "state_seal_url": "https://cdn.civil.services/us-states/seals/alabama-large.png", 17 | "map_image_url": "https://cdn.civil.services/us-states/maps/alabama-large.png", 18 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/alabama.jpg", 19 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/alabama.jpg", 20 | "twitter_url": "https://twitter.com/alabamagov", 21 | "facebook_url": "https://www.facebook.com/alabamagov" 22 | }, 23 | { 24 | "state": "Alaska", 25 | "slug": "alaska", 26 | "code": "AK", 27 | "nickname": "The Last Frontier", 28 | "website": "http://alaska.gov", 29 | "admission_date": "1959-01-03", 30 | "admission_number": 49, 31 | "capital_city": "Juneau", 32 | "capital_url": "http://www.juneau.org", 33 | "population": 735132, 34 | "population_rank": 47, 35 | "constitution_url": "http://www.legis.state.ak.us/basis/folioproxy.asp?url=http://wwwjnu01.legis.state.ak.us/cgi-bin/folioisa.dll/acontxt/query=*/doc/{t1}?", 36 | "state_flag_url": "https://cdn.civil.services/us-states/flags/alaska-large.png", 37 | "state_seal_url": "https://cdn.civil.services/us-states/seals/alaska-large.png", 38 | "map_image_url": "https://cdn.civil.services/us-states/maps/alaska-large.png", 39 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/alaska.jpg", 40 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/alaska.jpg", 41 | "twitter_url": "https://twitter.com/alaska", 42 | "facebook_url": "https://www.facebook.com/AlaskaLocalGovernments" 43 | }, 44 | { 45 | "state": "Arizona", 46 | "slug": "arizona", 47 | "code": "AZ", 48 | "nickname": "The Grand Canyon State", 49 | "website": "https://az.gov", 50 | "admission_date": "1912-02-14", 51 | "admission_number": 48, 52 | "capital_city": "Phoenix", 53 | "capital_url": "https://www.phoenix.gov", 54 | "population": 6626624, 55 | "population_rank": 15, 56 | "constitution_url": "http://www.azleg.gov/Constitution.asp", 57 | "state_flag_url": "https://cdn.civil.services/us-states/flags/arizona-large.png", 58 | "state_seal_url": "https://cdn.civil.services/us-states/seals/arizona-large.png", 59 | "map_image_url": "https://cdn.civil.services/us-states/maps/arizona-large.png", 60 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/arizona.jpg", 61 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/arizona.jpg", 62 | "twitter_url": null, 63 | "facebook_url": null 64 | }, 65 | { 66 | "state": "Arkansas", 67 | "slug": "arkansas", 68 | "code": "AR", 69 | "nickname": "The Natural State", 70 | "website": "http://arkansas.gov", 71 | "admission_date": "1836-06-15", 72 | "admission_number": 25, 73 | "capital_city": "Little Rock", 74 | "capital_url": "http://www.littlerock.org", 75 | "population": 2959373, 76 | "population_rank": 32, 77 | "constitution_url": "http://www.arkleg.state.ar.us/assembly/Summary/ArkansasConstitution1874.pdf", 78 | "state_flag_url": "https://cdn.civil.services/us-states/flags/arkansas-large.png", 79 | "state_seal_url": "https://cdn.civil.services/us-states/seals/arkansas-large.png", 80 | "map_image_url": "https://cdn.civil.services/us-states/maps/arkansas-large.png", 81 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/arkansas.jpg", 82 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/arkansas.jpg", 83 | "twitter_url": "https://twitter.com/arkansasgov", 84 | "facebook_url": "https://www.facebook.com/Arkansas.gov" 85 | }, 86 | { 87 | "state": "California", 88 | "slug": "california", 89 | "code": "CA", 90 | "nickname": "Golden State", 91 | "website": "http://www.ca.gov", 92 | "admission_date": "1850-09-09", 93 | "admission_number": 31, 94 | "capital_city": "Sacramento", 95 | "capital_url": "http://www.cityofsacramento.org", 96 | "population": 38332521, 97 | "population_rank": 1, 98 | "constitution_url": "http://www.leginfo.ca.gov/const-toc.html", 99 | "state_flag_url": "https://cdn.civil.services/us-states/flags/california-large.png", 100 | "state_seal_url": "https://cdn.civil.services/us-states/seals/california-large.png", 101 | "map_image_url": "https://cdn.civil.services/us-states/maps/california-large.png", 102 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/california.jpg", 103 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/california.jpg", 104 | "twitter_url": "https://twitter.com/cagovernment", 105 | "facebook_url": null 106 | }, 107 | { 108 | "state": "Colorado", 109 | "slug": "colorado", 110 | "code": "CO", 111 | "nickname": "The Centennial State", 112 | "website": "https://www.colorado.gov", 113 | "admission_date": "1876-08-01", 114 | "admission_number": 38, 115 | "capital_city": "Denver", 116 | "capital_url": "http://www.denvergov.org", 117 | "population": 5268367, 118 | "population_rank": 22, 119 | "constitution_url": "https://www.colorado.gov/pacific/archives/government", 120 | "state_flag_url": "https://cdn.civil.services/us-states/flags/colorado-large.png", 121 | "state_seal_url": "https://cdn.civil.services/us-states/seals/colorado-large.png", 122 | "map_image_url": "https://cdn.civil.services/us-states/maps/colorado-large.png", 123 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/colorado.jpg", 124 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/colorado.jpg", 125 | "twitter_url": "https://twitter.com/coloradogov", 126 | "facebook_url": "https://www.facebook.com/Colorado.gov" 127 | }, 128 | { 129 | "state": "Connecticut", 130 | "slug": "connecticut", 131 | "code": "CT", 132 | "nickname": "Constitution State", 133 | "website": "http://www.ct.gov", 134 | "admission_date": "1788-01-09", 135 | "admission_number": 5, 136 | "capital_city": "Hartford", 137 | "capital_url": "http://www.hartford.gov", 138 | "population": 3596080, 139 | "population_rank": 29, 140 | "constitution_url": "http://www.ct.gov/sots/cwp/view.asp?a=3188&q=392288", 141 | "state_flag_url": "https://cdn.civil.services/us-states/flags/connecticut-large.png", 142 | "state_seal_url": "https://cdn.civil.services/us-states/seals/connecticut-large.png", 143 | "map_image_url": "https://cdn.civil.services/us-states/maps/connecticut-large.png", 144 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/connecticut.jpg", 145 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/connecticut.jpg", 146 | "twitter_url": null, 147 | "facebook_url": null 148 | }, 149 | { 150 | "state": "Delaware", 151 | "slug": "delaware", 152 | "code": "DE", 153 | "nickname": "The First State / The Diamond State", 154 | "website": "http://delaware.gov", 155 | "admission_date": "1787-12-07", 156 | "admission_number": 1, 157 | "capital_city": "Dover", 158 | "capital_url": "http://www.cityofdover.com", 159 | "population": 925749, 160 | "population_rank": 45, 161 | "constitution_url": "http://www.state.de.us/facts/constit/welcome.htm", 162 | "state_flag_url": "https://cdn.civil.services/us-states/flags/delaware-large.png", 163 | "state_seal_url": "https://cdn.civil.services/us-states/seals/delaware-large.png", 164 | "map_image_url": "https://cdn.civil.services/us-states/maps/delaware-large.png", 165 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/delaware.jpg", 166 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/delaware.jpg", 167 | "twitter_url": "https://twitter.com/delaware_gov", 168 | "facebook_url": "https://www.facebook.com/delaware.gov" 169 | }, 170 | { 171 | "state": "Florida", 172 | "slug": "florida", 173 | "code": "FL", 174 | "nickname": "Sunshine State", 175 | "website": "http://www.myflorida.com", 176 | "admission_date": "1845-03-03", 177 | "admission_number": 27, 178 | "capital_city": "Tallahassee", 179 | "capital_url": "https://www.talgov.com/Main/Home.aspx", 180 | "population": 19552860, 181 | "population_rank": 4, 182 | "constitution_url": "http://www.leg.state.fl.us/Statutes/index.cfm", 183 | "state_flag_url": "https://cdn.civil.services/us-states/flags/florida-large.png", 184 | "state_seal_url": "https://cdn.civil.services/us-states/seals/florida-large.png", 185 | "map_image_url": "https://cdn.civil.services/us-states/maps/florida-large.png", 186 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/florida.jpg", 187 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/florida.jpg", 188 | "twitter_url": null, 189 | "facebook_url": null 190 | }, 191 | { 192 | "state": "Georgia", 193 | "slug": "georgia", 194 | "code": "GA", 195 | "nickname": "Peach State", 196 | "website": "http://georgia.gov", 197 | "admission_date": "1788-01-02", 198 | "admission_number": 4, 199 | "capital_city": "Atlanta", 200 | "capital_url": "http://www.atlantaga.gov", 201 | "population": 9992167, 202 | "population_rank": 8, 203 | "constitution_url": "http://sos.ga.gov/admin/files/Constitution_2013_Final_Printed.pdf", 204 | "state_flag_url": "https://cdn.civil.services/us-states/flags/georgia-large.png", 205 | "state_seal_url": "https://cdn.civil.services/us-states/seals/georgia-large.png", 206 | "map_image_url": "https://cdn.civil.services/us-states/maps/georgia-large.png", 207 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/georgia.jpg", 208 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/georgia.jpg", 209 | "twitter_url": "http://twitter.com/georgiagov", 210 | "facebook_url": "http://www.facebook.com/pages/georgiagov/29760668054" 211 | }, 212 | { 213 | "state": "Hawaii", 214 | "slug": "hawaii", 215 | "code": "HI", 216 | "nickname": "Aloha State", 217 | "website": "https://www.ehawaii.gov", 218 | "admission_date": "1959-08-21", 219 | "admission_number": 50, 220 | "capital_city": "Honolulu", 221 | "capital_url": "http://www.co.honolulu.hi.us", 222 | "population": 1404054, 223 | "population_rank": 40, 224 | "constitution_url": "http://lrbhawaii.org/con", 225 | "state_flag_url": "https://cdn.civil.services/us-states/flags/hawaii-large.png", 226 | "state_seal_url": "https://cdn.civil.services/us-states/seals/hawaii-large.png", 227 | "map_image_url": "https://cdn.civil.services/us-states/maps/hawaii-large.png", 228 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/hawaii.jpg", 229 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/hawaii.jpg", 230 | "twitter_url": "https://twitter.com/ehawaiigov", 231 | "facebook_url": "https://www.facebook.com/ehawaii.gov" 232 | }, 233 | { 234 | "state": "Idaho", 235 | "slug": "idaho", 236 | "code": "ID", 237 | "nickname": "Gem State", 238 | "website": "https://www.idaho.gov", 239 | "admission_date": "1890-07-03", 240 | "admission_number": 43, 241 | "capital_city": "Boise", 242 | "capital_url": "http://www.cityofboise.org", 243 | "population": 1612136, 244 | "population_rank": 39, 245 | "constitution_url": "http://www.legislature.idaho.gov/idstat/IC/Title003.htm", 246 | "state_flag_url": "https://cdn.civil.services/us-states/flags/idaho-large.png", 247 | "state_seal_url": "https://cdn.civil.services/us-states/seals/idaho-large.png", 248 | "map_image_url": "https://cdn.civil.services/us-states/maps/idaho-large.png", 249 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/idaho.jpg", 250 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/idaho.jpg", 251 | "twitter_url": "https://twitter.com/IDAHOgov", 252 | "facebook_url": null 253 | }, 254 | { 255 | "state": "Illinois", 256 | "slug": "illinois", 257 | "code": "IL", 258 | "nickname": "Prairie State", 259 | "website": "https://www.illinois.gov", 260 | "admission_date": "1818-12-03", 261 | "admission_number": 21, 262 | "capital_city": "Springfield", 263 | "capital_url": "http://www.springfield.il.us", 264 | "population": 12882135, 265 | "population_rank": 5, 266 | "constitution_url": "http://www.ilga.gov/commission/lrb/conmain.htm", 267 | "state_flag_url": "https://cdn.civil.services/us-states/flags/illinois-large.png", 268 | "state_seal_url": "https://cdn.civil.services/us-states/seals/illinois-large.png", 269 | "map_image_url": "https://cdn.civil.services/us-states/maps/illinois-large.png", 270 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/illinois.jpg", 271 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/illinois.jpg", 272 | "twitter_url": null, 273 | "facebook_url": null 274 | }, 275 | { 276 | "state": "Indiana", 277 | "slug": "indiana", 278 | "code": "IN", 279 | "nickname": "Hoosier State", 280 | "website": "http://www.in.gov", 281 | "admission_date": "1816-12-11", 282 | "admission_number": 19, 283 | "capital_city": "Indianapolis", 284 | "capital_url": "http://www.indy.gov/Pages/Home.aspx", 285 | "population": 6570902, 286 | "population_rank": 16, 287 | "constitution_url": "http://www.law.indiana.edu/uslawdocs/inconst.html", 288 | "state_flag_url": "https://cdn.civil.services/us-states/flags/indiana-large.png", 289 | "state_seal_url": "https://cdn.civil.services/us-states/seals/indiana-large.png", 290 | "map_image_url": "https://cdn.civil.services/us-states/maps/indiana-large.png", 291 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/indiana.jpg", 292 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/indiana.jpg", 293 | "twitter_url": "https://twitter.com/in_gov", 294 | "facebook_url": "https://www.facebook.com/IndianaGovernment" 295 | }, 296 | { 297 | "state": "Iowa", 298 | "slug": "iowa", 299 | "code": "IA", 300 | "nickname": "Hawkeye State", 301 | "website": "https://www.iowa.gov", 302 | "admission_date": "1846-12-28", 303 | "admission_number": 29, 304 | "capital_city": "Des Moines", 305 | "capital_url": "http://www.ci.des-moines.ia.us", 306 | "population": 3090416, 307 | "population_rank": 30, 308 | "constitution_url": "http://publications.iowa.gov/135/1/history/7-7.html", 309 | "state_flag_url": "https://cdn.civil.services/us-states/flags/iowa-large.png", 310 | "state_seal_url": "https://cdn.civil.services/us-states/seals/iowa-large.png", 311 | "map_image_url": "https://cdn.civil.services/us-states/maps/iowa-large.png", 312 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/iowa.jpg", 313 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/iowa.jpg", 314 | "twitter_url": "https://twitter.com/IAGOVTWEETS", 315 | "facebook_url": null 316 | }, 317 | { 318 | "state": "Kansas", 319 | "slug": "kansas", 320 | "code": "KS", 321 | "nickname": "Sunflower State", 322 | "website": "https://www.kansas.gov", 323 | "admission_date": "1861-01-29", 324 | "admission_number": 34, 325 | "capital_city": "Topeka", 326 | "capital_url": "http://www.topeka.org", 327 | "population": 2893957, 328 | "population_rank": 34, 329 | "constitution_url": "https://kslib.info/405/Kansas-Constitution", 330 | "state_flag_url": "https://cdn.civil.services/us-states/flags/kansas-large.png", 331 | "state_seal_url": "https://cdn.civil.services/us-states/seals/kansas-large.png", 332 | "map_image_url": "https://cdn.civil.services/us-states/maps/kansas-large.png", 333 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/kansas.jpg", 334 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/kansas.jpg", 335 | "twitter_url": "http://www.twitter.com/ksgovernment", 336 | "facebook_url": "http://www.facebook.com/pages/Topeka-KS/Kansasgov-Kansas-Government-Online/52068474220" 337 | }, 338 | { 339 | "state": "Kentucky", 340 | "slug": "kentucky", 341 | "code": "KY", 342 | "nickname": "Bluegrass State", 343 | "website": "http://kentucky.gov", 344 | "admission_date": "1792-06-01", 345 | "admission_number": 15, 346 | "capital_city": "Frankfort", 347 | "capital_url": "http://frankfort.ky.gov", 348 | "population": 4395295, 349 | "population_rank": 26, 350 | "constitution_url": "http://www.lrc.state.ky.us/Legresou/Constitu/intro.htm", 351 | "state_flag_url": "https://cdn.civil.services/us-states/flags/kentucky-large.png", 352 | "state_seal_url": "https://cdn.civil.services/us-states/seals/kentucky-large.png", 353 | "map_image_url": "https://cdn.civil.services/us-states/maps/kentucky-large.png", 354 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/kentucky.jpg", 355 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/kentucky.jpg", 356 | "twitter_url": "https://twitter.com/kygov", 357 | "facebook_url": "https://www.facebook.com/kygov" 358 | }, 359 | { 360 | "state": "Louisiana", 361 | "slug": "louisiana", 362 | "code": "LA", 363 | "nickname": "Pelican State", 364 | "website": "http://louisiana.gov", 365 | "admission_date": "1812-04-30", 366 | "admission_number": 18, 367 | "capital_city": "Baton Rouge", 368 | "capital_url": "http://brgov.com", 369 | "population": 4625470, 370 | "population_rank": 25, 371 | "constitution_url": "http://senate.legis.state.la.us/Documents/Constitution", 372 | "state_flag_url": "https://cdn.civil.services/us-states/flags/louisiana-large.png", 373 | "state_seal_url": "https://cdn.civil.services/us-states/seals/louisiana-large.png", 374 | "map_image_url": "https://cdn.civil.services/us-states/maps/louisiana-large.png", 375 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/louisiana.jpg", 376 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/louisiana.jpg", 377 | "twitter_url": null, 378 | "facebook_url": null 379 | }, 380 | { 381 | "state": "Maine", 382 | "slug": "maine", 383 | "code": "ME", 384 | "nickname": "Pine Tree State", 385 | "website": "http://www.maine.gov", 386 | "admission_date": "1820-03-15", 387 | "admission_number": 23, 388 | "capital_city": "Augusta", 389 | "capital_url": "http://www.augustamaine.gov", 390 | "population": 1328302, 391 | "population_rank": 41, 392 | "constitution_url": "http://www.maine.gov/legis/const", 393 | "state_flag_url": "https://cdn.civil.services/us-states/flags/maine-large.png", 394 | "state_seal_url": "https://cdn.civil.services/us-states/seals/maine-large.png", 395 | "map_image_url": "https://cdn.civil.services/us-states/maps/maine-large.png", 396 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/maine.jpg", 397 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/maine.jpg", 398 | "twitter_url": "https://twitter.com/mainegov_news", 399 | "facebook_url": "http://www.facebook.com/pages/Augusta-ME/Mainegov/98519328240" 400 | }, 401 | { 402 | "state": "Maryland", 403 | "slug": "maryland", 404 | "code": "MD", 405 | "nickname": "Old Line State", 406 | "website": "http://www.maryland.gov", 407 | "admission_date": "1788-04-28", 408 | "admission_number": 7, 409 | "capital_city": "Annapolis", 410 | "capital_url": "http://www.annapolis.gov", 411 | "population": 5928814, 412 | "population_rank": 19, 413 | "constitution_url": "http://msa.maryland.gov/msa/mdmanual/43const/html/const.html", 414 | "state_flag_url": "https://cdn.civil.services/us-states/flags/maryland-large.png", 415 | "state_seal_url": "https://cdn.civil.services/us-states/seals/maryland-large.png", 416 | "map_image_url": "https://cdn.civil.services/us-states/maps/maryland-large.png", 417 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/maryland.jpg", 418 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/maryland.jpg", 419 | "twitter_url": "https://twitter.com/statemaryland", 420 | "facebook_url": "https://www.facebook.com/statemaryland" 421 | }, 422 | { 423 | "state": "Massachusetts", 424 | "slug": "massachusetts", 425 | "code": "MA", 426 | "nickname": "Bay State", 427 | "website": "http://www.mass.gov", 428 | "admission_date": "1788-02-06", 429 | "admission_number": 6, 430 | "capital_city": "Boston", 431 | "capital_url": "http://www.ci.boston.ma.us", 432 | "population": 6692824, 433 | "population_rank": 14, 434 | "constitution_url": "http://www.state.ma.us/legis/const.htm", 435 | "state_flag_url": "https://cdn.civil.services/us-states/flags/massachusetts-large.png", 436 | "state_seal_url": "https://cdn.civil.services/us-states/seals/massachusetts-large.png", 437 | "map_image_url": "https://cdn.civil.services/us-states/maps/massachusetts-large.png", 438 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/massachusetts.jpg", 439 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/massachusetts.jpg", 440 | "twitter_url": "http://twitter.com/massgov", 441 | "facebook_url": "https://www.facebook.com/massgov" 442 | }, 443 | { 444 | "state": "Michigan", 445 | "slug": "michigan", 446 | "code": "MI", 447 | "nickname": "Wolverine State / Great Lakes State", 448 | "website": "http://www.michigan.gov", 449 | "admission_date": "1837-01-26", 450 | "admission_number": 26, 451 | "capital_city": "Lansing", 452 | "capital_url": "http://cityoflansingmi.com", 453 | "population": 9895622, 454 | "population_rank": 9, 455 | "constitution_url": "http://www.legislature.mi.gov/(S(hrowl12tg05hemnnkidim1jb))/mileg.aspx?page=GetObject&objectname=mcl-Constitution", 456 | "state_flag_url": "https://cdn.civil.services/us-states/flags/michigan-large.png", 457 | "state_seal_url": "https://cdn.civil.services/us-states/seals/michigan-large.png", 458 | "map_image_url": "https://cdn.civil.services/us-states/maps/michigan-large.png", 459 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/michigan.jpg", 460 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/michigan.jpg", 461 | "twitter_url": "https://twitter.com/migov", 462 | "facebook_url": "https://www.facebook.com/MIgovernment" 463 | }, 464 | { 465 | "state": "Minnesota", 466 | "slug": "minnesota", 467 | "code": "MN", 468 | "nickname": "North Star State / Land of 10,000 Lakes", 469 | "website": "https://mn.gov", 470 | "admission_date": "1858-05-11", 471 | "admission_number": 32, 472 | "capital_city": "Saint Paul", 473 | "capital_url": "http://www.stpaul.gov", 474 | "population": 5420380, 475 | "population_rank": 21, 476 | "constitution_url": "http://www.house.leg.state.mn.us/cco/rules/mncon/preamble.htm", 477 | "state_flag_url": "https://cdn.civil.services/us-states/flags/minnesota-large.png", 478 | "state_seal_url": "https://cdn.civil.services/us-states/seals/minnesota-large.png", 479 | "map_image_url": "https://cdn.civil.services/us-states/maps/minnesota-large.png", 480 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/minnesota.jpg", 481 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/minnesota.jpg", 482 | "twitter_url": null, 483 | "facebook_url": null 484 | }, 485 | { 486 | "state": "Mississippi", 487 | "slug": "mississippi", 488 | "code": "MS", 489 | "nickname": "Magnolia State", 490 | "website": "http://www.ms.gov", 491 | "admission_date": "1817-12-10", 492 | "admission_number": 20, 493 | "capital_city": "Jackson", 494 | "capital_url": "http://www.city.jackson.ms.us", 495 | "population": 2991207, 496 | "population_rank": 31, 497 | "constitution_url": "http://law.justia.com/constitution/mississippi", 498 | "state_flag_url": "https://cdn.civil.services/us-states/flags/mississippi-large.png", 499 | "state_seal_url": "https://cdn.civil.services/us-states/seals/mississippi-large.png", 500 | "map_image_url": "https://cdn.civil.services/us-states/maps/mississippi-large.png", 501 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/mississippi.jpg", 502 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/mississippi.jpg", 503 | "twitter_url": "https://twitter.com/msdotgov", 504 | "facebook_url": "https://www.facebook.com/msdotgov" 505 | }, 506 | { 507 | "state": "Missouri", 508 | "slug": "missouri", 509 | "code": "MO", 510 | "nickname": "Show Me State", 511 | "website": "https://www.mo.gov", 512 | "admission_date": "1821-08-10", 513 | "admission_number": 24, 514 | "capital_city": "Jefferson City", 515 | "capital_url": "http://www.jeffcitymo.org", 516 | "population": 6044171, 517 | "population_rank": 18, 518 | "constitution_url": "http://www.moga.mo.gov/mostatutes/moconstn.html", 519 | "state_flag_url": "https://cdn.civil.services/us-states/flags/missouri-large.png", 520 | "state_seal_url": "https://cdn.civil.services/us-states/seals/missouri-large.png", 521 | "map_image_url": "https://cdn.civil.services/us-states/maps/missouri-large.png", 522 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/missouri.jpg", 523 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/missouri.jpg", 524 | "twitter_url": "https://twitter.com/MoGov", 525 | "facebook_url": "https://www.facebook.com/mogov" 526 | }, 527 | { 528 | "state": "Montana", 529 | "slug": "montana", 530 | "code": "MT", 531 | "nickname": "Treasure State", 532 | "website": "http://mt.gov", 533 | "admission_date": "1889-11-08", 534 | "admission_number": 41, 535 | "capital_city": "Helena", 536 | "capital_url": "http://www.ci.helena.mt.us", 537 | "population": 1015165, 538 | "population_rank": 44, 539 | "constitution_url": "http://courts.mt.gov/content/library/docs/72constit.pdf", 540 | "state_flag_url": "https://cdn.civil.services/us-states/flags/montana-large.png", 541 | "state_seal_url": "https://cdn.civil.services/us-states/seals/montana-large.png", 542 | "map_image_url": "https://cdn.civil.services/us-states/maps/montana-large.png", 543 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/montana.jpg", 544 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/montana.jpg", 545 | "twitter_url": null, 546 | "facebook_url": null 547 | }, 548 | { 549 | "state": "Nebraska", 550 | "slug": "nebraska", 551 | "code": "NE", 552 | "nickname": "Cornhusker State", 553 | "website": "http://www.nebraska.gov", 554 | "admission_date": "1867-03-01", 555 | "admission_number": 37, 556 | "capital_city": "Lincoln", 557 | "capital_url": "http://lincoln.ne.gov", 558 | "population": 1868516, 559 | "population_rank": 37, 560 | "constitution_url": "http://www.state.ne.us/legislative/statutes/C", 561 | "state_flag_url": "https://cdn.civil.services/us-states/flags/nebraska-large.png", 562 | "state_seal_url": "https://cdn.civil.services/us-states/seals/nebraska-large.png", 563 | "map_image_url": "https://cdn.civil.services/us-states/maps/nebraska-large.png", 564 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/nebraska.jpg", 565 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/nebraska.jpg", 566 | "twitter_url": "https://twitter.com/Nebraskagov", 567 | "facebook_url": "https://www.facebook.com/nebraska.gov" 568 | }, 569 | { 570 | "state": "Nevada", 571 | "slug": "nevada", 572 | "code": "NV", 573 | "nickname": "The Silver State", 574 | "website": "http://nv.gov", 575 | "admission_date": "1864-10-31", 576 | "admission_number": 36, 577 | "capital_city": "Carson City", 578 | "capital_url": "http://www.carson.org", 579 | "population": 2790136, 580 | "population_rank": 35, 581 | "constitution_url": "http://www.leg.state.nv.us/Const/NvConst.html", 582 | "state_flag_url": "https://cdn.civil.services/us-states/flags/nevada-large.png", 583 | "state_seal_url": "https://cdn.civil.services/us-states/seals/nevada-large.png", 584 | "map_image_url": "https://cdn.civil.services/us-states/maps/nevada-large.png", 585 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/nevada.jpg", 586 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/nevada.jpg", 587 | "twitter_url": null, 588 | "facebook_url": null 589 | }, 590 | { 591 | "state": "New Hampshire", 592 | "slug": "new-hampshire", 593 | "code": "NH", 594 | "nickname": "Granite State", 595 | "website": "https://www.nh.gov", 596 | "admission_date": "1788-06-21", 597 | "admission_number": 9, 598 | "capital_city": "Concord", 599 | "capital_url": "http://www.concordnh.gov", 600 | "population": 1323459, 601 | "population_rank": 42, 602 | "constitution_url": "http://www.state.nh.us/constitution/constitution.html", 603 | "state_flag_url": "https://cdn.civil.services/us-states/flags/new-hampshire-large.png", 604 | "state_seal_url": "https://cdn.civil.services/us-states/seals/new-hampshire-large.png", 605 | "map_image_url": "https://cdn.civil.services/us-states/maps/new-hampshire-large.png", 606 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/new-hampshire.jpg", 607 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/new-hampshire.jpg", 608 | "twitter_url": "https://twitter.com/nhgov", 609 | "facebook_url": null 610 | }, 611 | { 612 | "state": "New Jersey", 613 | "slug": "new-jersey", 614 | "code": "NJ", 615 | "nickname": "Garden State", 616 | "website": "http://www.state.nj.us", 617 | "admission_date": "1787-12-18", 618 | "admission_number": 3, 619 | "capital_city": "Trenton", 620 | "capital_url": "http://www.trentonnj.org", 621 | "population": 8899339, 622 | "population_rank": 11, 623 | "constitution_url": "http://www.njleg.state.nj.us/lawsconstitution/consearch.asp", 624 | "state_flag_url": "https://cdn.civil.services/us-states/flags/new-jersey-large.png", 625 | "state_seal_url": "https://cdn.civil.services/us-states/seals/new-jersey-large.png", 626 | "map_image_url": "https://cdn.civil.services/us-states/maps/new-jersey-large.png", 627 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/new-jersey.jpg", 628 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/new-jersey.jpg", 629 | "twitter_url": null, 630 | "facebook_url": null 631 | }, 632 | { 633 | "state": "New Mexico", 634 | "slug": "new-mexico", 635 | "code": "NM", 636 | "nickname": "Land of Enchantment", 637 | "website": "http://www.newmexico.gov", 638 | "admission_date": "1912-01-06", 639 | "admission_number": 47, 640 | "capital_city": "Santa Fe", 641 | "capital_url": "http://www.santafenm.gov", 642 | "population": 2085287, 643 | "population_rank": 36, 644 | "constitution_url": "http://www.loc.gov/law/guide/us-nm.html", 645 | "state_flag_url": "https://cdn.civil.services/us-states/flags/new-mexico-large.png", 646 | "state_seal_url": "https://cdn.civil.services/us-states/seals/new-mexico-large.png", 647 | "map_image_url": "https://cdn.civil.services/us-states/maps/new-mexico-large.png", 648 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/new-mexico.jpg", 649 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/new-mexico.jpg", 650 | "twitter_url": null, 651 | "facebook_url": null 652 | }, 653 | { 654 | "state": "New York", 655 | "slug": "new-york", 656 | "code": "NY", 657 | "nickname": "Empire State", 658 | "website": "http://www.ny.gov", 659 | "admission_date": "1788-07-26", 660 | "admission_number": 11, 661 | "capital_city": "Albany", 662 | "capital_url": "http://www.albanyny.org", 663 | "population": 19651127, 664 | "population_rank": 3, 665 | "constitution_url": "https://www.dos.ny.gov/info/constitution.htm", 666 | "state_flag_url": "https://cdn.civil.services/us-states/flags/new-york-large.png", 667 | "state_seal_url": "https://cdn.civil.services/us-states/seals/new-york-large.png", 668 | "map_image_url": "https://cdn.civil.services/us-states/maps/new-york-large.png", 669 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/new-york.jpg", 670 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/new-york.jpg", 671 | "twitter_url": "https://twitter.com/nygov", 672 | "facebook_url": null 673 | }, 674 | { 675 | "state": "North Carolina", 676 | "slug": "north-carolina", 677 | "code": "NC", 678 | "nickname": "Old North State / Tar Heel State", 679 | "website": "http://www.nc.gov", 680 | "admission_date": "1789-11-21", 681 | "admission_number": 12, 682 | "capital_city": "Raleigh", 683 | "capital_url": "http://www.raleigh-nc.org", 684 | "population": 9848060, 685 | "population_rank": 10, 686 | "constitution_url": "http://statelibrary.dcr.state.nc.us/nc/stgovt/preconst.htm", 687 | "state_flag_url": "https://cdn.civil.services/us-states/flags/north-carolina-large.png", 688 | "state_seal_url": "https://cdn.civil.services/us-states/seals/north-carolina-large.png", 689 | "map_image_url": "https://cdn.civil.services/us-states/maps/north-carolina-large.png", 690 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/north-carolina.jpg", 691 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/north-carolina.jpg", 692 | "twitter_url": "https://twitter.com/NCdotGov", 693 | "facebook_url": null 694 | }, 695 | { 696 | "state": "North Dakota", 697 | "slug": "north-dakota", 698 | "code": "ND", 699 | "nickname": "Peace Garden State / Flickertail State / Roughrider State", 700 | "website": "http://www.nd.gov", 701 | "admission_date": "1889-11-02", 702 | "admission_number": 39, 703 | "capital_city": "Bismarck", 704 | "capital_url": "http://www.bismarck.org", 705 | "population": 723393, 706 | "population_rank": 48, 707 | "constitution_url": "http://www.legis.nd.gov/information/statutes/const-laws.html", 708 | "state_flag_url": "https://cdn.civil.services/us-states/flags/north-dakota-large.png", 709 | "state_seal_url": "https://cdn.civil.services/us-states/seals/north-dakota-large.png", 710 | "map_image_url": "https://cdn.civil.services/us-states/maps/north-dakota-large.png", 711 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/north-dakota.jpg", 712 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/north-dakota.jpg", 713 | "twitter_url": "https://twitter.com/ExperienceND", 714 | "facebook_url": "https://www.facebook.com/ExperienceND" 715 | }, 716 | { 717 | "state": "Ohio", 718 | "slug": "ohio", 719 | "code": "OH", 720 | "nickname": "Buckeye State", 721 | "website": "https://ohio.gov", 722 | "admission_date": "1803-03-01", 723 | "admission_number": 17, 724 | "capital_city": "Columbus", 725 | "capital_url": "http://ci.columbus.oh.us", 726 | "population": 11570808, 727 | "population_rank": 7, 728 | "constitution_url": "http://www.legislature.state.oh.us/constitution.cfm", 729 | "state_flag_url": "https://cdn.civil.services/us-states/flags/ohio-large.png", 730 | "state_seal_url": "https://cdn.civil.services/us-states/seals/ohio-large.png", 731 | "map_image_url": "https://cdn.civil.services/us-states/maps/ohio-large.png", 732 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/ohio.jpg", 733 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/ohio.jpg", 734 | "twitter_url": "https://twitter.com/ohgov", 735 | "facebook_url": null 736 | }, 737 | { 738 | "state": "Oklahoma", 739 | "slug": "oklahoma", 740 | "code": "OK", 741 | "nickname": "Sooner State", 742 | "website": "https://www.ok.gov", 743 | "admission_date": "1907-11-16", 744 | "admission_number": 46, 745 | "capital_city": "Oklahoma City", 746 | "capital_url": "http://www.okc.gov", 747 | "population": 3850568, 748 | "population_rank": 28, 749 | "constitution_url": "http://oklegal.onenet.net/okcon", 750 | "state_flag_url": "https://cdn.civil.services/us-states/flags/oklahoma-large.png", 751 | "state_seal_url": "https://cdn.civil.services/us-states/seals/oklahoma-large.png", 752 | "map_image_url": "https://cdn.civil.services/us-states/maps/oklahoma-large.png", 753 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/oklahoma.jpg", 754 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/oklahoma.jpg", 755 | "twitter_url": "https://twitter.com/okgov", 756 | "facebook_url": "https://www.facebook.com/okgov" 757 | }, 758 | { 759 | "state": "Oregon", 760 | "slug": "oregon", 761 | "code": "OR", 762 | "nickname": "Beaver State", 763 | "website": "http://www.oregon.gov", 764 | "admission_date": "1859-02-14", 765 | "admission_number": 33, 766 | "capital_city": "Salem", 767 | "capital_url": "http://www.cityofsalem.net/Pages/default.aspx", 768 | "population": 3930065, 769 | "population_rank": 27, 770 | "constitution_url": "http://bluebook.state.or.us/state/constitution/constitution.htm", 771 | "state_flag_url": "https://cdn.civil.services/us-states/flags/oregon-large.png", 772 | "state_seal_url": "https://cdn.civil.services/us-states/seals/oregon-large.png", 773 | "map_image_url": "https://cdn.civil.services/us-states/maps/oregon-large.png", 774 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/oregon.jpg", 775 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/oregon.jpg", 776 | "twitter_url": null, 777 | "facebook_url": null 778 | }, 779 | { 780 | "state": "Pennsylvania", 781 | "slug": "pennsylvania", 782 | "code": "PA", 783 | "nickname": "Keystone State", 784 | "website": "http://www.pa.gov", 785 | "admission_date": "1787-12-12", 786 | "admission_number": 2, 787 | "capital_city": "Harrisburg", 788 | "capital_url": "http://harrisburgpa.gov", 789 | "population": 12773801, 790 | "population_rank": 6, 791 | "constitution_url": "http://sites.state.pa.us/PA_Constitution.html", 792 | "state_flag_url": "https://cdn.civil.services/us-states/flags/pennsylvania-large.png", 793 | "state_seal_url": "https://cdn.civil.services/us-states/seals/pennsylvania-large.png", 794 | "map_image_url": "https://cdn.civil.services/us-states/maps/pennsylvania-large.png", 795 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/pennsylvania.jpg", 796 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/pennsylvania.jpg", 797 | "twitter_url": "https://www.facebook.com/visitPA", 798 | "facebook_url": "https://twitter.com/visitPA" 799 | }, 800 | { 801 | "state": "Rhode Island", 802 | "slug": "rhode-island", 803 | "code": "RI", 804 | "nickname": "The Ocean State", 805 | "website": "https://www.ri.gov", 806 | "admission_date": "1790-05-29", 807 | "admission_number": 13, 808 | "capital_city": "Providence", 809 | "capital_url": "http://www.providenceri.com", 810 | "population": 1051511, 811 | "population_rank": 43, 812 | "constitution_url": "http://webserver.rilin.state.ri.us/RiConstitution", 813 | "state_flag_url": "https://cdn.civil.services/us-states/flags/rhode-island-large.png", 814 | "state_seal_url": "https://cdn.civil.services/us-states/seals/rhode-island-large.png", 815 | "map_image_url": "https://cdn.civil.services/us-states/maps/rhode-island-large.png", 816 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/rhode-island.jpg", 817 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/rhode-island.jpg", 818 | "twitter_url": "https://twitter.com/rigov", 819 | "facebook_url": "https://www.facebook.com/RIgov-Rhode-Island-Government-Online-24056655991" 820 | }, 821 | { 822 | "state": "South Carolina", 823 | "slug": "south-carolina", 824 | "code": "SC", 825 | "nickname": "Palmetto State", 826 | "website": "http://www.sc.gov", 827 | "admission_date": "1788-05-23", 828 | "admission_number": 8, 829 | "capital_city": "Columbia", 830 | "capital_url": "http://www.columbiasc.net", 831 | "population": 4774839, 832 | "population_rank": 24, 833 | "constitution_url": "http://www.scstatehouse.gov/scconstitution/scconst.php", 834 | "state_flag_url": "https://cdn.civil.services/us-states/flags/south-carolina-large.png", 835 | "state_seal_url": "https://cdn.civil.services/us-states/seals/south-carolina-large.png", 836 | "map_image_url": "https://cdn.civil.services/us-states/maps/south-carolina-large.png", 837 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/south-carolina.jpg", 838 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/south-carolina.jpg", 839 | "twitter_url": "https://twitter.com/scgov", 840 | "facebook_url": "http://www.facebook.com/pages/SCgov/12752057990" 841 | }, 842 | { 843 | "state": "South Dakota", 844 | "slug": "south-dakota", 845 | "code": "SD", 846 | "nickname": "Mount Rushmore State", 847 | "website": "http://sd.gov", 848 | "admission_date": "1889-11-02", 849 | "admission_number": 40, 850 | "capital_city": "Pierre", 851 | "capital_url": "http://ci.pierre.sd.us", 852 | "population": 844877, 853 | "population_rank": 46, 854 | "constitution_url": "http://legis.sd.gov/statutes/Constitution", 855 | "state_flag_url": "https://cdn.civil.services/us-states/flags/south-dakota-large.png", 856 | "state_seal_url": "https://cdn.civil.services/us-states/seals/south-dakota-large.png", 857 | "map_image_url": "https://cdn.civil.services/us-states/maps/south-dakota-large.png", 858 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/south-dakota.jpg", 859 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/south-dakota.jpg", 860 | "twitter_url": null, 861 | "facebook_url": null 862 | }, 863 | { 864 | "state": "Tennessee", 865 | "slug": "tennessee", 866 | "code": "TN", 867 | "nickname": "Volunteer State", 868 | "website": "https://www.tn.gov", 869 | "admission_date": "1796-06-01", 870 | "admission_number": 16, 871 | "capital_city": "Nashville", 872 | "capital_url": "http://www.nashville.gov", 873 | "population": 6495978, 874 | "population_rank": 17, 875 | "constitution_url": "http://www.capitol.tn.gov/about/docs/TN-Constitution.pdf", 876 | "state_flag_url": "https://cdn.civil.services/us-states/flags/tennessee-large.png", 877 | "state_seal_url": "https://cdn.civil.services/us-states/seals/tennessee-large.png", 878 | "map_image_url": "https://cdn.civil.services/us-states/maps/tennessee-large.png", 879 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/tennessee.jpg", 880 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/tennessee.jpg", 881 | "twitter_url": "https://twitter.com/TNVacation", 882 | "facebook_url": "https://www.facebook.com/tnvacation" 883 | }, 884 | { 885 | "state": "Texas", 886 | "slug": "texas", 887 | "code": "TX", 888 | "nickname": "Lone Star State", 889 | "website": "https://www.texas.gov", 890 | "admission_date": "1845-12-29", 891 | "admission_number": 28, 892 | "capital_city": "Austin", 893 | "capital_url": "http://www.austintexas.gov", 894 | "population": 26448193, 895 | "population_rank": 2, 896 | "constitution_url": "http://www.constitution.legis.state.tx.us", 897 | "state_flag_url": "https://cdn.civil.services/us-states/flags/texas-large.png", 898 | "state_seal_url": "https://cdn.civil.services/us-states/seals/texas-large.png", 899 | "map_image_url": "https://cdn.civil.services/us-states/maps/texas-large.png", 900 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/texas.jpg", 901 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/texas.jpg", 902 | "twitter_url": "https://twitter.com/texasgov", 903 | "facebook_url": "http://www.facebook.com/Texas.gov" 904 | }, 905 | { 906 | "state": "Utah", 907 | "slug": "utah", 908 | "code": "UT", 909 | "nickname": "The Beehive State", 910 | "website": "https://utah.gov", 911 | "admission_date": "1896-01-04", 912 | "admission_number": 45, 913 | "capital_city": "Salt Lake City", 914 | "capital_url": "http://www.slcgov.com", 915 | "population": 2900872, 916 | "population_rank": 33, 917 | "constitution_url": "http://le.utah.gov/UtahCode/chapter.jsp?code=Constitution", 918 | "state_flag_url": "https://cdn.civil.services/us-states/flags/utah-large.png", 919 | "state_seal_url": "https://cdn.civil.services/us-states/seals/utah-large.png", 920 | "map_image_url": "https://cdn.civil.services/us-states/maps/utah-large.png", 921 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/utah.jpg", 922 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/utah.jpg", 923 | "twitter_url": "https://twitter.com/UtahGov", 924 | "facebook_url": "https://www.facebook.com/utahgov" 925 | }, 926 | { 927 | "state": "Vermont", 928 | "slug": "vermont", 929 | "code": "VT", 930 | "nickname": "Green Mountain State", 931 | "website": "http://vermont.gov", 932 | "admission_date": "1791-03-04", 933 | "admission_number": 14, 934 | "capital_city": "Montpelier", 935 | "capital_url": "http://www.montpelier-vt.org", 936 | "population": 626630, 937 | "population_rank": 49, 938 | "constitution_url": "http://www.leg.state.vt.us/statutes/const2.htm", 939 | "state_flag_url": "https://cdn.civil.services/us-states/flags/vermont-large.png", 940 | "state_seal_url": "https://cdn.civil.services/us-states/seals/vermont-large.png", 941 | "map_image_url": "https://cdn.civil.services/us-states/maps/vermont-large.png", 942 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/vermont.jpg", 943 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/vermont.jpg", 944 | "twitter_url": "https://twitter.com/vermontgov", 945 | "facebook_url": "https://www.facebook.com/MyVermont" 946 | }, 947 | { 948 | "state": "Virginia", 949 | "slug": "virginia", 950 | "code": "VA", 951 | "nickname": "Old Dominion State", 952 | "website": "https://www.virginia.gov", 953 | "admission_date": "1788-06-25", 954 | "admission_number": 10, 955 | "capital_city": "Richmond", 956 | "capital_url": "http://www.richmondgov.com", 957 | "population": 8260405, 958 | "population_rank": 12, 959 | "constitution_url": "http://hodcap.state.va.us/publications/Constitution-01-13.pdf", 960 | "state_flag_url": "https://cdn.civil.services/us-states/flags/virginia-large.png", 961 | "state_seal_url": "https://cdn.civil.services/us-states/seals/virginia-large.png", 962 | "map_image_url": "https://cdn.civil.services/us-states/maps/virginia-large.png", 963 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/virginia.jpg", 964 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/virginia.jpg", 965 | "twitter_url": null, 966 | "facebook_url": null 967 | }, 968 | { 969 | "state": "Washington", 970 | "slug": "washington", 971 | "code": "WA", 972 | "nickname": "The Evergreen State", 973 | "website": "http://www.wa.gov", 974 | "admission_date": "1889-11-11", 975 | "admission_number": 42, 976 | "capital_city": "Olympia", 977 | "capital_url": "http://www.ci.olympia.wa.us", 978 | "population": 6971406, 979 | "population_rank": 13, 980 | "constitution_url": "http://www.leg.wa.gov/lawsandagencyrules/pages/constitution.aspx", 981 | "state_flag_url": "https://cdn.civil.services/us-states/flags/washington-large.png", 982 | "state_seal_url": "https://cdn.civil.services/us-states/seals/washington-large.png", 983 | "map_image_url": "https://cdn.civil.services/us-states/maps/washington-large.png", 984 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/washington.jpg", 985 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/washington.jpg", 986 | "twitter_url": "https://twitter.com/wagov", 987 | "facebook_url": "" 988 | }, 989 | { 990 | "state": "West Virginia", 991 | "slug": "west-virginia", 992 | "code": "WV", 993 | "nickname": "Mountain State", 994 | "website": "http://www.wv.gov", 995 | "admission_date": "1863-06-20", 996 | "admission_number": 35, 997 | "capital_city": "Charleston", 998 | "capital_url": "http://www.cityofcharleston.org", 999 | "population": 1854304, 1000 | "population_rank": 38, 1001 | "constitution_url": "http://www.legis.state.wv.us/WVCODE/WV_CON.cfm", 1002 | "state_flag_url": "https://cdn.civil.services/us-states/flags/west-virginia-large.png", 1003 | "state_seal_url": "https://cdn.civil.services/us-states/seals/west-virginia-large.png", 1004 | "map_image_url": "https://cdn.civil.services/us-states/maps/west-virginia-large.png", 1005 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/west-virginia.jpg", 1006 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/west-virginia.jpg", 1007 | "twitter_url": "https://twitter.com/wvgov", 1008 | "facebook_url": "https://www.facebook.com/wvgov" 1009 | }, 1010 | { 1011 | "state": "Wisconsin", 1012 | "slug": "wisconsin", 1013 | "code": "WI", 1014 | "nickname": "Badger State", 1015 | "website": "https://www.wisconsin.gov", 1016 | "admission_date": "1848-05-29", 1017 | "admission_number": 30, 1018 | "capital_city": "Madison", 1019 | "capital_url": "http://www.ci.madison.wi.us", 1020 | "population": 5742713, 1021 | "population_rank": 20, 1022 | "constitution_url": "http://www.legis.state.wi.us/rsb/2wiscon.html", 1023 | "state_flag_url": "https://cdn.civil.services/us-states/flags/wisconsin-large.png", 1024 | "state_seal_url": "https://cdn.civil.services/us-states/seals/wisconsin-large.png", 1025 | "map_image_url": "https://cdn.civil.services/us-states/maps/wisconsin-large.png", 1026 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/wisconsin.jpg", 1027 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/wisconsin.jpg", 1028 | "twitter_url": null, 1029 | "facebook_url": null 1030 | }, 1031 | { 1032 | "state": "Wyoming", 1033 | "slug": "wyoming", 1034 | "code": "WY", 1035 | "nickname": "Equality State", 1036 | "website": "http://www.wyo.gov", 1037 | "admission_date": "1890-07-10", 1038 | "admission_number": 44, 1039 | "capital_city": "Cheyenne", 1040 | "capital_url": "http://www.cheyennecity.org", 1041 | "population": 582658, 1042 | "population_rank": 50, 1043 | "constitution_url": "http://legisweb.state.wy.us/statutes/constitution.aspx?file=titles/97Title97.htm", 1044 | "state_flag_url": "https://cdn.civil.services/us-states/flags/wyoming-large.png", 1045 | "state_seal_url": "https://cdn.civil.services/us-states/seals/wyoming-large.png", 1046 | "map_image_url": "https://cdn.civil.services/us-states/maps/wyoming-large.png", 1047 | "landscape_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/landscape/wyoming.jpg", 1048 | "skyline_background_url": "https://cdn.civil.services/us-states/backgrounds/1280x720/skyline/wyoming.jpg", 1049 | "twitter_url": null, 1050 | "facebook_url": null 1051 | } 1052 | ] -------------------------------------------------------------------------------- /data/us-state-capitals.csv: -------------------------------------------------------------------------------- 1 | name,description,latitude,longitude 2 | Alabama,Montgomery,32.377716,-86.300568 3 | Alaska,Juneau,58.301598,-134.420212 4 | Arizona,Phoenix,33.448143,-112.096962 5 | Arkansas,Little Rock,34.746613,-92.288986 6 | California,Sacramento,38.576668,-121.493629 7 | Colorado,Denver,39.739227,-104.984856 8 | Connecticut,Hartford
,41.764046,-72.682198 9 | Delaware,Dover,39.157307,-75.519722 10 | Hawaii,Honolulu,21.307442,-157.857376 11 | Florida,Tallahassee,30.438118,-84.281296 12 | Georgia,Atlanta
,33.749027,-84.388229 13 | Idaho,Boise,43.617775,-116.199722 14 | Illinois,Springfield,39.798363,-89.654961 15 | Indiana,Indianapolis,39.768623,-86.162643 16 | Iowa,Des Moines,41.591087,-93.603729 17 | Kansas,Topeka,39.048191,-95.677956 18 | Kentucky,Frankfort,38.186722,-84.875374 19 | Louisiana,Baton Rouge,30.457069,-91.187393 20 | Maine,Augusta,44.307167,-69.781693 21 | Maryland,Annapolis,38.978764,-76.490936 22 | Massachusetts,Boston,42.358162,-71.063698 23 | Michigan,Lansing,42.733635,-84.555328 24 | Minnesota,St. Paul,44.955097,-93.102211 25 | Mississippi,Jackson,32.303848,-90.182106 26 | Missouri,Jefferson City,38.579201,-92.172935 27 | Montana,Helena,46.585709,-112.018417 28 | Nebraska,Lincoln,40.808075,-96.699654 29 | Nevada,Carson City,39.163914,-119.766121 30 | New Hampshire,Concord,43.206898,-71.537994 31 | New Jersey,Trenton,40.220596,-74.769913 32 | New Mexico,Santa Fe,35.68224,-105.939728 33 | North Carolina,Raleigh,35.78043,-78.639099 34 | North Dakota,Bismarck,46.82085,-100.783318 35 | New York,Albany,42.652843,-73.757874 36 | Ohio,Columbus,39.961346,-82.999069 37 | Oklahoma,Oklahoma City,35.492207,-97.503342 38 | Oregon,Salem,44.938461,-123.030403 39 | Pennsylvania,Harrisburg,40.264378,-76.883598 40 | Rhode Island,Providence,41.830914,-71.414963 41 | South Carolina,Columbia,34.000343,-81.033211 42 | South Dakota,Pierre,44.367031,-100.346405 43 | Tennessee,Nashville,36.16581,-86.784241 44 | Texas,Austin,30.27467,-97.740349 45 | Utah,Salt Lake City,40.777477,-111.888237 46 | Vermont,Montpelier,44.262436,-72.580536 47 | Virginia,Richmond,37.538857,-77.43364 48 | Washington,Olympia,47.035805,-122.905014 49 | West Virginia,Charleston,38.336246,-81.612328 50 | Wisconsin,Madison,43.074684,-89.384445 51 | Wyoming,Cheyenne,41.140259,-104.820236 -------------------------------------------------------------------------------- /db.py: -------------------------------------------------------------------------------- 1 | """FastImporter App - Database Setup""" 2 | from fastapi import FastAPI 3 | import asyncpg 4 | 5 | import config 6 | 7 | async def connect_to_db(app: FastAPI) -> None: 8 | """ 9 | Connect to all databases. 10 | """ 11 | app.state.databases = {} 12 | for database in config.DATABASES.items(): 13 | app.state.databases[f'{database[0]}_pool'] = await asyncpg.create_pool( 14 | dsn=f"postgres://{database[1]['username']}:{database[1]['password']}@{database[1]['host']}:{database[1]['port']}/{database[0]}", 15 | min_size=1, 16 | max_size=10, 17 | max_queries=50000, 18 | max_inactive_connection_lifetime=300, 19 | timeout=180 # 3 Minutes 20 | ) 21 | 22 | async def close_db_connection(app: FastAPI) -> None: 23 | """ 24 | Close connection for all databases. 25 | """ 26 | for database in config.DATABASES: 27 | await app.state.databases[f'{database}_pool'].close() -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI 2 | from fastapi.middleware.cors import CORSMiddleware 3 | from prometheus_fastapi_instrumentator import Instrumentator 4 | 5 | from routers import imports 6 | import db 7 | 8 | DESCRIPTION = """ 9 | A lightweight python api to import data PostGIS from a multitude of locations. 10 | """ 11 | 12 | app = FastAPI( 13 | title="FastVector", 14 | description=DESCRIPTION, 15 | version="0.0.1", 16 | contact={ 17 | "name": "Michael Keller", 18 | "email": "michaelkeller03@gmail.com", 19 | }, 20 | license_info={ 21 | "name": "The MIT License (MIT)", 22 | "url": "https://mit-license.org/", 23 | }, 24 | ) 25 | 26 | app.add_middleware( 27 | CORSMiddleware, 28 | allow_origins=["*"], 29 | allow_credentials=True, 30 | allow_methods=["*"], 31 | allow_headers=["*"], 32 | ) 33 | 34 | app.include_router( 35 | imports.router, 36 | prefix="/api/v1/import", 37 | tags=["import"], 38 | ) 39 | 40 | @app.on_event("startup") 41 | async def startup_event(): 42 | """Application startup: register the database connection and create table list.""" 43 | await db.connect_to_db(app) 44 | 45 | 46 | @app.on_event("shutdown") 47 | async def shutdown_event(): 48 | """Application shutdown: de-register the database connection.""" 49 | await db.close_db_connection(app) 50 | 51 | Instrumentator().instrument(app).expose(app) 52 | -------------------------------------------------------------------------------- /models.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, Field 2 | 3 | class BaseResponseModel(BaseModel): 4 | process_id: str = Field( 5 | default="472e29dc-91a8-41d3-b05f-cee34006e3f7" 6 | ) 7 | url: str = Field( 8 | default="http://127.0.0.1:8000/api/v1/analysis/status/472e29dc-91a8-41d3-b05f-cee34006e3f7" 9 | ) 10 | 11 | class ArcgisModel(BaseModel): 12 | url: str = Field( 13 | default=None, title="The url that contains the service to download." 14 | ) 15 | token: str = Field( 16 | default=None, title="If endpoint is authenticated, token will be used to download the service." 17 | ) 18 | database: str = Field( 19 | default=None, title="Name of the database the table belongs to." 20 | ) 21 | 22 | class PointJsonUrl(BaseModel): 23 | database: str = Field( 24 | default=None, title="Name of the database the table belongs to." 25 | ) 26 | latitude: str 27 | longitude: str 28 | table_columns: list 29 | url: str 30 | 31 | class GeographicJsonUrl(BaseModel): 32 | database: str = Field( 33 | default=None, title="Name of the database the table belongs to." 34 | ) 35 | map: str 36 | map_column: str 37 | map_columns: list 38 | table_columns: list 39 | table_column: str 40 | url: str 41 | 42 | class GeojsonUrl(BaseModel): 43 | database: str = Field( 44 | default=None, title="Name of the database the table belongs to." 45 | ) 46 | url: str -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | fastapi==0.68.2 2 | asyncpg==0.25.0 3 | uvicorn==0.17.6 4 | prometheus_fastapi_instrumentator==5.8.2 5 | aiohttp==3.8.1 6 | pandas==1.4.2 7 | requests==2.27.1 -------------------------------------------------------------------------------- /routers/imports.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from typing import List 4 | from fastapi import APIRouter, BackgroundTasks, Request, Form 5 | from fastapi import File, UploadFile 6 | import requests 7 | 8 | import utilities 9 | import models 10 | 11 | router = APIRouter() 12 | 13 | import_processes = {} 14 | 15 | @router.get("/status/{process_id}", tags=["import"]) 16 | def status(process_id: str): 17 | if process_id not in import_processes: 18 | return {"status": "UNKNOWN", "error": "This process_id does not exist."} 19 | return import_processes[process_id] 20 | 21 | @router.post("/arcgis_service/", tags=["import"], response_model=models.BaseResponseModel) 22 | async def import_arcgis_service(info: models.ArcgisModel, request: Request, background_tasks: BackgroundTasks): 23 | new_table_id = utilities.get_new_table_id() 24 | 25 | process_id = utilities.get_new_process_id() 26 | 27 | process_url = str(request.base_url) 28 | 29 | process_url += f"api/v1/import/status/{process_id}" 30 | 31 | import_processes[process_id] = { 32 | "status": "PENDING" 33 | } 34 | 35 | background_tasks.add_task( 36 | utilities.get_arcgis_data, 37 | url=info.url, 38 | token=info.token, 39 | new_table_id=new_table_id, 40 | database=info.database, 41 | process_id=process_id 42 | ) 43 | 44 | return { 45 | "process_id": process_id, 46 | "url": process_url 47 | } 48 | 49 | @router.post("/geographic_data_from_geographic_file/", tags=["import"], response_model=models.BaseResponseModel) 50 | async def import_geographic_data_from_geographic_file( 51 | request: Request, 52 | background_tasks: BackgroundTasks, 53 | database: str = Form(...), 54 | files: List[UploadFile] = File(...) 55 | ): 56 | new_table_id = utilities.get_new_table_id() 57 | 58 | process_id = utilities.get_new_process_id() 59 | 60 | process_url = str(request.base_url) 61 | 62 | process_url += f"api/v1/import/status/{process_id}" 63 | 64 | file_path = "" 65 | 66 | for file in files: 67 | try: 68 | file_path = f"{os.getcwd()}/media/{new_table_id}_{file.filename}" 69 | with open(file_path, 'wb') as f: 70 | [f.write(chunk) for chunk in iter(lambda: file.file.read(1000), b'')] 71 | except Exception: 72 | media_directory = os.listdir(f"{os.getcwd()}/media/") 73 | for file in media_directory: 74 | if new_table_id in file: 75 | os.remove(f"{os.getcwd()}/media/{file}") 76 | 77 | return {"message": "There was an error uploading the file(s)"} 78 | finally: 79 | await file.close() 80 | 81 | import_processes[process_id] = { 82 | "status": "PENDING" 83 | } 84 | 85 | background_tasks.add_task( 86 | utilities.upload_geographic_file, 87 | file_path=file_path, 88 | new_table_id=new_table_id, 89 | database=database, 90 | process_id=process_id 91 | ) 92 | 93 | return { 94 | "process_id": process_id, 95 | "url": process_url 96 | } 97 | 98 | @router.post("/geographic_data_from_csv/", tags=["import"], response_model=models.BaseResponseModel) 99 | async def import_geographic_data_from_csv( 100 | request: Request, 101 | background_tasks: BackgroundTasks, 102 | database: str = Form(...), 103 | map: str = Form(...), 104 | map_column: str = Form(...), 105 | map_columns: List = Form(...), 106 | table_column: str = Form(...), 107 | table_columns: List = Form(...), 108 | files: List[UploadFile] = File(...) 109 | ): 110 | new_table_id = utilities.get_new_table_id() 111 | 112 | process_id = utilities.get_new_process_id() 113 | 114 | process_url = str(request.base_url) 115 | 116 | process_url += f"api/v1/import/status/{process_id}" 117 | 118 | file_path = "" 119 | 120 | for file in files: 121 | try: 122 | file_path = f"{os.getcwd()}/media/{new_table_id}_{file.filename}" 123 | with open(file_path, 'wb') as f: 124 | [f.write(chunk) for chunk in iter(lambda: file.file.read(1000), b'')] 125 | except Exception: 126 | media_directory = os.listdir(f"{os.getcwd()}/media/") 127 | for file in media_directory: 128 | if new_table_id in file: 129 | os.remove(f"{os.getcwd()}/media/{file}") 130 | 131 | return {"message": "There was an error uploading the file(s)"} 132 | finally: 133 | await file.close() 134 | 135 | import_processes[process_id] = { 136 | "status": "PENDING" 137 | } 138 | 139 | background_tasks.add_task( 140 | utilities.import_geographic_data_from_csv, 141 | file_path=file_path, 142 | new_table_id=new_table_id, 143 | database=database, 144 | map_column=map_column, 145 | table_column=table_column, 146 | map_columns=json.loads(map_columns[0]), 147 | table_columns=json.loads(table_columns[0]), 148 | map=map, 149 | process_id=process_id, 150 | app=request.app 151 | ) 152 | 153 | return { 154 | "process_id": process_id, 155 | "url": process_url 156 | } 157 | 158 | @router.post("/point_data_from_csv/", tags=["import"], response_model=models.BaseResponseModel) 159 | async def import_point_data_from_csv( 160 | request: Request, 161 | background_tasks: BackgroundTasks, 162 | database: str = Form(...), 163 | latitude: str = Form(...), 164 | longitude: str = Form(...), 165 | table_columns: List = Form(...), 166 | files: List[UploadFile] = File(...) 167 | ): 168 | new_table_id = utilities.get_new_table_id() 169 | 170 | process_id = utilities.get_new_process_id() 171 | 172 | process_url = str(request.base_url) 173 | 174 | process_url += f"api/v1/import/status/{process_id}" 175 | 176 | file_path = "" 177 | 178 | for file in files: 179 | try: 180 | file_path = f"{os.getcwd()}/media/{new_table_id}_{file.filename}" 181 | with open(file_path, 'wb') as f: 182 | [f.write(chunk) for chunk in iter(lambda: file.file.read(1000), b'')] 183 | except Exception: 184 | media_directory = os.listdir(f"{os.getcwd()}/media/") 185 | for file in media_directory: 186 | if new_table_id in file: 187 | os.remove(f"{os.getcwd()}/media/{file}") 188 | 189 | return {"message": "There was an error uploading the file(s)"} 190 | finally: 191 | await file.close() 192 | 193 | import_processes[process_id] = { 194 | "status": "PENDING" 195 | } 196 | 197 | background_tasks.add_task( 198 | utilities.import_point_data_from_csv, 199 | file_path=file_path, 200 | new_table_id=new_table_id, 201 | database=database, 202 | latitude=latitude, 203 | longitude=longitude, 204 | table_columns=json.loads(table_columns[0]), 205 | process_id=process_id, 206 | app=request.app 207 | ) 208 | 209 | return { 210 | "process_id": process_id, 211 | "url": process_url 212 | } 213 | 214 | @router.post("/geographic_data_from_json_file/", tags=["import"], response_model=models.BaseResponseModel) 215 | async def import_geographic_data_from_json_file( 216 | request: Request, 217 | background_tasks: BackgroundTasks, 218 | database: str = Form(...), 219 | map: str = Form(...), 220 | map_column: str = Form(...), 221 | map_columns: List = Form(...), 222 | table_column: str = Form(...), 223 | table_columns: List = Form(...), 224 | files: List[UploadFile] = File(...) 225 | ): 226 | new_table_id = utilities.get_new_table_id() 227 | 228 | process_id = utilities.get_new_process_id() 229 | 230 | process_url = str(request.base_url) 231 | 232 | process_url += f"api/v1/import/status/{process_id}" 233 | 234 | file_path = "" 235 | 236 | for file in files: 237 | try: 238 | file_path = f"{os.getcwd()}/media/{new_table_id}_{file.filename}" 239 | with open(file_path, 'wb') as f: 240 | [f.write(chunk) for chunk in iter(lambda: file.file.read(1000), b'')] 241 | except Exception: 242 | media_directory = os.listdir(f"{os.getcwd()}/media/") 243 | for file in media_directory: 244 | if new_table_id in file: 245 | os.remove(f"{os.getcwd()}/media/{file}") 246 | 247 | return {"message": "There was an error uploading the file(s)"} 248 | finally: 249 | await file.close() 250 | 251 | import_processes[process_id] = { 252 | "status": "PENDING" 253 | } 254 | 255 | background_tasks.add_task( 256 | utilities.import_geographic_data_from_json_file, 257 | file_path=file_path, 258 | new_table_id=new_table_id, 259 | database=database, 260 | map_column=map_column, 261 | table_column=table_column, 262 | map_columns=json.loads(map_columns[0]), 263 | table_columns=json.loads(table_columns[0]), 264 | map=map, 265 | process_id=process_id, 266 | app=request.app 267 | ) 268 | 269 | return { 270 | "process_id": process_id, 271 | "url": process_url 272 | } 273 | 274 | @router.post("/point_data_from_json_file/", tags=["import"], response_model=models.BaseResponseModel) 275 | async def import_point_data_from_json_file( 276 | request: Request, 277 | background_tasks: BackgroundTasks, 278 | database: str = Form(...), 279 | latitude: str = Form(...), 280 | longitude: str = Form(...), 281 | table_columns: List = Form(...), 282 | files: List[UploadFile] = File(...) 283 | ): 284 | new_table_id = utilities.get_new_table_id() 285 | 286 | process_id = utilities.get_new_process_id() 287 | 288 | process_url = str(request.base_url) 289 | 290 | process_url += f"api/v1/import/status/{process_id}" 291 | 292 | file_path = "" 293 | 294 | for file in files: 295 | try: 296 | file_path = f"{os.getcwd()}/media/{new_table_id}_{file.filename}" 297 | with open(file_path, 'wb') as f: 298 | [f.write(chunk) for chunk in iter(lambda: file.file.read(1000), b'')] 299 | except Exception: 300 | media_directory = os.listdir(f"{os.getcwd()}/media/") 301 | for file in media_directory: 302 | if new_table_id in file: 303 | os.remove(f"{os.getcwd()}/media/{file}") 304 | 305 | return {"message": "There was an error uploading the file(s)"} 306 | finally: 307 | await file.close() 308 | 309 | import_processes[process_id] = { 310 | "status": "PENDING" 311 | } 312 | 313 | background_tasks.add_task( 314 | utilities.import_point_data_from_json_file, 315 | file_path=file_path, 316 | new_table_id=new_table_id, 317 | database=database, 318 | latitude=latitude, 319 | longitude=longitude, 320 | table_columns=json.loads(table_columns[0]), 321 | process_id=process_id, 322 | app=request.app 323 | ) 324 | 325 | return { 326 | "process_id": process_id, 327 | "url": process_url 328 | } 329 | 330 | @router.post("/geographic_data_from_json_url/", tags=["import"], response_model=models.BaseResponseModel) 331 | async def import_geographic_data_from_json_url( 332 | request: Request, 333 | background_tasks: BackgroundTasks, 334 | info: models.GeographicJsonUrl 335 | ): 336 | new_table_id = utilities.get_new_table_id() 337 | 338 | process_id = utilities.get_new_process_id() 339 | 340 | process_url = str(request.base_url) 341 | 342 | process_url += f"api/v1/import/status/{process_id}" 343 | 344 | resp = requests.get(info.url) 345 | 346 | file_path = f"{os.getcwd()}/media/{new_table_id}.json" 347 | 348 | with open(f"{os.getcwd()}/media/{new_table_id}.json", "w") as my_file: 349 | my_file.write(resp.text) 350 | 351 | import_processes[process_id] = { 352 | "status": "PENDING" 353 | } 354 | 355 | background_tasks.add_task( 356 | utilities.import_geographic_data_from_json_file, 357 | file_path=file_path, 358 | new_table_id=new_table_id, 359 | database=info.database, 360 | map_column=info.map_column, 361 | table_column=info.table_column, 362 | map_columns=info.map_columns, 363 | table_columns=info.table_columns, 364 | map=info.map, 365 | process_id=process_id, 366 | app=request.app 367 | ) 368 | 369 | return { 370 | "process_id": process_id, 371 | "url": process_url 372 | } 373 | 374 | @router.post("/point_data_from_json_url/", tags=["import"], response_model=models.BaseResponseModel) 375 | async def import_point_data_from_json_url( 376 | request: Request, 377 | background_tasks: BackgroundTasks, 378 | info: models.PointJsonUrl 379 | ): 380 | new_table_id = utilities.get_new_table_id() 381 | 382 | process_id = utilities.get_new_process_id() 383 | 384 | process_url = str(request.base_url) 385 | 386 | process_url += f"api/v1/import/status/{process_id}" 387 | 388 | resp = requests.get(info.url) 389 | 390 | file_path = f"{os.getcwd()}/media/{new_table_id}.json" 391 | 392 | with open(f"{os.getcwd()}/media/{new_table_id}.json", "w") as my_file: 393 | my_file.write(resp.text) 394 | 395 | import_processes[process_id] = { 396 | "status": "PENDING" 397 | } 398 | 399 | background_tasks.add_task( 400 | utilities.import_point_data_from_json_file, 401 | file_path=file_path, 402 | new_table_id=new_table_id, 403 | database=info.database, 404 | latitude=info.latitude, 405 | longitude=info.longitude, 406 | table_columns=info.table_columns, 407 | process_id=process_id, 408 | app=request.app 409 | ) 410 | 411 | return { 412 | "process_id": process_id, 413 | "url": process_url 414 | } 415 | 416 | @router.post("/geojson_from_url/", tags=["import"], response_model=models.BaseResponseModel) 417 | async def import_geojson_from_url( 418 | request: Request, 419 | background_tasks: BackgroundTasks, 420 | info: models.GeojsonUrl 421 | ): 422 | new_table_id = utilities.get_new_table_id() 423 | 424 | process_id = utilities.get_new_process_id() 425 | 426 | process_url = str(request.base_url) 427 | 428 | process_url += f"api/v1/import/status/{process_id}" 429 | 430 | resp = requests.get(info.url) 431 | 432 | file_path = f"{os.getcwd()}/media/{new_table_id}.geojson" 433 | 434 | with open(f"{os.getcwd()}/media/{new_table_id}.geojson", "w") as my_file: 435 | my_file.write(resp.text) 436 | 437 | import_processes[process_id] = { 438 | "status": "PENDING" 439 | } 440 | 441 | background_tasks.add_task( 442 | utilities.upload_geographic_file, 443 | file_path=file_path, 444 | new_table_id=new_table_id, 445 | database=info.database, 446 | process_id=process_id 447 | ) 448 | 449 | return { 450 | "process_id": process_id, 451 | "url": process_url 452 | } -------------------------------------------------------------------------------- /utilities.py: -------------------------------------------------------------------------------- 1 | import random 2 | import os 3 | import re 4 | import string 5 | import uuid 6 | import datetime 7 | import subprocess 8 | import json 9 | import aiohttp 10 | import pandas as pd 11 | from fastapi import FastAPI 12 | 13 | import config 14 | from routers import imports 15 | 16 | def get_new_table_id() -> str: 17 | """ 18 | Method to return a new table id 19 | """ 20 | letters = string.ascii_lowercase 21 | 22 | return ''.join(random.choice(letters) for i in range(50)) 23 | 24 | def get_new_process_id() -> str: 25 | """ 26 | Method to return a new process id 27 | """ 28 | 29 | return str(uuid.uuid4()) 30 | 31 | def remove_bad_characters(string: str) -> str: 32 | regex = re.compile('[^a-zA-Z0-9_]') 33 | return regex.sub('_', string).lower() 34 | 35 | async def upload_csv_to_db_with_latitude_and_longitude(file_path: str, new_table_id: str, database: str, 36 | latitude: str, longitude: str, table_columns: list, app: FastAPI): 37 | """ 38 | Method to upload data from from a csv file with latitude and longitude columns into db. 39 | 40 | """ 41 | 42 | pd.options.display.max_rows = 10 43 | 44 | df = pd.read_csv(file_path) 45 | 46 | columns = "" 47 | 48 | formatted_table_columns = "" 49 | 50 | for col in table_columns: 51 | formatted_table_columns += f"{remove_bad_characters(col)}," 52 | 53 | formatted_table_columns = formatted_table_columns[:-1] 54 | 55 | create_table_sql = f"CREATE TABLE {new_table_id} (" 56 | 57 | for name, dtype in df.dtypes.iteritems(): 58 | columns += f"{remove_bad_characters(name)}," 59 | create_table_sql += f'"{remove_bad_characters(name)}"' 60 | if dtype == "object" or dtype == "datetime64": 61 | create_table_sql += " text," 62 | if dtype == "int64": 63 | create_table_sql += " integer," 64 | if dtype == "float64": 65 | create_table_sql += " double precision," 66 | 67 | create_table_sql = create_table_sql[:-1] 68 | 69 | columns = columns[:-1] 70 | 71 | create_table_sql += ");" 72 | 73 | pool = app.state.databases[f'{database}_pool'] 74 | 75 | async with pool.acquire() as con: 76 | await con.fetch(f"""DROP TABLE IF EXISTS "{new_table_id}";""") 77 | 78 | await con.fetch(create_table_sql) 79 | 80 | insert_sql = f"""COPY {new_table_id}({columns}) 81 | FROM '{file_path}' 82 | DELIMITER ',' 83 | CSV HEADER;""" 84 | 85 | await con.fetch(insert_sql) 86 | 87 | add_geom_sql = f""" 88 | SELECT AddGeometryColumn ('public','{new_table_id}','geom',4326,'POINT',2); 89 | """ 90 | 91 | await con.fetch(add_geom_sql) 92 | 93 | update_geom_sql = f""" 94 | UPDATE "{new_table_id}" 95 | SET geom = ST_SetSRID(ST_MakePoint({longitude},{latitude}), 4326); 96 | """ 97 | 98 | await con.fetch(update_geom_sql) 99 | 100 | delete_bad_geom_sql = f""" 101 | DELETE FROM "{new_table_id}" WHERE geom IS NULL; 102 | """ 103 | 104 | await con.fetch(delete_bad_geom_sql) 105 | 106 | async def upload_csv_to_db_with_geographic_data(file_path: str, new_table_id: str, database: str, 107 | map: str, map_column: str, table_column: str, table_columns: list, map_columns: list, app: FastAPI): 108 | """ 109 | Method to upload data from from a csv file with geographic data into db. 110 | 111 | """ 112 | 113 | pd.options.display.max_rows = 10 114 | 115 | df = pd.read_csv(file_path) 116 | 117 | columns = "" 118 | 119 | formatted_table_columns = "" 120 | 121 | formatted_map_columns = "" 122 | 123 | for col in table_columns: 124 | if col not in map_columns: 125 | formatted_table_columns += f"a.{remove_bad_characters(col)}," 126 | 127 | for column in map_columns: 128 | formatted_map_columns += f"b.{remove_bad_characters(column)}," 129 | 130 | create_table_sql = f"CREATE TABLE {new_table_id}_temp (" 131 | 132 | for name, dtype in df.dtypes.iteritems(): 133 | columns += f"{remove_bad_characters(name)}," 134 | create_table_sql += f'"{remove_bad_characters(name)}"' 135 | if dtype == "object" or dtype == "datetime64": 136 | create_table_sql += " text," 137 | if dtype == "int64": 138 | create_table_sql += " integer," 139 | if dtype == "float64": 140 | create_table_sql += " double precision," 141 | 142 | create_table_sql = create_table_sql[:-1] 143 | columns = columns[:-1] 144 | 145 | create_table_sql += ");" 146 | 147 | pool = app.state.databases[f'{database}_pool'] 148 | 149 | async with pool.acquire() as con: 150 | await con.fetch(f"""DROP TABLE IF EXISTS "{new_table_id}_temp";""") 151 | 152 | await con.fetch(create_table_sql) 153 | 154 | insert_sql = f"""COPY {new_table_id}_temp({columns}) 155 | FROM '{file_path}' 156 | DELIMITER ',' 157 | CSV HEADER;""" 158 | 159 | await con.fetch(insert_sql) 160 | 161 | join_sql = f"""CREATE TABLE "{new_table_id}" AS 162 | SELECT {formatted_table_columns} {formatted_map_columns} geom 163 | FROM "{new_table_id}_temp" as a 164 | LEFT JOIN "{map}" as b 165 | ON a."{table_column}" = b."{map_column}"; 166 | """ 167 | 168 | await con.fetch(join_sql) 169 | 170 | await con.fetch(f"""DROP TABLE IF EXISTS "{new_table_id}_temp";""") 171 | 172 | async def get_arcgis_data(url: str, new_table_id: str, process_id: str, database: str, token: str=None): 173 | """ 174 | Method get arcgis data from a given url and load it into a database. 175 | """ 176 | 177 | start = datetime.datetime.now() 178 | 179 | try: 180 | service_url = f"{url}?f=json" 181 | 182 | if token is not None: 183 | service_url += f"&token={token}" 184 | 185 | async with aiohttp.ClientSession() as session: 186 | 187 | async with session.get(service_url) as resp: 188 | 189 | data = await resp.json() 190 | 191 | max_number_of_features_per_query = data['maxRecordCount'] 192 | 193 | feature_stats_url = f"{url}/query?where=1%3D1&returnGeometry=false&returnIdsOnly=true&f=json&resultRecordCount=10" 194 | 195 | async with session.get(feature_stats_url) as feature_resp: 196 | 197 | data = await feature_resp.text() 198 | 199 | data = json.loads(data) 200 | 201 | object_ids = data['objectIds'] 202 | 203 | number_of_features = len(data['objectIds']) 204 | 205 | if number_of_features <= max_number_of_features_per_query: 206 | 207 | async with session.get(f"{url}/query?where=1=1&outFields=*&returnGeometry=true&geometryPrecision=6&outSR=4326&f=geojson") as resp: 208 | 209 | data = await resp.json() 210 | 211 | with open(f'{new_table_id}.geojson', 'w') as json_file: 212 | json.dump(data, json_file) 213 | 214 | load_geographic_data_to_server( 215 | table_id=new_table_id, 216 | file_path=f'{new_table_id}.geojson', 217 | database=database 218 | ) 219 | 220 | else: 221 | start_counter = 0 222 | 223 | feature_collection = { 224 | "type": "FeatureCollection", 225 | "features": [] 226 | } 227 | 228 | for x in range( start_counter, number_of_features, max_number_of_features_per_query ): 229 | ids_requested = object_ids[x: x + max_number_of_features_per_query ] 230 | payload = { 'f': 'geojson', 'where': '1=1', 231 | 'objectIds': str( ids_requested )[1:-1], 'outSR': '4326', 232 | 'returnGeometry': 'true', 'outFields': '*', 233 | 'geometryPrecision': '4'} 234 | 235 | async with session.post( f"{url}/query", data=payload ) as resp: 236 | 237 | data = await resp.text() 238 | 239 | data = json.loads(data) 240 | 241 | if 'error' in data: 242 | print(data['error']) 243 | 244 | feature_collection['features'] += data['features'] 245 | 246 | with open(f'{new_table_id}.geojson', 'w') as json_file: 247 | json.dump(feature_collection, json_file) 248 | 249 | load_geographic_data_to_server( 250 | table_id=new_table_id, 251 | file_path=f'{new_table_id}.geojson', 252 | database=database 253 | ) 254 | os.remove(f'{new_table_id}.geojson') 255 | imports.import_processes[process_id]['status'] = "SUCCESS" 256 | imports.import_processes[process_id]['new_table_id'] = new_table_id 257 | imports.import_processes[process_id]['completion_time'] = datetime.datetime.now() 258 | imports.import_processes[process_id]['run_time_in_seconds'] = datetime.datetime.now()-start 259 | except Exception as error: 260 | if os.path.exists(f'{new_table_id}.geojson'): 261 | os.remove(f'{new_table_id}.geojson') 262 | imports.import_processes[process_id]['status'] = "FAILURE" 263 | imports.import_processes[process_id]['error'] = str(error) 264 | imports.import_processes[process_id]['completion_time'] = datetime.datetime.now() 265 | imports.import_processes[process_id]['run_time_in_seconds'] = datetime.datetime.now()-start 266 | 267 | async def upload_geographic_file(file_path: str, new_table_id: str, process_id: str, database: str): 268 | """ 269 | Method to upload data from geographic file. 270 | 271 | """ 272 | 273 | start = datetime.datetime.now() 274 | 275 | try: 276 | load_geographic_data_to_server( 277 | table_id=new_table_id, 278 | file_path=file_path, 279 | database=database 280 | ) 281 | media_directory = os.listdir(f"{os.getcwd()}/media/") 282 | for file in media_directory: 283 | if new_table_id in file: 284 | os.remove(f"{os.getcwd()}/media/{file}") 285 | imports.import_processes[process_id]['status'] = "SUCCESS" 286 | imports.import_processes[process_id]['new_table_id'] = new_table_id 287 | imports.import_processes[process_id]['completion_time'] = datetime.datetime.now() 288 | imports.import_processes[process_id]['run_time_in_seconds'] = datetime.datetime.now()-start 289 | except Exception as error: 290 | media_directory = os.listdir(f"{os.getcwd()}/media/") 291 | for file in media_directory: 292 | if new_table_id in file: 293 | os.remove(f"{os.getcwd()}/media/{file}") 294 | imports.import_processes[process_id]['status'] = "FAILURE" 295 | imports.import_processes[process_id]['error'] = str(error) 296 | imports.import_processes[process_id]['completion_time'] = datetime.datetime.now() 297 | imports.import_processes[process_id]['run_time_in_seconds'] = datetime.datetime.now()-start 298 | 299 | async def import_geographic_data_from_csv(file_path: str, new_table_id: str, process_id: str, database: str, 300 | map: str, map_column: str, table_column: str, table_columns: list, map_columns: list, app: FastAPI): 301 | """ 302 | Method to upload data from from a csv file with geographic data. 303 | 304 | """ 305 | 306 | start = datetime.datetime.now() 307 | 308 | try: 309 | await upload_csv_to_db_with_geographic_data( 310 | file_path=file_path, 311 | new_table_id=new_table_id, 312 | database=database, 313 | map=map, 314 | map_column=map_column, 315 | table_column=table_column, 316 | table_columns=table_columns, 317 | map_columns=map_columns, 318 | app=app 319 | ) 320 | 321 | media_directory = os.listdir(f"{os.getcwd()}/media/") 322 | for file in media_directory: 323 | if new_table_id in file: 324 | os.remove(f"{os.getcwd()}/media/{file}") 325 | imports.import_processes[process_id]['status'] = "SUCCESS" 326 | imports.import_processes[process_id]['new_table_id'] = new_table_id 327 | imports.import_processes[process_id]['completion_time'] = datetime.datetime.now() 328 | imports.import_processes[process_id]['run_time_in_seconds'] = datetime.datetime.now()-start 329 | except Exception as error: 330 | media_directory = os.listdir(f"{os.getcwd()}/media/") 331 | for file in media_directory: 332 | if new_table_id in file: 333 | os.remove(f"{os.getcwd()}/media/{file}") 334 | imports.import_processes[process_id]['status'] = "FAILURE" 335 | imports.import_processes[process_id]['error'] = str(error) 336 | imports.import_processes[process_id]['completion_time'] = datetime.datetime.now() 337 | imports.import_processes[process_id]['run_time_in_seconds'] = datetime.datetime.now()-start 338 | 339 | async def import_point_data_from_csv(file_path: str, new_table_id: str, process_id: str, database: str, 340 | latitude: str, longitude: str, table_columns: list, app: FastAPI): 341 | """ 342 | Method to upload data from csv with lat lng columns. 343 | 344 | """ 345 | 346 | start = datetime.datetime.now() 347 | 348 | try: 349 | await upload_csv_to_db_with_latitude_and_longitude( 350 | file_path=file_path, 351 | new_table_id=new_table_id, 352 | database=database, 353 | latitude=latitude, 354 | longitude=longitude, 355 | table_columns=table_columns, 356 | app=app 357 | ) 358 | 359 | media_directory = os.listdir(f"{os.getcwd()}/media/") 360 | for file in media_directory: 361 | if new_table_id in file: 362 | os.remove(f"{os.getcwd()}/media/{file}") 363 | 364 | imports.import_processes[process_id]['status'] = "SUCCESS" 365 | imports.import_processes[process_id]['new_table_id'] = new_table_id 366 | imports.import_processes[process_id]['completion_time'] = datetime.datetime.now() 367 | imports.import_processes[process_id]['run_time_in_seconds'] = datetime.datetime.now()-start 368 | except Exception as error: 369 | media_directory = os.listdir(f"{os.getcwd()}/media/") 370 | for file in media_directory: 371 | if new_table_id in file: 372 | os.remove(f"{os.getcwd()}/media/{file}") 373 | imports.import_processes[process_id]['status'] = "FAILURE" 374 | imports.import_processes[process_id]['error'] = str(error) 375 | imports.import_processes[process_id]['completion_time'] = datetime.datetime.now() 376 | imports.import_processes[process_id]['run_time_in_seconds'] = datetime.datetime.now()-start 377 | 378 | async def import_point_data_from_json_file(file_path: str, new_table_id: str, process_id: str, database: str, 379 | latitude: str, longitude: str, table_columns: list, app: FastAPI): 380 | """ 381 | Method to upload data from csv with lat lng columns. 382 | 383 | """ 384 | 385 | start = datetime.datetime.now() 386 | 387 | try: 388 | df = pd.read_json(file_path) 389 | 390 | df.to_csv(f"{os.getcwd()}/media/{new_table_id}.csv", index=False, sep=',', encoding="utf-8") 391 | 392 | await upload_csv_to_db_with_latitude_and_longitude( 393 | file_path=f"{os.getcwd()}/media/{new_table_id}.csv", 394 | new_table_id=new_table_id, 395 | database=database, 396 | latitude=latitude, 397 | longitude=longitude, 398 | table_columns=table_columns, 399 | app=app 400 | ) 401 | 402 | media_directory = os.listdir(f"{os.getcwd()}/media/") 403 | for file in media_directory: 404 | if new_table_id in file: 405 | os.remove(f"{os.getcwd()}/media/{file}") 406 | imports.import_processes[process_id]['status'] = "SUCCESS" 407 | imports.import_processes[process_id]['new_table_id'] = new_table_id 408 | imports.import_processes[process_id]['completion_time'] = datetime.datetime.now() 409 | imports.import_processes[process_id]['run_time_in_seconds'] = datetime.datetime.now()-start 410 | except Exception as error: 411 | media_directory = os.listdir(f"{os.getcwd()}/media/") 412 | for file in media_directory: 413 | if new_table_id in file: 414 | os.remove(f"{os.getcwd()}/media/{file}") 415 | imports.import_processes[process_id]['status'] = "FAILURE" 416 | imports.import_processes[process_id]['error'] = str(error) 417 | imports.import_processes[process_id]['completion_time'] = datetime.datetime.now() 418 | imports.import_processes[process_id]['run_time_in_seconds'] = datetime.datetime.now()-start 419 | 420 | async def import_geographic_data_from_json_file(file_path: str, new_table_id: str, process_id: str, database: str, 421 | map: str, map_column: str, table_column: str, table_columns: list, map_columns: list, app: FastAPI): 422 | """ 423 | Method to upload data from from a json file with geographic data. 424 | 425 | """ 426 | 427 | start = datetime.datetime.now() 428 | 429 | try: 430 | df = pd.read_json(file_path) 431 | 432 | df.to_csv(f"{os.getcwd()}/media/{new_table_id}.csv", index=False, sep=',', encoding="utf-8") 433 | 434 | await upload_csv_to_db_with_geographic_data( 435 | file_path=f"{os.getcwd()}/media/{new_table_id}.csv", 436 | new_table_id=new_table_id, 437 | database=database, 438 | map=map, 439 | map_column=map_column, 440 | table_column=table_column, 441 | table_columns=table_columns, 442 | map_columns=map_columns, 443 | app=app 444 | ) 445 | 446 | media_directory = os.listdir(f"{os.getcwd()}/media/") 447 | for file in media_directory: 448 | if new_table_id in file: 449 | os.remove(f"{os.getcwd()}/media/{file}") 450 | imports.import_processes[process_id]['status'] = "SUCCESS" 451 | imports.import_processes[process_id]['new_table_id'] = new_table_id 452 | imports.import_processes[process_id]['completion_time'] = datetime.datetime.now() 453 | imports.import_processes[process_id]['run_time_in_seconds'] = datetime.datetime.now()-start 454 | except Exception as error: 455 | media_directory = os.listdir(f"{os.getcwd()}/media/") 456 | for file in media_directory: 457 | if new_table_id in file: 458 | os.remove(f"{os.getcwd()}/media/{file}") 459 | imports.import_processes[process_id]['status'] = "FAILURE" 460 | imports.import_processes[process_id]['error'] = str(error) 461 | imports.import_processes[process_id]['completion_time'] = datetime.datetime.now() 462 | imports.import_processes[process_id]['run_time_in_seconds'] = datetime.datetime.now()-start 463 | 464 | def load_geographic_data_to_server(table_id: str, file_path: str, database: object): 465 | 466 | db = config.DATABASES[database] 467 | host = db['host'] 468 | username = db['username'] 469 | password = db['password'] 470 | database = db['database'] 471 | subprocess.call(f'ogr2ogr -f "PostgreSQL" "PG:host={host} user={username} dbname={database} password={password}" "{file_path}" -lco GEOMETRY_NAME=geom -lco FID=gid -lco PRECISION=no -nln {table_id} -overwrite', shell=True) 472 | --------------------------------------------------------------------------------