├── .gitignore ├── Py ├── primer │ ├── config.py │ ├── census_tracts_poly.zip │ ├── environment.yml │ ├── morans_i.py │ ├── README.md │ └── rest_api.py ├── stocks │ ├── sp500.xlsx │ ├── README.md │ ├── stocks.py │ └── stocktotalreturnsp500.py ├── chime │ ├── screenshot.png │ ├── CHIMEDataSet_Example.csv │ ├── README.md │ └── CHIME.ipynb ├── 3d-plots │ ├── README.md │ ├── 3Dsurfaceplot.py │ └── 3Dvolumetricplot.py ├── x-y-plot │ ├── README.md │ └── plot.py ├── parks │ ├── README.md │ ├── parks.py │ └── parks.csv ├── addresses │ ├── address_dataframe.py │ └── README.md ├── horsemeat │ ├── README.md │ ├── Countries.csv │ ├── horsemeatexport.py │ ├── transform_table.py │ └── HorsemeatSimple.csv ├── snowfall │ ├── README.md │ ├── snowfall.py │ └── snowfalloriginal.py ├── top-and-bottom │ └── README.md ├── watershed-river-flow │ ├── README.md │ └── GRCA_river_flow.ipynb ├── electric-vehicles │ └── README.md └── database_connections │ └── README.md ├── diagrams ├── connection.png ├── post_kernel.png ├── pre_kernel.png ├── scripting_icon.png ├── jkg-client-diagram.png ├── jkg-cloud-diagram.png ├── jkg-colocated-diagram.png ├── jkg-dedicated-diagram.png ├── jkg-desktop-diagram.png ├── run.svg ├── restart.svg ├── import-file.svg ├── delete-cell.svg ├── switch-connections.svg ├── create-card.svg ├── help.svg ├── export-save.svg ├── insert-cell.svg ├── scripting-console.svg ├── add-model.svg └── run-all.svg ├── assets ├── Chrome │ ├── chrome-1.png │ ├── chrome-2.png │ └── chrome-3.png ├── Safari │ ├── safari-1.png │ ├── safari-2.png │ ├── safari-3.png │ ├── safari-4.png │ └── safari-5.png ├── Firefox │ ├── firefox-1.png │ ├── firefox-2.png │ └── firefox-3.png └── download_repo_as_zip.png ├── R ├── drag-n-drop-charting │ ├── plot_R.png │ ├── qq_plot_R.png │ ├── bar_plot_R.png │ ├── histogram_R.png │ ├── pie_chart_R.png │ └── README.md ├── orange-tree-growth │ ├── README.md │ └── orange_tree_growth.r └── miles-per-gallon-scatter-plot │ ├── README.md │ └── mpg_mapping.r ├── gateway ├── insights-base.yml ├── insights-latest.yml ├── insights-2022.yml ├── README.md └── selfsign.py ├── diagrams.md ├── LICENSE ├── README.md └── README_OLD.md /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store -------------------------------------------------------------------------------- /Py/primer/config.py: -------------------------------------------------------------------------------- 1 | apptoken="yourapikey" -------------------------------------------------------------------------------- /Py/stocks/sp500.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/Py/stocks/sp500.xlsx -------------------------------------------------------------------------------- /Py/chime/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/Py/chime/screenshot.png -------------------------------------------------------------------------------- /diagrams/connection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/diagrams/connection.png -------------------------------------------------------------------------------- /diagrams/post_kernel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/diagrams/post_kernel.png -------------------------------------------------------------------------------- /diagrams/pre_kernel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/diagrams/pre_kernel.png -------------------------------------------------------------------------------- /assets/Chrome/chrome-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/assets/Chrome/chrome-1.png -------------------------------------------------------------------------------- /assets/Chrome/chrome-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/assets/Chrome/chrome-2.png -------------------------------------------------------------------------------- /assets/Chrome/chrome-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/assets/Chrome/chrome-3.png -------------------------------------------------------------------------------- /assets/Safari/safari-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/assets/Safari/safari-1.png -------------------------------------------------------------------------------- /assets/Safari/safari-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/assets/Safari/safari-2.png -------------------------------------------------------------------------------- /assets/Safari/safari-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/assets/Safari/safari-3.png -------------------------------------------------------------------------------- /assets/Safari/safari-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/assets/Safari/safari-4.png -------------------------------------------------------------------------------- /assets/Safari/safari-5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/assets/Safari/safari-5.png -------------------------------------------------------------------------------- /assets/Firefox/firefox-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/assets/Firefox/firefox-1.png -------------------------------------------------------------------------------- /assets/Firefox/firefox-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/assets/Firefox/firefox-2.png -------------------------------------------------------------------------------- /assets/Firefox/firefox-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/assets/Firefox/firefox-3.png -------------------------------------------------------------------------------- /diagrams/scripting_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/diagrams/scripting_icon.png -------------------------------------------------------------------------------- /assets/download_repo_as_zip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/assets/download_repo_as_zip.png -------------------------------------------------------------------------------- /diagrams/jkg-client-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/diagrams/jkg-client-diagram.png -------------------------------------------------------------------------------- /diagrams/jkg-cloud-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/diagrams/jkg-cloud-diagram.png -------------------------------------------------------------------------------- /Py/primer/census_tracts_poly.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/Py/primer/census_tracts_poly.zip -------------------------------------------------------------------------------- /R/drag-n-drop-charting/plot_R.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/R/drag-n-drop-charting/plot_R.png -------------------------------------------------------------------------------- /diagrams/jkg-colocated-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/diagrams/jkg-colocated-diagram.png -------------------------------------------------------------------------------- /diagrams/jkg-dedicated-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/diagrams/jkg-dedicated-diagram.png -------------------------------------------------------------------------------- /diagrams/jkg-desktop-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/diagrams/jkg-desktop-diagram.png -------------------------------------------------------------------------------- /R/drag-n-drop-charting/qq_plot_R.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/R/drag-n-drop-charting/qq_plot_R.png -------------------------------------------------------------------------------- /R/drag-n-drop-charting/bar_plot_R.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/R/drag-n-drop-charting/bar_plot_R.png -------------------------------------------------------------------------------- /R/drag-n-drop-charting/histogram_R.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/R/drag-n-drop-charting/histogram_R.png -------------------------------------------------------------------------------- /R/drag-n-drop-charting/pie_chart_R.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Esri/insights-scripting-guide/HEAD/R/drag-n-drop-charting/pie_chart_R.png -------------------------------------------------------------------------------- /Py/chime/CHIMEDataSet_Example.csv: -------------------------------------------------------------------------------- 1 | countyname,unacast_ch,hospitaliz,population,hospital_1 2 | string,20,1,18000,2.99725 3 | string,21,3,76500,3.06657232 -------------------------------------------------------------------------------- /Py/3d-plots/README.md: -------------------------------------------------------------------------------- 1 | ## Create 3D charts using matplotlib 2 | 3 | The python scripts within this folder create different types of 3D charts. For simplicity, data values have been hard coded in each script. 4 | -------------------------------------------------------------------------------- /Py/x-y-plot/README.md: -------------------------------------------------------------------------------- 1 | ## Python Matplotlib chart 2 | 3 | This is a beginner sample. Using the matplotlib, the sample plots values to create a line graph. It also creates a label on the y-axis. Data values are hard coded in this script. -------------------------------------------------------------------------------- /Py/parks/README.md: -------------------------------------------------------------------------------- 1 | ## Charts for National Parks, stack plot sample 2 | 3 | The [parks.py](parks.py) script reads a CSV file and create a stack plot using Matplotlib. The stack plot shows trends over time, specifically the number of visitors to 3 different National Parks. 4 | -------------------------------------------------------------------------------- /Py/primer/environment.yml: -------------------------------------------------------------------------------- 1 | name: insights-primer 2 | channels: 3 | - conda-forge 4 | - defaults 5 | dependencies: 6 | - geopandas 7 | - msgpack-python 8 | - arcgis 9 | - pandas 10 | - shapely 11 | - requests 12 | - jupyter_kernel_gateway 13 | - numpy 14 | - python=3.7 15 | - libpysal 16 | - esda 17 | -------------------------------------------------------------------------------- /R/orange-tree-growth/README.md: -------------------------------------------------------------------------------- 1 | ## Orange Tree Growth 2 | 3 | 4 | This R script creates trend lines on a graphic. It shows age of tree and its relationship to circumference. The script also creates a legend and a title. 5 | 6 | * This sample depends on R-Essentials being available within the Jupyter Kernel Gateway. R-Essentials includes the Orange tree sample data. 7 | -------------------------------------------------------------------------------- /diagrams/run.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /gateway/insights-base.yml: -------------------------------------------------------------------------------- 1 | # This yaml file is for the 2020.2, 2020.3 and 2021.1 versions 2 | name: insights-base 3 | channels: 4 | - conda-forge 5 | - esri 6 | dependencies: 7 | - python=3.7 8 | - jupyter_kernel_gateway 9 | - pandas 10 | - numpy 11 | - shapely 12 | - requests 13 | - msgpack-python 14 | - matplotlib 15 | - geopandas 16 | - arcgis 17 | - r-itertools 18 | - r-essentials -------------------------------------------------------------------------------- /R/miles-per-gallon-scatter-plot/README.md: -------------------------------------------------------------------------------- 1 | ## Create Scatter Plot with LOESS Regression Curve 2 | 3 | The sample plots points using geom_point() and creates a smooth LOESS regression curve using geom_smooth(). The miles per gallon data is included with R-Essentials. 4 | 5 | * This sample depends on R-Essentials being available within the Jupyter Kernel Gateway. R-Essentials includes the mpg sample data. 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /diagrams/restart.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /gateway/insights-latest.yml: -------------------------------------------------------------------------------- 1 | # This yaml file is used for the ArcGIS Insights 2023.1, 2023.2, 2023.3 or above versions. 2 | name: insights-latest 3 | channels: 4 | - conda-forge 5 | dependencies: 6 | - python=3.9 7 | - jupyter_kernel_gateway=2.5.1 8 | - pandas=1.5.1 9 | - shapely=1.8.5 10 | - requests 11 | - msgpack-python=1.0.4 12 | - matplotlib 13 | - geopandas=0.11.1 14 | - pyspark=3.3.1 15 | - r-itertools 16 | - r-essentials -------------------------------------------------------------------------------- /gateway/insights-2022.yml: -------------------------------------------------------------------------------- 1 | # This yaml file can be used for ArcGIS Insights versions 2021.2, 2021.3, 2022.1, 2022.2, and 2022.3. 2 | name: insights-2022 3 | channels: 4 | - conda-forge 5 | dependencies: 6 | - python=3.7 7 | - jupyter_kernel_gateway 8 | - pandas 9 | - numpy 10 | - shapely 11 | - requests 12 | - msgpack-python 13 | - matplotlib 14 | - geopandas 15 | - r-itertools 16 | - r-essentials 17 | - pyspark 18 | - koalas -------------------------------------------------------------------------------- /diagrams/import-file.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /Py/addresses/address_dataframe.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | 4 | df = pd.DataFrame(np.array([ 5 | ["1001 19th St North", "Arlington", "VA", "Esri R&D"], 6 | ["380 New York St", "Redlands", "CA", "Esri Headquarters"], 7 | ["920 SW #rd Avenue", "Portland", "OR", "Esri R&D"], 8 | ["75 Broad St", "New York City", "NY", "Esri Regional Office"] 9 | ]), columns=["Address", "City", "State", "Office"]) 10 | 11 | df -------------------------------------------------------------------------------- /diagrams/delete-cell.svg: -------------------------------------------------------------------------------- 1 | trash-16 -------------------------------------------------------------------------------- /Py/parks/parks.py: -------------------------------------------------------------------------------- 1 | %matplotlib inline 2 | import matplotlib 3 | import pandas as pd 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | 7 | parks = pd.read_csv("https://raw.githubusercontent.com/Esri/insights-scripting-guide/master/Py/parks/parks.csv") 8 | 9 | x = parks["Year"] 10 | y = np.vstack([parks["Badlands"], parks["GrandCanyon"], parks["BryceCanyon"]]) 11 | labels = ["Badlands", "GrandCanyon", "BryceCanyon"] 12 | colors = ["green", "red", "blue"] 13 | plt.stackplot(x, y, labels=labels, colors=colors, edgecolor="black") 14 | plt.legend(loc=2) 15 | 16 | plt.show() 17 | -------------------------------------------------------------------------------- /diagrams/switch-connections.svg: -------------------------------------------------------------------------------- 1 | switch-16 -------------------------------------------------------------------------------- /gateway/README.md: -------------------------------------------------------------------------------- 1 | ## Important Note 2 | 3 | If you are using __2023.1, 2023.2, 2023.3 or above versions of the ArcGIS Insights__, use ```insights-latest.yml``` to create a conda environment. 4 | 5 | If you are using any of the versions __2021.2, 2021.3, 2022.1, 2022.2, or 2022.3 of the ArcGIS Insights__, use ```insights-2022.yml``` to create a conda environment. 6 | 7 | For older versions of __ArcGIS Insights (2020.2, 2020.3 or 2021.1)__, use ```insights-base.yml``` to create a conda environment. (_Not recommended_) 8 | 9 | The recommended channel for installing packages in conda environment is ```conda-forge```. 10 | -------------------------------------------------------------------------------- /diagrams/create-card.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /Py/horsemeat/README.md: -------------------------------------------------------------------------------- 1 | ## Python data scrubbing for Link Analysis 2 | 3 | >**This example has been refactored as [horsemeatexport.py](horsemeatexport.py). The refactored [horsemeatexport.py](horsemeatexport.py) only requires the [HorsemeatSimple.csv](HorsemeatSimple.csv) file.** 4 | 5 | This sample scubs data so that the resulting data set is ready for a link analysis within Insights. The script creates a new data file which puts importer and exporter locations on the same row, along with other attributes explaining global trade. 6 | 7 | * _This sample includes CSV data which needs to be placed in the same folder where the Jupyter Kernel Gateway was started._ 8 | 9 | -------------------------------------------------------------------------------- /diagrams/help.svg: -------------------------------------------------------------------------------- 1 | help-16 -------------------------------------------------------------------------------- /Py/snowfall/README.md: -------------------------------------------------------------------------------- 1 | ## Snowfall analysis 2 | 3 | This sample comes from Esri's International Developer Summit. The snowfalloriginal.py file is the original script used in the demo. The [snowfall.py](snowfall.py) file is a refactoring of the original. 4 | 5 | The scripts download 3 years of snowfall data from the web, then clean up and merge the files together. The resulting output is a comprehensive data frame of snowfall observations, which can be brought into Insights to do further analysis. 6 | 7 | 8 | Data in this sample was copied from this NOAA site 9 | [https://www.ncdc.noaa.gov/snow-and-ice/daily-snow/CA-snowfall-201902.csv](https://www.ncdc.noaa.gov/snow-and-ice/daily-snow/CA-snowfall-201902.csv) to S3. 10 | -------------------------------------------------------------------------------- /diagrams/export-save.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /Py/x-y-plot/plot.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2019 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | """ 22 | 23 | 24 | %matplotlib inline 25 | import matplotlib.pyplot as plt 26 | plt.plot([1,2,3,4]) 27 | plt.ylabel('some numbers') -------------------------------------------------------------------------------- /Py/stocks/README.md: -------------------------------------------------------------------------------- 1 | ## Stock analysis samples 2 | 3 | The [stocks.py](stocks.py) script reads a CSV file of stock data and includes functions for selected analyses. The stock data can be exported to the Insights data pane for further analysis and visualization or used in creating custom visualizations and continued analysis in the scripting environment. 4 | 5 | The [stocktotalreturnsp500.py](stocktotalreturnsp500.py) script and the [stocktotalreturnsp500.ipynb](stocktotalreturnsp500.ipynb) notebook read an Excel file of stock portfolio tickers and downloads stock and SP500 data used to compare portfolio gains/losses with the SP500 index. The stock data can be exported to the Insights data pane for further analysis and visualization or used in creating custom visualizations and continued analysis in the scripting environment 6 | -------------------------------------------------------------------------------- /Py/horsemeat/Countries.csv: -------------------------------------------------------------------------------- 1 | Country,Lat,Lon 2 | AUSTRIA,47.33,13.33 3 | BELGIUM,50.83,4 4 | BULGARIA,43,25 5 | CYPRUS,35,33 6 | CZECH REPUBLIC,49.75,15.5 7 | DENMARK,56,10 8 | ESTONIA,59,26 9 | FINLAND,64,26 10 | FRANCE,46,2 11 | GERMANY,51,9 12 | GREECE,39,22 13 | HUNGARY,47,20 14 | IRELAND,53,-8 15 | ITALY,42.83,12.83 16 | LATVIA,57,25 17 | LITHUANIA,56,24 18 | LUXEMBOURG,49.75,6.17 19 | MALTA,35.83,14.58 20 | NETHERLANDS,52.5,5.75 21 | POLAND,52,20 22 | PORTUGAL,39.5,-8 23 | ROMANIA,46,25 24 | SLOVAKIA,48.67,19.5 25 | SLOVENIA,46,15 26 | SPAIN,40,-4 27 | SWEDEN,62,15 28 | UNITED KINGDOM,54,-2 29 | ANDORRA,42.5,1.6 30 | FORMER YUGOSLAV REPUBLIC OF MACEDONIA,41.83,22 31 | GIBRALTAR,36.18,-5.37 32 | GHANA,8,-2 33 | GREENLAND,72,-40 34 | JAPAN,36,138 35 | KAZAKHSTAN,48,68 36 | RUSSIAN FEDERATION,60,100 37 | SAO TOME AND PRINCIPE,1,7 38 | SWITZERLAND,47,8 39 | THAILAND,15,100 40 | URUGUAY,-33,-56 41 | US,38,-97 -------------------------------------------------------------------------------- /R/miles-per-gallon-scatter-plot/mpg_mapping.r: -------------------------------------------------------------------------------- 1 | 2 | # Copyright 2019 Esri 3 | 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | # you may not use this file except in compliance with the License. 7 | 8 | # You may obtain a copy of the License at 9 | 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | # Unless required by applicable law or agreed to in writing, software 13 | 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | # See the License for the specific language governing permissions and 19 | 20 | # limitations under the License.​ 21 | 22 | 23 | 24 | ggplot(data = mpg, mapping = aes(x = displ, y = hwy)) + 25 | geom_point() + 26 | geom_smooth() 27 | 28 | ggplot() + 29 | geom_point(data = mpg, mapping = aes(x = displ, y = hwy)) + 30 | geom_smooth(data = mpg, mapping = aes(x = displ, y = hwy)) -------------------------------------------------------------------------------- /diagrams.md: -------------------------------------------------------------------------------- 1 | ## Kernel Gateway Deployment Planning 2 | 3 | Want to learn more about deploying a Jupyter Kernel Gateway? These conceptual diagrams were designed to help organizations visualize different Jupyter Kernel Gateway configurations. 4 | 5 | ### Insights Desktop 6 | 7 | ![Insights Desktop and Kernel Gateway](diagrams/jkg-desktop-diagram.png) 8 | 9 | 10 | ### ArcGIS Enterprise with Insights and Kernel Gateway 11 | 12 | #### Dedicated 13 | 14 | ![Dedicated Kernel Gateway](diagrams/jkg-dedicated-diagram.png) 15 | 16 | #### Co-Located 17 | 18 | ![Co-Located Kernel Gateway](diagrams/jkg-colocated-diagram.png) 19 | 20 | #### Client Kernel Gateway System Design 21 | 22 | ![Client Kernel Gateway](diagrams/jkg-client-diagram.png) 23 | 24 | 25 | ### On-premises ArcGIS Enterprise with Insights Insights and Cloud Kernel Gateway 26 | 27 | 28 | ![Cloud Kernel Gateway](diagrams/jkg-cloud-diagram.png) 29 | 30 | 31 | *Other configurations and designs may exisit. -------------------------------------------------------------------------------- /Py/primer/morans_i.py: -------------------------------------------------------------------------------- 1 | from libpysal.weights import Queen 2 | from esda.moran import Moran 3 | 4 | '''Make sure to replace the string by drag and dropping your data into line 5 before you run the script''' 5 | gdf = "REPLACEWITHYOURFILE" 6 | gdf.rename(columns = {"shape":"geometry"} , inplace = True) 7 | gdf = gdf.dropna() #drop polys with no income data 8 | 9 | '''Contiguity weights matrices define spatial connections through the existence of common boundaries. 10 | According to the queen method, two polygons need to share a point or more to be considered neighbors''' 11 | w_queen = Queen.from_dataframe(gdf) 12 | w_queen.transform = 'R' 13 | 14 | '''Morans I is a number that will tell us if there is a spatial pattern or not. 15 | Closer to 1 indicates positive or -1 indicates negative relationship. 0 indicates random. Return results to dataframe''' 16 | mi = Moran(gdf['b19013_001e'], w_queen) 17 | result = pd.DataFrame([["Moran's I", mi.I],["P-value", mi.p_sim]], columns = ["Value", "Name"]) 18 | 19 | %insights_return(result) -------------------------------------------------------------------------------- /Py/primer/README.md: -------------------------------------------------------------------------------- 1 | # A primer for Python Scripting in ArcGIS Insights (Part 1) 2 | This repository contains the Python code samples used in the ESRI blog *A primer for Python Scripting in ArcGIS Insights*. 3 | 4 | ## To work with the scripts, you will also need: 5 | - [Census API Key](https://www.census.gov/data/developers/guidance/api-user-guide.html) 6 | - Kernel gateway to an environment that has the required Python3 dependencies installed (Documentation on how to setup a Kernel gateway [here]((https://github.com/Esri/insights-scripting-guide)) and list of dependencies [here](environment.yml)) 7 | - Zipped, cleaned census tract shapefile from this repo (Data sourced from [Census.gov](https://www.census.gov/geographies/mapping-files/time-series/geo/carto-boundary-file.html)) 8 | - Working knowledge of Python, and the Python libraries Pandas and Geopandas 9 | 10 | ## Repository content 11 | - `rest_api.py` 12 | - `config.py` (replace the key with your personal API key) 13 | - `census_tracts_poly.zip` Cleaned census tract shapefile 14 | - `morans_i.py` 15 | - `environment.yml` 16 | -------------------------------------------------------------------------------- /Py/3d-plots/3Dsurfaceplot.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2019 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | """ 22 | 23 | 24 | %matplotlib inline 25 | import numpy as np 26 | import matplotlib.pyplot as plt 27 | from matplotlib import cm 28 | from mpl_toolkits.mplot3d import Axes3D 29 | 30 | X = np.arange(-5, 5, 0.25) 31 | Y = np.arange(-5, 5, 0.25) 32 | X, Y = np.meshgrid(X, Y) 33 | R = np.sqrt(X**2 + Y**2) 34 | Z = np.sin(R) 35 | 36 | fig = plt.figure() 37 | ax = Axes3D(fig) 38 | ax.plot_surface(X, Y, Z, rstride=1, cstride=1, cmap=cm.viridis) 39 | 40 | plt.show() -------------------------------------------------------------------------------- /Py/primer/rest_api.py: -------------------------------------------------------------------------------- 1 | import config 2 | 3 | '''Define config variables for API. B19013_001E is the Census label for Median Household Income (estimate).''' 4 | STATE = "state:36" 5 | TRACT = "tract:*" 6 | KEY = config.apptoken 7 | VAR = "B19013_001E" 8 | URL ="https://api.census.gov/data/2019/acs/acs5?" 9 | 10 | '''Census API call using the Requests library that gets the reponse and does some very basic error catching''' 11 | try: 12 | response = requests.get(URL, params = {"get" : VAR, "for" : TRACT, "in" : STATE, "key" : KEY}) 13 | response.raise_for_status() #raise exception 14 | df = pd.DataFrame.from_dict(response.json()) 15 | except requests.exceptions.HTTPError as err_http: #catches Http errors 16 | print(err_http) 17 | except requests.exceptions.TooManyRedirects as err_redir: 18 | print(err_redir) 19 | 20 | '''Minimal reformatting using Pandas to drop unnecesary columns, change datatypes and assign NaN to non-existent values''' 21 | header = df.iloc[0] 22 | df = df[1:] 23 | df.columns = header 24 | df = df.drop("state", axis=1) 25 | df["B19013_001E"] = pd.to_numeric(df["B19013_001E"]) 26 | df.loc[df["B19013_001E"] < 0, "B19013_001E"] = np.NaN 27 | 28 | '''Insights magic return function. Make sure you place this command in a separate cell''' 29 | %insights_return(df) -------------------------------------------------------------------------------- /R/orange-tree-growth/orange_tree_growth.r: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Esri 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | 5 | # you may not use this file except in compliance with the License. 6 | 7 | # You may obtain a copy of the License at 8 | 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | # Unless required by applicable law or agreed to in writing, software 12 | 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | 17 | # See the License for the specific language governing permissions and 18 | 19 | # limitations under the License.​ 20 | 21 | 22 | Orange$Tree <- as.numeric(Orange$Tree) 23 | ntrees <- max(Orange$Tree) 24 | 25 | xrange <- range(Orange$age) 26 | yrange <- range(Orange$circumference) 27 | 28 | plot(xrange, yrange, type="n", xlab="Age (days)", 29 | ylab="Circumference (mm)" ) 30 | colors <- rainbow(ntrees) 31 | linetype <- c(1:ntrees) 32 | plotchar <- seq(18,18+ntrees,1) 33 | 34 | for (i in 1:ntrees) { 35 | tree <- subset(Orange, Tree==i) 36 | lines(tree$age, tree$circumference, type="b", lwd=1.5, 37 | lty=linetype[i], col=colors[i], pch=plotchar[i]) 38 | } 39 | 40 | title("Tree Growth") 41 | 42 | legend(xrange[1], yrange[2], 1:ntrees, cex=0.8, col=colors, 43 | pch=plotchar, lty=linetype, title="Tree") -------------------------------------------------------------------------------- /gateway/selfsign.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import random 3 | from OpenSSL import crypto 4 | 5 | CERT_FILE = "server.crt" 6 | KEY_FILE = "server.key" 7 | 8 | def _generate_cert_openssl(): 9 | pkey = crypto.PKey() 10 | pkey.generate_key(crypto.TYPE_RSA, 2048) 11 | 12 | x509 = crypto.X509() 13 | subject = x509.get_subject() 14 | subject.commonName = socket.gethostname() 15 | x509.set_issuer(subject) 16 | x509.gmtime_adj_notBefore(0) 17 | x509.gmtime_adj_notAfter(5*365*24*60*60) 18 | x509.set_pubkey(pkey) 19 | x509.set_serial_number(random.randrange(100000)) 20 | x509.set_version(2) 21 | x509.add_extensions([ 22 | crypto.X509Extension(b'subjectAltName', False, 23 | ','.join([ 24 | 'DNS:%s' % socket.gethostname(), 25 | 'DNS:*.%s' % socket.gethostname(), 26 | 'DNS:localhost', 27 | 'DNS:*.localhost']).encode()), 28 | crypto.X509Extension(b"basicConstraints", True, b"CA:false")]) 29 | 30 | x509.sign(pkey, 'SHA256') 31 | 32 | with open(CERT_FILE, "w") as f: 33 | f.write(crypto.dump_certificate(crypto.FILETYPE_PEM, x509).decode('utf-8')) 34 | 35 | with open(KEY_FILE, "w") as f: 36 | f.write(crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey).decode('utf-8')) 37 | 38 | 39 | if __name__ == '__main__': 40 | _generate_cert_openssl() -------------------------------------------------------------------------------- /Py/horsemeat/horsemeatexport.py: -------------------------------------------------------------------------------- 1 | # import pandas 2 | import pandas as pd 3 | 4 | # Read horsemeat csv into a dataframe 5 | df = pd.read_csv("https://raw.githubusercontent.com/Esri/insights-scripting-guide/master/Py/horsemeat/HorsemeatSimple.csv") 6 | 7 | # Create dataframe of exporter country with lat and lon 8 | geo = df.loc[:, ["Country", "Lat", "Lon"]].rename(columns={"Lat": "Exporter_Lat", 9 | "Lon": "Exporter_Lon"}) 10 | 11 | # Drop Received and Exported columns 12 | df = df.drop(["Received", "Exported"], axis=1) 13 | 14 | # Get columns for multi-index 15 | idx = [col for col in df.columns if col in ("Country", "Lat", "Lon")] 16 | 17 | # Set index to multi-index 18 | m_df = df.set_index(idx) 19 | 20 | # Stack dataframe columns and drop NAs if present 21 | s_df = m_df.stack(dropna=True) 22 | 23 | # Reset index to original to remove multi-index 24 | l_df = s_df.reset_index() 25 | 26 | # Rename columns for Exporter_Country and Amount 27 | hm_df = l_df.rename(columns={"level_3": "Exporter_Country", 0: "Amount"}) 28 | 29 | # Remove all rows with zero amounts 30 | hm_non_zero = hm_df[(hm_df != 0).all(1)] 31 | 32 | # Join Exporter_Lat and Exporter_Lon 33 | hm_export_df = hm_non_zero.join(geo.set_index("Country"), on="Exporter_Country") 34 | 35 | # Display horse meat dataframe with Countries and Exporter Countries 36 | hm_export_df 37 | 38 | # Bring horsemeat data into Insights 39 | %insights_return(hm_export_df) 40 | -------------------------------------------------------------------------------- /diagrams/insert-cell.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Library / Icons / 16px / Insert 2 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /Py/addresses/README.md: -------------------------------------------------------------------------------- 1 | ## Create a Panda Data Frame with Geocode Ready Data 2 | 3 | Creates a Panda data frame that can be imported and persisted into Insights workbooks for further analysis. 4 | 5 | __Steps__ 6 | 7 | 1) In the first cell paste this code and then click Run (or _Shift-Enter_) 8 | 9 | ```python 10 | import pandas as pd 11 | import numpy as np 12 | 13 | df = pd.DataFrame(np.array([ 14 | ["1001 19th St North", "Arlington", "VA", "Esri R&D"], 15 | ["380 New York St", "Redlands", "CA", "Esri Headquarters"], 16 | ["920 SW #rd Avenue", "Portland", "OR", "Esri R&D"], 17 | ["75 Broad St", "New York City", "NY", "Esri Regional Office"] 18 | ]), columns=["Address", "City", "State", "Office"]) 19 | 20 | df 21 | ``` 22 | 23 | 2) In second cell paste this code (or use the magic command _Ctrl-Alt-B_ passing in ```df```) and then click Run. Notice the layer named ```Layer``` is created in the _Data Pane_ 24 | 25 | ```python 26 | %insights_return(df) 27 | ``` 28 | 29 | 3) Next find the layer named ```Layer``` in the _Data Pane_ and click the ellipse. From the context menu select __Enable Location__. Choose _Address_ and _Esri World Geocoder_. Then click Run. 30 | 31 | 32 | 4) Notice a new field _Location_ is created in the layer. This is a geometry field. Next drag the layer onto the canvas to create a Map (by dropping the layer in a _drop zone_ named _Map_). Once the map is created you can visualise and style each location and dive deeper into spatial and non-spatial analytics. -------------------------------------------------------------------------------- /Py/top-and-bottom/README.md: -------------------------------------------------------------------------------- 1 | ## Slice and dice geographic data 2 | 3 | **Review top and bottom items in your data** 4 | 5 | Suppose you need to put together a few crude lists showing the top and bottom 25 places. Let's say the task is related to total human population in zip codes. Here's what you could do using python and r scripting in ArcGIS for Insights. 6 | 7 | 1) Open Insights for ArcGIS 8 | 9 | 2) Create a new Workbook 10 | 11 | 3) Click add and select Living Atlas and search for USA Zip Code Points 12 | 13 | 4) Add USA Zip Code Points to the Workbook 14 | 15 | 5) Open a new Script Console Window and choose Python 16 | 17 | 6) Now using the console create a variable named ```df = ``` and drag the Zip code layer dropping it to the right of the equal sign 18 | 19 | 20 | ``` df = ``` 21 | 22 | 7) Next examine how the dataframe looks 23 | 24 | 25 | ``` df.head() ``` 26 | 27 | 8) Note there may be some unrealistic population values, such a population less than zero. Let's clean that up. 28 | 29 | ``` clean_df = df[df['POPULATION'] > 0] ``` 30 | 31 | 9) Now find the top 25 zip codes with the largest population 32 | 33 | ``` clean_df.nlargest(25, 'POPULATION') ``` 34 | 35 | 10) Now find the bottom 25 zip codes with the smallest population 36 | 37 | ``` clean_df.nsmallest(25, 'POPULATION') ``` 38 | 39 | 40 | > To convert the data frame into an Insights dataset, press ``` Ctrl/control + Alt/option + B ``` in a new cell. When the Insights magic command appears type the name of the data frame inside the parentheses and press ```Shift + Enter```. 41 | -------------------------------------------------------------------------------- /Py/3d-plots/3Dvolumetricplot.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2019 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | """ 22 | 23 | 24 | %matplotlib inline 25 | import matplotlib.pyplot as plt 26 | import numpy as np 27 | 28 | # This import registers the 3D projection, but is otherwise unused. 29 | from mpl_toolkits.mplot3d import Axes3D # noqa: F401 unused import 30 | 31 | 32 | # prepare some coordinates 33 | x, y, z = np.indices((8, 8, 8)) 34 | 35 | # draw cuboids in the top left and bottom right corners, and a link between them 36 | cube1 = (x < 3) & (y < 3) & (z < 3) 37 | cube2 = (x >= 5) & (y >= 5) & (z >= 5) 38 | link = abs(x - y) + abs(y - z) + abs(z - x) <= 2 39 | 40 | # combine the objects into a single boolean array 41 | voxels = cube1 | cube2 | link 42 | 43 | # set the colors of each object 44 | colors = np.empty(voxels.shape, dtype=object) 45 | colors[link] = 'red' 46 | colors[cube1] = 'blue' 47 | colors[cube2] = 'green' 48 | 49 | # plot it 50 | fig = plt.figure() 51 | ax = fig.gca(projection='3d') 52 | ax.voxels(voxels, facecolors=colors, edgecolor='k') 53 | 54 | plt.show() 55 | -------------------------------------------------------------------------------- /Py/watershed-river-flow/README.md: -------------------------------------------------------------------------------- 1 | # watershed-river-flow 2 | This sample demonstrates the workflow for creating an ArcGIS Insights workbook from river flow (QR) monitoring data. This sample includes a brief walkthrough for creating a data assembly script to collate datasets downloaded in subsets. 3 | 4 | The data collected for this exercise was downloaded from the [Grand River Conservation Authority (GRCA)](https://www.grandriver.ca/en/index.aspx) and their [monitoring datasets](https://data.grandriver.ca/downloads.html). This data is subject to their [Open Data License](https://data.grandriver.ca/docs/GRCA%20Open%20Data%20Licence%20v2.pdf). 5 | 6 | ## Requirements 7 | To activate the ArcGIS Insights scripting environment, dig into this [annoucement](https://www.esri.com/arcgis-blog/products/insights/announcements/scripting-insights-arcgis/) and the associated [repo](https://github.com/Esri/insights-scripting-guide). 8 | 9 | # Data Preparation 10 | The purpose of this script is to collect and prepare the data required to conduct the analysis and author our workbook. The blog post walkthrough can be read [here](https://www.esri.com/arcgis-blog/products/insights/analytics/make-this-watershed-workbook-data-prep/). 11 | 12 | The Jupyter Notebook that performs the data collation and prep is described below. 13 | 14 | ## Script Functions 15 | 1. The script processes a collection of River Flow `.csv` datasets downloaded to a local folder directory. 16 | 2. Transposes some of the metadata attributes from the file header into new columns. 17 | 3. Orders the fields. 18 | 4. Sets the column datatypes. 19 | 5. Sorts the dataframe. 20 | 6. Outputs the dataframe to the Insights datapane. 21 | -------------------------------------------------------------------------------- /R/drag-n-drop-charting/README.md: -------------------------------------------------------------------------------- 1 | ## Drag and drop data to the R console 2 | 3 | When you drag and drop a data from the Insights data pane to the Insights console, the dropped data is transformed to a R data frame list type. The data frame column names will match field names from the dropped Insights dataset. Below are examples, showing how to create charts via drag and drop with a few lines of R code. 4 | 5 | When dragging data you can choose an entier layer or just choose fields you want. In the console, when dropped data is represented using the below convention. 6 | 7 | ```layer.field``` 8 | 9 | ie. ```Allwear.salesamtho``` 10 | 11 | 12 | * Sample data named 'AllWear' is included in this folder. To load the data, go to the Insights home page. Click Datasets (on side navigation) and then click 'New Dataset' and select the AllWear.geojson file. 13 | 14 | 15 | ### Histogram 16 | 17 | ```data_from_pane <- Allwear.salesamtho``` 18 | 19 | ```hist(data_from_pane$salesamtho)``` 20 | 21 | 22 | 23 | ### QQPlot 24 | 25 | ```data_from_pane <- Allwear.salesamtho``` 26 | 27 | ```qqnorm(data_from_pane$salesamtho)``` 28 | 29 | 30 | 31 | ### BarPlot 32 | 33 | ```data_from_pane <- Allwear.salesamtho``` 34 | 35 | ```barplot(data_from_pane$salesamtho)``` 36 | 37 | 38 | 39 | ### Plot 40 | 41 | ```data_from_pane <- Allwear.salesamtho``` 42 | 43 | ```data_from_pane_qty <- Allwear.qty``` 44 | 45 | ```plot(data_from_pane$salesamtho, data_from_pane_qty$qty)``` 46 | 47 | 48 | 49 | ### Pie Chart 50 | 51 | ```data_from_pane <- Allwear.salesrep``` 52 | 53 | ```pie(table(data_from_pane$salesrep))``` 54 | 55 | 56 | -------------------------------------------------------------------------------- /diagrams/scripting-console.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Library / Buttons / Icons / 48px - with 16px - square Copy 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /Py/electric-vehicles/README.md: -------------------------------------------------------------------------------- 1 | ## Explore Electric Vehicle Charging Stations 2 | 3 | **Scrub nested JSON to desired format** 4 | 5 | Data included in this sample contains automobile charging stations in JSON. While the original authors intended the data for https://openchargemap.org, it's useful to know how to scrub and clean data for use in other systems such as Insights for ArcGIS. 6 | 7 | This sample contains two scripts and sample data. The first script _pretty prints_ JSON to the console to show the nested data. 8 | 9 | ``` 10 | import json 11 | import pprint 12 | 13 | with open("https://raw.githubusercontent.com/Esri/insights-scripting-guide/master/Py/electric-vehicles/ev_stations.json", 'r') as f: 14 | data = f.read() 15 | json_data = json.loads(data) 16 | 17 | pprint.pprint(json_data) 18 | 19 | ``` 20 | 21 | 22 | The second script flattens, manipulates, and simplifies the data in the nested JSON. It then converts the cleaned data into a panda data frame. 23 | 24 | ``` 25 | import urllib.request 26 | import pandas as pd 27 | 28 | with urllib.request.urlopen("https://raw.githubusercontent.com/Esri/insights-scripting-guide/master/Py/electric-vehicles/ev_stations.json") as url: 29 | data = url.read() 30 | column_filter = ["ID", "Title", "Latitude", "Longitude"] 31 | column_rename = {"Latitude": "Lat", "Longitude": "Lon"} 32 | df = pd.read_json(data, orient="records") 33 | ev_stations = pd.DataFrame.from_dict(elem for elem in df["AddressInfo"]).filter(items=column_filter).rename(index=str, columns=column_rename) 34 | %insights_return(ev_stations) 35 | 36 | ``` 37 | 38 | > To convert the data frame into an Insights dataset press ``` Ctrl/control + Alt/option + B ``` in a new cell. When the Insights magic command appears type the name of the data frame inside the parentheses and press ```Shift + Enter```. 39 | -------------------------------------------------------------------------------- /Py/parks/parks.csv: -------------------------------------------------------------------------------- 1 | Year,Badlands,GrandCanyon,BryceCanyon 2 | 1961,833300,1253000,264800 3 | 1962,1044800,1447400,251000 4 | 1963,1074000,1539500,289500 5 | 1964,1079800,1576600,300300 6 | 1965,1091300,1689200,366800 7 | 1966,1094800,1806000,396600 8 | 1967,1188700,1804900,295000 9 | 1968,1194600,1986300,320800 10 | 1969,1221300,2192600,366900 11 | 1970,1303100,2258200,345900 12 | 1971,1282000,2402100,377800 13 | 1972,1274600,2698300,424830 14 | 1973,1387900,1909700,429700 15 | 1974,1205300,1888600,408800 16 | 1975,1153200,2625100,579200 17 | 1976,1210100,2791600,625600 18 | 1977,1305400,2627200,611500 19 | 1978,1234915,2748642,679260 20 | 1979,858000,2131716,558095 21 | 1980,952652,2304973,571541 22 | 1981,1175952,2472270,474092 23 | 1982,1030484,2293127,471517 24 | 1983,1026981,2248082,472633 25 | 1984,1113675,2173584,495104 26 | 1985,950242,2711529,500782 27 | 1986,1025630,3035787,578018 28 | 1987,1174398,3513030,718342 29 | 1988,1110040,3859886,791348 30 | 1989,1237956,3966209,808045 31 | 1990,1326475,3776685,862659 32 | 1991,1518396,3886031,929067 33 | 1992,1205297,4203545,1018174 34 | 1993,1179458,4575602,1107951 35 | 1994,1130459,4364316,1028134 36 | 1995,1075569,4557645,994548 37 | 1996,1024705,4537703,1269600 38 | 1997,970696,4791668,1174824 39 | 1998,1021049,4239682,1166331 40 | 1999,950453,4575124,1081521 41 | 2000,1105824,4460228,1099275 42 | 2001,955469,4104809,1068619 43 | 2002,908898,4001974,886436 44 | 2003,871034,4124900,903760 45 | 2004,936030,4326234,987253 46 | 2005,909146,4401522,1017681 47 | 2006,840118,4279439,890676 48 | 2007,886654,4413668,1012563 49 | 2008,845734,4425314,1043321 50 | 2009,933918,4348068,1216377 51 | 2010,977778,4388386,1285492 52 | 2011,870741,4298178,1296000 53 | 2012,883406,4421352,1385352 54 | 2013,892372,4564840,1311875 55 | 2014,868094,4756771,1435741 56 | 2015,989354,5520736,1745804 57 | 2016,996263,5969811,2365110 58 | 2017,1054325,6254238,2571684 59 | -------------------------------------------------------------------------------- /Py/database_connections/README.md: -------------------------------------------------------------------------------- 1 | ## Database Connection Examples 2 | 3 | **Microsoft Access using pyodbc** 4 | 5 | 6 | ``` 7 | # make sure pyodbc package is installed 8 | # import pyodbc 9 | # make sure python and access versions are the same 10 | # both should be 32 bit or 64 bit 11 | import pyodbc 12 | # import pandas to create dataframe 13 | import pandas as pd 14 | 15 | # create connection string for driver and path to access database 16 | conn = pyodbc.connect(r"Driver={Microsoft Access Driver (*.mdb)};DBQ=C:\test_data\AAPL.mdb;") 17 | # query to get all records from AAPL table in access db 18 | query = 'select * from table_name;' 19 | # create pandas dataframe from table data 20 | df = pd.read_sql(query, conn) 21 | df 22 | 23 | ``` 24 | 25 | 26 | **PostgreSQL using psycopg2** 27 | 28 | ``` 29 | # make sure psycopg2 is installed 30 | # import psycopg2 31 | import psycopg2 32 | # import pandas to create dataframe 33 | import pandas as pd 34 | 35 | # change named objects to fit specific instance 36 | host = "database_host_name" 37 | user = "user" 38 | dbname = "database_name" 39 | password = "password" 40 | sslmode = "require" 41 | # build database connection string 42 | conn_string = "host={} user={} dbname={} password={} sslmode={}".format(host, user, dbname, password, sslmode) 43 | # make database connection 44 | conn = psycopg2.connect(conn_string) 45 | # create pandas dataframe from table data 46 | df = pd.read_sql('select * from table_name', con=conn) 47 | df 48 | ``` 49 | 50 | **Terradata using SQLAlchemy** 51 | 52 | ``` 53 | # make sure proper Teradata driver is installed 54 | # make sure SQLAlchemy is installed 55 | # import sqlalchemy create engine 56 | from sqlalchemy import create_engine 57 | # import pandas to create dataframe 58 | import pandas as pd 59 | 60 | # change named objects to fit specific instance 61 | user = "user" 62 | password = "password" 63 | host = "hostname" 64 | # make database connection 65 | engine = create_engine("teradata://{}:{}@{}:22/".format(user, password, host)) 66 | # execute sql to get records from table 67 | query = 'select * from table_name' 68 | query_result = engine.execute(query) 69 | 70 | # create pandas dataframe from table data 71 | df = pd.read_sql(query, engine) 72 | df 73 | ``` 74 | -------------------------------------------------------------------------------- /Py/horsemeat/transform_table.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2019 Esri 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | 6 | you may not use this file except in compliance with the License. 7 | 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | 18 | See the License for the specific language governing permissions and 19 | 20 | limitations under the License.​ 21 | """ 22 | 23 | 24 | import csv 25 | import pandas 26 | 27 | country = [] 28 | exporter = [] 29 | recieved = [] 30 | lat = [] 31 | lng = [] 32 | exporterLat = [] 33 | exporterLng = [] 34 | geoDict = {} 35 | 36 | def createFlatCSV(): 37 | df = pandas.DataFrame(data={"Country": country, 38 | "Amount": recieved, 39 | "Lat": lat, 40 | "Lng": lng, 41 | "Exporter": exporter, 42 | "ExporterLat": exporterLat, 43 | "ExporterFromLng": exporterLng}) 44 | 45 | df.to_csv("HorsemeatLinkTable.csv", sep=',',index=False) 46 | return df 47 | 48 | with open('Countries.csv') as csv_file: 49 | csv_reader = csv.reader(csv_file, delimiter=',') 50 | row_num = 0 51 | for row in csv_reader: 52 | if(row_num != 0): 53 | geoDict.update({row[0]:{'lat':row[1], 'lng':row[2]}}) 54 | row_num += 1 55 | 56 | with open('HorsemeatSimple.csv') as csv_file: 57 | csv_reader = csv.reader(csv_file, delimiter=',') 58 | row_num = 0 59 | header = [] 60 | for row in csv_reader: 61 | j = 0 62 | if row_num == 0: 63 | header = row 64 | row_num += 1 65 | else: 66 | for j in range(43): 67 | if j >= 3: 68 | country.append(row[0]) 69 | exporter.append(header[j]) 70 | recieved.append(row[j]) 71 | lat.append(row[43]) 72 | lng.append(row[44]) 73 | exporterLat.append(geoDict[header[j]]['lat']) 74 | exporterLng.append(geoDict[header[j]]['lng']) 75 | j = j + 1 76 | row_num += 1 77 | df = createFlatCSV() 78 | 79 | print(f'Processed {row_num} lines.') 80 | print(df) 81 | -------------------------------------------------------------------------------- /Py/snowfall/snowfall.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datetime as dt 3 | import pandas as pd 4 | import numpy as np 5 | 6 | # Download all years of snowfall data into a dictionary of dataframes 7 | dfs_dict = {} 8 | # URL to the snowfall data 9 | url = "https://s3-us-west-1.amazonaws.com/python-and-r/ca_snow_feb_{}.csv" 10 | # Years of interest 11 | years = ["2017", "2018", "2019"] 12 | # Get snowfall csv files for all years 13 | csv_files = [url.format(year) for year in years] 14 | # Build names for the dataframes for all the years 15 | dfs = ["{}_{}".format("df", year) for year in years] 16 | 17 | # List to hold the new column names 18 | column_rename = {"GHCN ID": "GHCN_ID", "Station Name": "Station_Name"} 19 | 20 | # Build the dict of dataframes for each year of interest 21 | for df, file in zip(dfs, csv_files): 22 | # Read csv snowfall data into dataframe; skip first row containg title and description 23 | dfs_dict[df] = pd.read_csv(file, skiprows=1).rename(index=str, columns=column_rename) 24 | # Handle the date fields for later melting into dataframe rows 25 | dfs_dict[df].columns = ["{} {}".format(col, "".join(df.split("_")[1])) if col.startswith("Feb") 26 | else col for col in dfs_dict[df].columns] 27 | # Replace "T" and "M" values with NaNs 28 | dfs_dict[df] = dfs_dict[df].replace("T", np.nan) 29 | dfs_dict[df] = dfs_dict[df].replace("M", np.nan) 30 | 31 | # Concatenate all the year dataframes into one 32 | snow_df = pd.concat([v for k, v in dfs_dict.items()], ignore_index=True, sort=False) 33 | 34 | # Collect the id variable columns for the melt of dates to rows 35 | id_vars = [col for col in snow_df.columns if not col.startswith("Feb")] 36 | # Collect the date variables for the melt of dates to rows 37 | date_vars = [col for col in snow_df.columns if col.startswith("Feb")] 38 | 39 | # Create new dataframe with the dates to rows; Create Date and Snowfall columns 40 | # and populate the corresponding values 41 | new_snow_df = snow_df.melt(id_vars=id_vars, value_vars=date_vars, var_name="Date", value_name="Snowfall") 42 | 43 | # Set Date column to datetime and do intitial format 44 | new_snow_df["Date"] = pd.to_datetime(new_snow_df["Date"], format="%b %d %Y") 45 | 46 | # Final format of date values 47 | new_snow_df["Date"] = new_snow_df["Date"].dt.strftime("%m/%d/%Y") 48 | 49 | # Get columns of interest for final snowfall dataframe 50 | final_columns = ["Station_Name", "Snowfall", "Date"] 51 | 52 | # Create final snowfall dataframe with new columns of interest 53 | final_snow_df = new_snow_df[final_columns].copy() 54 | 55 | # Drop NaNs from snowfall dataframe 56 | final_snow_df.dropna(inplace=True) 57 | 58 | # View final snowfall dataframe 59 | final_snow_df 60 | 61 | # Bring snowfall data into Insights 62 | %insights_return(final_snow_df) 63 | 64 | 65 | # Once in Insights you can convert the latitude and longitude values to geometry values (that are map ready) using enable location. 66 | -------------------------------------------------------------------------------- /diagrams/add-model.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Library / Icons / 16px / Tools 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Py/chime/README.md: -------------------------------------------------------------------------------- 1 | # ArcGIS Insights CHIME Script 2 | 3 | Penn Medicine’s Predictive Healthcare Team adapted the susceptible, infected, and recovered (SIR) mathematical model, to create a new model it calls CHIME (COVID-19 Hospital Impact Model for Epidemics). The CHIME model provides up-to-date estimates of how many people will need to be hospitalized, and of that number how many will need ICU beds and ventilators. 4 | 5 | The model uses a number of input parameters such as population size, percentage of infections, doubling time of the infection and, additionally can include impacts of physical distancing measure such as stay-at-home directives. 6 | 7 | This implementation, for ArcGIS Insights, allows for modelling of a single region, in which one value is required for these fields: 8 | 9 | ### Fields / Column Names 10 | 11 | | FIELD NAME | DESCRIPTION | 12 | | ---------------------|:-------------:| 13 | | countyname | County name | 14 | | unacast_ch | Doubling time in days (Contact Rate) | 15 | | hospitaliz | Currently hospitalized COVID-19 patients | 16 | | population | Regional population | 17 | | hospital_1 | Hospital market share (%) | 18 | 19 | 20 | * Implementing physical distancing and start date is optional 21 | 22 | 23 | This script can also be used to model multiple regions at once, for example, all counties in a state (see, cells 4 and 5). 24 | 25 | If you don't have ArcGIS Insights, you can download Insights Desktop [here](https://www.esri.com/en-us/arcgis/products/arcgis-insights/resources/desktop-client-download). 26 | 27 | 1. Open Insights and create a new workbook 28 | 2. Use the __Add__ button to upload a CSV containing the required fields and data _(see table above)_ 29 | 3. Rename the uploaded dataset using CHIME as a suffex, ie. DataSet_CHIME 30 | 4. Open the scripting console and make a Jupyter Kernel Gateway connection 31 | 5. Import _CHIME.ipynb_ into the scripting console 32 | 6. Using the data in the DataSet_CHIME select these fields _countyname, unacast_ch, hospitaliz, population, hospital_1_ from the data pane and drag them into the second cell, setting the variable named __chime_df__ 33 | 7. Once step 5 is complete, run cell 1 and 2 to completion 34 | 8. Next execute cell 3 35 | 1. This cell uses the magic command to load the merged dataset into Insights, ie ``` %insights_return(county_run_df) ``` 36 | 9. After cell 3 executes a new layer will be added to the data pane, named "Layer". Rename _Layer_ to _Chime Estimates_. 37 | 10. Optional - If your workbook includes a spatial layer containing county names _(ie. County_Name)_, enable location on _Chime Estimates_, by clicking the ellipse and selecting __Enable Location__. With the enable location dialog open, select your spatial layer and choose the _County_Name_ field to match. 38 | 11. Next to create interactive maps, charts, and tables follow the step-by-step instructions in this [blog post](https://www.esri.com/arcgis-blog/products/insights/analytics/use-chime-arcgis-insights/) 39 | 40 | 41 | Please log an issue, if you need assistance using this script or would like to offer feedback on how to make this resource more useful for people working in the front lines of epidemics. 42 | 43 | 44 | __Example workbook with CHIME Estimates__ 45 | 46 | ![Insights Scripting w/ CHIME ](screenshot.png) -------------------------------------------------------------------------------- /Py/stocks/stocks.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | 5 | def low_spread(stock_hist, price_type, num_spread=5): 6 | """ Get the lowest historical price for input spread of records.""" 7 | return stock_hist.sort_values(by=[price_type]).head(num_spread) 8 | 9 | 10 | def high_spread(stock_hist, price_type, num_spread=5): 11 | """ Get the lowest historical price for input spread of records.""" 12 | return stock_hist.sort_values(by=[price_type]).tail(num_spread).sort_values(by=[price_type], ascending=False) 13 | 14 | 15 | def adj_close_above_target(stock_hist, target_price, adj_close_column): 16 | """ Get the above target records for input target adjusted close price.""" 17 | if isinstance(adj_close_column, int): 18 | adj_close_column = stock_hist.dtypes.index[adj_close_column] 19 | above_trg = stock_hist.loc[stock_hist[adj_close_column] > target_price] 20 | above_target = above_trg.sort_values(by=[adj_close_column], ascending=False) 21 | return above_target 22 | 23 | 24 | def monthly_aggregate(stock_hist, adj_close_column, date_column): 25 | """ Get the monthly aggregate records for adjusted close price.""" 26 | if isinstance(adj_close_column, int): 27 | adj_close_column = stock_hist.dtypes.index[adj_close_column] 28 | if isinstance(date_column, int): 29 | date_column = stock_hist.dtypes.index[date_column] 30 | date_split = stock_hist[[adj_close_column, date_column]].copy() 31 | date_split.insert(1, "year", pd.DatetimeIndex(date_split[date_column]).year) 32 | date_split.insert(2, "month", pd.DatetimeIndex(date_split[date_column]).month) 33 | index_set = date_split.set_index(["year", "month"]) 34 | year_month_group = index_set.groupby(level=["year", "month"]) 35 | return year_month_group.agg([np.mean, np.std, np.min, np.max]) 36 | 37 | 38 | def simple_daily_perc_change(stock_hist, adj_close_column, date_column): 39 | """ Get the simple daily percent change records for adjusted close price.""" 40 | if isinstance(adj_close_column, int): 41 | adj_close_column = stock_hist.dtypes.index[adj_close_column] 42 | if isinstance(date_column, int): 43 | date_column = stock_hist.dtypes.index[date_column] 44 | per_change = stock_hist.set_index([date_column]) 45 | per_change["daily_perc"] = (per_change[adj_close_column].pct_change() * 100).round(2) 46 | per_change.fillna(0, inplace=True) 47 | return per_change.loc[:, [adj_close_column, "daily_perc"]] 48 | 49 | 50 | # Open AAPL stock historical data 51 | aapl_hist = pd.read_csv("https://raw.githubusercontent.com/Esri/insights-scripting-guide/master/Py/stocks/AAPL.csv") 52 | # Set dataframe index 53 | aapl_hist.set_index("trade_date") 54 | # Set price column 55 | price_type = "adj_close_price" 56 | 57 | ten_low = low_spread(aapl_hist, price_type, num_spread=10) 58 | 59 | ten_high = high_spread(aapl_hist, price_type, num_spread=10) 60 | 61 | adj_above = adj_close_above_target(aapl_hist, 185, "adj_close_price") 62 | 63 | month_agg = monthly_aggregate(aapl_hist, "adj_close_price", "trade_date") 64 | 65 | daily_perc_delta = simple_daily_perc_change(aapl_hist, "adj_close_price", "trade_date") 66 | 67 | aapl_hist['trade_date'] = pd.to_datetime(aapl_hist["trade_date"]) 68 | 69 | start_date = "2013-01-01" 70 | 71 | end_date = "2013-08-01" 72 | 73 | mask = (aapl_hist["trade_date"] > start_date) & (aapl_hist["trade_date"] <= end_date) 74 | 75 | aapl_hist.loc[mask] 76 | 77 | aapl_hist 78 | 79 | %insights_return(appl_hist) 80 | -------------------------------------------------------------------------------- /diagrams/run-all.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Library / Icons / 16px / Run all 2 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /Py/snowfall/snowfalloriginal.py: -------------------------------------------------------------------------------- 1 | ## To see how it works, each section below should be run sequentially. 2 | ## The first section needs to be run 3 times updating the year variable to 2019, 2018, and 2017. 3 | 4 | import datetime as dt 5 | import pandas as pd 6 | import os 7 | from six.moves import urllib 8 | 9 | year = "2019" 10 | 11 | ## Get the data from the web 12 | path = "ca_snow_feb_{}.csv".format(year) 13 | url = "https://s3-us-west-1.amazonaws.com/python-and-r/ca_snow_feb_{}.csv".format(year) 14 | urllib.request.urlretrieve(url,path) 15 | 16 | with open(path,'r') as f: 17 | text = f.readlines()[1:] 18 | text = ''.join([i for i in text]) 19 | 20 | ## Write cleaned data to new CSV file 21 | cleanedPath = "cln_{}".format(path) 22 | with open(cleanedPath, "w+") as txtFile: 23 | print(text, file=txtFile) 24 | 25 | df = pd.read_csv(cleanedPath, sep=',') 26 | 27 | ndf = df.rename(index=str, columns={"GHCN ID": "GHCN_ID", "Station Name": "Station_Name"}) 28 | stations = ndf[['GHCN_ID','Station_Name', 'County', 'Elevation', 'Latitude', 'Longitude']] 29 | stations 30 | 31 | ## Get yearly snowfall observations (find and replace, update columns and reshape the table) 32 | import datetime as dt 33 | import pandas as pd 34 | import os 35 | from six.moves import urllib 36 | 37 | ## Set year and state value 38 | year = 2019 39 | 40 | ## Get the data from the web 41 | path = "ca_snow_feb_{}.csv".format(year) 42 | url = "https://s3-us-west-1.amazonaws.com/python-and-r/ca_snow_feb_{}.csv".format(year) 43 | urllib.request.urlretrieve(url,path) 44 | 45 | ## Do find and replace on T and M values 46 | with open(path,'r') as f: 47 | text = f.readlines()[1:] 48 | text = ''.join([i for i in text]).replace(",M", ",") 49 | text = ''.join([i for i in text]).replace(",T", ",") 50 | 51 | ## Write cleaned data to new CSV file 52 | cleanedPath = "cln_{}".format(path) 53 | with open(cleanedPath, "w+") as txtFile: 54 | print(text, file=txtFile) 55 | 56 | ## Read CSV file into a dataframe 57 | df = pd.read_csv(cleanedPath, sep=',') 58 | 59 | ## Start reshaping the dataframe, begin by gathering the columns we want to use 60 | newCols = [] 61 | existingCols = df.columns.values.tolist() 62 | for i in range (df.columns.size): 63 | if i < 6: 64 | if df.columns[i] == "Station Name": 65 | newCols.append("Station_Name") 66 | newCols.append("Snowfall") 67 | newCols.append("Date") 68 | 69 | ## Create a table with each observation and put it in it's own row 70 | table = [] 71 | cols = df.columns.values.tolist() 72 | for index, series in df.iterrows(): 73 | attributes = [] 74 | for idx, column in enumerate(df.columns): 75 | if idx < 6: 76 | if column == "Station Name": 77 | attributes.append(series[idx]) 78 | if idx >= 6: 79 | date = dt.datetime.strptime("{}, {}".format(cols[idx], year),'%b %d, %Y') 80 | date = "{}/{}/{}".format(date.month, date.day, date.year) 81 | value = series[idx] 82 | row = attributes + [value, date] 83 | table.append(row) 84 | 85 | ## Now create the final reshaped table 86 | df = pd.DataFrame(table, columns=newCols) 87 | 88 | # Save table out as CSV 89 | snow = os.path.join(os.getcwd(), "snow") 90 | if not os.path.exists(snow): 91 | os.mkdir(snow) 92 | filename = "final_{}".format(path) 93 | path = os.path.join(os.getcwd(), "snow", filename) 94 | df.to_csv(path, sep=',', encoding='utf-8') 95 | df 96 | 97 | ## Merge all the data together (so we have 3 years of observations) 98 | all_files = list(glob.glob('./snow/*.csv')) 99 | df_list = [] 100 | for csv_file in all_files: 101 | df = pd.read_csv(csv_file) 102 | df_list.append(df) 103 | concat_df = pd.concat(df_list, ignore_index=True, sort=False) 104 | concat_csv = concat_df.to_csv('./snowall.csv', index=False) 105 | outdf = pd.read_csv('./snowall.csv') 106 | observations = outdf[['Station_Name','Snowfall', 'Date']] 107 | observations 108 | 109 | ## Next load the data into Insights to do further analysis 110 | %insights_return(observations) 111 | 112 | ## Once in Insights you can convert the latitude and longitude values to geometry values (that are map ready) using enable location. 113 | -------------------------------------------------------------------------------- /Py/horsemeat/HorsemeatSimple.csv: -------------------------------------------------------------------------------- 1 | Country,Received,Exported,AUSTRIA,BELGIUM,BULGARIA,CYPRUS,CZECH REPUBLIC,DENMARK,ESTONIA,FINLAND,FRANCE,GERMANY,GREECE,HUNGARY,IRELAND,ITALY,LATVIA,LITHUANIA,LUXEMBOURG,MALTA,NETHERLANDS,POLAND,PORTUGAL,ROMANIA,SLOVAKIA,SLOVENIA,SPAIN,SWEDEN,UNITED KINGDOM,ANDORRA,FORMER YUGOSLAV REPUBLIC OF MACEDONIA,GIBRALTAR,GHANA,GREENLAND,JAPAN,KAZAKHSTAN,RUSSIAN FEDERATION,SAO TOME AND PRINCIPE,SWITZERLAND,THAILAND,URUGUAY,US,Lat,Lon 2 | AUSTRIA,361900,1200,0,134600,0,0,0,0,0,0,0,3700,0,13600,0,93800,0,0,0,0,109800,0,0,6400,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,47.33,13.33 3 | BELGIUM,6368400,22123600,0,0,35400,0,0,79500,0,0,2387100,191200,600,0,654900,225700,0,0,400,0,1408400,0,0,1228600,0,0,5200,0,151400,0,0,0,0,0,0,0,0,0,0,0,0,0,50.83,4 4 | BULGARIA,2994800,1822700,0,181900,0,0,0,0,0,0,42000,0,0,827200,0,71300,0,0,0,0,0,0,0,1870800,0,0,1600,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,43,25 5 | CYPRUS,291800,0,0,200600,0,0,0,0,0,0,84800,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,6300,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,35,33 6 | CZECH REPUBLIC,234500,6600,0,38200,0,0,0,0,0,0,32500,0,0,0,63400,84000,0,0,0,0,12300,1100,0,0,0,0,3000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49.75,15.5 7 | DENMARK,206600,242000,0,76100,0,0,0,0,0,0,0,0,0,0,44600,40700,0,0,0,0,40900,0,0,0,0,0,4300,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,56,10 8 | ESTONIA,157900,31100,0,34300,0,0,0,0,0,0,0,0,0,110900,0,0,0,3500,0,0,0,0,0,0,0,0,2500,6700,0,0,0,0,0,0,0,0,0,0,0,0,0,0,59,26 9 | FINLAND,1534600,8800,0,832800,0,0,0,25400,28600,0,0,0,0,0,0,32700,0,0,0,0,615100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,64,26 10 | FRANCE,11840000,5183400,0,5898900,0,0,0,0,0,0,0,51000,0,0,528600,20700,0,0,1411100,0,2102500,0,0,0,0,0,24500,0,1802700,0,0,0,0,0,0,0,0,0,0,0,0,0,46,2 11 | GERMANY,1440800,329300,0,1108000,0,0,0,0,0,0,23700,0,0,0,95800,34000,0,0,2700,0,38200,75300,0,58800,0,0,3300,0,1000,0,0,0,0,0,0,0,0,0,0,0,0,0,51,9 12 | GREECE,74300,700,0,0,34300,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,15900,0,0,24000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,39,22 13 | HUNGARY,1348700,1071500,0,43000,0,0,0,0,0,0,0,0,0,0,0,1238900,0,0,0,0,0,0,0,16700,0,0,50100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,47,20 14 | IRELAND,11500,2926800,0,11500,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,53,-8 15 | ITALY,23321300,2527900,0,2573100,0,0,0,137100,0,0,2154300,16000,0,0,1172500,0,0,585400,0,0,973400,10571100,55600,1034700,0,6400,4028900,0,12800,0,0,0,0,0,0,0,0,0,0,0,0,0,42.83,12.83 16 | LATVIA,116100,0,0,0,0,0,0,0,0,5100,0,0,0,108300,0,0,0,2700,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,57,25 17 | LITHUANIA,2600,591900,0,1700,0,0,0,0,900,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,56,24 18 | LUXEMBOURG,130900,1414200,0,113700,0,0,0,0,0,0,11400,100,0,0,0,0,0,0,0,0,5700,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49.75,6.17 19 | MALTA,58800,0,0,55700,0,0,0,0,0,0,0,0,0,0,0,1200,0,0,0,0,700,1200,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,35.83,14.58 20 | NETHERLANDS,8098200,5330800,0,7265200,0,0,0,0,0,0,5100,42500,0,0,89100,286900,0,0,0,0,0,0,0,409400,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,52.5,5.75 21 | POLAND,1064100,10659200,0,3700,0,0,300,0,0,0,0,0,0,3000,0,71000,0,0,0,0,0,0,0,851800,0,0,0,5200,129100,0,0,0,0,0,0,0,0,0,0,0,0,0,52,20 22 | PORTUGAL,200,55700,0,100,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,39.5,-8 23 | ROMANIA,248500,5493100,0,21000,226300,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,1100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,46,25 24 | SLOVAKIA,7200,0,0,0,0,0,6300,0,0,0,0,0,0,0,0,0,0,0,0,0,0,900,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48.67,19.5 25 | SLOVENIA,15300,6400,1200,8700,0,0,0,0,0,0,0,0,0,500,0,4800,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,46,15 26 | SPAIN,1000,4155600,0,0,0,0,0,0,0,0,1000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,40,-4 27 | SWEDEN,336500,11900,0,219000,0,0,0,0,1600,3700,0,24800,0,0,73700,0,0,300,0,0,3800,9600,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,62,15 28 | UNITED KINGDOM,274200,2097000,0,4100,0,0,0,0,0,0,94900,0,0,8000,167200,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,54,-2 29 | ANDORRA,800,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,700,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,42.5,1.6 30 | FORMER YUGOSLAV REPUBLIC OF MACEDONIA,7700,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7700,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,41.83,22 31 | GIBRALTAR,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,36.18,-5.37 32 | GHANA,671600,0,0,671600,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,-2 33 | GREENLAND,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,72,-40 34 | JAPAN,2687100,0,0,2573100,0,0,0,0,0,0,1900,0,0,0,0,112100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,36,138 35 | KAZAKHSTAN,982900,0,0,0,982900,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48,68 36 | RUSSIAN FEDERATION,543800,0,0,0,543800,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,60,100 37 | SAO TOME AND PRINCIPE,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,7 38 | SWITZERLAND,628700,0,0,53000,0,0,0,0,0,0,344500,0,0,0,37000,174200,0,0,0,0,20000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,47,8 39 | THAILAND,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,15,100 40 | URUGUAY,28000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,28000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-33,-56 41 | US,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,38,-97 -------------------------------------------------------------------------------- /Py/stocks/stocktotalreturnsp500.py: -------------------------------------------------------------------------------- 1 | ! pip install pandas_datareader 2 | ! pip install plotly 3 | ! pip install xlrd 4 | ! pip install yfinance --upgrade --no-cache-dir 5 | import datetime 6 | import pandas as pd 7 | import numpy as np 8 | import yfinance as yf 9 | import matplotlib.pyplot as plt 10 | import plotly.graph_objs as go 11 | import pandas_datareader.data as pdr 12 | from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot 13 | 14 | init_notebook_mode(connected=True) 15 | % matplotlib inline 16 | 17 | 18 | def get_stock(tickers, start_date, end_date): 19 | def data(ticker): 20 | return pdr.get_data_yahoo(ticker, start=start_date, end=end_date) 21 | data_set = map(data, tickers) 22 | return pd.concat(data_set, keys=tickers, names=["Ticker", "Date"]) 23 | 24 | 25 | portfolio_df = pd.read_excel("sp500.xlsx") 26 | start_sp = datetime.datetime(2013, 1, 1) 27 | end_sp = datetime.datetime(2018, 3, 9) 28 | end_of_year = datetime.datetime(2017, 12, 29) 29 | yf.pdr_override() 30 | sp500 = pdr.get_data_yahoo("^GSPC", start_sp, end_sp) 31 | sp_500_adj_close = sp500[["Adj Close"]].reset_index() 32 | sp_500_adj_close_start = sp_500_adj_close[sp_500_adj_close["Date"] == end_of_year] 33 | tickers = portfolio_df["Ticker"].unique() 34 | 35 | all_data = get_stock(tickers, start_sp, end_sp) 36 | adj_close = all_data[["Adj Close"]].reset_index() 37 | adj_close_start = adj_close[adj_close["Date"] == end_of_year] 38 | adj_close_latest = adj_close[adj_close["Date"] == end_sp] 39 | adj_close_latest.set_index("Ticker", inplace=True) 40 | portfolio_df.set_index(["Ticker"], inplace=True) 41 | merge_folio = pd.merge(portfolio_df, adj_close_latest, left_index=True, right_index=True) 42 | merge_folio["ticker return"] = merge_folio["Adj Close"] / merge_folio["Unit Cost"] - 1 43 | merge_folio.reset_index(inplace=True) 44 | merge_folio_sp = pd.merge(merge_folio, sp_500_adj_close, left_on="Acquisition Date", right_on="Date") 45 | del merge_folio_sp["Date_y"] 46 | 47 | merge_folio_sp.rename(columns={"Date_x": "Latest Date", "Adj Close_x": "Ticker Adj Close", 48 | "Adj Close_y": "SP 500 Initial Close"}, inplace=True) 49 | merge_folio_sp["Equiv SP Shares"] = merge_folio_sp["Cost Basis"] / merge_folio_sp["SP 500 Initial Close"] 50 | merge_folio_sp_lat = pd.merge(merge_folio_sp, sp_500_adj_close, left_on="Latest Date", right_on="Date") 51 | del merge_folio_sp_lat["Date"] 52 | 53 | merge_folio_sp_lat.rename(columns={"Adj Close": "SP 500 Latest Close"}, inplace=True) 54 | merge_folio_sp_lat["SP Return"] = merge_folio_sp_lat["SP 500 Latest Close"] / \ 55 | merge_folio_sp_lat["SP 500 Initial Close"] - 1 56 | merge_folio_sp_lat["Abs Return Compare"] = merge_folio_sp_lat["ticker return"] - \ 57 | merge_folio_sp_lat["SP Return"] 58 | merge_folio_sp_lat["Ticker Share Value"] = merge_folio_sp_lat["Quantity"] * \ 59 | merge_folio_sp_lat["Ticker Adj Close"] 60 | merge_folio_sp_lat["SP 500 Value"] = merge_folio_sp_lat["Equiv SP Shares"] * \ 61 | merge_folio_sp_lat["SP 500 Latest Close"] 62 | merge_folio_sp_lat["Abs Value Compare"] = merge_folio_sp_lat["Ticker Share Value"] - \ 63 | merge_folio_sp_lat["SP 500 Value"] 64 | merge_folio_sp_lat["Stock Gain / Loss"] = merge_folio_sp_lat["Ticker Share Value"] - \ 65 | merge_folio_sp_lat["Cost Basis"] 66 | merge_folio_sp_lat["SP 500 Gain / Loss"] = merge_folio_sp_lat["SP 500 Value"] - \ 67 | merge_folio_sp_lat["Cost Basis"] 68 | 69 | merge_folio_sp_lat_YTD = pd.merge(merge_folio_sp_lat, adj_close_start, on="Ticker") 70 | del merge_folio_sp_lat_YTD["Date"] 71 | merge_folio_sp_lat_YTD.rename(columns={"Adj Close": "Ticker Start Year Close"}, inplace=True) 72 | merge_folio_sp_lat_YTD_sp = pd.merge(merge_folio_sp_lat_YTD, 73 | sp_500_adj_close_start, 74 | left_on="Start of Year", right_on="Date") 75 | del merge_folio_sp_lat_YTD_sp["Date"] 76 | merge_folio_sp_lat_YTD_sp.rename(columns={"Adj Close": "SP Start Year Close"}, inplace=True) 77 | merge_folio_sp_lat_YTD_sp["Share YTD"] = merge_folio_sp_lat_YTD_sp["Ticker Adj Close"] / \ 78 | merge_folio_sp_lat_YTD_sp["Ticker Start Year Close"] - 1 79 | merge_folio_sp_lat_YTD_sp["SP 500 YTD"] = merge_folio_sp_lat_YTD_sp["SP 500 Latest Close"] / \ 80 | merge_folio_sp_lat_YTD_sp["SP Start Year Close"] - 1 81 | merge_folio_sp_lat_YTD_sp = merge_folio_sp_lat_YTD_sp.sort_values(by="Ticker", ascending=True) 82 | merge_folio_sp_lat_YTD_sp["Cum Invst"] = merge_folio_sp_lat_YTD_sp["Cost Basis"].cumsum() 83 | merge_folio_sp_lat_YTD_sp["Cum Ticker Returns"] = merge_folio_sp_lat_YTD_sp["Ticker Share Value"].cumsum() 84 | merge_folio_sp_lat_YTD_sp["Cum SP Returns"] = merge_folio_sp_lat_YTD_sp["SP 500 Value"].cumsum() 85 | merge_folio_sp_lat_YTD_sp["Cum Ticker ROI Mult"] = merge_folio_sp_lat_YTD_sp["Cum Ticker Returns"] / \ 86 | merge_folio_sp_lat_YTD_sp["Cum Invst"] 87 | 88 | portfolio_df.reset_index(inplace=True) 89 | adj_close_acq_date = pd.merge(adj_close, portfolio_df, on="Ticker") 90 | del adj_close_acq_date["Quantity"] 91 | del adj_close_acq_date["Unit Cost"] 92 | del adj_close_acq_date["Cost Basis"] 93 | del adj_close_acq_date["Start of Year"] 94 | adj_close_acq_date.sort_values(by=["Ticker", "Acquisition Date", "Date"], ascending=[True, True, True], inplace=True) 95 | adj_close_acq_date["Date Delta"] = adj_close_acq_date["Date"] - adj_close_acq_date["Acquisition Date"] 96 | adj_close_acq_date["Date Delta"] = adj_close_acq_date[["Date Delta"]].apply(pd.to_numeric) 97 | adj_close_acq_date_modified = adj_close_acq_date[adj_close_acq_date["Date Delta"] >= 0] 98 | 99 | adj_close_pivot = adj_close_acq_date_modified.pivot_table(index=["Ticker", "Acquisition Date"], values="Adj Close", 100 | aggfunc=np.max) 101 | adj_close_pivot.reset_index(inplace=True) 102 | adj_close_pivot_merge = pd.merge(adj_close_pivot, adj_close, on=["Ticker", "Adj Close"]) 103 | merge_folio_sp_lat_YTD_sp_close_high = pd.merge(merge_folio_sp_lat_YTD_sp, 104 | adj_close_pivot_merge, 105 | on=["Ticker", "Acquisition Date"]) 106 | merge_folio_sp_lat_YTD_sp_close_high.rename(columns={"Adj Close": "Closing High Adj Close", 107 | "Date": "Closing High Adj Close Date"}, inplace=True) 108 | merge_folio_sp_lat_YTD_sp_close_high["Pct off High"] = merge_folio_sp_lat_YTD_sp_close_high["Ticker Adj Close"] / \ 109 | merge_folio_sp_lat_YTD_sp_close_high["Closing High Adj Close"] - 1 110 | 111 | trace_one = go.Bar(x=merge_folio_sp_lat_YTD_sp_close_high["Ticker"][0:10], 112 | y=merge_folio_sp_lat_YTD_sp_close_high["Stock Gain / Loss"][0:10], 113 | name="Ticker Total Return ($)") 114 | trace_two = go.Bar(x=merge_folio_sp_lat_YTD_sp_close_high["Ticker"][0:10], 115 | y=merge_folio_sp_lat_YTD_sp_close_high["SP 500 Gain / Loss"][0:10], 116 | name="SP 500 Total Return ($)") 117 | trace_three = go.Scatter(x=merge_folio_sp_lat_YTD_sp_close_high["Ticker"][0:10], 118 | y=merge_folio_sp_lat_YTD_sp_close_high["ticker return"][0:10], 119 | name="Ticker Total Return %", yaxis="y2") 120 | data = [trace_one, trace_two, trace_three] 121 | layout = go.Layout(title="Gain / Loss -- Total Return vs S&P 500", 122 | barmode="group", 123 | yaxis=dict(title="Gain / Loss ($)"), 124 | yaxis2=dict(title="Ticker Return", overlaying="y", 125 | side="right", tickformat=".2%"), 126 | xaxis=dict(title="Ticker"), 127 | legend=dict(x=.75, y=1)) 128 | fig = go.Figure(data=data, layout=layout) 129 | 130 | iplot(fig) 131 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Py/watershed-river-flow/GRCA_river_flow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import pandas as pd\n", 10 | "import os" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 2, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "# Create a list to contain a reference name for each csv\n", 20 | "file_names = []\n", 21 | "# Create a list to contain the path for each csv, this will be used to read the csvs\n", 22 | "file_paths = []\n", 23 | "# Create a dictionary to contain the dataframes that will be created for each csv\n", 24 | "dfs_dict = {}\n", 25 | "\n", 26 | "# Specify the path to search (where your csv downloads exist)\n", 27 | "path = r\"/Data_folder\"\n", 28 | "\n", 29 | "# Create a file counter (gets plugged into the reference name of the dataframe)\n", 30 | "f = 0\n", 31 | "\n", 32 | "# Walk through the path provided and build a list of all files with \".csv\" suffix\n", 33 | "for root, dirs, files in os.walk(path):\n", 34 | " for name in files:\n", 35 | " if \".csv\" in name:\n", 36 | " # Commit the file name to a list\n", 37 | " file_names.append(\"csv_{}\".format(f))\n", 38 | " # Commit the file path to a list (for reading later)\n", 39 | " file_paths.append(os.path.join(path, name))\n", 40 | " f += 1\n", 41 | " else:\n", 42 | " pass\n", 43 | "\n", 44 | "# For each csv found within the folder read the csv into a dataframe\n", 45 | "for n, csv in zip(file_names, file_paths):\n", 46 | " # Read csv and specify some new field names\n", 47 | " dfs_dict[n] = pd.read_csv(csv, sep=\",\", header=None, names=[\"timestamp\", \"river_flow_M3_S\"])\n", 48 | " # There\"s a big header block in each file. Since we're collating many sensors we'll transpose\n", 49 | " # some of the header info into useful fields. \n", 50 | " dfs_dict[n][\"station_name\"] = dfs_dict[n].iloc[1][\"river_flow_M3_S\"]\n", 51 | " dfs_dict[n][\"station_lat\"] = dfs_dict[n].iloc[2][\"river_flow_M3_S\"]\n", 52 | " dfs_dict[n][\"station_long\"] = dfs_dict[n].iloc[3][\"river_flow_M3_S\"]\n", 53 | " dfs_dict[n][\"station_no\"] = dfs_dict[n].iloc[8][\"river_flow_M3_S\"]\n", 54 | " dfs_dict[n][\"station_id\"] = dfs_dict[n].iloc[9][\"river_flow_M3_S\"]\n", 55 | " # All done transposing the header so we'll drop those rows from our dataframe\n", 56 | " dfs_dict[n] = dfs_dict[n].drop([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])\n", 57 | "\n", 58 | " # Re-order the columns we created\n", 59 | " column_names = [\"station_no\", \"station_id\", \"station_name\", \"timestamp\", \"river_flow_M3_S\",\n", 60 | " \"station_lat\", \"station_long\"]\n", 61 | " dfs_dict[n] = dfs_dict[n].reindex(columns = column_names)\n", 62 | "\n", 63 | "# For all the files we gathered and formatted append them to one dataframe\n", 64 | "df = pd.concat([v for k, v in dfs_dict.items()], ignore_index=True, sort=False)" 65 | ] 66 | }, 67 | { 68 | "cell_type": "code", 69 | "execution_count": 3, 70 | "metadata": {}, 71 | "outputs": [ 72 | { 73 | "data": { 74 | "text/html": [ 75 | "
\n", 76 | "\n", 89 | "\n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | "
station_nostation_idstation_nametimestampriver_flow_M3_Sstation_latstation_long
08514827York2006-01-01T00:00:00103.2999999999999743.021702537-79.89126664
18514827York2006-01-01T01:00:00103.043.021702537-79.89126664
28514827York2006-01-01T02:00:00103.2999999999999743.021702537-79.89126664
38514827York2006-01-01T03:00:00103.2999999999999743.021702537-79.89126664
48514827York2006-01-01T04:00:00103.2999999999999743.021702537-79.89126664
58514827York2006-01-01T05:00:00103.2999999999999743.021702537-79.89126664
68514827York2006-01-01T06:00:00103.2999999999999743.021702537-79.89126664
78514827York2006-01-01T07:00:00103.2999999999999743.021702537-79.89126664
88514827York2006-01-01T08:00:00103.2999999999999743.021702537-79.89126664
98514827York2006-01-01T09:00:00103.043.021702537-79.89126664
\n", 205 | "
" 206 | ], 207 | "text/plain": [ 208 | " station_no station_id station_name timestamp river_flow_M3_S \\\n", 209 | "0 85 14827 York 2006-01-01T00:00:00 103.29999999999997 \n", 210 | "1 85 14827 York 2006-01-01T01:00:00 103.0 \n", 211 | "2 85 14827 York 2006-01-01T02:00:00 103.29999999999997 \n", 212 | "3 85 14827 York 2006-01-01T03:00:00 103.29999999999997 \n", 213 | "4 85 14827 York 2006-01-01T04:00:00 103.29999999999997 \n", 214 | "5 85 14827 York 2006-01-01T05:00:00 103.29999999999997 \n", 215 | "6 85 14827 York 2006-01-01T06:00:00 103.29999999999997 \n", 216 | "7 85 14827 York 2006-01-01T07:00:00 103.29999999999997 \n", 217 | "8 85 14827 York 2006-01-01T08:00:00 103.29999999999997 \n", 218 | "9 85 14827 York 2006-01-01T09:00:00 103.0 \n", 219 | "\n", 220 | " station_lat station_long \n", 221 | "0 43.021702537 -79.89126664 \n", 222 | "1 43.021702537 -79.89126664 \n", 223 | "2 43.021702537 -79.89126664 \n", 224 | "3 43.021702537 -79.89126664 \n", 225 | "4 43.021702537 -79.89126664 \n", 226 | "5 43.021702537 -79.89126664 \n", 227 | "6 43.021702537 -79.89126664 \n", 228 | "7 43.021702537 -79.89126664 \n", 229 | "8 43.021702537 -79.89126664 \n", 230 | "9 43.021702537 -79.89126664 " 231 | ] 232 | }, 233 | "execution_count": 3, 234 | "metadata": {}, 235 | "output_type": "execute_result" 236 | } 237 | ], 238 | "source": [ 239 | "df.head(10)" 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "execution_count": 4, 245 | "metadata": {}, 246 | "outputs": [], 247 | "source": [ 248 | "# Now that all of our data is assembled lets format the columns\n", 249 | "# Format the column datatypes\n", 250 | "df[\"timestamp\"] = pd.to_datetime(arg = df[\"timestamp\"], format=\"%Y-%m-%dT%H:%M:%S\")\n", 251 | "df[\"river_flow_M3_S\"] = pd.to_numeric(arg = df[\"river_flow_M3_S\"], errors = \"coerce\")\n", 252 | "df[\"station_lat\"] = pd.to_numeric(arg = df[\"station_lat\"], errors = \"coerce\")\n", 253 | "df[\"station_long\"] = pd.to_numeric(arg = df[\"station_long\"], errors = \"coerce\")\n", 254 | "\n", 255 | "# Sort the dataset by the timestamp field so it's in logical order\n", 256 | "df = df.sort_values(by=\"timestamp\")" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": 5, 262 | "metadata": {}, 263 | "outputs": [ 264 | { 265 | "data": { 266 | "text/html": [ 267 | "
\n", 268 | "\n", 281 | "\n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | " \n", 296 | " \n", 297 | " \n", 298 | " \n", 299 | " \n", 300 | " \n", 301 | " \n", 302 | " \n", 303 | " \n", 304 | " \n", 305 | " \n", 306 | " \n", 307 | " \n", 308 | " \n", 309 | " \n", 310 | " \n", 311 | " \n", 312 | " \n", 313 | " \n", 314 | " \n", 315 | " \n", 316 | " \n", 317 | " \n", 318 | " \n", 319 | " \n", 320 | " \n", 321 | " \n", 322 | " \n", 323 | " \n", 324 | " \n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | " \n", 352 | " \n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | "
station_nostation_idstation_nametimestampriver_flow_M3_Sstation_latstation_long
08514827York2006-01-01 00:00:00103.30000043.021703-79.891267
42459214414575Aberfoyle2006-01-01 00:00:000.46835843.454656-80.162625
826829115056Leggatt2006-01-01 00:00:007.11385543.967280-80.354884
4864439315051Keldon2006-01-01 00:00:003.23721044.070820-80.370170
9011021615157Elmira2006-01-01 00:00:000.96240043.601811-80.556390
6269513315012Clair Creek2006-01-01 00:00:000.43387543.461917-80.541856
34834218414983Victoria Road2006-01-01 00:00:005.74225843.587903-80.274031
4588708114434Upper Belwood2006-01-01 00:00:0032.30840343.829211-80.298853
18514827York2006-01-01 01:00:00103.00000043.021703-79.891267
826839115056Leggatt2006-01-01 01:00:007.03852943.967280-80.354884
\n", 397 | "
" 398 | ], 399 | "text/plain": [ 400 | " station_no station_id station_name timestamp \\\n", 401 | "0 85 14827 York 2006-01-01 00:00:00 \n", 402 | "424592 144 14575 Aberfoyle 2006-01-01 00:00:00 \n", 403 | "82682 91 15056 Leggatt 2006-01-01 00:00:00 \n", 404 | "486443 93 15051 Keldon 2006-01-01 00:00:00 \n", 405 | "901102 16 15157 Elmira 2006-01-01 00:00:00 \n", 406 | "626951 33 15012 Clair Creek 2006-01-01 00:00:00 \n", 407 | "348342 184 14983 Victoria Road 2006-01-01 00:00:00 \n", 408 | "458870 81 14434 Upper Belwood 2006-01-01 00:00:00 \n", 409 | "1 85 14827 York 2006-01-01 01:00:00 \n", 410 | "82683 91 15056 Leggatt 2006-01-01 01:00:00 \n", 411 | "\n", 412 | " river_flow_M3_S station_lat station_long \n", 413 | "0 103.300000 43.021703 -79.891267 \n", 414 | "424592 0.468358 43.454656 -80.162625 \n", 415 | "82682 7.113855 43.967280 -80.354884 \n", 416 | "486443 3.237210 44.070820 -80.370170 \n", 417 | "901102 0.962400 43.601811 -80.556390 \n", 418 | "626951 0.433875 43.461917 -80.541856 \n", 419 | "348342 5.742258 43.587903 -80.274031 \n", 420 | "458870 32.308403 43.829211 -80.298853 \n", 421 | "1 103.000000 43.021703 -79.891267 \n", 422 | "82683 7.038529 43.967280 -80.354884 " 423 | ] 424 | }, 425 | "execution_count": 5, 426 | "metadata": {}, 427 | "output_type": "execute_result" 428 | } 429 | ], 430 | "source": [ 431 | "df.head(10)" 432 | ] 433 | }, 434 | { 435 | "cell_type": "code", 436 | "execution_count": null, 437 | "metadata": {}, 438 | "outputs": [], 439 | "source": [ 440 | "# This looks much better, lets bring our monitoring data into Insights for further analysis\n", 441 | "%insights_return(df)" 442 | ] 443 | } 444 | ], 445 | "metadata": { 446 | "kernelspec": { 447 | "display_name": "Python 3", 448 | "language": "python", 449 | "name": "python3" 450 | }, 451 | "language_info": { 452 | "codemirror_mode": { 453 | "name": "ipython", 454 | "version": 3 455 | }, 456 | "file_extension": ".py", 457 | "mimetype": "text/x-python", 458 | "name": "python", 459 | "nbconvert_exporter": "python", 460 | "pygments_lexer": "ipython3", 461 | "version": "3.8.5" 462 | } 463 | }, 464 | "nbformat": 4, 465 | "nbformat_minor": 4 466 | } 467 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DEPRECATED 2 | This repository has reached the end of its journey. It is now officially deprecated and will no longer be maintained. Thank you to all the users. Farewell, and onward to new endeavors. Read the announcement [here](https://support.esri.com/en-us/knowledge-base/deprecation-arcgis-insights-000034240) 3 | 4 | # Insights Scripting Guide 5 | 6 | This guide offers a reference for creating custom features in ArcGIS Insights using Python and R. It is the definitive guide for Insights scripting topics and a resource for implementing Jupyter's Kernel Gateway. 7 | 8 | Check out the wiki [here](https://github.com/Esri/insights-scripting-guide/wiki) 9 | 10 | 11 | ## Prerequisites 12 | 13 | * ArcGIS Insights (version 2020.x or above). Refer [readme](gateway/README.md) for details on versions. 14 | * Anaconda (with Python version 3.7 or above) 15 | * See needed Python and R [dependencies](gateway/insights-latest.yml) 16 | 17 | _Note: Scripting is not supported in Insights running in ArcGIS Online. Download [Insights Desktop](https://www.esri.com/en-us/arcgis/products/arcgis-insights/resources/desktop-client-download) for this instead, which supports ArcGIS Online connections, ArcGIS Enterprise connections, database and scripting features._ 18 | 19 | _Note: Plots created with pandas i.e., pandas.DataFrame.plot() are not displayed in the cards when added to a script model._ 20 | 21 | You can access an archived version of this documentation [here](README_OLD.md). 22 | 23 | 24 | ## Setup Kernel Gateway 25 | 26 | Insights supports connections to Jupyter's Kernel Gateway, which is an open source web server distributed through ```conda-forge``` and other repository channels. To setup a Kernel Gateway, with the required dependencies follow the deployment instructions below. 27 | 28 | * [Deploy with Anaconda](#Deploy-with-Anaconda) 29 | 30 | Check out [Deployment Patterns](#Deployment-Patterns) for system planning recommendations. 31 | 32 | 33 | ### Deploy with Anaconda 34 | 35 | _Note: If you have already created the conda environment, skip to this section [here](#Starting-the-kernel-gateway)_ 36 | 37 | 1) Install [Anaconda](https://www.anaconda.com/distribution/#download-section) 38 | 2) Create a folder named ```gateway``` 39 | 3) Copy ```selfsign.py``` into ```gateway``` folder 40 | 4) Copy the ```.yml``` file into the ```gateway``` folder. 41 | 5) Open _Anaconda's command prompt_ and CD into the ```gateway``` folder 42 | 6) If you are using __2023.1, 2023.2, 2023.3 or above versions of the ArcGIS Insights__, run the following commands: 43 | 44 | ```shell 45 | conda env create -f insights-latest.yml 46 | conda activate insights-latest 47 | python selfsign.py 48 | ``` 49 | 50 | _Note: If the process of creating a conda environment is taking too long, follow these instructions to create the environment [here](#alternative-method-to-create-a-conda-environment)_ 51 | 52 | 7) If you are using any of the versions __2021.2, 2021.3, 2022.1, 2022.2, or 2022.3 of the ArcGIS Insights__, run the following commands: 53 | 54 | ```shell 55 | conda env create -f insights-2022.yml 56 | conda activate insights-2022 57 | python selfsign.py 58 | ``` 59 | 60 | 8) If you are using any of the versions __2020.2, 2020.3 or 2021.1 of the ArcGIS Insights__, run the following commands: 61 | 62 | ```shell 63 | conda env create -f insights-base.yml 64 | conda activate insights-base 65 | python selfsign.py 66 | ``` 67 | 68 | 9) Start the Kernel Gateway: 69 | 70 | * If you are using __Insights in ArcGIS Enterprise__, run the following command: 71 | 72 | ```shell 73 | jupyter kernelgateway --KernelGatewayApp.ip=0.0.0.0 --KernelGatewayApp.port=9999 --KernelGatewayApp.allow_origin='*' --KernelGatewayApp.allow_credentials='*' --KernelGatewayApp.allow_headers='*' --KernelGatewayApp.allow_methods='*' --JupyterWebsocketPersonality.list_kernels=True --certfile=./server.crt --keyfile=./server.key --KernelGatewayApp.kernel_manager_class=notebook.services.kernels.kernelmanager.AsyncMappingKernelManager 74 | ``` 75 | 76 | * If you are using __Insights Desktop__, run the following command: 77 | 78 | ```shell 79 | jupyter kernelgateway --KernelGatewayApp.ip=0.0.0.0 --KernelGatewayApp.port=9999 --KernelGatewayApp.allow_origin='*' --KernelGatewayApp.allow_credentials='*' --KernelGatewayApp.allow_headers='*' --KernelGatewayApp.allow_methods='*' --JupyterWebsocketPersonality.list_kernels=True --KernelGatewayApp.kernel_manager_class=notebook.services.kernels.kernelmanager.AsyncMappingKernelManager 80 | ``` 81 | 82 | 10) Open the kernel gateway url in the browser before using it to connect in the Insights web application. For example: If you are using Insights in Chrome, open the gateway url in a new tab in Chrome, and bypass the security exceptions. Similarly, if you are using Insights in Safari or Firefox, open the gateway url and bypass the security exceptions. For more detailed instructions on bypassing security exceptions, refer [wiki](https://github.com/Esri/insights-scripting-guide/wiki/Bypassing-security-exceptions) 83 | 84 | 11) When you open the gateway url (ex: https://pickle.esri.com:9999), you will see a JSON response as ```{"reason": "Not Found", "message": ""}```. This is a valid message and it indicates that our kernel is up and running and it is ready to use within Insights. This message is just a default swagger-api's response. If you are interested in more detailed response regarding the kernel, you can navigate to ```api/kernelspecs``` page (ex: https://pickle.esri.com:9999/api/kernelspecs) 85 | 86 | 12) Do not close the Anaconda prompt or the terminal window after starting the kernel gateway. Just minimize it. Closing the window will kill the kernel gateway. 87 | 88 | 13) _Optional:_ Stop Kernel Gateway by pressing _Control-C_ in the running window or close the window. 89 | 90 | _Note:_ If you would like to access your data (.csv,.xls, etc.,) in the scripting environment, create a ```data``` folder within ```gateway``` folder and put your files in it. Then, activate your conda environment after CD'ng into the ```data``` folder and run the appropriate gateway command to start the gateway. 91 | 92 | 93 | ## Alternative method to create a conda environment 94 | 95 | In some cases, creating a conda environment using a yml file takes a long time, and the process might get stuck as conda is trying to solve the environment by resolving the conflicts between some of the dependencies. Instead of waiting for the process to resolve, we can install the packages one by one. Follow the instructions below to quickly create your environment. 96 | 97 | 1) Open _Anaconda's command prompt_ and CD into the ```gateway``` folder. 98 | 2) Creating a new environment and installing dependencies: 99 | 100 | * If you are using __2023.1, 2023.2, 2023.3 or above versions of the ArcGIS Insights__, run the following commands: 101 | 102 | ```shell 103 | # Creates a new environment named `insights-latest` with Python 3.9.14 installed. 104 | conda create -n insights-latest -c conda-forge python=3.9.14 105 | ``` 106 | 107 | ```shell 108 | # Activates the insights-latest environment. 109 | conda activate insights-latest 110 | ``` 111 | 112 | ```shell 113 | # Installs the packages from conda-forge channel. 114 | conda install -c conda-forge jupyter_kernel_gateway=2.5.1 pandas=1.5.1 shapely=1.8.5 msgpack-python=1.0.4 matplotlib 115 | conda install -c conda-forge geopandas=0.11.1 116 | conda install -c conda-forge pyspark=3.3.1 117 | # You can skip the following if you don't need R kernel. 118 | conda install -c conda-forge r-itertools 119 | conda install -c conda-forge r-essentials 120 | ``` 121 | 122 | ```shell 123 | # Creates certificates in the gateway folder. 124 | python selfsign.py 125 | ``` 126 | 127 | * If you are using any of the versions __2021.2, 2021.3, 2022.1, 2022.2, or 2022.3 of the ArcGIS Insights__, run the following commands: 128 | 129 | ```shell 130 | # Creates a new environment named `insights-2022` with Python 3.7.12 installed. 131 | conda create -n insights-2022 -c conda-forge python=3.7.12 132 | ``` 133 | 134 | ```shell 135 | # Activates the insights-2022 environment. 136 | conda activate insights-2022 137 | ``` 138 | 139 | ```shell 140 | # Installs the packages from conda-forge channel. 141 | conda install -c conda-forge jupyter_kernel_gateway=2.5.1 pandas=1.5.1 shapely=1.8.5 msgpack-python=1.0.4 matplotlib 142 | conda install -c conda-forge geopandas 143 | conda install -c conda-forge pyspark=3.1.0 144 | # You can skip the following if you don't need R kernel. 145 | conda install -c conda-forge r-itertools 146 | conda install -c conda-forge r-essentials 147 | ``` 148 | 149 | ```shell 150 | # Creates certificates in the gateway folder. 151 | python selfsign.py 152 | ``` 153 | 154 | 3) Start the kernel gateway: 155 | 156 | * If you are using __Insights in ArcGIS Enterprise__, run the following command: 157 | 158 | ```shell 159 | jupyter kernelgateway --KernelGatewayApp.ip=0.0.0.0 --KernelGatewayApp.port=9999 --KernelGatewayApp.allow_origin='*' --KernelGatewayApp.allow_credentials='*' --KernelGatewayApp.allow_headers='*' --KernelGatewayApp.allow_methods='*' --JupyterWebsocketPersonality.list_kernels=True --certfile=./server.crt --keyfile=./server.key 160 | --KernelGatewayApp.kernel_manager_class=notebook.services.kernels.kernelmanager.AsyncMappingKernelManager 161 | ``` 162 | 163 | * If you are using __Insights Desktop__, run the following command: 164 | 165 | ```shell 166 | jupyter kernelgateway --KernelGatewayApp.ip=0.0.0.0 --KernelGatewayApp.port=9999 --KernelGatewayApp.allow_origin='*' --KernelGatewayApp.allow_credentials='*' --KernelGatewayApp.allow_headers='*' --KernelGatewayApp.allow_methods='*' --JupyterWebsocketPersonality.list_kernels=True --KernelGatewayApp.kernel_manager_class=notebook.services.kernels.kernelmanager.AsyncMappingKernelManager 167 | ``` 168 | 169 | 170 | ## Starting the kernel gateway 171 | 172 | _Note:_ If you haven't created a conda environment. Refer to the instructions [here](#Deploy-with-Anaconda) 173 | 174 | 1) Open _Anaconda's command prompt_ and CD into the ```gateway``` folder. 175 | 2) Activate the conda environment: 176 | 177 | * If you are using __2023.1, 2023.2, 2023.3 or above versions of the ArcGIS Insights__, run the following command: 178 | 179 | ```shell 180 | conda activate insights-latest 181 | ``` 182 | 183 | * If you are using any of the versions __2021.2, 2021.3, 2022.1, 2022.2, or 2022.3 of the ArcGIS Insights__, run the following command: 184 | 185 | ```shell 186 | conda activate insights-2022 187 | ``` 188 | 189 | * If you are using any of the versions __2020.2, 2020.3 or 2021.1 of the ArcGIS Insights__, run the following command: 190 | 191 | ```shell 192 | conda activate insights-base 193 | ``` 194 | 195 | 3) Start the Kernel Gateway: 196 | 197 | * If you are using __Insights in ArcGIS Enterprise__, run the following command: 198 | 199 | ```shell 200 | jupyter kernelgateway --KernelGatewayApp.ip=0.0.0.0 --KernelGatewayApp.port=9999 --KernelGatewayApp.allow_origin='*' --KernelGatewayApp.allow_credentials='*' --KernelGatewayApp.allow_headers='*' --KernelGatewayApp.allow_methods='*' --JupyterWebsocketPersonality.list_kernels=True --certfile=./server.crt --keyfile=./server.key 201 | --KernelGatewayApp.kernel_manager_class=notebook.services.kernels.kernelmanager.AsyncMappingKernelManager 202 | ``` 203 | 204 | * If you are using __Insights Desktop__, run the following command: 205 | 206 | ```shell 207 | jupyter kernelgateway --KernelGatewayApp.ip=0.0.0.0 --KernelGatewayApp.port=9999 --KernelGatewayApp.allow_origin='*' --KernelGatewayApp.allow_credentials='*' --KernelGatewayApp.allow_headers='*' --KernelGatewayApp.allow_methods='*' --JupyterWebsocketPersonality.list_kernels=True --KernelGatewayApp.kernel_manager_class=notebook.services.kernels.kernelmanager.AsyncMappingKernelManager 208 | ``` 209 | 210 | ## Create a connection 211 | 212 | To create a connection to your Kernel Gateway follow these steps: 213 | 214 | 215 | 1) Open Insights 216 | 2) Create a new workbook 217 | 4) Click the _Scripting_ icon ![Open console](diagrams/scripting-console.svg) 218 | 5) Complete Kernel Gateway connection form 219 | 220 | 221 | _Note:_ Connections must reference the Kernel Gateway root URL. For tips on what connections may look like see [Connection examples](#Connection-examples). 222 | 223 | 224 | ## Connection examples 225 | 226 | Urls may be HTTP or HTTPS. Hosts can be referenced in numerous ways, IP address, localhost, FQDN etc. You can use any available inbound port number that is not already in use. If using 443, a connection will not require the port number. Here are some examples. __Yes__ means connection schema is supported. No, means that URL connection will likely fail (not work). 227 | 228 | 229 | | Connection URL | Insights in Enterprise | Insights Desktop | 230 | | ------------- |:-------------:|:-----:| 231 | | http://localhost:9999 | no | yes | 232 | | https://localhost:9999 | no | no | 233 | | http://pickle:9999| no | yes | 234 | | https://pickle:9999| no | no | 235 | | http://12.120.95.153:9999 | no | yes | 236 | | https://12.120.95.153:9999| yes | no | 237 | | http://pickle.esri.com:9999| no | yes | 238 | | https://pickle.esri.com:9999| yes | no 1 | 239 | 240 | 1 Insights Desktop can make connections to HTTPS Kernel Gateway endpoints, if the Kernel Gateway uses a domain or a certificate authority certificate. 241 | 242 | 2 If using port 443, the connection url will look like this ``` https://pickle.esri.com ```. Here ```pickle``` is the machine name. 243 | 244 | _Note:_ In Chrome, when trying to access the gateway url, it will give you a "Your connection is not private" warning. Click somewhere on the page and then blindly type the word `thisisunsafe`. This will instantly bypass the warning. 245 | 246 | ## General Features 247 | 248 | Python and R scripting features are distributed across the app. Shared scripts are accessed from the _Add_ dialog. Script modules and module options are accessed via the _Data Pane_. Lastly, the _Console_ itself has many script features. Refer to this table for an overview of tools and capabilities. 249 | 250 | | Icon | Tool Name | Description | 251 | | :-------------: |:-------------:| :-----| 252 | | ![Open console](diagrams/scripting-console.svg) | Open console | Opens the Python and R scripting console or Kernel Gateway connection dialog. If no Kernel Gateway connection exists within the page, this is when the connection dialog openes. | 253 | | ![Create module](diagrams/add-model.svg) | Create module | Creates a script from selected cells then adds a module to the data pane. | 254 | | ![Create card](diagrams/create-card.svg)| Create card | Takes the active cell and creates a card. | 255 | | ![Delete cell](diagrams/delete-cell.svg)| Delete cell | Deletes the active cell. | 256 | | ![Export script](diagrams/export-save.svg) | Export script | Enables saving of cell (or cells) to common formats like Python, R, or Jupyter Notebook files. | 257 | | ![Import file](diagrams/import-file.svg) | Import file | Enables importing of scripts into the console from common files like Python, R or Jupyter Notebook files. | 258 | | ![Insert cell](diagrams/insert-cell.svg) | Insert cell | Inserts a new scripting cell. | 259 | | ![Restart kernel](diagrams/restart.svg)| Restart kernel | Restarts the execution kernel within the Kernel Gateway. Restarting stops running scripts and clears the namespace and any data held in memory. | 260 | | ![Run](diagrams/run.svg)| Run | Runs script in active cell. | 261 | | ![Run all](diagrams/run-all.svg) | Run all | Runs all scripts in active cell. | 262 | | ![Switch connection](diagrams/switch-connections.svg)| Switch connection | Enables connection changing from one Kernel Gateway to another. | 263 | 264 | 265 | ### Shortcuts 266 | 267 | The console enables keyboard shortcuts to perform routine tasks quickly. 268 | 269 | | Shortcut | Description | 270 | |:-------------:|:-------------| 271 | | __Ctrl + B__ | Create comments for selected code. | 272 | | __Shift + Enter__ | Executes code in current cell. | 273 | | __Ctrl + Alt + B__ | Adds ```%insights_return()``` magic command to cell | 274 | 275 | 276 | ### Magic commands 277 | 278 | The console supports the following magic command. This magic command must be placed in it's own cell. 279 | 280 | | Magic command | Description | 281 | |:-------------:|:-------------| 282 | | ```%insights_return()``` | Converts Python or R data frames into Insights datasets. When ```%insights_return(df)``` is run it will generate an Insights dataset from the ```df``` object. Data will be persisted in the workbook (when the workbook is saved) and will appear in the data pane after execution. | 283 | 284 | 285 | ## Deployment Patterns 286 | 287 | There are various configurations to choose from when planning a Jupyter Kernel Gateway with Insights. It should be noted that some configurations may have tactical advantages over others. Additionally, each configuration will offer different end user experiences and varying degrees of effort regarding setup and maintenance. 288 | 289 | These conceptual diagrams were designed to help organizations visualize different kinds of Jupyter Kernel Gateway configurations next to different kinds of Insights deployments. 290 | 291 | ### Insights Desktop and Kernel Gateway 292 | 293 | ![Insights Desktop and Kernel Gateway](diagrams/jkg-desktop-diagram.png) 294 | 295 | 296 | * This configuration entails low newtworking and firewall considerations 297 | * Data files may live on personal computer or file server 298 | 299 | 300 | ### Insights in ArcGIS Enterprise and Kernel Gateway 301 | 302 | #### Dedicated 303 | 304 | ![Dedicated Kernel Gateway](diagrams/jkg-dedicated-diagram.png) 305 | 306 | * This configuration entails moderate networking and firewall considerations and skills 307 | * Data files should live on file server or Kernel Gateway machine 308 | 309 | 310 | #### Co-Located 311 | 312 | ![Co-Located Kernel Gateway](diagrams/jkg-colocated-diagram.png) 313 | 314 | * This configuration entails moderate networking and firewall considerations and skills 315 | * Data files should live on file server or Kernel Gateway machine 316 | 317 | 318 | #### Client Kernel Gateway System Design 319 | 320 | ![Client Kernel Gateway](diagrams/jkg-client-diagram.png) 321 | 322 | * This configuration entails moderate networking and firewall considerations and skills 323 | * Data files may live on personal computer or file server 324 | 325 | 326 | 327 | ### Cloud Kernel Gateway 328 | 329 | * Data files may need to be accessible from the cloud 330 | * This configuration entails advanced networking and firewall skills and considerations 331 | 332 | ![Cloud Kernel Gateway](diagrams/jkg-cloud-diagram.png) 333 | 334 | 335 | ## What is ArcGIS Insights? 336 | 337 | Part of the Esri Geospatial Cloud, ArcGIS Insights is data analytics made for advanced location intelligence. Using Insights you can ask questions you did not know to ask, analyze data completely, and tell powerful data stories. Connect to your data directly, then use maps, charts, tables and reuseable models and scripts to perform basic to complex analyses that scale based on skill level and business need. 338 | 339 | * [Case studies and testimonials](https://www.esri.com/en-us/arcgis/products/insights-for-arcgis/overview) 340 | * [Product and analytical tool documentation](https://doc.arcgis.com/en/insights/) 341 | 342 | 343 | ## FAQs and Troubleshooting 344 | 345 | #### How do I install additional Python libraries using the console that are not in my Kernel Gateway? 346 | 347 | You can do this by putting an explanation point in front of a _pip install_ command. Like, 348 | 349 | ``` 350 | !pip install BeautifulSoup4 351 | ``` 352 | 353 | If all goes well (after running the command), download activity will apear in the output cell. When the command finishes, you can then import your library and run scripts like normal. 354 | 355 | ```py 356 | from bs4 import BeautifulSoup 357 | soup = BeautifulSoup("

Hello Insights!

") 358 | print(soup.prettify()) 359 | ``` 360 | 361 | 362 | #### Insights is running in the web browser and when making a Kernel Gateway connection it says "_Not able to add this connection. Try with a different URL or web socket or check if your gateway is running._" 363 | 364 | 365 | If you've followed the guide (and ran the selfsign.py file), you have created a self signed SSL certificate. It may be possible that Insights cannot make a connection because the web browser itself does not trust the certificate. To work around this problem open the kernel gateway URL in the web browser and accept the browser warning. Then try connecting again. 366 | 367 | 368 | #### WebSocket connection to gateway url failed: Error during WebSocket handshake: Incorrect 'Sec-WebSocket-Accept' header value? 369 | 370 | This error can occur on a windows machine and when this happens, you will see an indefinite progress spinner in the scripting environment. This error can be found in the browser console and can be resolved by installing Websockets on IIS. Go to the Server Manager > Manage > Add Roles and Features > Server Roles > Web Server > Application Development > WebSocket Protocol. Make sure this is checked. Refer: [Link](https://github.com/aspnet/SignalR/issues/1585#issuecomment-373049766) 371 | 372 | 373 | #### My Kernel Gateway is on a different machine and I am having trouble making a connection using Insights? 374 | 375 | A fundamental way to troubleshoot this problem is confirm that all needed computers can talk to each other. If you are running Insights in Enterprise this means each ArcGIS Server machine, plus your Kernel Gateway and personal computer must all be able to communicate with each other. Insights Desktop entails less troubleshooting. For Insights Desktop deployments, only the Kernel Gateway and your personal computer need to talk to each other. 376 | 377 | Try getting the IP address of: 378 | 379 | * Your personal computer machine 380 | * Your kernel gateway machine 381 | * Your ArcGIS Server machine(s) 382 | 383 | and then from each machine run the ```ping``` command to see if ping messages are received. 384 | 385 | Tip: On windows, run ```ipconfig``` and reference the Iv4 address to get the IP address. On mac, run ```ipconfig getifaddr en0``` and note the address. 386 | 387 | #### Seeing any other setup or connection issues? 388 | 389 | If you encounter any issue which you are unable to troubleshoot, open your developer tools and see if there are any error messages in the console and create an issue on the repo with the error message. We will get back to you as soon as possible with a possible solution. 390 | 391 | Tip: To open Developer tools of the web browsers, use ```Shift + Ctrl + I``` on Windows and ```Option + Cmd + I``` on Mac. 392 | 393 | ## Get Insights Desktop 394 | 395 | [Download Insights Desktop](https://www.esri.com/en-us/arcgis/products/arcgis-insights/resources/desktop-client-download) 396 | 397 | 398 | ## Licensing 399 | Copyright 2020 Esri 400 | 401 | Licensed under the Apache License, Version 2.0 (the "License"); 402 | you may not use this file except in compliance with the License. 403 | You may obtain a copy of the License at 404 | 405 | http://www.apache.org/licenses/LICENSE-2.0 406 | 407 | Unless required by applicable law or agreed to in writing, software 408 | distributed under the License is distributed on an "AS IS" BASIS, 409 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 410 | See the License for the specific language governing permissions and 411 | limitations under the License. 412 | 413 | A copy of the license is available in the repository's [license.txt]( https://raw.github.com/Esri/quickstart-map-js/master/license.txt) file. 414 | -------------------------------------------------------------------------------- /README_OLD.md: -------------------------------------------------------------------------------- 1 | # ArcGIS Insights scripting guide 2 | 3 | Welcome to the ArcGIS Insights scripting guide. 4 | 5 | This repo contains information on getting started with scripting in ArcGIS Insights. This feature enables the execution of Python and R code using a bring-your-own scripting environment. Among many topics, we outline how to deploy a Jupyter Kernel Gateway, how to connect to a Jupyter Kernel Gateway, and tips and tricks for using the ArcGIS Insights scripting environment with other ArcGIS Insights features. In addition, the repo serves as a place to find and share useful Python and R scripts and creates a community around those who prefer to write code to advance data science and knowledge sharing. 6 | 7 | We welcome any contributions for improving this guide. If you find a bug or would like to report a problem in the ArcGIS Insights scripting environment, please log an issue in this repo. 8 | 9 | ## Overview 10 | 11 | * Learn how to configure a Jupyter Kernel Gateway (this is a requirement for using the ArcGIS Insights scripting environment). 12 | * Pick a [Windows](#windows-instructions) or [macOS](#macos-instructions) machine, such as a desktop or laptop, to host your Jupyter Kernel Gateway. 13 | * Generate Transport Layer Security (TLS) or Secure Sockets Layer (SSL) certificates. 14 | * Follow the guide for installing and deploying a Jupyter Kernel Gateway. 15 | * Learn tips about keyboard shortcuts and important scripting options. 16 | * Search for Python and R code. 17 | * Contribute Python and R code. 18 | * Read ArcGIS Insights use cases and product documentation. 19 | * Try ArcGIS Insights in Enterprise. 20 | * Use the ArcGIS Insights scripting environment in Enterprise. 21 | 22 | 23 | ### Required packages to export data from the scripting environment to the data pane 24 | To export data from the ArcGIS Insights scripting environment to the ArcGIS Insights data pane, the following Python and R packages are used: 25 | * The Python Kernel uses ```pandas```, ```geopandas```, ```numpy```, ```matplotlib```, ```msgpack```, ```shapely``` and ```requests```. 26 | * The R Kernel uses ```jsonlite```, ```data.table```, and ```itertools```. 27 | 28 | ## Install and deploy a Jupyter Kernel Gateway 29 | 30 | The ArcGIS Insights scripting environment uses a Jupyter Kernel Gateway. This section includes guidance on how to configure a Jupyter Kernel Gateway. It requires using either Docker or Anaconda and enabling TLS or SSL for security. TLS and SSL permit ArcGIS Insights to access your gateway over HTTPS and WSS. 31 | 32 | ## Windows instructions 33 | 34 | ### Create a TLS or SSL certificate 35 | > The instructions below will create a self-signed certificate. If you are able to create a trusted certificate in your enterprise, you can skip these steps and use the trusted enterprise certificate. 36 | 37 | 1) Clone the insights-scripting-guide repo to your local machine. 38 | 2) Use [Anaconda](https://www.anaconda.com/distribution/#windows) to install ``` openssl ``` for creating a self-signed certificate. 39 | 3) Open the Anaconda Command Prompt window. 40 | 4) Create a folder called __insightsgw__ on your local machine. For example, ``` C:\insightsgw ```. 41 | 5) Copy and paste the __insightsgw.cnf__ file from this repo's __Cert__ folder into the __insightsgw__ folder and edit each placeholder value (__host_name__ and __domain__) within the file. 42 | 43 | Example: 44 | 45 | ``` 46 | [dn] 47 | CN=insights.esri.com 48 | [req] 49 | distinguished_name=dn 50 | [EXT] 51 | subjectAltName=DNS:insights.esri.com 52 | keyUsage=digitalSignature 53 | extendedKeyUsage=serverAuth 54 | ``` 55 | 56 | > When editing the __insightsgw.cnf__ file, if you are unsure of your __host_name__ run ``` hostname ``` from a command prompt to get the correct value. If you are unsure of your __domain__ run ``` set user ``` from a command prompt to get the correct value. 57 | 58 | 6) Using the Anaconda Command Prompt, change directories to __insightsgw__. For example, ``` cd C:\insightsgw ```. 59 | 7) Change the placeholders for ____ and ____ and run the following command: 60 | 61 | ``` openssl req -x509 -days 365 -out .crt -keyout .key -newkey rsa:2048 -nodes -sha256 -subj "/CN=." -extensions EXT -config insightsgw.cnf ``` 62 | 63 | Example: 64 | 65 | ``` openssl req -x509 -days 365 -out insights.crt -keyout insights.key -newkey rsa:2048 -nodes -sha256 -subj "/CN=insights.esri.com" -extensions EXT -config insightsgw.cnf ``` 66 | 67 | 8) Close the Anaconda Command Prompt when the command has finished running. There should now be a __.key__ and a __.crt__ file in your __insightsgw__ folder. 68 | 9) Using administrative privileges, import the .crt file in your __insightsgw__ folder into the Windows Certificate Store's Trusted Root Certification Authorities. Use the following steps: 69 | * Open Manage computer certificates. 70 | * Open Trusted Root Certification Authorities. 71 | * Right click Certificates. 72 | * Click All Tasks > Import. 73 | * Select the .crt file created using ``` openssl ```. 74 | * Flush DNS using by running ```ipconfig /flushdns``` from the command prompt. 75 | 10) Add new _Inbound_ and _Outbound_ rules for port 9999 using Windows Defender Firewall with Advanced Security. 76 | 77 | ### Create a gateway 78 | 79 | _When creating a Jupyter Kernel Gateway choose either the Docker or Anaconda section below (not both)._ 80 | 81 | #### Create a gateway using Docker 82 | 83 | 1) Install [Docker](https://www.docker.com/products/docker-desktop). 84 | 2) Create a folder called __insightsgw__ on your local machine for example, ``` c:\insightsgw ```. 85 | 3) Copy and paste ``` Dockerfile ``` from this repo's _Docker_ folder into the __insightsgw__ folder. 86 | 4) Edit ``` Dockerfile ``` to fit your requirements. 87 | 88 | > The ENV KG_ALLOW_ORIGIN and ENV KG_LIST_KERNELS parameters are required for the Jupyter Kernel Gateway 89 | 90 | > If you want to load data into the container to work with inside the ArcGIS Insights console, create a local folder and add the data files to it and then set that folder to be copied from the local machine to the container in the ```Dockerfile```. For example, create a folder in your local project folder __insightsgw__ named __data__ and add working data files to it. Then, add ```COPY /data/* /data/``` to the ```Dockerfile``` to copy the local __data__ folder with the files to the container. Adding ```WORKDIR /data``` to the ```Dockerfile``` will set the container working folder to the __data__ folder. 91 | 92 | 5) Run ``` docker build -t insightsgw . ``` from a command prompt in the folder created in step 2. For example, ``` c:\insightsgw>docker build -t insightsgw . ``` 93 | 6) Run ``` docker run -p 9999:9999 insightsgw ``` from a command prompt in the folder created in step 2. For example, ``` c:\insightsgw>docker run -p 9999:9999 insightsgw ``` 94 | 95 | > If there is a database on the local host machine with data to be used in the ArcGIS Insights console, a connection to it can be created in the docker run command. For example, connecting to PostgreSQL in Windows ``` docker run -p 9999:9999 -e DB_PORT=5432 -e DB_HOST=docker.for.win.host.internal insightsgw ``` 96 | 97 |     Upon success the prompt will show the following output: 98 | 99 |       ``` [KernelGatewayApp] Jupyter Kernel Gateway at https://0.0.0.0:9999 ``` 100 | 101 | 7) Leave the command prompt open while the Jupyter Kernel Gateway is running. 102 | 8) To test that the gateway is running as expected, open a web browser and navigate to ``` https://.:9999/api ```. For example, https://insights.esri.com:9999/api. 103 | 104 |     You should see JSON text on the web page showing the api version information. For example, 105 | ``` {"version": "5.7.4"} ```. 106 | 107 | > The default configuration of Docker is limiting so it is best practice to use Docker's Advanced Setting tab to configure additional CPU and memory resources. 108 | 109 | > In addition to configuring resources for the container from the Docker application, they can be configured within ``` Dockerfile ``` as well. See the Docker documentation for configuration details. 110 | 111 | #### Create a gateway using Anaconda 112 | 113 | 1) Install [Anaconda](https://www.anaconda.com/distribution/). 114 | 2) Open the Anaconda Command Prompt window. 115 | 3) Create an ArcGIS Insights Python environment using the command ``` conda create -n my_insights_env python=3.7 ```. 116 | 4) Activate the ArcGIS Insights Python environment using the command ``` conda activate my_insights_env ```. 117 | 5) Install Jupyter Kernel Gateway using the command ``` conda install -c anaconda jupyter_kernel_gateway=2.1.0 ```. 118 | 6) Install pandas and numpy packages using the command ``` conda install -c anaconda numpy pandas```. 119 | 7) Install geopandas and matplotlib package using the command ``` conda install -c conda-forge geopandas matplotlib ```. 120 | 8) Install msgpack package using the command ``` conda install -c conda-forge msgpack-python ```. 121 | 9) Install shapely package using the command ``` conda install -c conda-forge shapely ```. 122 | 10) Install requests package using the command ``` conda install -c conda-forge requests ```. 123 | 11) Add the ArcGIS API for Python using the command ``` conda install -c esri arcgis ```. 124 | 12) Install essential R packages using the command ``` conda install -c r r-essentials ```. 125 | 13) Add the itertools package using the command ``` conda install -c conda-forge r-itertools ```. 126 | 127 | > Consider what directory to use when launching the Jupyter Kernel Gateway. If you have a ``` C:\insightsgw\data ``` directory which contains ``` .csv ``` files, launching the Jupyter Kernel Gateway from this folder will enable easy access to those files within scripts by allowing the use of relative paths for file and folder access. 128 | 129 | > The ``` KernelGatewayApp.allow_origin ``` and ``` JupyterWebsocketPersonality.list_kernels ``` parameters are required. 130 | 131 | 10) Launch the Jupyter Kernel Gateway using the following command, changing the placeholders for ____ 132 | ``` 133 | jupyter kernelgateway --KernelGatewayApp.ip=0.0.0.0 --KernelGatewayApp.port=9999 --KernelGatewayApp.allow_origin='*' --KernelGatewayApp.allow_credentials='*' --KernelGatewayApp.allow_headers='*' --KernelGatewayApp.allow_methods='*' --JupyterWebsocketPersonality.list_kernels=True --certfile=C:\insightsgw\.crt --keyfile=C:\insightsgw\.key 134 | ``` 135 | 136 | Example: 137 | 138 | ``` 139 | jupyter kernelgateway --KernelGatewayApp.ip=0.0.0.0 --KernelGatewayApp.port=9999 --KernelGatewayApp.allow_origin='*' --KernelGatewayApp.allow_credentials='*' --KernelGatewayApp.allow_headers='*' --KernelGatewayApp.allow_methods='*' --JupyterWebsocketPersonality.list_kernels=True --certfile=C:\insightsgw\insights.crt --keyfile=C:\insightsgw\insights.key 140 | ``` 141 |     Upon success the prompt will show the following output: 142 | 143 |       ``` [KernelGatewayApp] Jupyter Kernel Gateway at https://0.0.0.0:9999 ``` 144 | 145 | 11) Leave the command prompt open while the Jupyter Kernel Gateway is running. 146 | 147 | 12) To test that the Gateway is running as expected, open a web browser and navigate to ``` https://.:9999/api ```. For example, https://insights.esri.com:9999/api. 148 | 149 |     You should see JSON text on the web page showing the api version information. For example, 150 | ``` {"version": "5.7.4"} ```. 151 | 152 | > If you are experiencing issues connecting to the console after creating a self-signed certificate, check the browser to make sure an exception does not need to be added for the site created in the certificate. In the example above, the site is ``` https://insights.esri.com ```. Navigate to ``` https://insights.esri.com ``` in the browser. If an exception is required for the self-signed certificate, a warning page will be shown: 153 | 154 | > 155 | 156 | > Click Advanced 157 | 158 | > 159 | 160 | > Click Add Exception... 161 | 162 | > 163 | 164 | > Click Confirm Security Exception 165 | 166 | ## macOS instructions 167 | 168 | ### Create a TLS or SSL cert 169 | > The instructions below will create a self-signed certificate. If you are able to create a trusted certificate in your enterprise, you can skip these steps and use the trusted enterprise certificate. 170 | 171 | 1) Clone the insights-scripting-guide repo to your local machine. 172 | 2) Create a directory called __insightsgw__ on your local machine. For example, ``` insightspc:insightsgw insightsuser$ ```. 173 | 3) Copy and paste the __insightsgw.cnf__ file from this repo's __Cert__ directory into the __insightsgw__ directory and edit each placeholder value (__host_name__ and __domain__) within the file. 174 | 175 | Example: 176 | 177 | ``` 178 | [dn] 179 | CN=insights.esri.com 180 | [req] 181 | distinguished_name=dn 182 | [EXT] 183 | subjectAltName=DNS:insights.esri.com 184 | keyUsage=digitalSignature 185 | extendedKeyUsage=serverAuth 186 | ``` 187 | 188 | > When editing the __insightsgw.cnf__ file, if you are unsure of your __host_name__ run ``` sudo scutil --get LocalHostName ``` from the terminal to get the correct value. If you are unsure of your __domain__ run ``` sudo scutil --get HostName ``` from the terminal to get the correct value. 189 | 190 | 4) Using the terminal, change directories to __insightsgw__. For example, ``` insightspc:Documents insightsuser$ cd insightsgw ```. 191 | 192 |     ``` openssl ``` is installed on macOS by default. 193 | 194 | 5) Change the placeholders for ____ and ____ and run the following command: 195 | 196 | ``` openssl req -x509 -days 365 -out .crt -keyout .key -newkey rsa:2048 -nodes -sha256 -subj '/CN=.' -extensions EXT -config ../insightsgw.cnf ``` 197 | 198 | Example: 199 | 200 | ``` openssl req -x509 -days 365 -out insights.crt -keyout insights.key -newkey rsa:2048 -nodes -sha256 -subj '/CN=insights.esri.com' -extensions EXT -config ../insightsgw.cnf ``` 201 | 202 |     The terminal can be closed when finished. 203 | 204 |     There should now be a __.key__ and a __.crt__ file in your __insightsgw__ directory. 205 | 206 | 6) Using administrative privileges, import the .crt file in your __insightsgw__ directory into the Mac Keychain. Use the following steps: 207 | * Open Keychain 208 | * Choose System from Keychains 209 | * Choose Certificates from Category 210 | * Click File > Import Items 211 | * Choose ```.crt``` file created using ```openssl``` 212 | * Flush the DNS using ```sudo killall -HUP mDNSResponder``` from terminal 213 | 214 | ### Create a gateway 215 | 216 | _When creating a Jupyter Kernel Gateway choose either the Docker or Anaconda section below (not both)._ 217 | 218 | #### Create a gateway using Docker 219 | 220 | 1) Install [Docker](https://www.docker.com/products/docker-desktop). 221 | 2) Create a directory called __insightsgw__ on your local machine. For example, ``` insightspc:insightsgw insightsuser$ ```. 222 | 3) Copy and paste ``` Dockerfile ``` from this repo's __Docker__ directory into the __insightsgw__ directory. 223 | 4) Edit ``` Dockerfile ``` to fit your requirements. 224 | 225 | > The ENV KG_ALLOW_ORIGIN and ENV KG_LIST_KERNELS parameters are required for the Jupyter Kernel Gateway. 226 | 227 | > If you want to load data into the container to work with inside the ArcGIS Insights console, create a local directory and add the data files to it and then set that directory to be copied from the local machine to the container in ```Dockerfile```. For example, create a directory in your local project directory __insightsgw__ named __data__ and add working data files to it. Then, add ```COPY /data/* /data/``` to ```Dockerfile``` to copy the local __data__ directory with the files to the container. Adding ```WORKDIR /data``` to ```Dockerfile``` will set the container working directory to the __data__ directory. 228 | 5) Run ``` docker build -t insightsgw . ``` from the terminal in the directory created in step 2. For example, ```insightspc:insightsgw insightsuser$docker build -t insightsgw . ``` 229 | 6) Run ``` docker run -p 9999:9999 insightsgw ``` from the terminal in the folder created in step 2. For example, ``` insightspc:insightsgw insightsuser$docker run -p 9999:9999 insightsgw ``` 230 | 231 |     Upon success the prompt will show the following output: 232 | 233 |       ``` [KernelGatewayApp] Jupyter Kernel Gateway at https://0.0.0.0:9999 ``` 234 | 235 |     Leave the terminal open while the Jupyter Kernel Gateway is running. 236 | 237 | 7) To test that the Gateway is running as expected, open a web browser and navigate to ``` https://.:9999/api ```. For example, https://insights.esri.com:9999/api. 238 |     You should see JSON text on the web page showing the api version information. For example, 239 | ``` {"version": "5.7.4"} ```. 240 | 241 | > The default configuration of Docker is limiting so it is highly recommended to use Docker's Advanced Setting tab to configure additional CPU and memory resources. 242 | 243 | > In addition to configuring resources for the container from the Docker application, they can be configured within the Dockerfile as well. See the Docker documentation for configuration details. 244 | 245 | #### Create a gateway using Anaconda 246 | 247 | 1) Install [Anaconda](https://www.anaconda.com/distribution/). 248 | 2) Open the terminal window. 249 | 3) Create an ArcGIS Insights Python environment using the command ``` conda create -n my_insights_env python=3.6 ```. 250 | 4) Activate the ArcGIS Insights Python environment using the command ``` source activate my_insights_env ```. 251 | 5) Install Jupyter Kernel Gateway using the command ``` conda install -c anaconda jupyter_kernel_gateway ```. 252 | 6) Install pandas and numpy packages using the command ``` conda install -c anaconda numpy pandas ```. 253 | 7) Install geopandas and matplotlib package using the command ``` conda install -c conda-forge geopandas matplotlib ```. 254 | 8) Install msgpack package using the command ``` conda install -c conda-forge msgpack-python ```. 255 | 9) Install shapely package using the command ``` conda install -c conda-forge shapely ```. 256 | 10) Install requests package using the command ``` conda install -c conda-forge requests ```. 257 | 11) Add the ArcGIS API for Python using the command ``` conda install -c esri arcgis ```. 258 | 12) Install essential R packages using the command ``` conda install -c r r-essentials ```. 259 | 13) Add the itertools package using the command ``` conda install -c conda-forge r-itertools ```. 260 | 261 | > Consider what directory to use when launching the Jupyter Kernel Gateway. If you have a ``` /Users/insightsuser/Documents/insightsgw/data ``` directory which contains ``` .csv ``` files, launching the Jupyter Kernel Gateway from this directory will enable easy access to those files within scripts by allowing the use of relative paths for file and directory access. 262 | 263 | > The ``` KernelGatewayApp.allow_origin ``` and ``` JupyterWebsocketPersonality.list_kernels ``` parameters are required. 264 | 265 | 10) Launch the Jupyter Kernel Gateway using the following command, changing the placeholders for ____ 266 | ``` 267 | jupyter kernelgateway --KernelGatewayApp.ip=0.0.0.0 --KernelGatewayApp.port=9999 --KernelGatewayApp.allow_origin='*' --KernelGatewayApp.allow_credentials='*' --KernelGatewayApp.allow_headers='*' --KernelGatewayApp.allow_methods='*' --JupyterWebsocketPersonality.list_kernels=True --certfile=../certs/.crt --keyfile=../certs/.key 268 | ``` 269 | 270 | Example: 271 | 272 | ``` 273 | jupyter kernelgateway --KernelGatewayApp.ip=0.0.0.0 --KernelGatewayApp.port=9999 --KernelGatewayApp.allow_origin='*' --KernelGatewayApp.allow_credentials='*' --KernelGatewayApp.allow_headers='*' --KernelGatewayApp.allow_methods='*' --JupyterWebsocketPersonality.list_kernels=True --certfile=../certs/insights.crt --keyfile=../certs/insights.key 274 | ``` 275 |     Upon success the prompt will show the following output: 276 | 277 |       ``` [KernelGatewayApp] Jupyter Kernel Gateway at https://0.0.0.0:9999 ``` 278 | 279 |     Leave the terminal open while the Jupyter Kernel Gateway is running. 280 | 281 | 11) To test that the gateway is running as expected, open a web browser and navigate to ``` https://.:9999/api ```. For example, https://insights.esri.com:9999/api. 282 | 283 | 14) You should see JSON text on the web page showing the api version information. For example, 284 | ``` {"version": "5.7.4"} ``` 285 | 286 | > If you are experiencing issues connecting to the console after creating a self-signed certificate, check the browser to make sure an exception does not need to be added for the site created in the certificate. In the example above, the site is ``` https://insights.esri.com ```. Navigate to ``` https://insights.esri.com ``` in the browser. If an exception is required for the self-signed certificate, a warning page will be shown: 287 | 288 | > 289 | 290 | > Click Advanced 291 | 292 | > 293 | 294 | > Click Add Exception... 295 | 296 | > 297 | 298 | > Click Confirm Security Exception 299 | 300 | ## Use the console in ArcGIS Insights 301 | 302 | The ArcGIS Insights Console is available within workbooks only in the Enterprise version, not in Online. You can launch the console by clicking the _Console_ button next to the _Basemap_ button in the workbook toolbar. 303 | 304 | Learn more about the scripting console in the ArcGIS Insights documentation. 305 | 306 | ### Connecting your Jupyter Kernel Gateway 307 | 308 | 1) To connect ArcGIS Insights to your Jupyter Kernel Gateway, click the _Console_ button . 309 | 310 |     The New Jupyter Kernel Gateway connection window opens. 311 | 312 | 2) Enter the URL and web socket connection information. 313 | 314 |     If using the examples in this guide, the __URL__ parameter will be ``` https://.:9999 ``` and the __Web socket__ parameter will be ``` wss://.:9999 ```. 315 | 316 | Example: 317 | 318 | * URL 319 | 320 |       ```https://insights.esri.com:9999``` 321 | 322 | * Web Socket 323 | 324 |       ```wss://insights.esri.com:9999``` 325 | 326 | ### Shortcuts and ArcGIS Insights magic commands 327 | 328 | The ArcGIS Insights console uses keyboard shortcuts and magic commands so that routine tasks can be performed quickly and efficiently. 329 | * Use the ```%insights_return()``` magic command to add an R data frame or Pandas DataFrame to the ArcGIS Insights data pane. 330 | 331 |     __The %insights_return magic command must be run on a single line in a single cell in the ArcGIS Insights Console__ 332 | 333 | * ``` Ctrl/control + Alt/option + B ``` Add ```%insights_return``` magic command to cell. 334 | * ``` Ctrl/control + / ``` Comment line. 335 | * ``` Ctrl/control + Spacebar ``` Enable IntelliSense. 336 | * ``` Shift/shift + Enter/return ``` Execute the code in the current cell. 337 | * ``` Shift/shift + Up Arrow or Down Arrow ``` Access the history of executed cells. 338 | 339 | 340 | # What is ArcGIS Insights? 341 | 342 | Part of the Esri Geospatial Cloud, ArcGIS Insights is web-based data analytics made for advanced location intelligence. Answer questions you didn’t know to ask, analyze data completely, and tell powerful data stories. Connect to your data directly, then use maps, charts and tables to perform basic to complex analyses that scale based on skill level and business need. 343 | 344 | * [Case studies and testimonials](https://www.esri.com/en-us/arcgis/products/insights-for-arcgis/overview) 345 | * [Product and analytical tool documentation](https://doc.arcgis.com/en/insights/) 346 | 347 | 348 | ## Why Get Involved? 349 | 350 | Send feedback to us concerning the ArcGIS Insights scripting console. Found an issue or have questions? Feel free to post questions or comments and report bugs to the issues section of this repo. 351 | 352 | ## Start using ArcGIS Insights with a Free Trial 353 | 354 | Sign-up to [start a free trial](https://www.esri.com/en-us/arcgis/products/insights-for-arcgis/trial?adumkts=product&adupro=Insights_for_ArcGIS&aduc=pr&adum=blogs&utm_Source=pr&aduca=arcgis_insights_existing_customers_promotions&aduat=blog&aduco=exploring-the-atlantic-ocean-in-insights&adupt=lead_gen&sf_id=70139000001eKGfAAM). 355 | 356 | 357 | ## Licensing 358 | Copyright 2019 Esri 359 | 360 | Licensed under the Apache License, Version 2.0 (the "License"); 361 | you may not use this file except in compliance with the License. 362 | You may obtain a copy of the License at 363 | 364 | http://www.apache.org/licenses/LICENSE-2.0 365 | 366 | Unless required by applicable law or agreed to in writing, software 367 | distributed under the License is distributed on an "AS IS" BASIS, 368 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 369 | See the License for the specific language governing permissions and 370 | limitations under the License. 371 | 372 | A copy of the license is available in the repository's [license.txt]( https://raw.github.com/Esri/quickstart-map-js/master/license.txt) file. 373 | -------------------------------------------------------------------------------- /Py/chime/CHIME.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import warnings\n", 10 | "warnings.simplefilter(action='ignore', category=FutureWarning)\n", 11 | "\n", 12 | "from collections import namedtuple\n", 13 | "from datetime import date\n", 14 | "from datetime import date, datetime, timedelta\n", 15 | "from logging import INFO, basicConfig, getLogger\n", 16 | "from sys import stdout\n", 17 | "from typing import Dict, Generator, Tuple, Sequence, Optional\n", 18 | "\n", 19 | "import numpy as np\n", 20 | "import pandas as pd\n", 21 | "import geopandas as gpd\n", 22 | "\n", 23 | "Disposition = namedtuple(\"Disposition\", (\"rate\", \"days\"))\n", 24 | "\n", 25 | "\n", 26 | "class Parameters:\n", 27 | " def __init__(self, **kwargs):\n", 28 | " self.population=3600000\n", 29 | " self.current_hospitalized=69\n", 30 | " self.date_first_hospitalized=None\n", 31 | " self.doubling_time=4.0\n", 32 | " self.hospitalized=Disposition(0.025, 7)\n", 33 | " self.icu=Disposition(0.0075, 9)\n", 34 | " self.infectious_days=14\n", 35 | " self.market_share=0.15\n", 36 | " self.n_days=100\n", 37 | " self.current_date=date.today()\n", 38 | " self.mitigation_date = None\n", 39 | " self.relative_contact_rate=0.3\n", 40 | " self.ventilated=Disposition(0.005, 10)\n", 41 | " self.recovered = 0\n", 42 | " \n", 43 | " if bool(kwargs):\n", 44 | " self.population = kwargs.get(\"population\",self.population)\n", 45 | " self.current_hospitalized = kwargs.get(\"current_hospitalized\",self.current_hospitalized)\n", 46 | " self.hospitalized=Disposition(kwargs.get(\"hospitalized\")/100, 7)\n", 47 | " self.relative_contact_rate=kwargs.get(\"relative_contact_rate\")/100\n", 48 | " \n", 49 | " self.dispositions = {\n", 50 | " \"hospitalized\": self.hospitalized,\n", 51 | " \"icu\": self.icu,\n", 52 | " \"ventilated\": self.ventilated,\n", 53 | " }\n", 54 | "\n", 55 | "\n", 56 | "basicConfig(\n", 57 | " level=INFO,\n", 58 | " format=\"%(asctime)s - %(name)s - %(levelname)s - %(message)s\",\n", 59 | " stream=stdout,\n", 60 | ")\n", 61 | "logger = getLogger(__name__)\n", 62 | "\n", 63 | "\n", 64 | "class SimSirModel:\n", 65 | "\n", 66 | " def __init__(self, p: Parameters):\n", 67 | "\n", 68 | " self.rates = {\n", 69 | " key: d.rate\n", 70 | " for key, d in p.dispositions.items()\n", 71 | " }\n", 72 | "\n", 73 | " self.days = {\n", 74 | " key: d.days\n", 75 | " for key, d in p.dispositions.items()\n", 76 | " }\n", 77 | "\n", 78 | " self.keys = (\"susceptible\", \"infected\", \"recovered\")\n", 79 | "\n", 80 | " # An estimate of the number of infected people on the day that\n", 81 | " # the first hospitalized case is seen\n", 82 | " #\n", 83 | " # Note: this should not be an integer.\n", 84 | " infected = (\n", 85 | " 1.0 / p.market_share / p.hospitalized.rate\n", 86 | " )\n", 87 | "\n", 88 | " susceptible = p.population - infected\n", 89 | "\n", 90 | " gamma = 1.0 / p.infectious_days\n", 91 | " self.gamma = gamma\n", 92 | "\n", 93 | " self.susceptible = susceptible\n", 94 | " self.infected = infected\n", 95 | " self.recovered = p.recovered\n", 96 | "\n", 97 | " if p.date_first_hospitalized is None and p.doubling_time is not None:\n", 98 | " # Back-projecting to when the first hospitalized case would have been admitted\n", 99 | " logger.info('Using doubling_time: %s', p.doubling_time)\n", 100 | "\n", 101 | " intrinsic_growth_rate = get_growth_rate(p.doubling_time)\n", 102 | "\n", 103 | " self.beta = get_beta(intrinsic_growth_rate, gamma, self.susceptible, 0.0)\n", 104 | " self.beta_t = get_beta(intrinsic_growth_rate, self.gamma, self.susceptible, p.relative_contact_rate)\n", 105 | "\n", 106 | " self.i_day = 0 # seed to the full length\n", 107 | " self.run_projection(p, [(self.beta, p.n_days)])\n", 108 | " self.i_day = i_day = int(get_argmin_ds(self.census_df, p.current_hospitalized))\n", 109 | "\n", 110 | " self.run_projection(p, self.gen_policy(p))\n", 111 | "\n", 112 | " logger.info('Set i_day = %s', i_day)\n", 113 | " p.date_first_hospitalized = p.current_date - timedelta(days=i_day)\n", 114 | " logger.info(\n", 115 | " 'Estimated date_first_hospitalized: %s; current_date: %s; i_day: %s',\n", 116 | " p.date_first_hospitalized,\n", 117 | " p.current_date,\n", 118 | " self.i_day)\n", 119 | "\n", 120 | " elif p.date_first_hospitalized is not None and p.doubling_time is None:\n", 121 | " # Fitting spread parameter to observed hospital census (dates of 1 patient and today)\n", 122 | " self.i_day = (p.current_date - p.date_first_hospitalized).days\n", 123 | " self.current_hospitalized = p.current_hospitalized\n", 124 | " logger.info(\n", 125 | " 'Using date_first_hospitalized: %s; current_date: %s; i_day: %s, current_hospitalized: %s',\n", 126 | " p.date_first_hospitalized,\n", 127 | " p.current_date,\n", 128 | " self.i_day,\n", 129 | " p.current_hospitalized,\n", 130 | " )\n", 131 | "\n", 132 | " # Make an initial coarse estimate\n", 133 | " dts = np.linspace(1, 15, 15)\n", 134 | " min_loss = self.get_argmin_doubling_time(p, dts)\n", 135 | "\n", 136 | " # Refine the coarse estimate\n", 137 | " for iteration in range(4):\n", 138 | " dts = np.linspace(dts[min_loss-1], dts[min_loss+1], 15)\n", 139 | " min_loss = self.get_argmin_doubling_time(p, dts)\n", 140 | "\n", 141 | " p.doubling_time = dts[min_loss]\n", 142 | "\n", 143 | " logger.info('Estimated doubling_time: %s', p.doubling_time)\n", 144 | " intrinsic_growth_rate = get_growth_rate(p.doubling_time)\n", 145 | " self.beta = get_beta(intrinsic_growth_rate, self.gamma, self.susceptible, 0.0)\n", 146 | " self.beta_t = get_beta(intrinsic_growth_rate, self.gamma, self.susceptible, p.relative_contact_rate)\n", 147 | " self.run_projection(p, self.gen_policy(p))\n", 148 | "\n", 149 | " self.population = p.population\n", 150 | " else:\n", 151 | " logger.info(\n", 152 | " 'doubling_time: %s; date_first_hospitalized: %s',\n", 153 | " p.doubling_time,\n", 154 | " p.date_first_hospitalized,\n", 155 | " )\n", 156 | " raise AssertionError('doubling_time or date_first_hospitalized must be provided.')\n", 157 | "\n", 158 | " logger.info('len(np.arange(-i_day, n_days+1)): %s', len(np.arange(-self.i_day, p.n_days+1)))\n", 159 | " logger.info('len(raw_df): %s', len(self.raw_df))\n", 160 | "\n", 161 | " self.infected = self.raw_df['infected'].values[self.i_day]\n", 162 | " self.susceptible = self.raw_df['susceptible'].values[self.i_day]\n", 163 | " self.recovered = self.raw_df['recovered'].values[self.i_day]\n", 164 | "\n", 165 | " self.intrinsic_growth_rate = intrinsic_growth_rate\n", 166 | "\n", 167 | " # r_t is r_0 after distancing\n", 168 | " self.r_t = self.beta_t / gamma * susceptible\n", 169 | " self.r_naught = self.beta / gamma * susceptible\n", 170 | "\n", 171 | " doubling_time_t = 1.0 / np.log2(\n", 172 | " self.beta_t * susceptible - gamma + 1)\n", 173 | " self.doubling_time_t = doubling_time_t\n", 174 | " \n", 175 | " #SIM_SIR_DF - 3rd Plot\n", 176 | " self.sim_sir_w_date_df = build_sim_sir_w_date_df(self.raw_df, p.current_date, self.keys)\n", 177 | "\n", 178 | " self.sim_sir_w_date_floor_df = build_floor_df(self.sim_sir_w_date_df, self.keys)\n", 179 | " self.admits_floor_df = build_floor_df(self.admits_df, p.dispositions.keys())\n", 180 | " self.census_floor_df = build_floor_df(self.census_df, p.dispositions.keys())\n", 181 | "\n", 182 | " self.daily_growth_rate = get_growth_rate(p.doubling_time)\n", 183 | " self.daily_growth_rate_t = get_growth_rate(self.doubling_time_t)\n", 184 | "\n", 185 | " def get_argmin_doubling_time(self, p: Parameters, dts):\n", 186 | " losses = np.full(dts.shape[0], np.inf)\n", 187 | " for i, i_dt in enumerate(dts):\n", 188 | " intrinsic_growth_rate = get_growth_rate(i_dt)\n", 189 | " self.beta = get_beta(intrinsic_growth_rate, self.gamma, self.susceptible, 0.0)\n", 190 | " self.beta_t = get_beta(intrinsic_growth_rate, self.gamma, self.susceptible, p.relative_contact_rate)\n", 191 | "\n", 192 | " self.run_projection(p, self.gen_policy(p))\n", 193 | "\n", 194 | " # Skip values the would put the fit past peak\n", 195 | " peak_admits_day = self.admits_df.hospitalized.argmax()\n", 196 | " if peak_admits_day < 0:\n", 197 | " continue\n", 198 | "\n", 199 | " loss = self.get_loss()\n", 200 | " losses[i] = loss\n", 201 | "\n", 202 | " min_loss = pd.Series(losses).argmin()\n", 203 | " return min_loss\n", 204 | "\n", 205 | " def gen_policy(self, p: Parameters) -> Sequence[Tuple[float, int]]:\n", 206 | " if p.mitigation_date is not None:\n", 207 | " mitigation_day = -(p.current_date - p.mitigation_date).days\n", 208 | " else:\n", 209 | " mitigation_day = 0\n", 210 | "\n", 211 | " total_days = self.i_day + p.n_days\n", 212 | "\n", 213 | " if mitigation_day < -self.i_day:\n", 214 | " mitigation_day = -self.i_day\n", 215 | "\n", 216 | " pre_mitigation_days = self.i_day + mitigation_day\n", 217 | " post_mitigation_days = total_days - pre_mitigation_days\n", 218 | "\n", 219 | " return [\n", 220 | " (self.beta, pre_mitigation_days),\n", 221 | " (self.beta_t, post_mitigation_days),\n", 222 | " ]\n", 223 | "\n", 224 | " def run_projection(self, p: Parameters, policy: Sequence[Tuple[float, int]]):\n", 225 | " self.raw_df = sim_sir_df(\n", 226 | " self.susceptible,\n", 227 | " self.infected,\n", 228 | " p.recovered,\n", 229 | " self.gamma,\n", 230 | " -self.i_day,\n", 231 | " policy\n", 232 | " )\n", 233 | "\n", 234 | " self.dispositions_df = build_dispositions_df(self.raw_df, self.rates, p.market_share, p.current_date)\n", 235 | " #Projected Admissions plot - Plot 1\n", 236 | " self.admits_df = build_admits_df(self.dispositions_df)\n", 237 | " #Projected Census plot - Plot 2\n", 238 | " self.census_df = build_census_df(self.admits_df, self.days)\n", 239 | " self.current_infected = self.raw_df.infected.loc[self.i_day]\n", 240 | "\n", 241 | " def get_loss(self) -> float:\n", 242 | " \"\"\"Squared error: predicted vs. actual current hospitalized.\"\"\"\n", 243 | " predicted = self.census_df.hospitalized.loc[self.i_day]\n", 244 | " return (self.current_hospitalized - predicted) ** 2.0\n", 245 | "\n", 246 | "\n", 247 | "def get_argmin_ds(census_df: pd.DataFrame, current_hospitalized: float) -> float:\n", 248 | " # By design, this forbids choosing a day after the peak\n", 249 | " # If that's a problem, see #381\n", 250 | " peak_day = census_df.hospitalized.argmax()\n", 251 | " losses_df = (census_df.hospitalized[:peak_day] - current_hospitalized) ** 2.0\n", 252 | " return losses_df.argmin()\n", 253 | "\n", 254 | "\n", 255 | "def get_beta(\n", 256 | " intrinsic_growth_rate: float,\n", 257 | " gamma: float,\n", 258 | " susceptible: float,\n", 259 | " relative_contact_rate: float\n", 260 | ") -> float:\n", 261 | " return (\n", 262 | " (intrinsic_growth_rate + gamma)\n", 263 | " / susceptible\n", 264 | " * (1.0 - relative_contact_rate)\n", 265 | " )\n", 266 | "\n", 267 | "\n", 268 | "def get_growth_rate(doubling_time: Optional[float]) -> float:\n", 269 | " \"\"\"Calculates average daily growth rate from doubling time.\"\"\"\n", 270 | " if doubling_time is None or doubling_time == 0.0:\n", 271 | " return 0.0\n", 272 | " return (2.0 ** (1.0 / doubling_time) - 1.0)\n", 273 | "\n", 274 | "\n", 275 | "def sir(\n", 276 | " s: float, i: float, r: float, beta: float, gamma: float, n: float\n", 277 | ") -> Tuple[float, float, float]:\n", 278 | " \"\"\"The SIR model, one time step.\"\"\"\n", 279 | " s_n = (-beta * s * i) + s\n", 280 | " i_n = (beta * s * i - gamma * i) + i\n", 281 | " r_n = gamma * i + r\n", 282 | "\n", 283 | " # TODO:\n", 284 | " # Post check dfs for negative values and\n", 285 | " # warn the user that their input data is bad.\n", 286 | " # JL: I suspect that these adjustments covered bugs.\n", 287 | "\n", 288 | " #if s_n < 0.0:\n", 289 | " # s_n = 0.0\n", 290 | " #if i_n < 0.0:\n", 291 | " # i_n = 0.0\n", 292 | " #if r_n < 0.0:\n", 293 | " # r_n = 0.0\n", 294 | " scale = n / (s_n + i_n + r_n)\n", 295 | " return s_n * scale, i_n * scale, r_n * scale\n", 296 | "\n", 297 | "\n", 298 | "def gen_sir(\n", 299 | " s: float, i: float, r: float, gamma: float, i_day: int, policies: Sequence[Tuple[float, int]]\n", 300 | ") -> Generator[Tuple[int, float, float, float], None, None]:\n", 301 | " \"\"\"Simulate SIR model forward in time yielding tuples.\n", 302 | " Parameter order has changed to allow multiple (beta, n_days)\n", 303 | " to reflect multiple changing social distancing policies.\n", 304 | " \"\"\"\n", 305 | " s, i, r = (float(v) for v in (s, i, r))\n", 306 | " n = s + i + r\n", 307 | " d = i_day\n", 308 | " for beta, n_days in policies:\n", 309 | " for _ in range(n_days):\n", 310 | " yield d, s, i, r\n", 311 | " s, i, r = sir(s, i, r, beta, gamma, n)\n", 312 | " d += 1\n", 313 | " yield d, s, i, r\n", 314 | "\n", 315 | "\n", 316 | "def sim_sir_df(\n", 317 | " s: float, i: float, r: float,\n", 318 | " gamma: float, i_day: int, policies: Sequence[Tuple[float, int]]\n", 319 | ") -> pd.DataFrame:\n", 320 | " \"\"\"Simulate the SIR model forward in time.\"\"\"\n", 321 | " return pd.DataFrame(\n", 322 | " data=gen_sir(s, i, r, gamma, i_day, policies),\n", 323 | " columns=(\"day\", \"susceptible\", \"infected\", \"recovered\"),\n", 324 | " )\n", 325 | "\n", 326 | "\n", 327 | "def build_sim_sir_w_date_df(\n", 328 | " raw_df: pd.DataFrame,\n", 329 | " current_date: datetime,\n", 330 | " keys: Sequence[str],\n", 331 | ") -> pd.DataFrame:\n", 332 | " day = raw_df.day\n", 333 | " return pd.DataFrame({\n", 334 | " \"day\": day,\n", 335 | " \"date\": day.astype('timedelta64[D]') + np.datetime64(current_date),\n", 336 | " **{\n", 337 | " key: raw_df[key]\n", 338 | " for key in keys\n", 339 | " }\n", 340 | " })\n", 341 | "\n", 342 | "\n", 343 | "def build_floor_df(df, keys):\n", 344 | " \"\"\"Build floor sim sir w date.\"\"\"\n", 345 | " return pd.DataFrame({\n", 346 | " \"day\": df.day,\n", 347 | " \"date\": df.date,\n", 348 | " **{\n", 349 | " key: np.floor(df[key])\n", 350 | " for key in keys\n", 351 | " }\n", 352 | " })\n", 353 | "\n", 354 | "\n", 355 | "def build_dispositions_df(\n", 356 | " raw_df: pd.DataFrame,\n", 357 | " rates: Dict[str, float],\n", 358 | " market_share: float,\n", 359 | " current_date: datetime,\n", 360 | ") -> pd.DataFrame:\n", 361 | " \"\"\"Build dispositions dataframe of patients adjusted by rate and market_share.\"\"\"\n", 362 | " patients = raw_df.infected + raw_df.recovered\n", 363 | " day = raw_df.day\n", 364 | " return pd.DataFrame({\n", 365 | " \"day\": day,\n", 366 | " \"date\": day.astype('timedelta64[D]') + np.datetime64(current_date),\n", 367 | " **{\n", 368 | " key: patients * rate * market_share\n", 369 | " for key, rate in rates.items()\n", 370 | " }\n", 371 | " })\n", 372 | "\n", 373 | "\n", 374 | "def build_admits_df(dispositions_df: pd.DataFrame) -> pd.DataFrame:\n", 375 | " \"\"\"Build admits dataframe from dispositions.\"\"\"\n", 376 | " admits_df = dispositions_df - dispositions_df.shift(1)\n", 377 | " admits_df.day = dispositions_df.day\n", 378 | " admits_df.date = dispositions_df.date\n", 379 | " return admits_df\n", 380 | "\n", 381 | "\n", 382 | "def build_census_df(\n", 383 | " admits_df: pd.DataFrame,\n", 384 | " lengths_of_stay: Dict[str, int],\n", 385 | ") -> pd.DataFrame:\n", 386 | " \"\"\"Average Length of Stay for each disposition of COVID-19 case (total guesses)\"\"\"\n", 387 | " return pd.DataFrame({\n", 388 | " 'day': admits_df.day,\n", 389 | " 'date': admits_df.date,\n", 390 | " **{\n", 391 | " key: (\n", 392 | " admits_df[key].cumsum()\n", 393 | " - admits_df[key].cumsum().shift(los).fillna(0)\n", 394 | " )\n", 395 | " for key, los in lengths_of_stay.items()\n", 396 | " }\n", 397 | " })\n", 398 | "\n", 399 | "\n", 400 | "#Convert the current dataframe to stacked groupby format\n", 401 | "def pivot(df, state=None):\n", 402 | " df_copy = df.copy()\n", 403 | " if state:\n", 404 | " return (df_copy.set_index([\"day\", \"date\", \"date_str\", \"state\", \"countyname\"]).stack().reset_index(name=\"Value\").rename(columns={'level_5': \"status\"}))\n", 405 | " return (df_copy.set_index([\"day\", \"date\", \"date_str\", \"countyname\"]).stack().reset_index(name=\"Value\").rename(columns={'level_4': \"status\"}))\n", 406 | "\n", 407 | "\n", 408 | "\n", 409 | "def run_geo_merge_df(df, geodf):\n", 410 | " return geodf.merge(df, on=\"countyname\")\n", 411 | "\n", 412 | "\n", 413 | "#Adds county, date_str (date as string), state (if exists) and calls the pivot definition\n", 414 | "def modify_df(df, county, state=None):\n", 415 | " if state:\n", 416 | " df['state'] = state\n", 417 | " df['countyname'] = county\n", 418 | " df['date_str'] = df['date'].astype('str')\n", 419 | " \n", 420 | " return pivot(df,state)\n", 421 | "\n", 422 | "\n", 423 | "\n", 424 | "# Runs the model for the county with or without geometries\n", 425 | "def run_model_for_county(df, county):\n", 426 | " df.columns = map(str.lower, df.columns)\n", 427 | " if all(col in df.columns.values for col in ['countyname', 'hospitaliz', 'population', 'unacast_ch', 'hospital_1']):\n", 428 | " if df.loc[df['countyname'] == county].shape[0]:\n", 429 | " c= df.loc[df['countyname'] == county].iloc[0]\n", 430 | " p = Parameters(population=c.population,\n", 431 | " current_hospitalized=c.hospitaliz,\n", 432 | " relative_contact_rate=c.unacast_ch,\n", 433 | " hospitalized=c.hospital_1)\n", 434 | " m = SimSirModel(p)\n", 435 | " pivot_admits = modify_df(m.admits_floor_df, county)\n", 436 | " pivot_census = modify_df(m.census_floor_df, county)\n", 437 | " pivot_sir = modify_df(m.sim_sir_w_date_floor_df, county)\n", 438 | " return pivot_admits, pivot_census, pivot_sir\n", 439 | " else:\n", 440 | " print(\"County {} doesn't exist\".format(county))\n", 441 | " \n", 442 | " else:\n", 443 | " print('Required column names do not exists')\n", 444 | "\n", 445 | "\n", 446 | "\n", 447 | "# Runs the model for the state/county with or without geometries\n", 448 | "def run_model_for_state_county(df, state, county):\n", 449 | " df.columns = map(str.lower, df.columns)\n", 450 | " if all(col in df.columns.values for col in ['statename','countyname', 'hospitaliz', 'population', 'unacast_ch', 'hospital_1']):\n", 451 | " if df.loc[(df['statename'] == state) & (df['countyname'] == county)].shape[0]:\n", 452 | " c= df.loc[(df['statename'] == state) & (df['countyname'] == county)].iloc[0]\n", 453 | " p = Parameters(population=c.population,\n", 454 | " current_hospitalized=c.hospitaliz,\n", 455 | " relative_contact_rate=c.unacast_ch,\n", 456 | " hospitalized=c.hospital_1)\n", 457 | " m = SimSirModel(p)\n", 458 | " pivot_admits = modify_df(m.admits_floor_df, county, state)\n", 459 | " pivot_census = modify_df(m.census_floor_df, county, state)\n", 460 | " pivot_sir = modify_df(m.sim_sir_w_date_floor_df, county, state)\n", 461 | " return pivot_admits, pivot_census, pivot_sir\n", 462 | " else:\n", 463 | " print(\"County {} doesn't exist\".format(county))\n", 464 | " \n", 465 | " else:\n", 466 | " print('Required column names do not exists')\n", 467 | "\n", 468 | "\n", 469 | "def merge_dfs(admits_df, census_df, sir_df):\n", 470 | " admits_df_cpy = admits_df.copy()\n", 471 | " census_df_cpy = census_df.copy()\n", 472 | " sir_df_cpy = sir_df.copy()\n", 473 | " admits_df_cpy['type'] = 'admits'\n", 474 | " census_df_cpy['type'] = 'census'\n", 475 | " sir_df_cpy['type'] = 'sim_sir'\n", 476 | " return pd.concat([admits_df_cpy, census_df_cpy, sir_df_cpy], ignore_index=True)\n", 477 | "\n", 478 | "#This runs the model for all the counties if you have the county name in the dataset\n", 479 | "#(Column name should be 'countyname')\n", 480 | "def run_model_for_all_counties(df):\n", 481 | " df.columns = map(str.lower, df.columns)\n", 482 | " all_df = pd.DataFrame()\n", 483 | " for index, row in df.iterrows():\n", 484 | " admits_df, census_df, sir_df = run_model_for_county(df,(row['countyname']))\n", 485 | " fin_df = merge_dfs(admits_df, census_df, sir_df)\n", 486 | " all_df = all_df.append(fin_df)\n", 487 | " return all_df\n", 488 | "\n", 489 | "\n", 490 | "#This runs the model for all the states if you have the statename and respective county name in the dataset\n", 491 | "#(Column names should be 'statename' and 'countyname')\n", 492 | "def run_model_for_all_states(df):\n", 493 | " df.columns = map(str.lower, df.columns)\n", 494 | " all_df = pd.DataFrame()\n", 495 | " for index, row in df.iterrows():\n", 496 | " admits_df, census_df, sir_df = run_model_for_state_county(df,(row['statename']),(row['countyname']))\n", 497 | " fin_df = merge_dfs(admits_df, census_df, sir_df)\n", 498 | " all_df = all_df.append(fin_df)\n", 499 | " return all_df\n" 500 | ] 501 | }, 502 | { 503 | "cell_type": "code", 504 | "execution_count": null, 505 | "metadata": {}, 506 | "outputs": [], 507 | "source": [ 508 | "#DND the 5 fields(Countyname, Hospitalz, Unacast_ch, Hospital_1, Population) from Chime DS\n", 509 | "chime_df = #pd.read_csv('CHIMEDataSet.csv')\n", 510 | "county_run_df = run_model_for_all_counties(chime_df)" 511 | ] 512 | }, 513 | { 514 | "cell_type": "code", 515 | "execution_count": null, 516 | "metadata": {}, 517 | "outputs": [], 518 | "source": [ 519 | "%insights_return(county_run_df)" 520 | ] 521 | }, 522 | { 523 | "cell_type": "code", 524 | "execution_count": null, 525 | "metadata": {}, 526 | "outputs": [], 527 | "source": [ 528 | "#DND the 6 fields(Statename, Countyname, Hospitalz, Unacast_ch, Hospital_1, Population) from USA State county DS\n", 529 | "#usa_state_county_df = #pd.read_csv('USStatesCHIME.csv')\n", 530 | "#state_run_df = run_model_for_all_states(usa_state_county_df)" 531 | ] 532 | }, 533 | { 534 | "cell_type": "code", 535 | "execution_count": null, 536 | "metadata": {}, 537 | "outputs": [], 538 | "source": [ 539 | "#%insights_return(state_run_df)" 540 | ] 541 | } 542 | ], 543 | "metadata": { 544 | "kernelspec": { 545 | "display_name": "Python 3", 546 | "language": "python", 547 | "name": "python3" 548 | }, 549 | "language_info": { 550 | "codemirror_mode": { 551 | "name": "ipython", 552 | "version": 3 553 | }, 554 | "file_extension": ".py", 555 | "mimetype": "text/x-python", 556 | "name": "python", 557 | "nbconvert_exporter": "python", 558 | "pygments_lexer": "ipython3", 559 | "version": "3.6.10" 560 | } 561 | }, 562 | "nbformat": 4, 563 | "nbformat_minor": 2 564 | } 565 | --------------------------------------------------------------------------------