├── wwo_hist.egg-info ├── dependency_links.txt ├── top_level.txt ├── SOURCES.txt └── PKG-INFO ├── dist ├── wwo_hist-0.0.7.tar.gz └── wwo_hist-0.0.7-py3-none-any.whl ├── doc └── screenshots │ ├── Example_01.PNG │ └── Example_02.PNG ├── setup.py ├── README.md └── wwo_hist └── __init__.py /wwo_hist.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /wwo_hist.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | wwo_hist 2 | -------------------------------------------------------------------------------- /dist/wwo_hist-0.0.7.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ekapope/WorldWeatherOnline/HEAD/dist/wwo_hist-0.0.7.tar.gz -------------------------------------------------------------------------------- /doc/screenshots/Example_01.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ekapope/WorldWeatherOnline/HEAD/doc/screenshots/Example_01.PNG -------------------------------------------------------------------------------- /doc/screenshots/Example_02.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ekapope/WorldWeatherOnline/HEAD/doc/screenshots/Example_02.PNG -------------------------------------------------------------------------------- /dist/wwo_hist-0.0.7-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ekapope/WorldWeatherOnline/HEAD/dist/wwo_hist-0.0.7-py3-none-any.whl -------------------------------------------------------------------------------- /wwo_hist.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | README.md 2 | setup.py 3 | wwo_hist/__init__.py 4 | wwo_hist.egg-info/PKG-INFO 5 | wwo_hist.egg-info/SOURCES.txt 6 | wwo_hist.egg-info/dependency_links.txt 7 | wwo_hist.egg-info/top_level.txt -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | This script is used to retrieve and transform weather data into single csv. 4 | example API explorer: https://www.worldweatheronline.com/developer/premium-api-explorer.aspx 5 | 6 | input: api_key, location_list, start_date, end_date, frequency 7 | 8 | output: location_name.csv' 9 | 10 | @author: Ekapope Viriyakovithya 11 | """ 12 | 13 | import setuptools 14 | 15 | with open("README.md", "r") as fh: 16 | long_description = fh.read() 17 | 18 | setuptools.setup( 19 | name="wwo_hist", 20 | version="0.0.7", 21 | author="Ekapope Viriyakovithya", 22 | author_email="ekapope.v@gmail.com", 23 | description="This package is used to retrieve and transform historical weather data from www.worldweatheronline.com into pandas dataframe and csv.", 24 | long_description=long_description, 25 | long_description_content_type="text/markdown", 26 | url="https://github.com/ekapope/WorldWeatherOnline", 27 | packages=setuptools.find_packages(), 28 | classifiers=[ 29 | "Programming Language :: Python :: 3", 30 | "License :: OSI Approved :: MIT License", 31 | "Operating System :: OS Independent", 32 | ], 33 | ) 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # WorldWeatherOnline historical weather data API wrapper 2 | 3 | [![Downloads](https://pepy.tech/badge/wwo-hist)](https://pepy.tech/project/wwo-hist) 4 | 5 | This package is used to retrieve and transform historical weather data from www.worldweatheronline.com into pandas dataframe and csv. 6 | 7 | You can get API key for free (free trial 500 requests/key/day for 60 days, as of 30-May-2019). 8 | 9 | example API explorer: https://www.worldweatheronline.com/developer/premium-api-explorer.aspx 10 | 11 | 12 | **Input**: api_key, location_list, start_date, end_date, frequency 13 | 14 | **Output**: location_name.csv 15 | 16 | **Output column names**: date_time, maxtempC, mintempC, totalSnow_cm, sunHour, uvIndex, uvIndex, moon_illumination, moonrise, moonset, sunrise, sunset, DewPointC, FeelsLikeC, HeatIndexC, WindChillC, WindGustKmph, cloudcover, humidity, precipMM, pressure, tempC, visibility, winddirDegree, windspeedKmph 17 | 18 | 19 | #### Install the package: 20 | ``` 21 | pip install wwo-hist 22 | ``` 23 | 24 | #### Import package 25 | ```python 26 | from wwo_hist import retrieve_hist_data 27 | ``` 28 | 29 | #### Set working directory to store output csv file(s) 30 | ```python 31 | import os 32 | os.chdir(".\YOUR_PATH") 33 | ``` 34 | 35 | 36 | #### Example code 37 | ```python 38 | frequency=3 39 | start_date = '11-DEC-2018' 40 | end_date = '11-MAR-2019' 41 | api_key = 'YOUR_API_KEY' 42 | location_list = ['singapore','california'] 43 | 44 | hist_weather_data = retrieve_hist_data(api_key, 45 | location_list, 46 | start_date, 47 | end_date, 48 | frequency, 49 | location_label = False, 50 | export_csv = True, 51 | store_df = True) 52 | ``` 53 | 54 | #### Parameters: 55 | 56 | 57 | **api_key**: *string* 58 | 59 | (Premium/ free trial) API key from https://www.worldweatheronline.com/developer/api/ 60 | 61 | 62 | **location_list**: *list of string* 63 | 64 | US Zipcode, UK Postcode, Canada Postalcode, IP address, Latitude/Longitude (decimal degree) or city name 65 | 66 | 67 | **start_date**: *string* 68 | 69 | Preferred date format: 'dd-mmm-yyyy' 70 | 71 | 72 | **end_date**: *string* 73 | 74 | Preferred date format: 'dd-mmm-yyyy' 75 | 76 | 77 | **frequency**: *integer* 78 | 79 | 1, 3, 6, 12, 24 80 | 81 | 1 hourly, 3 hourly, 6 hourly, 12 hourly (day/night) or 24 hourly (day average) 82 | 83 | 84 | **location_label**: *bool*, default = False 85 | 86 | If True, all column names will have city name as prefix. 87 | 88 | 89 | **export_csv**: *bool*, default = True 90 | 91 | If False, no csv file will be exported to current directory. 92 | 93 | 94 | **store_df**: *bool*, default = False 95 | 96 | If True, retrieved dataframe(s) will be stored as list in the work space. 97 | 98 | 99 | #### Console view 100 | ![IPython](/doc/screenshots/Example_01.PNG) 101 | 102 | 103 | #### Output file 104 | ![CSV file](/doc/screenshots/Example_02.PNG) 105 | -------------------------------------------------------------------------------- /wwo_hist.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 2.1 2 | Name: wwo-hist 3 | Version: 0.0.7 4 | Summary: This package is used to retrieve and transform historical weather data from www.worldweatheronline.com into pandas dataframe and csv. 5 | Home-page: https://github.com/ekapope/WorldWeatherOnline 6 | Author: Ekapope Viriyakovithya 7 | Author-email: ekapope.v@gmail.com 8 | License: UNKNOWN 9 | Description: # WorldWeatherOnline historical weather data API wrapper 10 | 11 | [![Downloads](https://pepy.tech/badge/wwo-hist)](https://pepy.tech/project/wwo-hist) 12 | 13 | This package is used to retrieve and transform historical weather data from www.worldweatheronline.com into pandas dataframe and csv. 14 | 15 | You can get API key for free (free trial 500 requests/key/day for 60 days, as of 30-May-2019). 16 | 17 | example API explorer: https://www.worldweatheronline.com/developer/premium-api-explorer.aspx 18 | 19 | 20 | **Input**: api_key, location_list, start_date, end_date, frequency 21 | 22 | **Output**: location_name.csv 23 | 24 | **Output column names**: date_time, maxtempC, mintempC, totalSnow_cm, sunHour, uvIndex, uvIndex, moon_illumination, moonrise, moonset, sunrise, sunset, DewPointC, FeelsLikeC, HeatIndexC, WindChillC, WindGustKmph, cloudcover, humidity, precipMM, pressure, tempC, visibility, winddirDegree, windspeedKmph 25 | 26 | 27 | #### Install the package: 28 | ``` 29 | pip install wwo-hist 30 | ``` 31 | 32 | #### Import package 33 | ```python 34 | from wwo_hist import retrieve_hist_data 35 | ``` 36 | 37 | #### Set working directory to store output csv file(s) 38 | ```python 39 | import os 40 | os.chdir(".\YOUR_PATH") 41 | ``` 42 | 43 | 44 | #### Example code 45 | ```python 46 | frequency=3 47 | start_date = '11-DEC-2018' 48 | end_date = '11-MAR-2019' 49 | api_key = 'YOUR_API_KEY' 50 | location_list = ['singapore','california'] 51 | 52 | hist_weather_data = retrieve_hist_data(api_key, 53 | location_list, 54 | start_date, 55 | end_date, 56 | frequency, 57 | location_label = False, 58 | export_csv = True, 59 | store_df = True) 60 | ``` 61 | 62 | #### Parameters: 63 | 64 | 65 | **api_key**: *string* 66 | 67 | (Premium/ free trial) API key from https://www.worldweatheronline.com/developer/api/ 68 | 69 | 70 | **location_list**: *list of string* 71 | 72 | US Zipcode, UK Postcode, Canada Postalcode, IP address, Latitude/Longitude (decimal degree) or city name 73 | 74 | 75 | **start_date**: *string* 76 | 77 | Preferred date format: 'dd-mmm-yyyy' 78 | 79 | 80 | **end_date**: *string* 81 | 82 | Preferred date format: 'dd-mmm-yyyy' 83 | 84 | 85 | **frequency**: *integer* 86 | 87 | 1, 3, 6, 12, 24 88 | 89 | 1 hourly, 3 hourly, 6 hourly, 12 hourly (day/night) or 24 hourly (day average) 90 | 91 | 92 | **location_label**: *bool*, default = False 93 | 94 | If True, all column names will have city name as prefix. 95 | 96 | 97 | **export_csv**: *bool*, default = True 98 | 99 | If False, no csv file will be exported to current directory. 100 | 101 | 102 | **store_df**: *bool*, default = False 103 | 104 | If True, retrieved dataframe(s) will be stored as list in the work space. 105 | 106 | 107 | #### Console view 108 | ![IPython](/doc/screenshots/Example_01.PNG) 109 | 110 | 111 | #### Output file 112 | ![CSV file](/doc/screenshots/Example_02.PNG) 113 | 114 | Platform: UNKNOWN 115 | Classifier: Programming Language :: Python :: 3 116 | Classifier: License :: OSI Approved :: MIT License 117 | Classifier: Operating System :: OS Independent 118 | Description-Content-Type: text/markdown 119 | -------------------------------------------------------------------------------- /wwo_hist/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | This script is used to retrieve and transform weather data into single csv. 4 | example API explorer: https://www.worldweatheronline.com/developer/premium-api-explorer.aspx 5 | 6 | input: api_key, location_list, start_date, end_date, frequency 7 | 8 | output: location_name.csv' 9 | 10 | @author: Ekapope Viriyakovithya 11 | """ 12 | 13 | import urllib.request 14 | import urllib.parse 15 | import json 16 | import pandas as pd 17 | from datetime import datetime 18 | import os 19 | 20 | 21 | ################################## 22 | # function to unnest json for each month 23 | def extract_monthly_data(data): 24 | num_days = len(data) 25 | # initialize df_month to store return data 26 | df_month = pd.DataFrame() 27 | for i in range(num_days): 28 | # extract this day 29 | d = data[i] 30 | # astronomy data is the same for the whole day 31 | astr_df = pd.DataFrame(d['astronomy']) 32 | # hourly data; temperature for each hour of the day 33 | hourly_df = pd.DataFrame(d['hourly']) 34 | # this wanted_key will be duplicated and use 'ffill' to fill up the NAs 35 | wanted_keys = ['date', 'maxtempC', 'mintempC', 'totalSnow_cm', 'sunHour', 'uvIndex'] # The keys you want 36 | subset_d = dict((k, d[k]) for k in wanted_keys if k in d) 37 | this_df = pd.DataFrame(subset_d, index=[0]) 38 | df = pd.concat([this_df.reset_index(drop=True), astr_df], axis=1) 39 | # concat selected astonomy columns with hourly data 40 | df = pd.concat([df, hourly_df], axis=1) 41 | df = df.fillna(method='ffill') 42 | # make date_time columm to proper format 43 | # fill leading zero for hours to 4 digits (0000-2400 hr) 44 | df['time'] = df['time'].apply(lambda x: x.zfill(4)) 45 | # keep only first 2 digit (00-24 hr) 46 | df['time'] = df['time'].str[:2] 47 | # convert to pandas datetime 48 | df['date_time'] = pd.to_datetime(df['date'] + ' ' + df['time']) 49 | # keep only interested columns 50 | col_to_keep = ['date_time', 'maxtempC', 'mintempC', 'totalSnow_cm', 'sunHour', 'uvIndex', 51 | 'moon_illumination', 'moonrise', 'moonset', 'sunrise', 'sunset', 52 | 'DewPointC', 'FeelsLikeC', 'HeatIndexC', 'WindChillC', 'WindGustKmph', 53 | 'cloudcover', 'humidity', 'precipMM', 'pressure', 'tempC', 'visibility', 54 | 'winddirDegree', 'windspeedKmph'] 55 | df = df[col_to_keep] 56 | df = df.loc[:,~df.columns.duplicated()] 57 | df_month = pd.concat([df_month, df]) 58 | return (df_month) 59 | 60 | 61 | ################################## 62 | # function to retrive data by date range and location 63 | # default frequency = 1 hr 64 | # each month costs 1 request (free trial 500 requests/key, as of 30-May-2019) 65 | def retrieve_this_location(api_key, location, start_date, end_date, frequency, response_cache_path): 66 | start_time = datetime.now() 67 | 68 | # create list of first day of month for range between start and end dates non-inclusive (open) 69 | list_mon_begin = pd.date_range(start_date, end_date, freq='MS', closed='right') 70 | # convert to Series and add start_date at beginning 71 | list_mon_begin = pd.concat([pd.Series(pd.to_datetime(start_date)), pd.Series(list_mon_begin)], ignore_index=True) 72 | 73 | # create list of month end dates for range between start and end dates non-inclusive (open) 74 | list_mon_end = pd.date_range(start_date, end_date, freq='M', closed='left') 75 | # convert to Series and add end_date at end 76 | list_mon_end = pd.concat([pd.Series(list_mon_end), pd.Series(pd.to_datetime(end_date))], ignore_index=True) 77 | 78 | # count number of months to be retrieved 79 | total_months = len(list_mon_begin) 80 | 81 | # initialize df_hist to store return data 82 | df_hist = pd.DataFrame() 83 | for m in range(total_months): 84 | start_d = str(list_mon_begin[m])[:10] 85 | end_d = str(list_mon_end[m])[:10] 86 | file_path = f'{response_cache_path}/{location}_{start_d}_{end_d}' 87 | if response_cache_path and os.path.exists(file_path): 88 | print('Reading cached data for ' + location + ': from ' + start_d + ' to ' + end_d) 89 | with open(f'{response_cache_path}/{location}_{start_d}_{end_d}', 'r') as f: 90 | json_data = json.load(f) 91 | else: 92 | print('Currently retrieving data for ' + location + ': from ' + start_d + ' to ' + end_d) 93 | url_page = 'http://api.worldweatheronline.com/premium/v1/past-weather.ashx?key=' + api_key + '&q=' + location + '&format=json&date=' + start_d + '&enddate=' + end_d + '&tp=' + str( 94 | frequency) 95 | json_page = urllib.request.urlopen(url_page, timeout=10) 96 | json_data = json.loads(json_page.read().decode()) 97 | 98 | if response_cache_path: 99 | with open(f'{response_cache_path}/{location}_{start_d}_{end_d}', 'w') as f: 100 | json.dump(json_data, f) 101 | data = json_data['data']['weather'] 102 | # call function to extract json object 103 | df_this_month = extract_monthly_data(data) 104 | df_this_month['location'] = location 105 | df_hist = pd.concat([df_hist, df_this_month]) 106 | 107 | time_elapsed = datetime.now() - start_time 108 | print('Time elapsed (hh:mm:ss.ms) {}'.format(time_elapsed)) 109 | return (df_hist) 110 | 111 | 112 | ################################## 113 | # main function to retrive the data by location list 114 | def retrieve_hist_data(api_key, location_list, start_date, end_date, frequency, location_label=False, export_csv=True, 115 | store_df=False, response_cache_path=None): 116 | result_list = [] 117 | for location in location_list: 118 | print('\n\nRetrieving weather data for ' + location + '\n\n') 119 | df_this_city = retrieve_this_location(api_key, location, start_date, end_date, frequency, response_cache_path) 120 | 121 | if (location_label == True): 122 | # add city name as prefix to the colnames 123 | df_this_city = df_this_city.add_prefix(location + '_') 124 | df_this_city.columns.values[0] = 'date_time' 125 | 126 | if (export_csv == True): 127 | df_this_city.to_csv('./' + location + '.csv', header=True, index=False) 128 | print('\n\nexport ' + location + ' completed!\n\n') 129 | 130 | if (store_df == True): 131 | # save result as object in the work space 132 | result_list.append(df_this_city) 133 | 134 | return (result_list) 135 | ################################## 136 | --------------------------------------------------------------------------------