├── config.ini
├── src
├── __init__.py
├── stats
│ ├── __init__.py
│ ├── stepsStats.py
│ ├── hbStats.py
│ ├── combinedStats.py
│ └── sleepStats.py
├── test
│ ├── __init__.py
│ ├── testHbStats.py
│ ├── testPlotting.py
│ └── testSleepStats.py
├── resources
│ ├── __init__.py
│ ├── unittest
│ │ ├── test_sleep_basic01.csv
│ │ ├── test_sleep_basic02.csv
│ │ ├── test_hb_basic01.csv
│ │ ├── test_sleep_intervals01.csv
│ │ ├── test_sleep_intervals02.csv
│ │ └── test_sleepStats.csv
│ └── intraday_sleep_sample.csv
└── util
│ ├── __init__.py
│ ├── gather_keys_oauth2.py
│ ├── scraper.py
│ ├── utils.py
│ └── plotting.py
├── requirements.txt
├── .gitignore
├── setup.py
├── README.md
├── heartBeatTesting.ipynb
├── LICENSE
└── Sleep Dashboard.ipynb
/config.ini:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/src/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/src/stats/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/src/test/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/src/resources/__init__.py:
--------------------------------------------------------------------------------
1 | import os
2 | RESOURCE_PATH = os.path.join(os.path.dirname(__file__))
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | numpy
2 | pandas
3 | matplotlib
4 | seaborn
5 | scipy
6 | scikit-learn
7 | requests
8 |
--------------------------------------------------------------------------------
/src/util/__init__.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | logging.basicConfig(level=logging.INFO)
4 | logger = logging.getLogger(__name__)
5 | logger.setLevel(logging.INFO)
6 |
--------------------------------------------------------------------------------
/src/stats/stepsStats.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import numpy as np
3 |
4 | NAME_VAL_COL = "value"
5 | NAME_DT_COL = "datetime"
6 |
7 | def groupAndSumByDate(stepsData):
8 | data = stepsData
9 | if isinstance(stepsData, list):
10 | data = pd.concat(stepsData, ignore_index=True)
11 | res = data.groupby(data[NAME_DT_COL].dt.date)[NAME_VAL_COL].sum()
12 | return res
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | __pycache__/
2 | *.py[cod]
3 |
4 | *.pydevproject
5 | .project
6 | .metadata
7 | bin/**
8 | tmp/**
9 | tmp/**/*
10 | *.tmp
11 | *.bak
12 | *.swp
13 | *~.nib
14 | local.properties
15 | .classpath
16 | .settings/
17 | .loadpath
18 | *.egg-info
19 |
20 | # CDT-specific
21 | .cproject
22 |
23 | #PYCHARM
24 | .idea/
25 |
26 | #IPYNB
27 | *.ipynb_checkpoints
28 | src/test/devTesting.ipynb
29 |
30 |
--------------------------------------------------------------------------------
/src/resources/unittest/test_sleep_basic01.csv:
--------------------------------------------------------------------------------
1 | 2016-03-21 23:18:00,2
2 | 2016-03-21 23:19:00,2
3 | 2016-03-21 23:20:00,2
4 | 2016-03-21 23:21:00,2
5 | 2016-03-21 23:22:00,2
6 | 2016-03-21 23:23:00,3
7 | 2016-03-21 23:24:00,2
8 | 2016-03-21 23:25:00,1
9 | 2016-03-21 23:26:00,1
10 | 2016-03-21 23:27:00,1
11 | 2016-03-21 23:28:00,1
12 | 2016-03-21 23:29:00,1
13 | 2016-03-21 23:30:00,1
14 | 2016-03-21 23:31:00,1
15 | 2016-03-21 23:32:00,1
16 | 2016-03-21 23:33:00,1
--------------------------------------------------------------------------------
/src/resources/unittest/test_sleep_basic02.csv:
--------------------------------------------------------------------------------
1 | 2016-03-23 00:18:00,2
2 | 2016-03-23 00:19:00,2
3 | 2016-03-23 00:20:00,2
4 | 2016-03-23 00:21:00,2
5 | 2016-03-23 00:22:00,2
6 | 2016-03-23 00:23:00,3
7 | 2016-03-23 00:24:00,2
8 | 2016-03-23 00:25:00,1
9 | 2016-03-23 00:26:00,1
10 | 2016-03-23 00:27:00,1
11 | 2016-03-23 00:28:00,1
12 | 2016-03-23 00:29:00,1
13 | 2016-03-23 00:30:00,1
14 | 2016-03-23 00:31:00,1
15 | 2016-03-23 00:32:00,1
16 | 2016-03-23 00:33:00,1
--------------------------------------------------------------------------------
/src/resources/unittest/test_hb_basic01.csv:
--------------------------------------------------------------------------------
1 | 2016-03-21 23:18:00,60
2 | 2016-03-21 23:19:00,60
3 | 2016-03-21 23:20:00,60
4 | 2016-03-21 23:21:00,60
5 | 2016-03-21 23:22:00,60
6 | 2016-03-21 23:23:00,60
7 | 2016-03-21 23:24:00,60
8 | 2016-03-21 23:25:00,60
9 | 2016-03-21 23:26:00,60
10 | 2016-03-21 23:27:00,60
11 | 2016-03-21 23:28:00,50
12 | 2016-03-21 23:29:00,60
13 | 2016-03-21 23:30:00,60
14 | 2016-03-21 23:31:00,60
15 | 2016-03-21 23:32:00,60
16 | 2016-03-21 23:33:00,70
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup, find_packages
2 |
3 |
4 | setup(
5 | name="fitbit_scraper",
6 | version="0.1.0",
7 | description="Utility to scrape personal Fitbit data",
8 | license="Apache 2.0",
9 | author="Alex Martinelli",
10 | packages=find_packages(),
11 | entry_points={
12 | 'console_scripts': ['fitbit-scraper=src.util.scraper:main'],
13 | },
14 | install_requires=[
15 | 'fitbit',
16 | 'cherrypy'
17 | ],
18 | )
19 |
--------------------------------------------------------------------------------
/src/stats/hbStats.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 |
3 | NAME_VAL_COL = "value"
4 | NAME_DT_COL = "datetime"
5 |
6 | def basicStats(hbData):
7 | return hbData.describe()
8 |
9 | def groupByBasicStats(groupByFun, data, indexName=None):
10 | hbData = data.groupby(groupByFun)
11 |
12 | stats = {'count' : hbData[NAME_VAL_COL].count(),
13 | 'mean' : hbData[NAME_VAL_COL].mean(),
14 | 'min' : hbData[NAME_VAL_COL].min(),
15 | 'max' : hbData[NAME_VAL_COL].max(),
16 | 'std' : hbData[NAME_VAL_COL].std()
17 | }
18 |
19 | df = pd.DataFrame.from_dict(stats)
20 |
21 | if indexName:
22 | df.index.name = indexName
23 |
24 | return df.reset_index()
25 |
26 | def getMaxValues(data, n):
27 | return data.nlargest(n, NAME_VAL_COL)
28 |
29 | def getMinValues(data, n):
30 | return data.nsmallest(n, NAME_VAL_COL)
--------------------------------------------------------------------------------
/src/resources/unittest/test_sleep_intervals01.csv:
--------------------------------------------------------------------------------
1 | 2016-03-21 23:18:00,2
2 | 2016-03-21 23:19:00,2
3 | 2016-03-21 23:20:00,2
4 | 2016-03-21 23:21:00,2
5 | 2016-03-21 23:22:00,2
6 | 2016-03-21 23:23:00,3
7 | 2016-03-21 23:24:00,2
8 | 2016-03-21 23:25:00,1
9 | 2016-03-21 23:26:00,1
10 | 2016-03-21 23:27:00,1
11 | 2016-03-21 23:28:00,1
12 | 2016-03-21 23:29:00,2
13 | 2016-03-21 23:30:00,2
14 | 2016-03-21 23:31:00,1
15 | 2016-03-21 23:32:00,1
16 | 2016-03-21 23:33:00,1
17 | 2016-03-21 23:34:00,1
18 | 2016-03-21 23:35:00,1
19 | 2016-03-21 23:36:00,2
20 | 2016-03-21 23:37:00,2
21 | 2016-03-21 23:38:00,1
22 | 2016-03-21 23:39:00,1
23 | 2016-03-21 23:40:00,1
24 | 2016-03-21 23:41:00,1
25 | 2016-03-21 23:42:00,1
26 | 2016-03-21 23:43:00,1
27 | 2016-03-21 23:44:00,1
28 | 2016-03-21 23:45:00,1
29 | 2016-03-21 23:46:00,2
30 | 2016-03-21 23:47:00,1
31 | 2016-03-21 23:48:00,1
32 | 2016-03-21 23:49:00,1
33 | 2016-03-21 23:50:00,2
34 | 2016-03-21 23:51:00,1
35 | 2016-03-21 23:52:00,1
36 | 2016-03-21 23:53:00,1
37 | 2016-03-21 23:54:00,1
38 | 2016-03-21 23:55:00,1
39 | 2016-03-21 23:56:00,1
40 | 2016-03-21 23:57:00,1
41 | 2016-03-21 23:58:00,1
42 | 2016-03-21 23:59:00,2
43 | 2016-03-22 00:00:00,1
44 | 2016-03-22 00:01:00,1
--------------------------------------------------------------------------------
/src/resources/unittest/test_sleep_intervals02.csv:
--------------------------------------------------------------------------------
1 | 2016-03-22 23:18:00,2
2 | 2016-03-22 23:19:00,2
3 | 2016-03-22 23:20:00,2
4 | 2016-03-22 23:21:00,2
5 | 2016-03-22 23:22:00,2
6 | 2016-03-22 23:23:00,1
7 | 2016-03-22 23:24:00,1
8 | 2016-03-22 23:25:00,1
9 | 2016-03-22 23:26:00,1
10 | 2016-03-22 23:27:00,1
11 | 2016-03-22 23:28:00,1
12 | 2016-03-22 23:29:00,2
13 | 2016-03-22 23:30:00,2
14 | 2016-03-22 23:31:00,1
15 | 2016-03-22 23:32:00,1
16 | 2016-03-22 23:33:00,1
17 | 2016-03-22 23:34:00,1
18 | 2016-03-22 23:35:00,1
19 | 2016-03-22 23:36:00,2
20 | 2016-03-22 23:37:00,2
21 | 2016-03-22 23:38:00,1
22 | 2016-03-22 23:39:00,1
23 | 2016-03-22 23:40:00,1
24 | 2016-03-22 23:41:00,1
25 | 2016-03-22 23:42:00,1
26 | 2016-03-22 23:43:00,1
27 | 2016-03-22 23:44:00,1
28 | 2016-03-22 23:45:00,1
29 | 2016-03-22 23:46:00,2
30 | 2016-03-22 23:47:00,1
31 | 2016-03-22 23:48:00,1
32 | 2016-03-22 23:49:00,1
33 | 2016-03-22 23:50:00,2
34 | 2016-03-22 23:51:00,1
35 | 2016-03-22 23:52:00,1
36 | 2016-03-22 23:53:00,1
37 | 2016-03-22 23:54:00,1
38 | 2016-03-22 23:55:00,1
39 | 2016-03-22 23:56:00,1
40 | 2016-03-22 23:57:00,1
41 | 2016-03-22 23:58:00,1
42 | 2016-03-22 23:59:00,1
43 | 2016-03-23 00:00:00,1
44 | 2016-03-23 00:01:00,1
--------------------------------------------------------------------------------
/src/test/testHbStats.py:
--------------------------------------------------------------------------------
1 | import unittest
2 |
3 | from src.resources import RESOURCE_PATH
4 | from src.stats import hbStats
5 | from src.util import utils
6 | import pandas as pd
7 |
8 | class HbStatsTestCase(unittest.TestCase):
9 | def test_basicStats(self):
10 | filepath = RESOURCE_PATH + "\\unittest\\test_hb_basic01.csv"
11 | data = utils.loadIntradayData(filepath).set_index('datetime')
12 | stats = hbStats.groupByBasicStats(pd.TimeGrouper(freq='d'), data)
13 |
14 | self.assertEqual(stats.iloc[0]['count'], 16)
15 | self.assertEqual(stats.iloc[0]['max'], 70)
16 | self.assertEqual(stats.iloc[0]['min'], 50)
17 | self.assertEqual(stats.iloc[0]['mean'], 60)
18 |
19 | def test_minMax(self):
20 | filepath = RESOURCE_PATH + "\\unittest\\test_hb_basic01.csv"
21 | data = utils.loadIntradayData(filepath).set_index('datetime')
22 | stats = hbStats.getMaxValues(data, 2)
23 |
24 | self.assertEqual(stats.iloc[0].value, 70)
25 | self.assertEqual(stats.iloc[1].value, 60)
26 |
27 | stats = hbStats.getMinValues(data, 2)
28 |
29 | self.assertEqual(stats.iloc[0].value, 50)
30 | self.assertEqual(stats.iloc[1].value, 60)
31 |
32 | if __name__ == '__main__':
33 | unittest.main()
34 |
--------------------------------------------------------------------------------
/src/test/testPlotting.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import pandas as pd
3 |
4 | from src.resources import RESOURCE_PATH
5 | from src.stats import sleepStats
6 | from src.util import utils
7 | from src.util import plotting as mplot
8 |
9 | class SleepStatsTestCase(unittest.TestCase):
10 | def test_plottingOnBasicStats(self):
11 | filepath = RESOURCE_PATH + "\\unittest\\test_sleep_basic01.csv"
12 | data1 = utils.loadIntradayData(filepath)
13 | filepath = RESOURCE_PATH + "\\unittest\\test_sleep_basic02.csv"
14 | data2 = utils.loadIntradayData(filepath)
15 | stats = sleepStats.generateStatsFrom([data1, data2],
16 | sleepStats.STATS_NAME_BASIC_AND_TIMING).reset_index()
17 | stats['date'] = pd.to_datetime(stats['date'])
18 |
19 | mplot.plotYearAndMonthStatsSleep(stats)
20 | mplot.plotPreliminaryStats(stats)
21 | mplot.plotWeekdayStatsSleep(stats)
22 | mplot.plotDailyStatsSleep(stats)
23 | mplot.plotMonthlyStatsSleep(stats)
24 |
25 |
26 | def test_plottingOnIntradayStats(self):
27 | filepath = RESOURCE_PATH + "\\unittest\\test_sleep_basic01.csv"
28 | data1 = utils.loadIntradayData(filepath)
29 | filepath = RESOURCE_PATH + "\\unittest\\test_sleep_basic02.csv"
30 | data2 = utils.loadIntradayData(filepath)
31 | stats = sleepStats.generateStatsFrom([data1, data2],
32 | sleepStats.STATS_NAME_INTRADAY)
33 |
34 | data = stats.apply(pd.value_counts)
35 | mplot.plotSleepValueHeatmap(data, sleepValue=1)
36 |
37 | if __name__ == '__main__':
38 | unittest.main()
39 |
--------------------------------------------------------------------------------
/src/stats/combinedStats.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 |
4 | def combineHbAndToggl(togglData, hbData):
5 | """
6 | Combine Toggl data and heart-rate data, creating a summary of heart-rate info
7 | for each Toggl entry (based on date and time correspondence)
8 | :param togglData: dataframe of toggl data
9 | :param hbData: heart-rate dataframe
10 | :return: resulting dataframe of original toggl data + hb stats
11 | """
12 |
13 | def computeHbStats(togglRow, hbData):
14 | hb = hbData[(hbData['datetime'] > togglRow['Start'])
15 | & (hbData['datetime'] < togglRow['End'])]['value']
16 | # hb = hb_data.between_time(toggl_row['Start'], toggl_row['End'])
17 | stats = {'count': hb.count(),
18 | 'mean': hb.mean(),
19 | 'min': hb.min(),
20 | 'max': hb.max(),
21 | 'std': hb.std()
22 | }
23 | return pd.Series(stats)
24 |
25 | # compute hb stats using previous function, and join to toggl data
26 | combinedStats = togglData.join(togglData.apply(lambda x: computeHbStats(x, hbData), axis=1))
27 |
28 | # remove entries for which no hb stats are present (any apart from count would work)
29 | combinedStats = combinedStats.dropna(how='all', subset=['max', 'mean', 'min', 'std'])
30 |
31 | return combinedStats
32 |
33 | def correlateSleepAndSteps(sleepStats, stepsStats, sleepStatName):
34 | return sleepStats[sleepStatName].corr(stepsStats)
35 |
36 | def tranformToGrowthFromStart(stats):
37 | res = stats.apply(lambda x: x / x[0])
38 | return res
39 |
40 | def tranformToGrowthFromPrevious(stats):
41 | res = stats.apply(lambda x: x - x.shift(1))
42 | return res
43 |
44 | def tranformToGrowthFromPreviousLog(stats):
45 | res = stats.apply(lambda x: np.log(x) - np.log(x.shift(1)))
46 | return res
47 |
48 | def normalize(row):
49 | return row - row.mean()
50 |
51 | def timeToInt(time):
52 | hour = time.hour
53 | return (hour * 60) + time.minute
54 |
55 | def pivotedTimeToInt(time, pivot, max_val=23):
56 | hour = time.hour
57 | if hour >= pivot:
58 | return ((hour - pivot) * 60) + time.minute
59 | else:
60 | return ((max_val - (pivot - hour) ) * 60) + time.minute
--------------------------------------------------------------------------------
/src/test/testSleepStats.py:
--------------------------------------------------------------------------------
1 | import unittest
2 |
3 | from src.resources import RESOURCE_PATH
4 | from src.stats import sleepStats
5 | from src.util import utils
6 | import datetime
7 |
8 | class SleepStatsTestCase(unittest.TestCase):
9 | def test_firstMinuteAsleep(self):
10 | filepath = RESOURCE_PATH + "\\unittest\\test_sleepStats.csv"
11 | data = utils.loadIntradayData(filepath)
12 | firstMinuteAsleep = sleepStats.getFirstMinuteAsleep(data)
13 | self.assertEqual(firstMinuteAsleep, 8)
14 |
15 | firstMinuteAsleep = sleepStats.getFirstMinuteAsleep(data, 10)
16 | self.assertEqual(firstMinuteAsleep, 8)
17 |
18 | firstMinuteAsleep = sleepStats.getFirstMinuteAsleep(data, 11)
19 | self.assertEqual(firstMinuteAsleep, 19)
20 |
21 | firstMinuteAsleep = sleepStats.getFirstMinuteAsleep(data, 500)
22 | self.assertEqual(firstMinuteAsleep, None)
23 |
24 | def test_basicStats(self):
25 | filepath = RESOURCE_PATH + "\\unittest\\test_sleep_basic01.csv"
26 | data1 = utils.loadIntradayData(filepath)
27 | filepath = RESOURCE_PATH + "\\unittest\\test_sleep_basic02.csv"
28 | data2 = utils.loadIntradayData(filepath)
29 | stats = sleepStats.generateBasicStats([data1, data2])
30 |
31 | self.assertEqual(stats.iloc[0].name, datetime.date(2016, 3, 21))
32 | self.assertEqual(stats.iloc[1].name, datetime.date(2016, 3, 22))
33 |
34 | def test_intradayStats(self):
35 | filepath = RESOURCE_PATH + "\\unittest\\test_sleep_basic01.csv"
36 | data1 = utils.loadIntradayData(filepath)
37 | filepath = RESOURCE_PATH + "\\unittest\\test_sleep_basic02.csv"
38 | data2 = utils.loadIntradayData(filepath)
39 | stats = sleepStats.generateIntradayStats([data1, data2])
40 |
41 | self.assertEqual(stats.iloc[0].name, datetime.date(2016, 3, 21))
42 | self.assertEqual(stats.iloc[1].name, datetime.date(2016, 3, 22))
43 |
44 | def test_intervalsStats(self):
45 | filepath = RESOURCE_PATH + "\\unittest\\test_sleep_intervals01.csv"
46 | data1 = utils.loadIntradayData(filepath)
47 | filepath = RESOURCE_PATH + "\\unittest\\test_sleep_intervals02.csv"
48 | data2 = utils.loadIntradayData(filepath)
49 | stats = sleepStats.generateIntervalsStats([data1, data2], 5)
50 |
51 | self.assertEqual(stats.iloc[0][0], 5)
52 | self.assertEqual(stats.iloc[0][1], 8)
53 | self.assertEqual(stats.iloc[0][2], 8)
54 | self.assertEqual(stats.iloc[1][0], 6)
55 | self.assertEqual(stats.iloc[1][1], 5)
56 | self.assertEqual(stats.iloc[1][2], 8)
57 | self.assertEqual(stats.iloc[1][3], 11)
58 |
59 | if __name__ == '__main__':
60 | unittest.main()
61 |
--------------------------------------------------------------------------------
/src/resources/unittest/test_sleepStats.csv:
--------------------------------------------------------------------------------
1 | 2016-03-21 23:18:00,2
2 | 2016-03-21 23:19:00,2
3 | 2016-03-21 23:20:00,2
4 | 2016-03-21 23:21:00,2
5 | 2016-03-21 23:22:00,2
6 | 2016-03-21 23:23:00,3
7 | 2016-03-21 23:24:00,2
8 | 2016-03-21 23:25:00,1
9 | 2016-03-21 23:26:00,1
10 | 2016-03-21 23:27:00,1
11 | 2016-03-21 23:28:00,1
12 | 2016-03-21 23:29:00,1
13 | 2016-03-21 23:30:00,1
14 | 2016-03-21 23:31:00,1
15 | 2016-03-21 23:32:00,1
16 | 2016-03-21 23:33:00,1
17 | 2016-03-21 23:34:00,1
18 | 2016-03-21 23:35:00,2
19 | 2016-03-21 23:36:00,1
20 | 2016-03-21 23:37:00,1
21 | 2016-03-21 23:38:00,1
22 | 2016-03-21 23:39:00,1
23 | 2016-03-21 23:40:00,1
24 | 2016-03-21 23:41:00,1
25 | 2016-03-21 23:42:00,1
26 | 2016-03-21 23:43:00,1
27 | 2016-03-21 23:44:00,1
28 | 2016-03-21 23:45:00,1
29 | 2016-03-21 23:46:00,1
30 | 2016-03-21 23:47:00,1
31 | 2016-03-21 23:48:00,1
32 | 2016-03-21 23:49:00,1
33 | 2016-03-21 23:50:00,1
34 | 2016-03-21 23:51:00,1
35 | 2016-03-21 23:52:00,1
36 | 2016-03-21 23:53:00,1
37 | 2016-03-21 23:54:00,1
38 | 2016-03-21 23:55:00,1
39 | 2016-03-21 23:56:00,1
40 | 2016-03-21 23:57:00,1
41 | 2016-03-21 23:58:00,1
42 | 2016-03-21 23:59:00,1
43 | 2016-03-22 00:00:00,1
44 | 2016-03-22 00:01:00,1
45 | 2016-03-22 00:02:00,1
46 | 2016-03-22 00:03:00,1
47 | 2016-03-22 00:04:00,1
48 | 2016-03-22 00:05:00,1
49 | 2016-03-22 00:06:00,1
50 | 2016-03-22 00:07:00,1
51 | 2016-03-22 00:08:00,1
52 | 2016-03-22 00:09:00,2
53 | 2016-03-22 00:10:00,2
54 | 2016-03-22 00:11:00,2
55 | 2016-03-22 00:12:00,2
56 | 2016-03-22 00:13:00,1
57 | 2016-03-22 00:14:00,1
58 | 2016-03-22 00:15:00,1
59 | 2016-03-22 00:16:00,1
60 | 2016-03-22 00:17:00,1
61 | 2016-03-22 00:18:00,2
62 | 2016-03-22 00:19:00,1
63 | 2016-03-22 00:20:00,1
64 | 2016-03-22 00:21:00,1
65 | 2016-03-22 00:22:00,1
66 | 2016-03-22 00:23:00,1
67 | 2016-03-22 00:24:00,1
68 | 2016-03-22 00:25:00,1
69 | 2016-03-22 00:26:00,1
70 | 2016-03-22 00:27:00,1
71 | 2016-03-22 00:28:00,2
72 | 2016-03-22 00:29:00,1
73 | 2016-03-22 00:30:00,1
74 | 2016-03-22 00:31:00,2
75 | 2016-03-22 00:32:00,1
76 | 2016-03-22 00:33:00,1
77 | 2016-03-22 00:34:00,1
78 | 2016-03-22 00:35:00,1
79 | 2016-03-22 00:36:00,1
80 | 2016-03-22 00:37:00,1
81 | 2016-03-22 00:38:00,1
82 | 2016-03-22 00:39:00,1
83 | 2016-03-22 00:40:00,1
84 | 2016-03-22 00:41:00,1
85 | 2016-03-22 00:42:00,1
86 | 2016-03-22 00:43:00,1
87 | 2016-03-22 00:44:00,1
88 | 2016-03-22 00:45:00,1
89 | 2016-03-22 00:46:00,1
90 | 2016-03-22 00:47:00,1
91 | 2016-03-22 00:48:00,1
92 | 2016-03-22 00:49:00,1
93 | 2016-03-22 00:50:00,1
94 | 2016-03-22 00:51:00,1
95 | 2016-03-22 00:52:00,1
96 | 2016-03-22 00:53:00,1
97 | 2016-03-22 00:54:00,1
98 | 2016-03-22 00:55:00,2
99 | 2016-03-22 00:56:00,1
100 | 2016-03-22 00:57:00,1
101 | 2016-03-22 00:58:00,1
102 | 2016-03-22 00:59:00,1
103 | 2016-03-22 01:00:00,1
104 | 2016-03-22 01:01:00,1
105 | 2016-03-22 01:02:00,1
106 | 2016-03-22 01:03:00,2
107 | 2016-03-22 01:04:00,1
108 | 2016-03-22 01:05:00,1
109 | 2016-03-22 01:06:00,1
110 | 2016-03-22 01:07:00,1
111 | 2016-03-22 01:08:00,1
--------------------------------------------------------------------------------
/src/util/gather_keys_oauth2.py:
--------------------------------------------------------------------------------
1 | # code from https://github.com/orcasgit/python-fitbit
2 | import cherrypy
3 | import sys
4 | import threading
5 | import traceback
6 | import webbrowser
7 |
8 | from fitbit.api import FitbitOauth2Client
9 | from oauthlib.oauth2.rfc6749.errors import MismatchingStateError, MissingTokenError
10 |
11 |
12 | class OAuth2Server:
13 | def __init__(self, client_id, client_secret,
14 | redirect_uri='http://127.0.0.1:8080/'):
15 | """ Initialize the FitbitOauth2Client """
16 | self.redirect_uri = redirect_uri
17 | self.success_html = """
18 | You are now authorized to access the Fitbit API!
19 |
You can close this window
"""
20 | self.failure_html = """
21 | ERROR: %s
You can close this window
%s"""
22 | self.oauth = FitbitOauth2Client(client_id, client_secret)
23 |
24 | def browser_authorize(self):
25 | """
26 | Open a browser to the authorization url and spool up a CherryPy
27 | server to accept the response
28 | """
29 | url, _ = self.oauth.authorize_token_url(redirect_uri=self.redirect_uri)
30 | # Open the web browser in a new thread for command-line browser support
31 | threading.Timer(1, webbrowser.open, args=(url,)).start()
32 | cherrypy.quickstart(self)
33 |
34 | @cherrypy.expose
35 | def index(self, state, code=None, error=None):
36 | """
37 | Receive a Fitbit response containing a verification code. Use the code
38 | to fetch the access_token.
39 | """
40 | error = None
41 | if code:
42 | try:
43 | self.oauth.fetch_access_token(code, self.redirect_uri)
44 | except MissingTokenError:
45 | error = self._fmt_failure(
46 | 'Missing access token parameter.Please check that '
47 | 'you are using the correct client_secret')
48 | except MismatchingStateError:
49 | error = self._fmt_failure('CSRF Warning! Mismatching state')
50 | else:
51 | error = self._fmt_failure('Unknown error while authenticating')
52 | # Use a thread to shutdown cherrypy so we can return HTML first
53 | self._shutdown_cherrypy()
54 | return error if error else self.success_html
55 |
56 | def _fmt_failure(self, message):
57 | tb = traceback.format_tb(sys.exc_info()[2])
58 | tb_html = '%s
' % ('\n'.join(tb)) if tb else ''
59 | return self.failure_html % (message, tb_html)
60 |
61 | def _shutdown_cherrypy(self):
62 | """ Shutdown cherrypy in one second, if it's running """
63 | if cherrypy.engine.state == cherrypy.engine.states.STARTED:
64 | threading.Timer(1, cherrypy.engine.exit).start()
65 |
66 |
67 | if __name__ == '__main__':
68 |
69 | if not (len(sys.argv) == 3):
70 | print("Arguments: client_id and client_secret")
71 | sys.exit(1)
72 |
73 | server = OAuth2Server(*sys.argv[1:])
74 | server.browser_authorize()
75 |
76 | profile = server.fitbit.user_profile_get()
77 | print('You are authorized to access data for the user: {}'.format(
78 | profile['user']['fullName']))
79 |
80 | print('TOKEN\n=====\n')
81 | for key, value in server.fitbit.client.session.token.items():
82 | print('{} = {}'.format(key, value))
83 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Fitbit Analyzer
2 | Analyzer and statistics generator for Fitbit data like sleep, steps count and heart rate. Check out also [my related introductory article](https://medium.com/@5agado/a-quest-for-better-sleep-with-fitbit-data-analysis-5f10b3f548a#.inflxkcln).
3 |
4 | Includes OAuth 2.0 **scraper** to download personal Fitbit data using the [Fitbit API Python Client Implementation](http://python-fitbit.readthedocs.io/en/latest/).
5 |
6 | ## Usage
7 | Run
8 |
9 | python setup.py install
10 |
11 | You will then be able to run the scraper via
12 |
13 | fitbit-scraper --id --secret --out dump/destination/folder/path
14 |
15 | The client-id and client-secret can be obtained by creating a Fitbit app on the official website. The scraper will than take care to download all data to the given folder, avoiding to re-download data already present (apart for the last present day, which will be re-scraped to avoid missing entries). See `fitbit-scraper --help` for a detailed usage description.
16 |
17 | ### Data Format
18 | In the *util* folder you can find some code for loading sleep, heartbeat and steps data as dumped by the scraper script.
19 | For the data-dump folder I have one folder per year, and then one sub-folder for each day, in which the different files are generated (e.g. *sleep.json*, *steps.json*).
20 |
21 | Example:
22 | ```
23 | ├───2016
24 | │ ├───2016-01-01
25 | │ │ sleep.json
26 | │ │ steps.json
27 | │ │
28 | │ └───2016-01-02
29 | │ heartbeat.json
30 | │
31 | └───2017
32 | └───2017-01-11
33 | calories.json
34 | distance.json
35 | elevation.json
36 | floors.json
37 | heartbeat.json
38 | sleep.json
39 | steps.json
40 | ```
41 |
42 | ## Analyzer and Stats
43 | Compared to basic scraping this phase has more dependencies; see the *requirements.txt* file and [here](http://conda.pydata.org/docs/using/envs.html#share-an-environment) for a guide on how to manage environments in Conda.
44 |
45 | The [Sleep Dashboard notebook](Sleep%20Dashboard.ipynb) can provide a quick way to explore your sleep data. See [related article](https://medium.com/towards-data-science/interactive-visualizations-in-jupyter-notebook-3be02ab2b8cd) and [video](https://www.youtube.com/watch?v=FYnM84TgzZU). I am currently working on notebooks for the other measures, so all feedback is welcome.
46 |
47 |
48 | ### Sleep Stats
49 | Sleep values are mapped like this: 0=none (no measure taken), 1=sleeping, 2=restless, 3=awake.
50 |
51 | * **Basic Stats** (sleep values count, sleep efficiency, hours of sleep, total minutes in bed, N max-values for each stat)
52 | * **Timing Stats** (first minute asleep, to bed time, wake up time, sleep interval min/max length)
53 | * **Intervals Stats** for each day all the sleep intervals lengths
54 | * **Intraday Stats** minute to minute report for each day, for the specified sleep values. Total value count, with normalization and centering on specific time.
55 |
56 |
57 | ### Heart Rate Stats
58 | * **Basic Stats** (count, mean, min, max, std) values for the entire dataset, or for aggregated subsets. Common aggregation features are date, year, month, day, day of week and hour
59 | * **Daily Stats** plotting of average heart-rate by day, month, year, weekday
60 | * **Min/Max Values** get the N min or max heartbeat values in the dataset
61 |
62 | ### Steps Stats
63 | * **Daily Count** steps count per day
64 |
65 | **NEW**
66 | ### Combined Stats
67 | * **Correlation** correlation coefficient between heterogeneous stats (e.g. steps and sleep)
68 | * **Combined TOGGL and heart-rate stats** explore relations between Toggl activities and projects and heart-rate
69 |
70 | ## TODO
71 | * Toggl data exploratory notebook
72 | * moving average sleep (days for achieving flatness)
73 | * correlation between constant exercise/running and decreased heart-rate
74 | * formal verification of causality (e.g. good sleep causes to walk, or walk cause good sleep)
75 | * derive sleep-states (REM/NREM) stats (e.g. identify, how long in each) from basic ones, or via correlation with heartbeat (as for now doesn't seem to be feasible actual ECG is still the more proper option)
76 | * max HR and zones (% of MHR)
77 |
78 | ## License
79 |
80 | Released under version 2.0 of the [Apache License].
81 |
82 | [Apache license]: http://www.apache.org/licenses/LICENSE-2.0
83 | [my related article]: https://medium.com/@5agado/conversation-analyzer-baa80c566d7b#.w20u1gltf
--------------------------------------------------------------------------------
/src/util/scraper.py:
--------------------------------------------------------------------------------
1 | import datetime
2 | import json
3 | import sys
4 | import argparse
5 | from pathlib import Path
6 |
7 | import fitbit
8 | from src.util import logger
9 | from src.util import gather_keys_oauth2 as Oauth2
10 |
11 |
12 | def dumpToFile(data_type, dumpDir: Path, date, data):
13 | directory = dumpDir / str(date.year) / str(date)
14 | directory.mkdir(parents=True, exist_ok=True)
15 | with (directory / "{}.json".format(data_type)).open(mode='w') as f:
16 | f.write(json.dumps(data, indent=True))
17 |
18 |
19 | def previouslyDumped(dumpDir: Path, date):
20 | return (dumpDir / str(date.year) / str(date)).is_dir()
21 |
22 |
23 | def dumpDay(client, dumpDir: Path, date):
24 | steps_data = client.intraday_time_series('activities/steps', date)
25 | intradayData = steps_data['activities-steps-intraday']['dataset']
26 | if not intradayData:
27 | logger.info("No {} measures for {}. Skipping the rest too".format('steps', date.split('\\')[-1]))
28 | return None
29 |
30 | dumpToFile("steps", dumpDir, date, steps_data)
31 | dumpToFile("sleep", dumpDir, date, client.get_sleep(date))
32 | dumpToFile("calories", dumpDir, date, client.intraday_time_series('activities/calories', date))
33 | dumpToFile("distance", dumpDir, date, client.intraday_time_series('activities/distance', date))
34 | dumpToFile("floors", dumpDir, date, client.intraday_time_series('activities/floors', date))
35 | dumpToFile("elevation", dumpDir, date, client.intraday_time_series('activities/elevation', date))
36 | dumpToFile("heartbeat", dumpDir, date, client.intraday_time_series('activities/heart', date))
37 |
38 |
39 | def scrapeFromDateOnward(startDate, dumpDir: Path, client):
40 | date = datetime.datetime.strptime(startDate, "%Y-%m-%d").date()
41 | todayDate = datetime.date.today()
42 | while previouslyDumped(dumpDir, date):
43 | logger.info("Already scraped {}".format(datetime.datetime.strftime(date, "%Y-%m-%d")))
44 | date += datetime.timedelta(days=1)
45 |
46 | date -= datetime.timedelta(days=1)
47 | logger.info("Will RE-Scrape data for {}".format(datetime.datetime.strftime(date, "%Y-%m-%d")))
48 | while date < todayDate:
49 | logger.info("Scraping data for {}".format(datetime.datetime.strftime(date, "%Y-%m-%d")))
50 | dumpDay(client, dumpDir, date)
51 | date += datetime.timedelta(days=1)
52 |
53 |
54 | def scrapeFromTodayAndBackward(dumpDir: Path, client, limit, stop_if_already_dumped=True):
55 | # dumping
56 | count = 1
57 | date = datetime.date.today()
58 | while count < limit:
59 | if previouslyDumped(dumpDir, date):
60 | logger.info("Already scraped {}".format(date.isoformat()))
61 | if stop_if_already_dumped:
62 | print("Stopping the scraping")
63 | break
64 | date -= datetime.timedelta(days=1)
65 | continue
66 | logger.info("Scraping data for {}".format(date.isoformat()))
67 | dumpDay(client, dumpDir, date)
68 | date -= datetime.timedelta(days=1)
69 | count += 1
70 | dumpDay(client, dumpDir, date)
71 |
72 |
73 | def main(_=None):
74 | parser = argparse.ArgumentParser(description='Fitbit Scraper')
75 | parser.add_argument('--id', metavar='clientId', dest='clientId', required=True,
76 | help="client-id of your Fitbit app")
77 | parser.add_argument('--secret', metavar='clientSecret', dest='clientSecret', required=True,
78 | help="client-secret of your Fitbit app")
79 | parser.add_argument('--out', metavar='outDir', dest='outDir', required=True,
80 | help="output data destination folder")
81 | parser.add_argument('--start', dest='startDate', default='2016-01-01',
82 | help="Date from which to start the forward scraping. Defaults to 2016-01-01")
83 | #parser.add_argument('--limit', type=int, dest='limit', default=400,
84 | # help="maximum number of days to scrape")
85 |
86 | args = parser.parse_args()
87 | clientId = args.clientId
88 | clientSecret = args.clientSecret
89 | dumpDir = Path(args.outDir)
90 | startDate = args.startDate
91 | #limit = args.limit
92 |
93 | server = Oauth2.OAuth2Server(clientId, clientSecret)
94 | server.browser_authorize()
95 |
96 | ACCESS_TOKEN = server.oauth.session.token['access_token']
97 | REFRESH_TOKEN = server.oauth.session.token['refresh_token']
98 |
99 | client = fitbit.Fitbit(clientId, clientSecret, oauth2=True,
100 | access_token=ACCESS_TOKEN, refresh_token=REFRESH_TOKEN)
101 |
102 | scrapeFromDateOnward(startDate, dumpDir, client)
103 |
104 |
105 | if __name__ == "__main__":
106 | main(sys.argv[1:])
--------------------------------------------------------------------------------
/heartBeatTesting.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "metadata": {
7 | "ExecuteTime": {
8 | "end_time": "2017-10-05T09:30:32.985827",
9 | "start_time": "2017-10-05T09:30:31.186724"
10 | }
11 | },
12 | "outputs": [],
13 | "source": [
14 | "import pandas as pd\n",
15 | "import numpy as np\n",
16 | "import seaborn as sns\n",
17 | "import datetime\n",
18 | "\n",
19 | "from pathlib import Path\n",
20 | "import os\n",
21 | "import sys\n",
22 | "sys.path.append(os.path.join(os.getcwd(), \"src\"))\n",
23 | "\n",
24 | "from util import utils\n",
25 | "from util import plotting\n",
26 | "from stats import hbStats\n",
27 | "\n",
28 | "import time\n",
29 | "\n",
30 | "%load_ext autoreload\n",
31 | "%autoreload 2"
32 | ]
33 | },
34 | {
35 | "cell_type": "markdown",
36 | "metadata": {},
37 | "source": [
38 | "### Load and prepare data"
39 | ]
40 | },
41 | {
42 | "cell_type": "code",
43 | "execution_count": null,
44 | "metadata": {},
45 | "outputs": [],
46 | "source": [
47 | "dataFolder = str(Path.home() / \"Documents/scripts_data/fitbit_analyzer/dataDump/\")"
48 | ]
49 | },
50 | {
51 | "cell_type": "code",
52 | "execution_count": null,
53 | "metadata": {
54 | "scrolled": true
55 | },
56 | "outputs": [],
57 | "source": [
58 | "start = time.time()\n",
59 | "hbData = utils.loadHBData(dataFolder)\n",
60 | "end = time.time()\n",
61 | "print(\"Data loaded in {:.2f}s\".format(end - start))\n",
62 | "data = pd.concat(hbData, ignore_index=True)\n",
63 | "print(\"Sample\")\n",
64 | "print(data.head())"
65 | ]
66 | },
67 | {
68 | "cell_type": "code",
69 | "execution_count": null,
70 | "metadata": {},
71 | "outputs": [],
72 | "source": [
73 | "data.set_index('datetime', inplace=True)"
74 | ]
75 | },
76 | {
77 | "cell_type": "code",
78 | "execution_count": null,
79 | "metadata": {},
80 | "outputs": [],
81 | "source": [
82 | "data[-2:]"
83 | ]
84 | },
85 | {
86 | "cell_type": "markdown",
87 | "metadata": {},
88 | "source": [
89 | "## Plot"
90 | ]
91 | },
92 | {
93 | "cell_type": "code",
94 | "execution_count": null,
95 | "metadata": {},
96 | "outputs": [],
97 | "source": [
98 | "DAYS = 7\n",
99 | "WEEKS = 5\n",
100 | "NB_VALUES = 60*24 # minutes per day\n",
101 | "assert len(test_data)==DAYS*WEEKS*NB_VALUES"
102 | ]
103 | },
104 | {
105 | "cell_type": "code",
106 | "execution_count": null,
107 | "metadata": {},
108 | "outputs": [],
109 | "source": [
110 | "min_val = 0\n",
111 | "max_val = 1\n",
112 | "test_data['value'] = min_val + (((test_data['value'] - test_data['value'].min()) * (max_val - min_val)) / (\n",
113 | " test_data['value'].max() - test_data['value'].min()))\n",
114 | "print(len(test_data))"
115 | ]
116 | },
117 | {
118 | "cell_type": "code",
119 | "execution_count": null,
120 | "metadata": {},
121 | "outputs": [],
122 | "source": [
123 | "%matplotlib notebook\n",
124 | "from matplotlib import pyplot as plt\n",
125 | "from matplotlib import animation\n",
126 | "\n",
127 | "fig, ax = plt.subplots(dpi=100, figsize=(5, 4))\n",
128 | "im = ax.imshow(grid, cmap='gray')\n",
129 | "#plt.axis('off')\n",
130 | "\n",
131 | "def animate(i, test_data, im, WEEKS, DAYS, NB_VALUES):\n",
132 | " grid = []\n",
133 | " for k in range(WEEKS):\n",
134 | " week = []\n",
135 | " for j in range(DAYS):\n",
136 | " week.append(test_data.loc[i+((DAYS*k+j)*NB_VALUES)].value)\n",
137 | " grid.append(week)\n",
138 | " im.set_data(np.array(grid))\n",
139 | "\n",
140 | "ani = animation.FuncAnimation(fig, animate, frames=10000, interval=10,\n",
141 | " fargs=[test_data, im, WEEKS, DAYS, NB_VALUES]) # be sure to pass the additional args needed for the animation\n",
142 | " #.save('anim.mp4', writer=animation.FFMpegFileWriter(fps=30))"
143 | ]
144 | },
145 | {
146 | "cell_type": "code",
147 | "execution_count": null,
148 | "metadata": {},
149 | "outputs": [],
150 | "source": [
151 | "for i in range(50):\n",
152 | " grid = []\n",
153 | " for k in range(WEEKS):\n",
154 | " week = []\n",
155 | " for j in range(DAYS):\n",
156 | " week.append(test_data.loc[i+((DAYS*k+j)*NB_VALUES)].value)\n",
157 | " grid.append(week)\n",
158 | " im.set_data(np.array(grid))\n",
159 | "print(np.array(grid).shape)"
160 | ]
161 | },
162 | {
163 | "cell_type": "code",
164 | "execution_count": null,
165 | "metadata": {},
166 | "outputs": [],
167 | "source": [
168 | "np.array(grid)"
169 | ]
170 | },
171 | {
172 | "cell_type": "code",
173 | "execution_count": null,
174 | "metadata": {},
175 | "outputs": [],
176 | "source": [
177 | "fig, ax = plt.subplots(dpi=100, figsize=(5, 4))\n",
178 | "im = ax.imshow(np.array(grid), cmap='gray')"
179 | ]
180 | },
181 | {
182 | "cell_type": "markdown",
183 | "metadata": {},
184 | "source": [
185 | "### Basic Overall Stats"
186 | ]
187 | },
188 | {
189 | "cell_type": "code",
190 | "execution_count": null,
191 | "metadata": {},
192 | "outputs": [],
193 | "source": [
194 | "hbStats.basicStats(data)"
195 | ]
196 | },
197 | {
198 | "cell_type": "markdown",
199 | "metadata": {},
200 | "source": [
201 | "### Basic Aggregated Stats"
202 | ]
203 | },
204 | {
205 | "cell_type": "code",
206 | "execution_count": null,
207 | "metadata": {},
208 | "outputs": [],
209 | "source": [
210 | "#hourStats = hbStats.groupByBasicStats(data['datetime'].dt.hour, data, 'hour')\n",
211 | "#plotting.plot(hourStats, ['count', 'max', 'min', 'std'], 'hour', 2, 2)\n",
212 | "dayStats = hbStats.groupByBasicStats(data['datetime'].dt.day, data, 'day')\n",
213 | "plotting.plot(dayStats, ['count', 'max', 'min', 'std'], 'day', 2, 2)\n",
214 | "#monthStats = hbStats.groupByBasicStats(data['datetime'].dt.month, data, 'month')\n",
215 | "#plotting.plotMonthlyStatsHb(monthStats)\n",
216 | "#dayOfWeekStats = hbStats.groupByBasicStats(data['datetime'].dt.dayofweek, data, 'weekday')\n",
217 | "#plotting.plotWeekdayStatsHb(dayOfWeekStats)\n",
218 | "#plotting.plotDailyStatsHb(data)"
219 | ]
220 | },
221 | {
222 | "cell_type": "code",
223 | "execution_count": null,
224 | "metadata": {},
225 | "outputs": [],
226 | "source": [
227 | "dayStats.head()"
228 | ]
229 | },
230 | {
231 | "cell_type": "markdown",
232 | "metadata": {},
233 | "source": [
234 | "### Min/Max Values"
235 | ]
236 | },
237 | {
238 | "cell_type": "code",
239 | "execution_count": null,
240 | "metadata": {},
241 | "outputs": [],
242 | "source": [
243 | "hbStats.getMaxValues(data, 5)"
244 | ]
245 | },
246 | {
247 | "cell_type": "code",
248 | "execution_count": null,
249 | "metadata": {},
250 | "outputs": [],
251 | "source": [
252 | "hbStats.getMinValues(data, 5)"
253 | ]
254 | },
255 | {
256 | "cell_type": "code",
257 | "execution_count": null,
258 | "metadata": {},
259 | "outputs": [],
260 | "source": []
261 | }
262 | ],
263 | "metadata": {
264 | "anaconda-cloud": {},
265 | "kernelspec": {
266 | "display_name": "Python [conda env:Anaconda3]",
267 | "language": "python",
268 | "name": "conda-env-Anaconda3-py"
269 | },
270 | "language_info": {
271 | "codemirror_mode": {
272 | "name": "ipython",
273 | "version": 3
274 | },
275 | "file_extension": ".py",
276 | "mimetype": "text/x-python",
277 | "name": "python",
278 | "nbconvert_exporter": "python",
279 | "pygments_lexer": "ipython3",
280 | "version": "3.6.3"
281 | },
282 | "toc": {
283 | "base_numbering": 1,
284 | "nav_menu": {
285 | "height": "104px",
286 | "width": "252px"
287 | },
288 | "number_sections": true,
289 | "sideBar": true,
290 | "skip_h1_title": false,
291 | "title_cell": "Table of Contents",
292 | "title_sidebar": "Contents",
293 | "toc_cell": false,
294 | "toc_position": {},
295 | "toc_section_display": "block",
296 | "toc_window_display": false
297 | }
298 | },
299 | "nbformat": 4,
300 | "nbformat_minor": 1
301 | }
302 |
--------------------------------------------------------------------------------
/src/stats/sleepStats.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import numpy as np
3 | from enum import Enum
4 | from datetime import time
5 |
6 | class SleepValue(Enum):
7 | none = 0
8 | sleeping = 1
9 | restless = 2
10 | awake = 3
11 |
12 | SLEEP_VALUES = {x.value:x.name for x in SleepValue}
13 |
14 | STATS_NAME_BASIC = 'basicStats'
15 | STATS_NAME_TIMING = 'timingStats'
16 | STATS_NAME_BASIC_AND_TIMING = 'basicAndTimingStats'
17 | STATS_NAME_INTERVALS = 'intervalsStats'
18 | STATS_NAME_INTRADAY = 'intradayStats'
19 |
20 | def generateStatsFrom(data, statsType, **kwargs):
21 | """
22 | Generic method for stats generation.
23 | :param data: sleep data. List of dataframes, one for each day
24 | :param statsType: name of the stat to be generated
25 | :return:
26 | """
27 |
28 | if statsType == STATS_NAME_BASIC:
29 | stats = generateBasicStats(data)
30 | elif statsType == STATS_NAME_TIMING:
31 | stats = generateTimingStats(data, **kwargs)
32 | elif statsType == STATS_NAME_BASIC_AND_TIMING:
33 | basicStats = generateBasicStats(data)
34 | timingStats = generateTimingStats(data, **kwargs)
35 | stats = pd.concat([basicStats, timingStats], axis=1)
36 | elif statsType == STATS_NAME_INTERVALS:
37 | stats = generateIntervalsStats(data, **kwargs)
38 | elif statsType == STATS_NAME_INTRADAY:
39 | stats = generateIntradayStats(data, **kwargs)
40 | else:
41 | raise Exception(statsType + ' Stat not implemented')
42 | return stats
43 |
44 | #Each row is a day with basic stats like sleep values count, efficiency and total minutes
45 | def generateBasicStats(filesData):
46 | #Create new df to store the stats
47 | basicStats = pd.DataFrame(columns=[1,2,3,'total_minutes','sleep_efficiency'])
48 | basicStats.index.rename('date', inplace=True)
49 |
50 | #For each file-data, extract the basic stats, and add them to the df
51 | for fileData in filesData:
52 | stats = fileData.groupby(['value'], as_index=False).size()
53 | stats['total_minutes'] = stats.sum()
54 | date = getSleepDate(fileData)
55 | basicStats.loc[date] = stats
56 |
57 | #Derive additional stats
58 | basicStats = basicStats.fillna(0).astype(int)
59 | basicStats['sleep_efficiency'] = (basicStats[SleepValue.sleeping.value]/basicStats['total_minutes'])*100
60 | basicStats['sleep_inefficiency'] = (basicStats['sleep_efficiency']-100).abs()
61 | basicStats['sleep_hours'] = (basicStats[SleepValue.sleeping.value]/60)
62 | basicStats['total_hours'] = basicStats['total_minutes']/60
63 |
64 | #Rename columns
65 | columns = SLEEP_VALUES
66 | columns['value'] = 'date'
67 | basicStats.rename(columns=columns, inplace=True)
68 | return basicStats
69 |
70 | #Each row is a day with timing stats about like first minute asleep, interval avg and max length
71 | def generateTimingStats(filesData, minSleepIntervalRequired=0):
72 | #Create new df to store the stats
73 | intervalsStats = pd.DataFrame(columns=['first_min_asleep',
74 | 'to_bed_time',
75 | 'wake_up_time',
76 | 'sleep_interval_max_len',
77 | 'sleep_interval_avg_len'])
78 | intervalsStats.index.rename('date', inplace=True)
79 |
80 | #For each file-data, extract the basic stats, and add them to the df
81 | for fileData in filesData:
82 | firstMinuteAsleep = getFirstMinuteAsleep(fileData, minSleepIntervalRequired)
83 | stats = getSleepIntervalsStats(fileData, minSleepIntervalRequired)
84 | date = getSleepDate(fileData)
85 | intervalsStats.loc[date] = [firstMinuteAsleep, fileData.iloc[0]['datetime'].time(),
86 | fileData.iloc[-1]['datetime'].time(),
87 | stats.len.max(), stats.len.mean()]
88 |
89 | return intervalsStats
90 |
91 | #Each row is a day with length of all its sleep intervals
92 | def generateIntervalsStats(filesData, minSleepIntervalRequired=0):
93 | #Create new df to store the stats
94 | intervalsStats = pd.DataFrame(columns=np.arange(20))
95 |
96 | #For each file-data, extract the basic stats, and add them to the df
97 | for fileData in filesData:
98 | stats = getSleepIntervalsStats(fileData, minSleepIntervalRequired)
99 | date = getSleepDate(fileData)
100 | intervalsStats.loc[date] = stats.len
101 |
102 | return intervalsStats
103 |
104 | #Each row is a complete intraday value array for the corresponding day
105 | def generateIntradayStats(filesData, useTime=True):
106 | #Create new df to store the stats
107 | if useTime:
108 | minutes = pd.date_range('00:00', '23:59', freq='1min')
109 | intradayStats = pd.DataFrame(columns=[x.time().strftime("%H:%M")
110 | for x in minutes])
111 | else:
112 | intradayStats = pd.DataFrame(columns=np.arange(600))
113 | intradayStats.index.rename('date', inplace=True)
114 |
115 | #For each file-data, extract the basic stats, and add them to the df
116 | for fileData in filesData:
117 | data = fileData.copy(deep=True)
118 | date = getSleepDate(data)
119 | data['time'] = [x.strftime("%H:%M") for x in data['datetime'].dt.time]
120 | data.set_index(['time'], inplace=True)
121 | intradayStats.loc[date] = data['value']
122 |
123 | return intradayStats
124 |
125 | #----------------------------#
126 | # SECOND LEVEL METHODS #
127 | #----------------------------#
128 |
129 | def normalizedIntradayCountStats(intradayStats, limitCount=5):
130 | # For each minute, number of days for which we have a valid measure (record)
131 | notNullCount = intradayStats.count()
132 | # Ignore minutes where we have low level of records
133 | notNullCount[notNullCount < limitCount] = None
134 | # Count how many times each value appears for each minute
135 | valueCount = intradayStats.apply(pd.value_counts)
136 | # Normalize each minute by records count
137 | res = valueCount.div(notNullCount, axis=1)
138 | return res
139 |
140 | def centerIntradayCountStats(intradayCountStats, timeToCenterOn="12:00"):
141 | columns = list(intradayCountStats.columns.values)
142 | k = columns.index(timeToCenterOn)
143 | columns = columns[k:] + columns[:k]
144 | return intradayCountStats[columns]
145 |
146 | def getSleepIntervalsStats(data, minSleepIntervalRequired=0):
147 | sleepIntervals = getSleepIntervals(data, minSleepIntervalRequired)
148 |
149 | #Create new df to store the stats
150 | stats = pd.DataFrame(columns=['minute_start','minute_end','time_start','time_end','len'])
151 |
152 | #For each interval get start, end, len stats, and add it to the df
153 | for i, interval in enumerate(sleepIntervals):
154 | #Substract one because interval indicated the minute, and starts from 1
155 | timeStart = data.iloc[interval[0]-1]['datetime']
156 | timeEnd = data.iloc[interval[1]-1]['datetime']
157 | stats.loc[i] = [interval[0], interval[1], timeStart, timeEnd, (interval[1] - interval[0])+1]
158 |
159 | return stats
160 |
161 | #Return the date associated with the provides sleep data.
162 | #It should be the date associated with the first recorded minute, but if this is after
163 | #midnight and before noon, we want to consider the previous day.
164 | def getSleepDate(data):
165 | firstDatetime = data.ix[0]['datetime']
166 | if time(12,00) <= firstDatetime.time()<=time(23,59):
167 | return firstDatetime.date()
168 | else:
169 | return (firstDatetime - pd.DateOffset(1)).date()
170 |
171 |
172 | def getFirstMinuteAsleep(data, minSleepIntervalRequired=0):
173 | firstAsleep = None
174 |
175 | values = data['value']
176 | indxs = values[values==SleepValue.sleeping.value].index
177 | for indx in indxs:
178 | if (len(values[indx:indx+minSleepIntervalRequired]) >= minSleepIntervalRequired and
179 | (values[indx:indx+minSleepIntervalRequired]==SleepValue.sleeping.value).all()):
180 | #Indexing starts from 0, minutes start from 1
181 | firstAsleep = indx+1
182 | break
183 |
184 | return firstAsleep
185 |
186 | def getSleepIntervals(data, minSleepIntervalRequired=0):
187 | values = data['value']
188 | splits = np.append(np.where(np.diff(values)!=0)[0], [len(values)-1])+1
189 |
190 | prev = 0
191 | intervals = []
192 | for split in splits:
193 | if values[prev] == SleepValue.sleeping.value and (split-prev)>=minSleepIntervalRequired:
194 | intervals.append((prev+1, split))
195 | prev = split
196 |
197 | return intervals
198 |
199 | def getMaxValues(basicStats, n, statName):
200 | return basicStats.nlargest(n, statName)[statName]
201 |
202 |
203 |
204 |
--------------------------------------------------------------------------------
/src/util/utils.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import json
3 | import os
4 | import sys
5 | import datetime
6 | from datetime import time
7 |
8 | from src.util import logger
9 |
10 | def loadIntradayData(filepath):
11 | data = pd.read_csv(filepath, parse_dates=[0], names=['datetime', 'value'])
12 | return data
13 |
14 | #TODO implement and test loading more than one intraday interval
15 | def loadSleepData(dumpDir):
16 | """
17 | Load sleep data from dumping done using the official Fitbit API.
18 | Check README file for further info on the scraping process and saved format
19 | :param dumpDir: the folder where the date has been dumped
20 | :return: a list of dataframes, one for each day, containing the intraday sleep data
21 | """
22 | SLEEP_VALUE_NONE = 0
23 | def loadFun(jsonData):
24 | sleeps = jsonData['sleep']
25 | if not sleeps:
26 | return None
27 | date = datetime.datetime.strptime(sleeps[0]['dateOfSleep'], "%Y-%m-%d").date()
28 | if len(sleeps)>1:
29 | logger.info("There are {} sleep records for {}, taking main sleep".format(len(sleeps), date))
30 | intradayData = None
31 | for sleep in sleeps:
32 | if sleep['isMainSleep']:
33 | intradayData = sleep['minuteData']
34 | break
35 | dayToSubstract = datetime.timedelta(days=1)
36 | df = pd.read_json(json.dumps(intradayData), convert_dates=['time'])
37 | if (df['value']==SLEEP_VALUE_NONE).all():
38 | logger.info("There are only none values for {}".format(date))
39 | return None
40 | df['datetime'] = df.apply(lambda x: datetime.datetime.combine(
41 | date - dayToSubstract if time(12,00) <= x['dateTime'].time() <=time(23,59) else date,
42 | x['dateTime'].time()), axis=1)
43 | df.drop('dateTime', inplace=True, axis=1)
44 | return df
45 |
46 | return _loadData(dumpDir, 'sleep', loadFun)
47 |
48 | def loadHBData(dumpDir, concat=False):
49 | """
50 | Load heart-rate data from dumping done using the official Fitbit API.
51 | Check README file for further info on the scraping process and saved format
52 | :param concat: specify when to concat all entries in a single dataframe
53 | :param dumpDir: the folder where the date has been dumped
54 | :return: a list of dataframes, one for each day, containing the intraday heart-rate data.
55 | or a single one, if concat is set to True.
56 | """
57 | def loadFun(jsonData):
58 | summaryData = jsonData['activities-heart']
59 | date = summaryData[0]['dateTime']
60 | if len(summaryData)!=1:
61 | logger.info("There are {} heart data entries for {}".format(len(summaryData), date))
62 | intradayData = jsonData['activities-heart-intraday']['dataset']
63 | if not intradayData:
64 | return None
65 | df = pd.read_json(json.dumps(intradayData))
66 | df['datetime'] = pd.to_datetime(date + ' ' + df['time'])
67 | df.drop('time', inplace=True, axis=1)
68 | return df
69 |
70 | if concat:
71 | return pd.concat(_loadData(dumpDir, 'heartbeat', loadFun), ignore_index=True)
72 | else:
73 | return _loadData(dumpDir, 'heartbeat', loadFun)
74 |
75 | # TODO load heartRateZones data
76 | def loadHBSummaryData(dumpDir):
77 | """
78 | Load heart-rate summary data from dumping done using the official Fitbit API.
79 | Check README file for further info on the scraping process and saved format
80 | :param dumpDir: the folder where the date has been dumped
81 | :return: a list of tuples, one for each day, containing summary heart-rate info
82 | """
83 | def loadFun(jsonData):
84 | summaryData = jsonData['activities-heart']
85 | date = datetime.datetime.strptime(summaryData[0]['dateTime'], "%Y-%m-%d").date()
86 | if len(summaryData)!=1:
87 | logger.info("There are {} heart data entries for {}".format(len(summaryData), date))
88 |
89 | try:
90 | restingHeartRate = summaryData[0]['value']['restingHeartRate']
91 | except KeyError:
92 | logger.info("No resting heart rate info for {}".format(date))
93 | return None
94 | return date, restingHeartRate
95 |
96 | entries = _loadData(dumpDir, 'heartbeat', loadFun)
97 | return pd.DataFrame(entries, columns=['date', 'rhr']).set_index(['date'])
98 |
99 | def loadStepsData(dumpDir):
100 | """
101 | Load steps data from dumping done using the official Fitbit API.
102 | Check README file for further info on the scraping process and saved format
103 | :param dumpDir: the folder where the date has been dumped
104 | :return: a list of dataframes, one for each day, containing the intraday steps data
105 | """
106 | def loadFun(jsonData):
107 | intradayData = jsonData['activities-steps-intraday']['dataset']
108 | date = jsonData['activities-steps'][0]['dateTime']
109 | if not intradayData:
110 | return None
111 | df = pd.read_json(json.dumps(intradayData))
112 | df['datetime'] = pd.to_datetime(date + ' ' + df['time'])
113 | df.drop('time', inplace=True, axis=1)
114 | return df
115 |
116 | return _loadData(dumpDir, 'steps', loadFun)
117 |
118 | def loadTotalSteps(dumpDir):
119 | """
120 | Load total steps count from dumping done using the official Fitbit API.
121 | Check README file for further info on the scraping process and saved format
122 | :param dumpDir: the folder where the date has been dumped
123 | :return: a dataframe containing the total steps count indexed by day
124 | """
125 | def loadFun(jsonData):
126 | data = jsonData['activities-steps']
127 | date = datetime.datetime.strptime(data[0]['dateTime'],
128 | "%Y-%m-%d").date()
129 | if len(data)!=1:
130 | logger.info("There are {} steps data entries for {}".format(len(data), date))
131 | totalSteps = int(data[0]['value'])
132 | return date, totalSteps
133 |
134 | entries = _loadData(dumpDir, 'steps', loadFun)
135 | return pd.DataFrame(entries, columns=['date', 'steps']).set_index(['date'])
136 |
137 | #TODO maybe better to return a dictionary? Or for some types it will not work
138 | def _loadData(dumpDir, dataType, loadFun):
139 | """
140 | Helper method.
141 | For the data-dump folder there should be one folder per year, and then one sub-folder for each day,
142 | in which the different files are generated (e.g. sleep.json, steps.json)
143 | :param dumpDir: the folder where the date has been dumped
144 | :param dataType: the type of data to be loaded, equivalent to the name of the corresponding file
145 | :param loadFun: function defining the procedure for the data loading
146 | :return: a list of objects as defined in loadFun
147 | """
148 | data = []
149 | # First level should be the year
150 | yearDirs = getAllSubDirsNamesOf(dumpDir)
151 | # Second level should be the date
152 | for year in yearDirs:
153 | dates = getAllSubDirsNamesOf(year)
154 | for date in dates:
155 | # Dumped files are named .json
156 | filename = os.path.join(date, dataType) + '.json'
157 | try:
158 | with open(filename) as fileData:
159 | jsonData = json.load(fileData)
160 | dayData = loadFun(jsonData)
161 | if dayData is None:
162 | logger.info("No {} measures for {}".format(dataType, date.split('\\')[-1]))
163 | continue
164 | else:
165 | data.append(dayData)
166 | except FileNotFoundError:
167 | logger.warning("{} not found. Might be cause last scraped day.".format(filename))
168 |
169 | return data
170 |
171 | def loadTogglData(reportsPaths, columnsToDrop=None):
172 | """
173 | Load a list of toggl reports in a dataframe
174 | :param reportsPaths: list of reports paths
175 | :param columnsToDrop: list of names of columns to drop, all need to be present
176 | :return: all reports unified in a single dataframe
177 | """
178 | reports = []
179 | # for each path, load and add report to list
180 | for path in reportsPaths:
181 | # lovely automatic parsing of dates as well as combination of date and time
182 | report = pd.read_csv(path, parse_dates=[['Start date', 'Start time'],
183 | ['End date', 'End time']])
184 | # rename datetime columns
185 | report = report.rename(index=str, columns={"Start date_Start time": "Start",
186 | "End date_End time": "End"})
187 | # drop unnecessary fields
188 | if columnsToDrop:
189 | report.drop(columnsToDrop, axis=1, inplace=True)
190 | reports.append(report)
191 | # concatenate reports in single dataframe
192 | return pd.concat(reports, ignore_index=True)
193 |
194 | def getAllSubDirsNamesOf(mainDir):
195 | subDirs = filter(os.path.isdir,
196 | [os.path.join(mainDir, subDir) for subDir in os.listdir(mainDir)])
197 | return subDirs
--------------------------------------------------------------------------------
/src/util/plotting.py:
--------------------------------------------------------------------------------
1 | import matplotlib.pyplot as plt
2 | import numpy as np
3 | import pandas as pd
4 | import seaborn as sns
5 | from scipy import stats as scipystats
6 |
7 | from src.stats import sleepStats, hbStats
8 |
9 | NAMES={'sleep_inefficiency':'Sleep Inefficiency (%)',
10 | 'sleep_efficiency':'Sleep Efficiency (%)',
11 | 'restless':'Restless (minutes)',
12 | 'awake':'Awake (minutes)',
13 | 'total_minutes':'Total Minutes',
14 | 'sleep_hours':'Hours of Sleep',
15 | 'first_min_asleep':'First Minute Asleep'}
16 |
17 | dayOfWeek = {0: 'Mon', 1: 'Tue', 2: 'Wed', 3: 'Thur', 4: 'Fri', 5: 'Sat', 6: 'Sun'}
18 | dayOfWeekOrder = ['Mon', 'Tue', 'Wed', 'Thur', 'Fri', 'Sat', 'Sun']
19 |
20 | months={1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', 7:'Jul', 8:'Aug',
21 | 9:'Sep', 10:'Oct', 11:'Nov', 12:'Dec'}
22 | monthsOrder = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']
23 |
24 | def plotPreliminaryStats(stats):
25 | """
26 | Plot measures distribution using histograms
27 | :param stats: data to plot
28 | """
29 | columns = ['sleep_efficiency', 'restless', 'awake', 'total_minutes', 'sleep_hours',
30 | 'first_min_asleep']
31 | plotStats = stats[columns]
32 | plotStats = plotStats.rename(columns=NAMES)
33 | return plotStats.hist()
34 | #sns.plt.show()
35 |
36 | def plotWeekdayStatsSleep(stats):
37 | columns = ['sleep_efficiency', 'restless', 'sleep_hours',
38 | 'first_min_asleep', 'awake', 'total_minutes']
39 | return _plotWeekdayStats(stats, columns, groupBy=True)
40 |
41 | def plotWeekdayStatsHb(stats):
42 | columns = ['count', 'mean', 'std']
43 | return _plotWeekdayStats(stats, columns, groupBy=False)
44 |
45 | def _plotWeekdayStats(stats, columns, groupBy=True):
46 | dataToPlot = stats.copy()
47 | # Group by weekday and rename date column
48 | if groupBy:
49 | dataToPlot = dataToPlot.groupby(stats['date'].dt.weekday).mean()
50 | dataToPlot = dataToPlot.reset_index().rename(columns={'date':'weekday'})
51 |
52 | # change stats from columns to row attribute
53 | dataToPlot = pd.melt(dataToPlot, id_vars=['weekday'], value_vars=columns,
54 | var_name='stats', value_name='val')
55 | # Rename stats and weekdays
56 | dataToPlot['stats'].replace(NAMES, inplace=True)
57 | dataToPlot['weekday'].replace(dayOfWeek, inplace=True)
58 | # Plot
59 | g = sns.factorplot(data=dataToPlot, x="weekday", y="val", col="stats",
60 | order=dayOfWeekOrder, kind="point", sharey=False, col_wrap=3)
61 | g.set_xticklabels(rotation=45)
62 | g.set(xlabel='')
63 | return g
64 | #sns.plt.show()
65 |
66 | def plotWeekdayStatsByMonthSleep(stats):
67 | stat_name = 'sleep_efficiency'
68 | return _plotWeekdayByMonthStats(stats, stat_name)
69 |
70 | def _plotWeekdayByMonthStats(stats, stat_name):
71 | dataToPlot = _prepareWeekdayByMonthStats(stats)
72 |
73 | # Plot
74 | g = sns.pointplot(data=dataToPlot, x="day", y=stat_name, hue="month", order=dayOfWeekOrder)
75 | g.set(xlabel='')
76 | g.set_ylabel(NAMES[stat_name])
77 | return g
78 | #sns.plt.show()
79 |
80 | def _prepareWeekdayByMonthStats(stats):
81 | # Add day and month columns, and groupby
82 | stats = stats.copy()
83 | stats['day'] = stats['date'].dt.weekday
84 | stats['month'] = stats['date'].dt.month
85 | dataToPlot = stats.groupby(['day', 'month']).mean()
86 |
87 | dataToPlot = dataToPlot.reset_index()
88 | dataToPlot['day'].replace(dayOfWeek, inplace=True)
89 | dataToPlot['month'].replace(months, inplace=True)
90 |
91 | return dataToPlot
92 |
93 | # def plotWeekdayStats(stats, columns):
94 | # """
95 | # Plot aggregated (mean) stats by dayOfWeek
96 | # :param stats: data to plot
97 | # :param columns: columns from stats to plot
98 | # """
99 | # MEASURE_NAME = 'weekday'
100 | # dayOfWeek={0:'Mon', 1:'Tue', 2:'Wed', 3:'Thur', 4:'Fri', 5:'Sat', 6:'Sun'}
101 | # order = ['Mon','Tue','Wed','Thur','Fri','Sat','Sun']
102 | # stats[MEASURE_NAME] = stats[MEASURE_NAME].map(dayOfWeek)
103 | #
104 | # f, axes = getAxes(2,2)
105 | # for i, c in enumerate(columns):
106 | # if c in NAMES:
107 | # c = NAMES[c]
108 | # g = sns.barplot(x=MEASURE_NAME, y=c, data=stats, order=order, ax=axes[i])
109 | # g.set_xlabel('')
110 | # sns.plt.show()
111 | # #plot(stats, columns, MEASURE_NAME, 2, 3, order=order)
112 |
113 | def plotMonthlyStatsSleep(stats, columns=None):
114 | if not columns:
115 | columns = ['sleep_inefficiency', 'restless', 'sleep_hours',
116 | 'first_min_asleep']
117 | return _plotMonthlyStats(stats, columns, groupBy=True)
118 |
119 | def plotMonthlyStatsHb(stats):
120 | columns = ['count', 'mean', 'max', 'min', 'std']
121 | return _plotMonthlyStats(stats, columns, groupBy=False)
122 |
123 | def _plotMonthlyStats(stats, columns, groupBy=True):
124 | dataToPlot = stats.copy()
125 | # Group by month and rename date column
126 | if groupBy:
127 | dataToPlot = dataToPlot.groupby(stats['date'].dt.month).mean()
128 | dataToPlot = dataToPlot.reset_index().rename(columns={'date': 'month'})
129 |
130 | # change stats from columns to row attribute
131 | dataToPlot = pd.melt(dataToPlot, id_vars=['month'], value_vars=columns,
132 | var_name='stats', value_name='val')
133 | # Rename stats and weekdays
134 | dataToPlot['stats'].replace(NAMES, inplace=True)
135 | dataToPlot['month'].replace(months, inplace=True)
136 | order = [m for m in monthsOrder if m in dataToPlot['month'].unique()]
137 | # Plot
138 | g = sns.factorplot(data=dataToPlot, x="month", y="val", col="stats", order=order, kind="bar", sharey=False)
139 | g.set_xticklabels(rotation=45)
140 | g.set(xlabel='')
141 | return g
142 | #sns.plt.show()
143 |
144 | # def _plotMonthlyStats(stats, columns):
145 | # """
146 | # Plot aggregated (mean) stats by month
147 | # :param stats: data to plot
148 | # :param columns: columns from stats to plot
149 | # """
150 | # MEASURE_NAME = 'month'
151 | # months={1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', 7:'Jul', 8:'Aug',
152 | # 9:'Sep', 10:'Oct', 11:'Nov', 12:'Dec'}
153 | # order = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']
154 | # stats[MEASURE_NAME] = stats[MEASURE_NAME].map(months)
155 | #
156 | # order = [m for m in order if m in stats[MEASURE_NAME].unique()]
157 | #
158 | # f, axes = getAxes(2,2)
159 | # for i, c in enumerate(columns):
160 | # if c in NAMES:
161 | # c = NAMES[c]
162 | # g = sns.barplot(x=MEASURE_NAME, y=c, data=stats, order=order, ax=axes[i])
163 | # g.set_xlabel('')
164 | # sns.plt.show()
165 |
166 | def _prepareYearAndMonthStats(stats, columns):
167 | # Group by month and change stats from columns to row attribute
168 | dataToPlot = stats.groupby(stats['date'].dt.to_period("M")).mean()
169 | dataToPlot = pd.melt(dataToPlot.reset_index(), id_vars=['date'], value_vars=columns,
170 | var_name='stats', value_name='val')
171 | # Rename stats
172 | dataToPlot['stats'].replace(NAMES, inplace=True)
173 | return dataToPlot
174 |
175 | def plotYearAndMonthStatsSleep(stats, columns=None):
176 | """
177 | Plot aggregated (mean) stats by year and month.
178 | :param stats: data to plot
179 | """
180 | if not columns:
181 | columns = ['sleep_efficiency', 'sleep_hours']
182 |
183 | dataToPlot = _prepareYearAndMonthStats(stats, columns)
184 | # Plot
185 | g = sns.factorplot(data=dataToPlot, x="date", y="val", row="stats", kind="point", sharey=False)
186 | g.set_xticklabels(rotation=45)
187 | for ax in g.axes.flat:
188 | ax.grid(b=True)
189 | return g
190 | #sns.plt.show()
191 |
192 | def _prepareDailyStats(stats, columns):
193 | dataToPlot = stats.rename(columns=NAMES)
194 |
195 | dates = pd.date_range(start=dataToPlot.date.iloc[0].date(), end=dataToPlot.date.iloc[-1].date())
196 | dataToPlot.set_index(['date'], inplace=True)
197 | dataToPlot = dataToPlot.reindex(dates)
198 | dataToPlot.reset_index(inplace=True)
199 | dataToPlot.rename(columns={'index': 'date'}, inplace=True)
200 | return dataToPlot
201 |
202 | def plotDailyStatsSleep(stats, columns=None):
203 | """
204 | Plot daily stats. Fill all data range, and put NaN for days without measures
205 | :param data: data to plot
206 | """
207 | MEASURE_NAME = 'date'
208 | if not columns:
209 | columns = ['sleep_inefficiency', 'sleep_hours']
210 | dataToPlot = _prepareDailyStats(stats, columns)
211 |
212 | f, axes = getAxes(2,1)
213 | xTicksDiv = min(10, len(dataToPlot))
214 | #xticks = [(x-pd.DateOffset(years=1, day=2)).date() for x in stats.date]
215 | xticks = [x.date() for x in dataToPlot.date]
216 | keptticks = xticks[::int(len(xticks)/xTicksDiv)]
217 | xticks = ['' for _ in xticks]
218 | xticks[::int(len(xticks)/xTicksDiv)] = keptticks
219 | for i, c in enumerate(columns):
220 | g =sns.pointplot(x=MEASURE_NAME, y=NAMES[c], data=dataToPlot, ax=axes[i])
221 | g.set_xticklabels([])
222 | g.set_xlabel('')
223 | g.set_xticklabels(xticks, rotation=45)
224 | sns.plt.show()
225 |
226 | def plotDailyStatsHb(data):
227 | ax = data.groupby(data[hbStats.NAME_DT_COL].dt.date).mean().plot()
228 | #data.groupby(data[hbStats.NAME_DT_COL].dt.date).mean().rolling(30).mean().plot(ax=ax)
229 | sns.plt.show()
230 |
231 | def plotYearMonthStatsHb(data):
232 | #pd.groupby(b,by=[b.index.month,b.index.year])
233 | data.groupby(pd.TimeGrouper(freq='M')).mean().plot()
234 | sns.plt.show()
235 |
236 | def plotSleepValueHeatmap(intradayStats, sleepValue=1):
237 | sns.set_context("poster")
238 | sns.set_style("darkgrid")
239 |
240 | xTicksDiv = 20
241 | #stepSize = int(len(xticks)/xTicksDiv)
242 | stepSize = 60
243 | xticks = [x for x in intradayStats.columns.values]
244 | keptticks = xticks[::stepSize]
245 | xticks = ['' for _ in xticks]
246 | xticks[::stepSize] = keptticks
247 | plt.figure(figsize=(16, 4.2))
248 | g = sns.heatmap(intradayStats.loc[sleepValue].reshape(1,-1))
249 | g.set_xticklabels(xticks, rotation=45)
250 | g.set_yticklabels([])
251 | g.set_ylabel(sleepStats.SLEEP_VALUES[sleepValue])
252 | plt.tight_layout()
253 | sns.plt.show()
254 |
255 | def plotCorrelation(stats):
256 | #columnsToDrop = ['sleep_interval_max_len', 'sleep_interval_min_len',
257 | # 'sleep_interval_avg_len', 'sleep_inefficiency',
258 | # 'sleep_hours', 'total_hours']
259 |
260 | #stats = stats.drop(columnsToDrop, axis=1)
261 |
262 | g = sns.PairGrid(stats)
263 | def corrfunc(x, y, **kws):
264 | r, p = scipystats.pearsonr(x, y)
265 | ax = plt.gca()
266 | ax.annotate("r = {:.2f}".format(r),xy=(.1, .9), xycoords=ax.transAxes)
267 | ax.annotate("p = {:.2f}".format(p),xy=(.2, .8), xycoords=ax.transAxes)
268 | if p>0.04:
269 | ax.patch.set_alpha(0.1)
270 |
271 | g.map_upper(plt.scatter)
272 | g.map_diag(plt.hist)
273 | g.map_lower(sns.kdeplot, cmap="Blues_d")
274 | g.map_upper(corrfunc)
275 | sns.plt.show()
276 |
277 | def getAxes(nrows, ncols):
278 | f, axes = plt.subplots(nrows=nrows, ncols=ncols)
279 | axes = axes.reshape(-1)
280 | return f, axes
281 |
282 | def plot(data, columns, measureName, nrows, ncols, order=None):
283 | f, axes = plt.subplots(nrows=nrows, ncols=ncols)
284 | axes = axes.reshape(-1)
285 | for i, c in enumerate(columns):
286 | sns.barplot(x=measureName, y=c, data=data, order=order, ax=axes[i])
287 | sns.plt.show()
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "{}"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright {yyyy} {name of copyright owner}
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
203 |
--------------------------------------------------------------------------------
/src/resources/intraday_sleep_sample.csv:
--------------------------------------------------------------------------------
1 | 2016-03-21 23:18:00,2
2 | 2016-03-21 23:19:00,2
3 | 2016-03-21 23:20:00,2
4 | 2016-03-21 23:21:00,2
5 | 2016-03-21 23:22:00,2
6 | 2016-03-21 23:23:00,3
7 | 2016-03-21 23:24:00,2
8 | 2016-03-21 23:25:00,1
9 | 2016-03-21 23:26:00,1
10 | 2016-03-21 23:27:00,1
11 | 2016-03-21 23:28:00,1
12 | 2016-03-21 23:29:00,1
13 | 2016-03-21 23:30:00,1
14 | 2016-03-21 23:31:00,1
15 | 2016-03-21 23:32:00,1
16 | 2016-03-21 23:33:00,1
17 | 2016-03-21 23:34:00,1
18 | 2016-03-21 23:35:00,2
19 | 2016-03-21 23:36:00,1
20 | 2016-03-21 23:37:00,1
21 | 2016-03-21 23:38:00,1
22 | 2016-03-21 23:39:00,1
23 | 2016-03-21 23:40:00,1
24 | 2016-03-21 23:41:00,1
25 | 2016-03-21 23:42:00,1
26 | 2016-03-21 23:43:00,1
27 | 2016-03-21 23:44:00,1
28 | 2016-03-21 23:45:00,1
29 | 2016-03-21 23:46:00,1
30 | 2016-03-21 23:47:00,1
31 | 2016-03-21 23:48:00,1
32 | 2016-03-21 23:49:00,1
33 | 2016-03-21 23:50:00,1
34 | 2016-03-21 23:51:00,1
35 | 2016-03-21 23:52:00,1
36 | 2016-03-21 23:53:00,1
37 | 2016-03-21 23:54:00,1
38 | 2016-03-21 23:55:00,1
39 | 2016-03-21 23:56:00,1
40 | 2016-03-21 23:57:00,1
41 | 2016-03-21 23:58:00,1
42 | 2016-03-21 23:59:00,1
43 | 2016-03-22 00:00:00,1
44 | 2016-03-22 00:01:00,1
45 | 2016-03-22 00:02:00,1
46 | 2016-03-22 00:03:00,1
47 | 2016-03-22 00:04:00,1
48 | 2016-03-22 00:05:00,1
49 | 2016-03-22 00:06:00,1
50 | 2016-03-22 00:07:00,1
51 | 2016-03-22 00:08:00,1
52 | 2016-03-22 00:09:00,1
53 | 2016-03-22 00:10:00,1
54 | 2016-03-22 00:11:00,1
55 | 2016-03-22 00:12:00,1
56 | 2016-03-22 00:13:00,1
57 | 2016-03-22 00:14:00,1
58 | 2016-03-22 00:15:00,1
59 | 2016-03-22 00:16:00,1
60 | 2016-03-22 00:17:00,1
61 | 2016-03-22 00:18:00,2
62 | 2016-03-22 00:19:00,1
63 | 2016-03-22 00:20:00,1
64 | 2016-03-22 00:21:00,1
65 | 2016-03-22 00:22:00,1
66 | 2016-03-22 00:23:00,1
67 | 2016-03-22 00:24:00,1
68 | 2016-03-22 00:25:00,1
69 | 2016-03-22 00:26:00,1
70 | 2016-03-22 00:27:00,1
71 | 2016-03-22 00:28:00,1
72 | 2016-03-22 00:29:00,1
73 | 2016-03-22 00:30:00,1
74 | 2016-03-22 00:31:00,2
75 | 2016-03-22 00:32:00,1
76 | 2016-03-22 00:33:00,1
77 | 2016-03-22 00:34:00,1
78 | 2016-03-22 00:35:00,1
79 | 2016-03-22 00:36:00,1
80 | 2016-03-22 00:37:00,1
81 | 2016-03-22 00:38:00,1
82 | 2016-03-22 00:39:00,1
83 | 2016-03-22 00:40:00,1
84 | 2016-03-22 00:41:00,1
85 | 2016-03-22 00:42:00,1
86 | 2016-03-22 00:43:00,1
87 | 2016-03-22 00:44:00,1
88 | 2016-03-22 00:45:00,1
89 | 2016-03-22 00:46:00,1
90 | 2016-03-22 00:47:00,1
91 | 2016-03-22 00:48:00,1
92 | 2016-03-22 00:49:00,1
93 | 2016-03-22 00:50:00,1
94 | 2016-03-22 00:51:00,1
95 | 2016-03-22 00:52:00,1
96 | 2016-03-22 00:53:00,1
97 | 2016-03-22 00:54:00,1
98 | 2016-03-22 00:55:00,1
99 | 2016-03-22 00:56:00,1
100 | 2016-03-22 00:57:00,1
101 | 2016-03-22 00:58:00,1
102 | 2016-03-22 00:59:00,1
103 | 2016-03-22 01:00:00,1
104 | 2016-03-22 01:01:00,1
105 | 2016-03-22 01:02:00,1
106 | 2016-03-22 01:03:00,1
107 | 2016-03-22 01:04:00,1
108 | 2016-03-22 01:05:00,1
109 | 2016-03-22 01:06:00,1
110 | 2016-03-22 01:07:00,1
111 | 2016-03-22 01:08:00,1
112 | 2016-03-22 01:09:00,1
113 | 2016-03-22 01:10:00,1
114 | 2016-03-22 01:11:00,1
115 | 2016-03-22 01:12:00,2
116 | 2016-03-22 01:13:00,2
117 | 2016-03-22 01:14:00,2
118 | 2016-03-22 01:15:00,1
119 | 2016-03-22 01:16:00,1
120 | 2016-03-22 01:17:00,1
121 | 2016-03-22 01:18:00,1
122 | 2016-03-22 01:19:00,1
123 | 2016-03-22 01:20:00,1
124 | 2016-03-22 01:21:00,1
125 | 2016-03-22 01:22:00,1
126 | 2016-03-22 01:23:00,1
127 | 2016-03-22 01:24:00,1
128 | 2016-03-22 01:25:00,1
129 | 2016-03-22 01:26:00,1
130 | 2016-03-22 01:27:00,1
131 | 2016-03-22 01:28:00,2
132 | 2016-03-22 01:29:00,2
133 | 2016-03-22 01:30:00,1
134 | 2016-03-22 01:31:00,2
135 | 2016-03-22 01:32:00,2
136 | 2016-03-22 01:33:00,1
137 | 2016-03-22 01:34:00,1
138 | 2016-03-22 01:35:00,1
139 | 2016-03-22 01:36:00,1
140 | 2016-03-22 01:37:00,1
141 | 2016-03-22 01:38:00,1
142 | 2016-03-22 01:39:00,1
143 | 2016-03-22 01:40:00,1
144 | 2016-03-22 01:41:00,1
145 | 2016-03-22 01:42:00,1
146 | 2016-03-22 01:43:00,1
147 | 2016-03-22 01:44:00,1
148 | 2016-03-22 01:45:00,1
149 | 2016-03-22 01:46:00,1
150 | 2016-03-22 01:47:00,1
151 | 2016-03-22 01:48:00,1
152 | 2016-03-22 01:49:00,2
153 | 2016-03-22 01:50:00,2
154 | 2016-03-22 01:51:00,1
155 | 2016-03-22 01:52:00,1
156 | 2016-03-22 01:53:00,1
157 | 2016-03-22 01:54:00,1
158 | 2016-03-22 01:55:00,1
159 | 2016-03-22 01:56:00,1
160 | 2016-03-22 01:57:00,1
161 | 2016-03-22 01:58:00,1
162 | 2016-03-22 01:59:00,1
163 | 2016-03-22 02:00:00,1
164 | 2016-03-22 02:01:00,1
165 | 2016-03-22 02:02:00,2
166 | 2016-03-22 02:03:00,1
167 | 2016-03-22 02:04:00,1
168 | 2016-03-22 02:05:00,1
169 | 2016-03-22 02:06:00,1
170 | 2016-03-22 02:07:00,1
171 | 2016-03-22 02:08:00,1
172 | 2016-03-22 02:09:00,1
173 | 2016-03-22 02:10:00,1
174 | 2016-03-22 02:11:00,1
175 | 2016-03-22 02:12:00,1
176 | 2016-03-22 02:13:00,1
177 | 2016-03-22 02:14:00,1
178 | 2016-03-22 02:15:00,1
179 | 2016-03-22 02:16:00,1
180 | 2016-03-22 02:17:00,1
181 | 2016-03-22 02:18:00,1
182 | 2016-03-22 02:19:00,1
183 | 2016-03-22 02:20:00,1
184 | 2016-03-22 02:21:00,1
185 | 2016-03-22 02:22:00,1
186 | 2016-03-22 02:23:00,1
187 | 2016-03-22 02:24:00,1
188 | 2016-03-22 02:25:00,1
189 | 2016-03-22 02:26:00,1
190 | 2016-03-22 02:27:00,1
191 | 2016-03-22 02:28:00,1
192 | 2016-03-22 02:29:00,1
193 | 2016-03-22 02:30:00,1
194 | 2016-03-22 02:31:00,1
195 | 2016-03-22 02:32:00,1
196 | 2016-03-22 02:33:00,1
197 | 2016-03-22 02:34:00,1
198 | 2016-03-22 02:35:00,2
199 | 2016-03-22 02:36:00,2
200 | 2016-03-22 02:37:00,2
201 | 2016-03-22 02:38:00,1
202 | 2016-03-22 02:39:00,1
203 | 2016-03-22 02:40:00,1
204 | 2016-03-22 02:41:00,1
205 | 2016-03-22 02:42:00,1
206 | 2016-03-22 02:43:00,1
207 | 2016-03-22 02:44:00,1
208 | 2016-03-22 02:45:00,1
209 | 2016-03-22 02:46:00,1
210 | 2016-03-22 02:47:00,1
211 | 2016-03-22 02:48:00,1
212 | 2016-03-22 02:49:00,1
213 | 2016-03-22 02:50:00,1
214 | 2016-03-22 02:51:00,1
215 | 2016-03-22 02:52:00,1
216 | 2016-03-22 02:53:00,1
217 | 2016-03-22 02:54:00,1
218 | 2016-03-22 02:55:00,1
219 | 2016-03-22 02:56:00,1
220 | 2016-03-22 02:57:00,1
221 | 2016-03-22 02:58:00,1
222 | 2016-03-22 02:59:00,1
223 | 2016-03-22 03:00:00,1
224 | 2016-03-22 03:01:00,1
225 | 2016-03-22 03:02:00,1
226 | 2016-03-22 03:03:00,1
227 | 2016-03-22 03:04:00,1
228 | 2016-03-22 03:05:00,2
229 | 2016-03-22 03:06:00,1
230 | 2016-03-22 03:07:00,1
231 | 2016-03-22 03:08:00,1
232 | 2016-03-22 03:09:00,1
233 | 2016-03-22 03:10:00,1
234 | 2016-03-22 03:11:00,1
235 | 2016-03-22 03:12:00,1
236 | 2016-03-22 03:13:00,1
237 | 2016-03-22 03:14:00,1
238 | 2016-03-22 03:15:00,1
239 | 2016-03-22 03:16:00,1
240 | 2016-03-22 03:17:00,1
241 | 2016-03-22 03:18:00,1
242 | 2016-03-22 03:19:00,1
243 | 2016-03-22 03:20:00,1
244 | 2016-03-22 03:21:00,1
245 | 2016-03-22 03:22:00,1
246 | 2016-03-22 03:23:00,1
247 | 2016-03-22 03:24:00,1
248 | 2016-03-22 03:25:00,1
249 | 2016-03-22 03:26:00,1
250 | 2016-03-22 03:27:00,1
251 | 2016-03-22 03:28:00,1
252 | 2016-03-22 03:29:00,1
253 | 2016-03-22 03:30:00,1
254 | 2016-03-22 03:31:00,1
255 | 2016-03-22 03:32:00,1
256 | 2016-03-22 03:33:00,1
257 | 2016-03-22 03:34:00,1
258 | 2016-03-22 03:35:00,1
259 | 2016-03-22 03:36:00,2
260 | 2016-03-22 03:37:00,1
261 | 2016-03-22 03:38:00,1
262 | 2016-03-22 03:39:00,1
263 | 2016-03-22 03:40:00,1
264 | 2016-03-22 03:41:00,1
265 | 2016-03-22 03:42:00,1
266 | 2016-03-22 03:43:00,1
267 | 2016-03-22 03:44:00,1
268 | 2016-03-22 03:45:00,1
269 | 2016-03-22 03:46:00,2
270 | 2016-03-22 03:47:00,1
271 | 2016-03-22 03:48:00,1
272 | 2016-03-22 03:49:00,1
273 | 2016-03-22 03:50:00,1
274 | 2016-03-22 03:51:00,1
275 | 2016-03-22 03:52:00,1
276 | 2016-03-22 03:53:00,1
277 | 2016-03-22 03:54:00,1
278 | 2016-03-22 03:55:00,1
279 | 2016-03-22 03:56:00,1
280 | 2016-03-22 03:57:00,1
281 | 2016-03-22 03:58:00,1
282 | 2016-03-22 03:59:00,1
283 | 2016-03-22 04:00:00,1
284 | 2016-03-22 04:01:00,1
285 | 2016-03-22 04:02:00,1
286 | 2016-03-22 04:03:00,1
287 | 2016-03-22 04:04:00,1
288 | 2016-03-22 04:05:00,1
289 | 2016-03-22 04:06:00,1
290 | 2016-03-22 04:07:00,1
291 | 2016-03-22 04:08:00,1
292 | 2016-03-22 04:09:00,1
293 | 2016-03-22 04:10:00,1
294 | 2016-03-22 04:11:00,1
295 | 2016-03-22 04:12:00,1
296 | 2016-03-22 04:13:00,1
297 | 2016-03-22 04:14:00,1
298 | 2016-03-22 04:15:00,1
299 | 2016-03-22 04:16:00,1
300 | 2016-03-22 04:17:00,1
301 | 2016-03-22 04:18:00,1
302 | 2016-03-22 04:19:00,1
303 | 2016-03-22 04:20:00,1
304 | 2016-03-22 04:21:00,1
305 | 2016-03-22 04:22:00,1
306 | 2016-03-22 04:23:00,1
307 | 2016-03-22 04:24:00,1
308 | 2016-03-22 04:25:00,1
309 | 2016-03-22 04:26:00,1
310 | 2016-03-22 04:27:00,2
311 | 2016-03-22 04:28:00,1
312 | 2016-03-22 04:29:00,1
313 | 2016-03-22 04:30:00,1
314 | 2016-03-22 04:31:00,1
315 | 2016-03-22 04:32:00,1
316 | 2016-03-22 04:33:00,1
317 | 2016-03-22 04:34:00,1
318 | 2016-03-22 04:35:00,1
319 | 2016-03-22 04:36:00,1
320 | 2016-03-22 04:37:00,1
321 | 2016-03-22 04:38:00,1
322 | 2016-03-22 04:39:00,2
323 | 2016-03-22 04:40:00,1
324 | 2016-03-22 04:41:00,1
325 | 2016-03-22 04:42:00,1
326 | 2016-03-22 04:43:00,1
327 | 2016-03-22 04:44:00,1
328 | 2016-03-22 04:45:00,1
329 | 2016-03-22 04:46:00,1
330 | 2016-03-22 04:47:00,1
331 | 2016-03-22 04:48:00,1
332 | 2016-03-22 04:49:00,1
333 | 2016-03-22 04:50:00,1
334 | 2016-03-22 04:51:00,1
335 | 2016-03-22 04:52:00,1
336 | 2016-03-22 04:53:00,1
337 | 2016-03-22 04:54:00,1
338 | 2016-03-22 04:55:00,1
339 | 2016-03-22 04:56:00,1
340 | 2016-03-22 04:57:00,1
341 | 2016-03-22 04:58:00,1
342 | 2016-03-22 04:59:00,1
343 | 2016-03-22 05:00:00,1
344 | 2016-03-22 05:01:00,1
345 | 2016-03-22 05:02:00,1
346 | 2016-03-22 05:03:00,1
347 | 2016-03-22 05:04:00,1
348 | 2016-03-22 05:05:00,1
349 | 2016-03-22 05:06:00,1
350 | 2016-03-22 05:07:00,1
351 | 2016-03-22 05:08:00,1
352 | 2016-03-22 05:09:00,1
353 | 2016-03-22 05:10:00,1
354 | 2016-03-22 05:11:00,2
355 | 2016-03-22 05:12:00,1
356 | 2016-03-22 05:13:00,1
357 | 2016-03-22 05:14:00,1
358 | 2016-03-22 05:15:00,2
359 | 2016-03-22 05:16:00,1
360 | 2016-03-22 05:17:00,1
361 | 2016-03-22 05:18:00,1
362 | 2016-03-22 05:19:00,1
363 | 2016-03-22 05:20:00,1
364 | 2016-03-22 05:21:00,1
365 | 2016-03-22 05:22:00,1
366 | 2016-03-22 05:23:00,1
367 | 2016-03-22 05:24:00,1
368 | 2016-03-22 05:25:00,1
369 | 2016-03-22 05:26:00,1
370 | 2016-03-22 05:27:00,1
371 | 2016-03-22 05:28:00,1
372 | 2016-03-22 05:29:00,1
373 | 2016-03-22 05:30:00,1
374 | 2016-03-22 05:31:00,1
375 | 2016-03-22 05:32:00,1
376 | 2016-03-22 05:33:00,1
377 | 2016-03-22 05:34:00,1
378 | 2016-03-22 05:35:00,1
379 | 2016-03-22 05:36:00,1
380 | 2016-03-22 05:37:00,1
381 | 2016-03-22 05:38:00,1
382 | 2016-03-22 05:39:00,1
383 | 2016-03-22 05:40:00,1
384 | 2016-03-22 05:41:00,1
385 | 2016-03-22 05:42:00,1
386 | 2016-03-22 05:43:00,1
387 | 2016-03-22 05:44:00,2
388 | 2016-03-22 05:45:00,1
389 | 2016-03-22 05:46:00,1
390 | 2016-03-22 05:47:00,1
391 | 2016-03-22 05:48:00,1
392 | 2016-03-22 05:49:00,1
393 | 2016-03-22 05:50:00,1
394 | 2016-03-22 05:51:00,1
395 | 2016-03-22 05:52:00,1
396 | 2016-03-22 05:53:00,1
397 | 2016-03-22 05:54:00,1
398 | 2016-03-22 05:55:00,1
399 | 2016-03-22 05:56:00,1
400 | 2016-03-22 05:57:00,1
401 | 2016-03-22 05:58:00,1
402 | 2016-03-22 05:59:00,1
403 | 2016-03-22 06:00:00,1
404 | 2016-03-22 06:01:00,1
405 | 2016-03-22 06:02:00,1
406 | 2016-03-22 06:03:00,1
407 | 2016-03-22 06:04:00,1
408 | 2016-03-22 06:05:00,1
409 | 2016-03-22 06:06:00,1
410 | 2016-03-22 06:07:00,1
411 | 2016-03-22 06:08:00,1
412 | 2016-03-22 06:09:00,1
413 | 2016-03-22 06:10:00,1
414 | 2016-03-22 06:11:00,1
415 | 2016-03-22 06:12:00,1
416 | 2016-03-22 06:13:00,1
417 | 2016-03-22 06:14:00,1
418 | 2016-03-22 06:15:00,1
419 | 2016-03-22 06:16:00,1
420 | 2016-03-22 06:17:00,1
421 | 2016-03-22 06:18:00,1
422 | 2016-03-22 06:19:00,2
423 | 2016-03-22 06:20:00,1
424 | 2016-03-22 06:21:00,1
425 | 2016-03-22 06:22:00,1
426 | 2016-03-22 06:23:00,1
427 | 2016-03-22 06:24:00,1
428 | 2016-03-22 06:25:00,1
429 | 2016-03-22 06:26:00,1
430 | 2016-03-22 06:27:00,1
431 | 2016-03-22 06:28:00,1
432 | 2016-03-22 06:29:00,1
433 | 2016-03-22 06:30:00,1
434 | 2016-03-22 06:31:00,1
435 | 2016-03-22 06:32:00,1
436 | 2016-03-22 06:33:00,1
437 | 2016-03-22 06:34:00,1
438 | 2016-03-22 06:35:00,1
439 | 2016-03-22 06:36:00,1
440 | 2016-03-22 06:37:00,1
441 | 2016-03-22 06:38:00,1
442 | 2016-03-22 06:39:00,1
443 | 2016-03-22 06:40:00,1
444 | 2016-03-22 06:41:00,1
445 | 2016-03-22 06:42:00,1
446 | 2016-03-22 06:43:00,1
447 | 2016-03-22 06:44:00,1
448 | 2016-03-22 06:45:00,1
449 | 2016-03-22 06:46:00,1
450 | 2016-03-22 06:47:00,1
451 | 2016-03-22 06:48:00,1
452 | 2016-03-22 06:49:00,1
453 | 2016-03-22 06:50:00,1
454 | 2016-03-22 06:51:00,1
455 | 2016-03-22 06:52:00,2
456 | 2016-03-22 06:53:00,2
457 | 2016-03-22 06:54:00,1
458 | 2016-03-22 06:55:00,1
459 | 2016-03-22 06:56:00,2
460 | 2016-03-22 06:57:00,1
--------------------------------------------------------------------------------
/Sleep Dashboard.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Table of Contents\n",
8 | "* [Intro](#Intro)\n",
9 | "* [Load Data](#Load-Data)\n",
10 | "* [Generate and Save Stats](#Generate-and-Save-Stats)\n",
11 | "* [Basic Stats](#Basic-Stats)\n",
12 | "\t* [Preliminary Stats](#Preliminary-Stats)\n",
13 | "\t* [Weekday Stats](#Weekday-Stats)\n",
14 | "\t* [Monthly Stats](#Monthly-Stats)\n",
15 | "\t* [Year-Month Stats](#Year-Month-Stats)\n",
16 | "\t* [Weekday Stats By Month](#Weekday-Stats-By-Month)\n",
17 | "\t* [Daily Stats](#Daily-Stats)\n",
18 | "* [Intraday Stats](#Intraday-Stats)\n",
19 | "\t* [Daily Stats](#Daily-Stats)\n"
20 | ]
21 | },
22 | {
23 | "cell_type": "markdown",
24 | "metadata": {},
25 | "source": [
26 | "# Intro"
27 | ]
28 | },
29 | {
30 | "cell_type": "markdown",
31 | "metadata": {},
32 | "source": [
33 | "This notebook explores analysis and visualization of Fitbit sleep data."
34 | ]
35 | },
36 | {
37 | "cell_type": "code",
38 | "execution_count": null,
39 | "metadata": {
40 | "ExecuteTime": {
41 | "end_time": "2017-10-05T09:45:06.073764",
42 | "start_time": "2017-10-05T09:45:04.280662"
43 | },
44 | "collapsed": false
45 | },
46 | "outputs": [],
47 | "source": [
48 | "import os\n",
49 | "import sys\n",
50 | "from os.path import join\n",
51 | "\n",
52 | "import pandas as pd\n",
53 | "import numpy as np\n",
54 | "import seaborn as sns\n",
55 | "import time\n",
56 | "import datetime\n",
57 | "import matplotlib.pyplot as plt\n",
58 | "\n",
59 | "sys.path.append(os.path.join(os.getcwd(), \"src\"))\n",
60 | "from resources import RESOURCE_PATH\n",
61 | "from stats import sleepStats, combinedStats\n",
62 | "from util import utils, plotting as mplot\n",
63 | "\n",
64 | "%load_ext autoreload\n",
65 | "%autoreload 2\n",
66 | "\n",
67 | "%matplotlib notebook\n",
68 | "sns.set_context(\"paper\")\n",
69 | "\n",
70 | "dataFolder = \"path_to_your_fitbit_JSON_export\"\n",
71 | "statsFolder = join(dataFolder, os.path.pardir, 'folder_name_for_generated_stats')"
72 | ]
73 | },
74 | {
75 | "cell_type": "code",
76 | "execution_count": null,
77 | "metadata": {
78 | "ExecuteTime": {
79 | "end_time": "2017-10-05T09:45:11.609081",
80 | "start_time": "2017-10-05T09:45:10.588023"
81 | },
82 | "collapsed": false
83 | },
84 | "outputs": [],
85 | "source": [
86 | "import plotly.plotly as py\n",
87 | "from plotly.offline import init_notebook_mode, enable_mpl_offline, iplot_mpl\n",
88 | "import cufflinks as cf\n",
89 | "init_notebook_mode(connected=True)\n",
90 | "cf.go_offline(connected=True)\n",
91 | "enable_mpl_offline()\n",
92 | "\n",
93 | "from ipywidgets import interact, widgets\n",
94 | "#from IPython.display import display, clear_output"
95 | ]
96 | },
97 | {
98 | "cell_type": "code",
99 | "execution_count": null,
100 | "metadata": {
101 | "ExecuteTime": {
102 | "end_time": "2017-09-04T10:14:26.728995",
103 | "start_time": "2017-09-04T10:14:26.329972"
104 | },
105 | "collapsed": false
106 | },
107 | "outputs": [],
108 | "source": [
109 | "# Enable logging from Fitbit Analyer code\n",
110 | "import logging\n",
111 | "logger = logging.getLogger()\n",
112 | "logger.setLevel(logging.ERROR)\n",
113 | "logger.handlers[0].stream = sys.stdout"
114 | ]
115 | },
116 | {
117 | "cell_type": "markdown",
118 | "metadata": {},
119 | "source": [
120 | "# Load Data"
121 | ]
122 | },
123 | {
124 | "cell_type": "code",
125 | "execution_count": null,
126 | "metadata": {
127 | "ExecuteTime": {
128 | "end_time": "2017-10-05T09:45:17.824436",
129 | "start_time": "2017-10-05T09:45:17.487417"
130 | },
131 | "collapsed": true
132 | },
133 | "outputs": [],
134 | "source": [
135 | "# helper method to load provided test data\n",
136 | "def load_test_sleep_data():\n",
137 | " filepath = RESOURCE_PATH + \"\\\\unittest\\\\test_sleep_basic01.csv\"\n",
138 | " data1 = utils.loadIntradayData(filepath)\n",
139 | " filepath = RESOURCE_PATH + \"\\\\unittest\\\\test_sleep_basic02.csv\"\n",
140 | " data2 = utils.loadIntradayData(filepath)\n",
141 | " return [data1, data2]"
142 | ]
143 | },
144 | {
145 | "cell_type": "markdown",
146 | "metadata": {},
147 | "source": [
148 | "Load sleep data from raw JSON export of Fitbit records."
149 | ]
150 | },
151 | {
152 | "cell_type": "code",
153 | "execution_count": null,
154 | "metadata": {
155 | "ExecuteTime": {
156 | "end_time": "2017-10-05T09:45:19.714545",
157 | "start_time": "2017-10-05T09:45:19.379525"
158 | },
159 | "collapsed": false,
160 | "scrolled": true
161 | },
162 | "outputs": [],
163 | "source": [
164 | "start = time.time()\n",
165 | "#sleepData = utils.loadSleepData(dataFolder) # use for loading your own data\n",
166 | "sleepData = load_test_sleep_data() #use for testing\n",
167 | "end = time.time()\n",
168 | "print(\"Data loaded in {:.2f}s\".format(end - start))\n",
169 | "print(\"Loaded {} dataframes\".format(len(sleepData)))\n",
170 | "print(\"{} total entries\".format(np.sum([df.size for df in sleepData])))\n",
171 | "print(\"Sample from first dataframe:\")\n",
172 | "print(sleepData[0].head())"
173 | ]
174 | },
175 | {
176 | "cell_type": "markdown",
177 | "metadata": {},
178 | "source": [
179 | "# Generate and Save Stats"
180 | ]
181 | },
182 | {
183 | "cell_type": "markdown",
184 | "metadata": {},
185 | "source": [
186 | "For the loaded sleep data generate all currently avaiable stats:\n",
187 | "* **Basic Stats** (sleep values count, sleep efficiency, hours of sleep, total minutes in bed, N max-values for each stat)\n",
188 | "* **Timing Stats** (first minute asleep, to bed time, wake up time, sleep interval min/max length)\n",
189 | "* **Intervals Stats** for each day all the sleep intervals lengths\n",
190 | "* **Intraday Stats** minute to minute report for each day, for the specified sleep values. Total value count, with normalization and centering on specific time."
191 | ]
192 | },
193 | {
194 | "cell_type": "code",
195 | "execution_count": null,
196 | "metadata": {
197 | "ExecuteTime": {
198 | "end_time": "2017-10-05T09:55:04.813010",
199 | "start_time": "2017-10-05T09:55:04.373985"
200 | },
201 | "collapsed": false
202 | },
203 | "outputs": [],
204 | "source": [
205 | "start = time.time()\n",
206 | "basicAndTimingStats = sleepStats.generateStatsFrom(sleepData, sleepStats.STATS_NAME_BASIC_AND_TIMING)\n",
207 | "end = time.time()\n",
208 | "print(\"Computed basicAndTimingStats in {:.2f}s\".format(end - start))\n",
209 | "start = time.time()\n",
210 | "intervalsStats = sleepStats.generateStatsFrom(sleepData, sleepStats.STATS_NAME_INTERVALS)\n",
211 | "end = time.time()\n",
212 | "print(\"Computed intervalsStats in {:.2f}s\".format(end - start))\n",
213 | "start = time.time()\n",
214 | "intradayStats = sleepStats.generateStatsFrom(sleepData, sleepStats.STATS_NAME_INTRADAY)\n",
215 | "end = time.time()\n",
216 | "print(\"Computed intradayStats in {:.2f}s\".format(end - start))"
217 | ]
218 | },
219 | {
220 | "cell_type": "code",
221 | "execution_count": null,
222 | "metadata": {
223 | "collapsed": false
224 | },
225 | "outputs": [],
226 | "source": [
227 | "#print(basicAndTimingStats.head())\n",
228 | "#print(intervalsStats.head())\n",
229 | "#print(intradayStats.head())"
230 | ]
231 | },
232 | {
233 | "cell_type": "markdown",
234 | "metadata": {},
235 | "source": [
236 | "Save generated stats to file"
237 | ]
238 | },
239 | {
240 | "cell_type": "code",
241 | "execution_count": null,
242 | "metadata": {
243 | "ExecuteTime": {
244 | "end_time": "2017-10-05T09:55:16.364671",
245 | "start_time": "2017-10-05T09:55:16.006650"
246 | },
247 | "collapsed": false
248 | },
249 | "outputs": [],
250 | "source": [
251 | "today = datetime.date.today().strftime(\"%Y_%m_%d\")\n",
252 | "basicAndTimingStatsFilepath = os.path.join(statsFolder, \"basicAndTimingStats_{}.csv\".format(today))\n",
253 | "intervalsStatsFilepath = os.path.join(statsFolder, \"intervalStats_{}.csv\".format(today))\n",
254 | "intradayStatsFilepath = os.path.join(statsFolder, \"intradayStats_{}.csv\".format(today))\n",
255 | "\n",
256 | "basicAndTimingStats.to_csv(basicAndTimingStatsFilepath, index=False)\n",
257 | "intervalsStats.to_csv(intervalsStatsFilepath, index=False)\n",
258 | "intradayStats.to_csv(intradayStatsFilepath, index=False)"
259 | ]
260 | },
261 | {
262 | "cell_type": "markdown",
263 | "metadata": {},
264 | "source": [
265 | "# Basic Stats"
266 | ]
267 | },
268 | {
269 | "cell_type": "code",
270 | "execution_count": null,
271 | "metadata": {
272 | "ExecuteTime": {
273 | "end_time": "2017-09-05T16:55:58.997528",
274 | "start_time": "2017-09-05T16:55:58.725513"
275 | },
276 | "collapsed": false
277 | },
278 | "outputs": [],
279 | "source": [
280 | "# load basic and timing stats\n",
281 | "basicAndTimingStatsFilename = \"basicAndTimingStats_{}.csv\".format(today)\n",
282 | "basicAndTimingStats = pd.read_csv(os.path.join(statsFolder, basicAndTimingStatsFilename), \n",
283 | " parse_dates=['date', 'to_bed_time', 'wake_up_time'])\n",
284 | "basicAndTimingStats.head()"
285 | ]
286 | },
287 | {
288 | "cell_type": "code",
289 | "execution_count": null,
290 | "metadata": {
291 | "ExecuteTime": {
292 | "end_time": "2017-10-05T09:52:58.611792",
293 | "start_time": "2017-10-05T09:52:58.296774"
294 | },
295 | "collapsed": false
296 | },
297 | "outputs": [],
298 | "source": [
299 | "basicAndTimingStats.info()"
300 | ]
301 | },
302 | {
303 | "cell_type": "markdown",
304 | "metadata": {},
305 | "source": [
306 | "## Preliminary Stats"
307 | ]
308 | },
309 | {
310 | "cell_type": "code",
311 | "execution_count": null,
312 | "metadata": {
313 | "ExecuteTime": {
314 | "end_time": "2017-10-05T09:48:45.892337",
315 | "start_time": "2017-10-05T09:48:45.315304"
316 | },
317 | "collapsed": false,
318 | "scrolled": false
319 | },
320 | "outputs": [],
321 | "source": [
322 | "# plot preliminary stats (static plot) \n",
323 | "plot = mplot.plotPreliminaryStats(basicAndTimingStats)"
324 | ]
325 | },
326 | {
327 | "cell_type": "markdown",
328 | "metadata": {},
329 | "source": [
330 | "## Weekday Stats"
331 | ]
332 | },
333 | {
334 | "cell_type": "code",
335 | "execution_count": null,
336 | "metadata": {
337 | "ExecuteTime": {
338 | "end_time": "2017-10-05T09:48:52.637723",
339 | "start_time": "2017-10-05T09:48:52.285703"
340 | },
341 | "collapsed": false
342 | },
343 | "outputs": [],
344 | "source": [
345 | "# plot preliminary stats (static plot) \n",
346 | "plot = mplot.plotWeekdayStatsSleep(basicAndTimingStats)\n",
347 | "plot"
348 | ]
349 | },
350 | {
351 | "cell_type": "code",
352 | "execution_count": null,
353 | "metadata": {
354 | "ExecuteTime": {
355 | "end_time": "2017-09-05T16:56:42.641025",
356 | "start_time": "2017-09-05T16:56:41.759974"
357 | },
358 | "collapsed": false
359 | },
360 | "outputs": [],
361 | "source": [
362 | "# transform static plot using Plotly\n",
363 | "iplot_mpl(plot.fig)"
364 | ]
365 | },
366 | {
367 | "cell_type": "markdown",
368 | "metadata": {},
369 | "source": [
370 | "## Monthly Stats"
371 | ]
372 | },
373 | {
374 | "cell_type": "code",
375 | "execution_count": null,
376 | "metadata": {
377 | "ExecuteTime": {
378 | "end_time": "2017-10-05T09:46:55.711035",
379 | "start_time": "2017-10-05T09:46:55.341014"
380 | },
381 | "collapsed": false
382 | },
383 | "outputs": [],
384 | "source": [
385 | "plot = mplot.plotMonthlyStatsSleep(basicAndTimingStats)\n",
386 | "plot"
387 | ]
388 | },
389 | {
390 | "cell_type": "markdown",
391 | "metadata": {},
392 | "source": [
393 | "## Year-Month Stats"
394 | ]
395 | },
396 | {
397 | "cell_type": "code",
398 | "execution_count": null,
399 | "metadata": {
400 | "ExecuteTime": {
401 | "end_time": "2017-10-05T09:47:27.207837",
402 | "start_time": "2017-10-05T09:47:26.808814"
403 | },
404 | "collapsed": false
405 | },
406 | "outputs": [],
407 | "source": [
408 | "# plot Year-Month stats (static plot)\n",
409 | "plot = mplot.plotYearAndMonthStatsSleep(basicAndTimingStats)"
410 | ]
411 | },
412 | {
413 | "cell_type": "code",
414 | "execution_count": null,
415 | "metadata": {
416 | "ExecuteTime": {
417 | "end_time": "2017-09-05T17:42:43.098914",
418 | "start_time": "2017-09-05T17:42:41.924847"
419 | },
420 | "collapsed": false
421 | },
422 | "outputs": [],
423 | "source": [
424 | "# interactive single year-month stat\n",
425 | "@interact(stat_name=mplot.NAMES.keys())\n",
426 | "def iplot_yearAndMonthStats(stat_name):\n",
427 | " data = basicAndTimingStats.groupby(basicAndTimingStats['date'].dt.to_period(\"M\"))[[stat_name]].mean()\n",
428 | " data.iplot(title='Year-Month Average - {}'.format(mplot.NAMES[stat_name]))"
429 | ]
430 | },
431 | {
432 | "cell_type": "markdown",
433 | "metadata": {},
434 | "source": [
435 | "## Weekday Stats By Month"
436 | ]
437 | },
438 | {
439 | "cell_type": "code",
440 | "execution_count": null,
441 | "metadata": {
442 | "ExecuteTime": {
443 | "end_time": "2017-09-05T17:51:11.524994",
444 | "start_time": "2017-09-05T17:51:10.967962"
445 | },
446 | "collapsed": false
447 | },
448 | "outputs": [],
449 | "source": [
450 | "# plot weekday stats by month (static plot) \n",
451 | "plot = mplot.plotWeekdayStatsByMonthSleep(basicAndTimingStats)"
452 | ]
453 | },
454 | {
455 | "cell_type": "code",
456 | "execution_count": null,
457 | "metadata": {
458 | "ExecuteTime": {
459 | "end_time": "2017-09-05T17:51:22.286610",
460 | "start_time": "2017-09-05T17:51:21.927589"
461 | },
462 | "collapsed": true
463 | },
464 | "outputs": [],
465 | "source": [
466 | "weekdayByMonthStats = mplot._prepareWeekdayByMonthStats(basicAndTimingStats)"
467 | ]
468 | },
469 | {
470 | "cell_type": "code",
471 | "execution_count": null,
472 | "metadata": {
473 | "ExecuteTime": {
474 | "end_time": "2017-09-05T18:06:22.766114",
475 | "start_time": "2017-09-05T18:06:22.357091"
476 | },
477 | "collapsed": false
478 | },
479 | "outputs": [],
480 | "source": [
481 | "@interact(stat_name=mplot.NAMES.keys())\n",
482 | "def iplot_weekdayByMonthStats(stat_name):\n",
483 | " weekdayByMonthStats.pivot('day', 'month', values=stat_name).iplot(title='Weekday By Month - {}'.format(mplot.NAMES[stat_name]),\n",
484 | " xTitle='Weekday', yTitle=mplot.NAMES[stat_name])"
485 | ]
486 | },
487 | {
488 | "cell_type": "markdown",
489 | "metadata": {},
490 | "source": [
491 | "## Daily Stats"
492 | ]
493 | },
494 | {
495 | "cell_type": "code",
496 | "execution_count": null,
497 | "metadata": {
498 | "ExecuteTime": {
499 | "end_time": "2017-09-05T18:03:12.380225",
500 | "start_time": "2017-09-05T18:03:11.974201"
501 | },
502 | "collapsed": false
503 | },
504 | "outputs": [],
505 | "source": [
506 | "@interact(stat_name=mplot.NAMES.keys())\n",
507 | "def iplot_weekdayByMonthStats(stat_name):\n",
508 | " data = basicAndTimingStats[['date', stat_name]].set_index(['date'])\n",
509 | " data.iplot(title='Daily Stats - {}'.format(mplot.NAMES[stat_name]), yTitle=mplot.NAMES[stat_name])"
510 | ]
511 | },
512 | {
513 | "cell_type": "markdown",
514 | "metadata": {},
515 | "source": [
516 | "# Intraday Stats"
517 | ]
518 | },
519 | {
520 | "cell_type": "code",
521 | "execution_count": null,
522 | "metadata": {
523 | "collapsed": false
524 | },
525 | "outputs": [],
526 | "source": [
527 | "intradayStatsFilename = \"intradayStats_{}.csv\".format(today)\n",
528 | "intradayStats = pd.read_csv(os.path.join(statsFolder, intradayStats), \n",
529 | " parse_dates=['date', 'to_bed_time', 'wake_up_time'])\n",
530 | "intradayStats.drop(\"date\", axis=1, inplace=True)\n",
531 | "data = intradayStats.apply(pd.value_counts)\n",
532 | "#mplot.plotSleepValueHeatmap(data, sleepValue=1)"
533 | ]
534 | },
535 | {
536 | "cell_type": "code",
537 | "execution_count": null,
538 | "metadata": {
539 | "collapsed": false
540 | },
541 | "outputs": [],
542 | "source": [
543 | "normIntradayCountStats = sleepStats.normalizedIntradayCountStats(intradayStats)\n",
544 | "centeredIntradayCountStats = sleepStats.centerIntradayCountStats(normIntradayCountStats)\n",
545 | "#mplot.plotSleepValueHeatmap(centeredIntradayCountStats, sleepValue=1)"
546 | ]
547 | },
548 | {
549 | "cell_type": "markdown",
550 | "metadata": {},
551 | "source": [
552 | "## Daily Stats\n"
553 | ]
554 | },
555 | {
556 | "cell_type": "code",
557 | "execution_count": null,
558 | "metadata": {
559 | "collapsed": false
560 | },
561 | "outputs": [],
562 | "source": [
563 | "#stats.set_index('date', inplace=True)\n",
564 | "stats['sleep_efficiency_rol_mean'] = stats['sleep_efficiency'].rolling(center=False,window=20).mean()\n",
565 | "stats['sleep_efficiency'].plot()\n",
566 | "stats['sleep_efficiency_rol_mean'].plot()\n",
567 | "sns.plt.show()"
568 | ]
569 | },
570 | {
571 | "cell_type": "code",
572 | "execution_count": null,
573 | "metadata": {
574 | "collapsed": false
575 | },
576 | "outputs": [],
577 | "source": [
578 | "testData = stats['restless']\n",
579 | "testData.resample('20D').mean().plot()\n",
580 | "testData.plot()\n",
581 | "sns.plt.show()"
582 | ]
583 | }
584 | ],
585 | "metadata": {
586 | "anaconda-cloud": {},
587 | "kernelspec": {
588 | "display_name": "Python [conda env:fitbit-analyzer]",
589 | "language": "python",
590 | "name": "conda-env-fitbit-analyzer-py"
591 | },
592 | "language_info": {
593 | "codemirror_mode": {
594 | "name": "ipython",
595 | "version": 3
596 | },
597 | "file_extension": ".py",
598 | "mimetype": "text/x-python",
599 | "name": "python",
600 | "nbconvert_exporter": "python",
601 | "pygments_lexer": "ipython3",
602 | "version": "3.5.1"
603 | },
604 | "toc": {
605 | "colors": {
606 | "hover_highlight": "#DAA520",
607 | "running_highlight": "#FF0000",
608 | "selected_highlight": "#FFD700"
609 | },
610 | "moveMenuLeft": true,
611 | "nav_menu": {
612 | "height": "99px",
613 | "width": "252px"
614 | },
615 | "navigate_menu": true,
616 | "number_sections": true,
617 | "sideBar": true,
618 | "threshold": 4,
619 | "toc_cell": false,
620 | "toc_section_display": "block",
621 | "toc_window_display": true
622 | }
623 | },
624 | "nbformat": 4,
625 | "nbformat_minor": 0
626 | }
627 |
--------------------------------------------------------------------------------