175 |
176 |
177 |
178 |
164 |
165 |
172 |
173 |
174 |
179 |
180 |
56 |
57 |
110 |
111 |
112 |
113 |
114 |
120 |
121 |
122 |
205 |
206 |
207 |
208 |
209 |
214 |
215 |
216 |
217 |
218 |
219 |
222 |
223 |
224 |
225 |
226 |
227 |
228 |
--------------------------------------------------------------------------------
/tests/test_muse.py:
--------------------------------------------------------------------------------
1 | """
2 | Tests for the muse module
3 | """
4 | import os
5 | import sys
6 | import unittest
7 |
8 | import numpy as np
9 | import pandas as pd
10 |
11 | import devicely
12 |
13 |
14 | class MuseTestCase(unittest.TestCase):
15 | READ_PATH = 'tests/Muse_test_data/read_test_data.csv'
16 | WRITE_PATH = 'tests/Muse_test_data/write_test_data.csv'
17 |
18 | expected_data_head = pd.DataFrame(
19 | {'AUX_RIGHT': [844.1392, np.nan, 861.06226, 1009.34064, 1006.52014],
20 | 'Accelerometer_X': [-0.23260498046875, np.nan, -0.23260498046875, -0.23260498046875, -0.23260498046875],
21 | 'Accelerometer_Y': [-0.04901123046875, np.nan, -0.04901123046875, -0.04901123046875, -0.04901123046875],
22 | 'Accelerometer_Z': [0.9664306640625, np.nan, 0.9664306640625, 0.9664306640625, 0.9664306640625],
23 | 'Alpha_AF7': [0.70684004, np.nan, 0.7046354, 0.7046354, 0.7046354],
24 | 'Alpha_AF8': [0.3738289, np.nan, 0.37960157, 0.37960157, 0.37960157],
25 | 'Alpha_TP10': [0.5247787, np.nan, 0.5247787, 0.5247787, 0.5247787],
26 | 'Alpha_TP9': [0.6818201, np.nan, 0.6818201, 0.6818201, 0.6818201],
27 | 'Battery': [100.0, np.nan, 100.0, 100.0, 100.0],
28 | 'Beta_AF7': [0.9047759, np.nan, 0.90967786, 0.90967786, 0.90967786],
29 | 'Beta_AF8': [-0.074241824, np.nan, -0.07182129, -0.07182129, -0.07182129],
30 | 'Beta_TP10': [0.24255054, np.nan, 0.24255054, 0.24255054, 0.24255054],
31 | 'Beta_TP9': [0.40113458, np.nan, 0.40113458, 0.40113458, 0.40113458],
32 | 'Delta_AF7': [0.65365446, np.nan, 0.6321334, 0.6321334, 0.6321334],
33 | 'Delta_AF8': [0.87098247, np.nan, 0.8560632, 0.8560632, 0.8560632],
34 | 'Delta_TP10': [0.7412762, np.nan, 0.7412762, 0.7412762, 0.7412762],
35 | 'Delta_TP9': [0.7765235, np.nan, 0.7765235, 0.7765235, 0.7765235],
36 | 'Elements': [np.nan, '/muse/elements/blink', np.nan, np.nan, np.nan],
37 | 'Gamma_AF7': [0.7472885, np.nan, 0.73841846, 0.73841846, 0.73841846],
38 | 'Gamma_AF8': [-0.29481375, np.nan, -0.29959682, -0.29959682, -0.29959682],
39 | 'Gamma_TP10': [-0.3348679, np.nan, -0.3348679, -0.3348679, -0.3348679],
40 | 'Gamma_TP9': [-0.0852012, np.nan, -0.0852012, -0.0852012, -0.0852012],
41 | 'Gyro_X': [2.998199462890625, np.nan, 2.998199462890625, 2.998199462890625, 2.998199462890625],
42 | 'Gyro_Y': [2.130889892578125, np.nan, 2.130889892578125, 2.130889892578125, 2.130889892578125],
43 | 'Gyro_Z': [1.966400146484375, np.nan, 1.966400146484375, 1.966400146484375, 1.966400146484375],
44 | 'HSI_AF7': [1.0, np.nan, 1.0, 1.0, 1.0],
45 | 'HSI_AF8': [1.0, np.nan, 1.0, 1.0, 1.0],
46 | 'HSI_TP10': [1.0, np.nan, 1.0, 1.0, 1.0],
47 | 'HSI_TP9': [1.0, np.nan, 1.0, 1.0, 1.0],
48 | 'HeadBandOn': [1.0, np.nan, 1.0, 1.0, 1.0],
49 | 'RAW_AF7': [798.60803, np.nan, 796.5934, 805.8608, 805.4579],
50 | 'RAW_AF8': [794.17584, np.nan, 791.75824, 789.7436, 784.5055],
51 | 'RAW_TP10': [835.6777, np.nan, 833.663, 827.2161, 820.3663],
52 | 'RAW_TP9': [832.4542, np.nan, 829.2308, 824.7985, 819.9634],
53 | 'Theta_AF7': [0.328677, np.nan, 0.33023703, 0.33023703, 0.33023703],
54 | 'Theta_AF8': [0.5665972, np.nan, 0.56282604, 0.56282604, 0.56282604],
55 | 'Theta_TP10': [0.5058985, np.nan, 0.5058985, 0.5058985, 0.5058985],
56 | 'Theta_TP9': [0.76928276, np.nan, 0.76928276, 0.76928276, 0.76928276]},
57 | index=pd.DatetimeIndex(['2021-04-05 15:48:04.834', '2021-04-05 15:48:04.840', '2021-04-05 15:48:04.836', '2021-04-05 15:48:04.842', '2021-04-05 15:48:04.843'], name='TimeStamp')
58 | )
59 |
60 | def setUp(self):
61 | self.reader = devicely.MuseReader(self.READ_PATH)
62 |
63 | def test_read(self):
64 | pd.testing.assert_frame_equal(self.reader.data.head(), self.expected_data_head, check_like=True)
65 |
66 | def test_write(self):
67 | self.reader.write(self.WRITE_PATH)
68 | new_reader = devicely.MuseReader(self.WRITE_PATH)
69 | pd.testing.assert_frame_equal(new_reader.data, self.reader.data)
70 | os.remove(self.WRITE_PATH)
71 |
72 | def test_timeshift_by_timedelta(self):
73 | shift = pd.Timedelta('1 days, 2 hours, 3 minutes, 4 seconds, 5 milliseconds')
74 | self.reader.timeshift(shift)
75 | expected_shifted_time_col = pd.DatetimeIndex(['2021-04-06 17:51:08.839000', '2021-04-06 17:51:08.845000',
76 | '2021-04-06 17:51:08.841000', '2021-04-06 17:51:08.847000',
77 | '2021-04-06 17:51:08.848000'],
78 | name='TimeStamp')
79 | pd.testing.assert_index_equal(self.reader.data.head().index, expected_shifted_time_col)
80 |
81 | def test_timeshift_to_timestamp(self):
82 | ts_shift = pd.Timestamp('01.02.2015, 11:56:32')
83 | self.reader.timeshift(ts_shift)
84 | expected_shifted_time_col = pd.DatetimeIndex(['2015-01-02 11:56:32', '2015-01-02 11:56:32.006000',
85 | '2015-01-02 11:56:32.002000', '2015-01-02 11:56:32.008000',
86 | '2015-01-02 11:56:32.009000'],
87 | name='TimeStamp')
88 | pd.testing.assert_index_equal(self.reader.data.head().index, expected_shifted_time_col)
89 |
90 | def test_random_timeshift(self):
91 | earliest_possible_time_col = pd.DatetimeIndex(['2019-04-06 15:48:04.834000', '2019-04-06 15:48:04.840000',
92 | '2019-04-06 15:48:04.836000', '2019-04-06 15:48:04.842000',
93 | '2019-04-06 15:48:04.843000'])
94 |
95 | latest_possible_time_col = pd.DatetimeIndex(['2021-03-06 15:48:04.834000', '2021-03-06 15:48:04.840000',
96 | '2021-03-06 15:48:04.836000', '2021-03-06 15:48:04.842000',
97 | '2021-03-06 15:48:04.843000'])
98 |
99 | self.reader.timeshift()
100 | self.assertTrue((earliest_possible_time_col <= self.reader.data.head().index).all())
101 | self.assertTrue((self.reader.data.head().index <= latest_possible_time_col).all())
102 |
103 |
104 | if __name__ == '__main__':
105 | unittest.main()
106 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | [](https://github.com/pyOpenSci/software-review/issues/37)
2 | [](https://joss.theoj.org/papers/3abafc8a04e02d7c61d0bf4fb714af28)
3 | [](https://badge.fury.io/py/devicely)
4 | [](https://pypi.python.org/pypi/devicely/)
5 | [](https://github.com/hpi-dhc/devicely/actions/workflows/test.yml)
6 | 
7 | [](https://zenodo.org/badge/latestdoi/279395106)
8 | [](https://www.repostatus.org/#active)
9 | [](https://mybinder.org/v2/gh/hpi-dhc/devicely-example/HEAD)
10 |
11 | 
12 |
13 | Devicely is a Python package for reading, de-identifying and writing data from various health monitoring sensors.
14 | With devicely, you can read sensor data and have it easily accessible in dataframes.
15 | You can also de-identify data and write them back using their original data format. This makes it convenient to share sensor data with other researchers while mantaining people's privacy.
16 |
17 | [Documentation](https://hpi-dhc.github.io/devicely/)
18 |
19 | [PyPi](https://pypi.org/project/devicely/)
20 |
21 | [Conda-forge](https://github.com/conda-forge/devicely-feedstock)
22 |
23 | ## Installation
24 |
25 | ### PyPi (current release)
26 |
27 | Installing `devicely` is as easy as executing:
28 |
29 | `pip install devicely`
30 |
31 | ### Conda-forge (current release)
32 |
33 | To install `devicely`through `conda-forge`:
34 |
35 | ```
36 | conda config --add channels conda-forge
37 | conda config --set channel_priority strict
38 | ```
39 |
40 | Once the `conda-forge` channel has been enabled, `devicely` can be installed with:
41 |
42 | `conda install devicely`
43 |
44 | List all of the versions of `devicely` available on your platform with:
45 |
46 | `conda search devicely --channel conda-forge`
47 |
48 | ### Locally (development version)
49 |
50 | ```
51 | git clone git@github.com:hpi-dhc/devicely.git
52 | cd devicely
53 | pip install .
54 | ```
55 |
56 | ## Sneak Peek
57 |
58 | All devices contain the following methods as exemplified through the `EmpaticaReader`:
59 |
60 | ```
61 | empatica_reader = devicely.EmpaticaReader(path_to_empatica_files)
62 | empatica_reader.timeshift()
63 | empatica_reader.write(path_to_write_files)
64 | ```
65 |
66 | You can also try this [notebook](https://github.com/hpi-dhc/devicely-example)
67 | with examples and sample data or check our binder:
68 |
69 | [](https://mybinder.org/v2/gh/hpi-dhc/devicely-example/HEAD)
70 |
71 | ## Quick Start
72 |
73 | To get started quickly, follow our [quick-start guide](https://hpi-dhc.github.io/devicely/examples.html#).
74 |
75 | Or check the full documentation: https://hpi-dhc.github.io/devicely/
76 |
77 |
78 | ## Supported Sensors
79 |
80 | - [Empatica E4](https://e4.empatica.com/e4-wristband) is a wearable device that offers real-time physiological data acquisition such as blood volume pulse, electrodermal activity (EDA), heart rate, interbeat intervals, 3-axis acceleration and skin temperature.
81 |
82 | - [Biovotion Everion](https://www.biovotion.com/everion/) is a wearable device used for the continuous monitoring of vital signs. Currently, it measures the following vital signs: heart rate, blood pulse wave, heart rate variability, activity, SPO2, blood perfusion, respiration rate, steps, energy expenditure, skin temperature, EDA / galvanic skin response (GSR), barometric pressure and sleep.
83 |
84 | - [1-lead ECG monitor FarosTM 180 from Bittium](https://shop.bittium.com/product/36/bittium-faros-180-solution-pack) is a one channel ECG monitor with sampling frequency up to 1000 Hz and a 3D acceleration sampling up to 100Hz.
85 |
86 | - [Spacelabs (SL 90217)](https://www.spacelabshealthcare.com/products/diagnostic-cardiology/abp-monitoring/90217a/) is an oscillometric blood pressure (BP) monitor which can be used to automatically track a person's BP in specificed time intervals.
87 |
88 | - [TimeStamp for Android](https://play.google.com/store/apps/details?id=gj.timestamp&hl=en) allows you to record the timestamp of an event at the time it occurs. It also allows you to create specific tags such as "Running" or "Walking" and timestamp those specific activities.
89 |
90 | - [Shimmer Consensys GSR](https://www.shimmersensing.com/products/gsr-optical-pulse-development-kit#specifications-tab) is a device that is used to collect sensor data in real time and it contains sensors such as GSR / EDA, photoplethysmography (PPG), 3-axis accelerometer, 3-axis gyroscope, 3-axis magnetometer & integrated altimeter.
91 |
92 | - [Muse S headband](https://choosemuse.com/muse-s/) is a consumer grade headband consisting of 4 electrodes electroencephalography (EEG) sensors, 3-axis accelerometer (ACC), gyroscope, and photoplethysmography (PPG) sensors.
93 |
94 | ## How to Contribute
95 |
96 | If you want to be part of this mission, please check our documentation on how to contribute [here](https://hpi-dhc.github.io/devicely/contribution.html).
97 |
98 | ## Authors
99 |
100 | ```
101 | * Ariane Sasso
102 | * Jost Morgenstern
103 | * Felix Musmann
104 | * Bert Arnrich
105 | ```
106 |
107 | ## Contributors
108 |
109 | ```
110 | * Arpita Kappattanavar
111 | * Bjarne Pfitzner
112 | * Lin Zhou
113 | * Pascal Hecker
114 | * Philipp Hildebrandt
115 | * Sidratul Moontaha
116 | ```
117 |
118 | ## Citation
119 |
120 | Sasso, A., Morgenstern, J., Musmann, F., & Arnrich, B. (2021). Devicely: A Python package for reading, timeshifting and writing sensor data. Journal of Open Source Software, 6(66), 3679. https://doi.org/10.21105/joss.03679
121 |
122 | ### BibTeX
123 | ```
124 | @article{Sasso2021,
125 | doi = {10.21105/joss.03679},
126 | url = {https://doi.org/10.21105/joss.03679},
127 | year = {2021},
128 | publisher = {The Open Journal},
129 | volume = {6},
130 | number = {66},
131 | pages = {3679},
132 | author = {Ariane Sasso and Jost Morgenstern and Felix Musmann and Bert Arnrich},
133 | title = {Devicely: A Python package for reading, timeshifting and writing sensor data},
134 | journal = {Journal of Open Source Software}
135 | }
136 | ```
137 |
--------------------------------------------------------------------------------
/inst/paper.bib:
--------------------------------------------------------------------------------
1 | @article{Kamisalic2018,
2 | abstract = {Wearable devices have recently received considerable interest due to their great promise for a plethora of applications. Increased research efforts are oriented towards a non-invasive monitoring of human health as well as activity parameters. A wide range of wearable sensors are being developed for real-time non-invasive monitoring. This paper provides a comprehensive review of sensors used in wrist-wearable devices, methods used for the visualization of parameters measured as well as methods used for intelligent analysis of data obtained from wrist-wearable devices. In line with this, the main features of commercial wrist-wearable devices are presented. As a result of this review, a taxonomy of sensors, functionalities, and methods used in non-invasive wrist-wearable devices was assembled.},
3 | author = {Kami\v{s}ali{\'{c}}, Aida and Fister, Iztok and Turkanovi{\'{c}}, Muhamed and Karakati\v{c}, Sa\v{s}o},
4 | doi = {10.3390/s18061714},
5 | issn = {14248220},
6 | journal = {Sensors (Switzerland)},
7 | keywords = {Intelligent analysis,Non-invasive,Sensor,Taxonomy,Visualization,Wrist-wearable device},
8 | number = {6},
9 | pmid = {29799504},
10 | title = {{Sensors and functionalities of non-invasive wrist-wearable devices: A review}},
11 | url = {www.mdpi.com/journal/sensors},
12 | volume = {18},
13 | year = {2018}
14 | }
15 |
16 | @misc{IDC2020,
17 | author = {IDC},
18 | booktitle = {2020},
19 | title = {{Shipments of Wearable Devices Leap to 125 Million Units, Up 35.1{\%} in the Third Quarter, According to IDC}},
20 | url = {https://www.idc.com/getdoc.jsp?containerId=prUS47067820},
21 | urldate = {2021-03-15},
22 | year = {2020}
23 | }
24 |
25 | @article{Bayoumy2021,
26 | author = {Bayoumy, Karim and Gaber, Mohammed and Elshafeey, Abdallah and Mhaimeed, Omar and Dineen, Elizabeth H and Marvel, Francoise A and Martin, Seth S and Muse, Evan D and Turakhia, Mintu P and Tarakji, Khaldoun G and Elshazly, Mohamed B},
27 | doi = {10.1038/s41569-021-00522-7},
28 | isbn = {0123456789},
29 | issn = {1759-5002},
30 | journal = {Nature Reviews Cardiology},
31 | month = {mar},
32 | title = {{Smart wearable devices in cardiovascular care: where we are and how to move forward}},
33 | url = {www.nature.com/nrcardio http://www.nature.com/articles/s41569-021-00522-7},
34 | year = {2021}
35 | }
36 |
37 | @article{Mishra2020,
38 | abstract = {Consumer wearable devices that continuously measure vital signs have been used to monitor the onset of infectious disease. Here, we show that data from consumer smartwatches can be used for the pre-symptomatic detection of coronavirus disease 2019 (COVID-19). We analysed physiological and activity data from 32 individuals infected with COVID-19, identified from a cohort of nearly 5,300 participants, and found that 26 of them (81{\%}) had alterations in their heart rate, number of daily steps or time asleep. Of the 25 cases of COVID-19 with detected physiological alterations for which we had symptom information, 22 were detected before (or at) symptom onset, with four cases detected at least nine days earlier. Using retrospective smartwatch data, we show that 63{\%} of the COVID-19 cases could have been detected before symptom onset in real time via a two-tiered warning system based on the occurrence of extreme elevations in resting heart rate relative to the individual baseline. Our findings suggest that activity tracking and health monitoring via consumer wearable devices may be used for the large-scale, real-time detection of respiratory infections, often pre-symptomatically.},
39 | author = {Mishra, Tejaswini and Wang, Meng and Metwally, Ahmed A and Bogu, Gireesh K and Brooks, Andrew W and Bahmani, Amir and Alavi, Arash and Celli, Alessandra and Higgs, Emily and Dagan-Rosenfeld, Orit and Fay, Bethany and Kirkpatrick, Susan and Kellogg, Ryan and Gibson, Michelle and Wang, Tao and Hunting, Erika M and Mamic, Petra and Ganz, Ariel B and Rolnik, Benjamin and Li, Xiao and Snyder, Michael P},
40 | doi = {10.1038/s41551-020-00640-6},
41 | issn = {2157846X},
42 | journal = {Nature Biomedical Engineering},
43 | number = {12},
44 | pages = {1208--1220},
45 | pmid = {33208926},
46 | title = {{Pre-symptomatic detection of COVID-19 from smartwatch data}},
47 | url = {https://doi.org/10.1038/s41551-020-00640-6},
48 | volume = {4},
49 | year = {2020}
50 | }
51 |
52 | @article{Snyder2020,
53 | doi = {10.21105/joss.02106},
54 | url = {https://doi.org/10.21105/joss.02106},
55 | year = {2020},
56 | publisher = {The Open Journal},
57 | volume = {5},
58 | number = {47},
59 | pages = {2106},
60 | author = {Phil Snyder and Meghasyam Tummalacherla and Thanneer Perumal and Larsson Omberg},
61 | title = {mhealthtools: A Modular R Package for Extracting Features from Mobile and Wearable Sensor Data},
62 | journal = {Journal of Open Source Software}
63 | }
64 |
65 | @article{Christakis2019,
66 | doi = {10.21105/joss.01663},
67 | url = {https://doi.org/10.21105/joss.01663},
68 | year = {2019},
69 | publisher = {The Open Journal},
70 | volume = {4},
71 | number = {44},
72 | pages = {1663},
73 | author = {Yiorgos Christakis and Nikhil Mahadevan and Shyamal Patel},
74 | title = {SleepPy: A python package for sleep analysis from accelerometer data},
75 | journal = {Journal of Open Source Software}
76 | }
77 |
78 | @article{Bartels2020,
79 | doi = {10.21105/joss.01867},
80 | url = {https://doi.org/10.21105/joss.01867},
81 | year = {2020},
82 | publisher = {The Open Journal},
83 | volume = {5},
84 | number = {51},
85 | pages = {1867},
86 | author = {Rhenan Bartels and Tiago Peçanha},
87 | title = {HRV: a Pythonic package for Heart Rate Variability Analysis},
88 | journal = {Journal of Open Source Software}
89 | }
90 |
91 | @article{Czech2019,
92 | doi = {10.21105/joss.01778},
93 | url = {https://doi.org/10.21105/joss.01778},
94 | year = {2019},
95 | publisher = {The Open Journal},
96 | volume = {4},
97 | number = {43},
98 | pages = {1778},
99 | author = {Matthew D. Czech and Shyamal Patel},
100 | title = {GaitPy: An Open-Source Python Package for Gait Analysis Using an Accelerometer on the Lower Back},
101 | journal = {Journal of Open Source Software}
102 | }
103 |
104 | @article{Foell2020,
105 | doi = {10.1016/j.cmpb.2021.106461},
106 | author = {Föll, Simon and Maritsch, Martin and Spinola, Federica and Mishra, Varun and Barata, Filipe and Kowatsch, Tobias and Fleisch, Elgar and Wortmann, Felix},
107 | title = {{{FLIRT}}: A {{Feature Generation Toolkit}} for {{Wearable Data}}},
108 | journal = {Computer Methods and Programs in Biomedicine},
109 | year = {2021}
110 | }
111 |
112 | @misc{Bent2020,
113 | author = {Bent, Brinnae},
114 | publisher = {DBDP (Digital Biomarker Discovery Pipeline)},
115 | title = {wearablecompute is an open source Python package containing over 50 data and domain-driven features},
116 | url = {https://github.com/brinnaebent/wearablecompute},
117 | year = {2020}
118 | }
119 |
--------------------------------------------------------------------------------
/docs/py-modindex.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
123 |
124 |
203 |
204 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
162 |
202 |
163 |
181 |
201 |
56 |
57 |
110 |
111 |
112 |
113 |
114 |
120 |
121 |
122 |
247 |
248 |
249 |
250 |
251 |
256 |
257 |
258 |
259 |
260 |
261 |
262 |
263 |
--------------------------------------------------------------------------------
/inst/paper.md:
--------------------------------------------------------------------------------
1 | ---
2 | title: 'Devicely: A Python package for reading, timeshifting and writing sensor data'
3 | tags:
4 | - Python
5 | - Wearables
6 | - Sensors
7 | - Health
8 | authors:
9 | - name: Ariane Sasso^[corresponding author]
10 | orcid: 0000-0002-3669-4599
11 | affiliation: 1
12 | - name: Jost Morgenstern
13 | orcid: 0000-0001-6268-9948
14 | affiliation: 1
15 | - name: Felix Musmann
16 | orcid: 0000-0001-5365-0785
17 | affiliation: 1
18 | - name: Bert Arnrich
19 | orcid: 0000-0001-8380-7667
20 | affiliation: 1
21 | affiliations:
22 | - name: Digital Health Center, Hasso Plattner Institute, University of Potsdam
23 | index: 1
24 | date: 25 August 2021
25 | bibliography: paper.bib
26 |
27 | ---
28 |
29 | # Summary
30 |
31 | Wearable devices can track a multitude of parameters such as heart rate, body
32 | temperature, blood oxygen saturation, acceleration, blood glucose and many more
33 | [@Kamisalic2018]. Moreover, they are becoming increasingly popular with a
34 | steep increase in market presence in 2020 alone [@IDC2020]. Applications
35 | for wearable
36 | devices vary from tracking cardiovascular risks [@Bayoumy2021] to identifying
37 | COVID-19 onset [@Mishra2020]. Therefore, there is a great need for scientists to
38 | easily go through data acquired from different wearables and to be able to share
39 | them while protecting user privacy.
40 | In order to solve this problem and empower scientists working with biosignals,
41 | we developed the **devicely** package. It processes the data into a tabular
42 | format and contains tools for data de-identification. It allows scientists to focus
43 | on what they want: the analysis of biosignals guided by privacy principles.
44 |
45 | # Related Work
46 |
47 | The first example of a package working with wearable data is mhealthtools [@Snyder2020], which is developed in R and focuses on
48 | extracting features from sensors such as inertial measurement units (IMUs). Its main difference from **devicely**
49 | is firstly the language (R versus Python) and secondly their complementary
50 | nature. Mhealthtools offers functionalities for feature extraction and
51 | **devicely** is
52 | intended to help users in a prior step by reading and writing data from
53 | wearables into standardized formats.
54 |
55 | There are also packages developed in Python, such as SleepPy [@Christakis2019] which uses raw
56 | accelerometer data for assessing sleep quantity and quality. The HRV [@Bartels2020] package uses CSV and text files or Python iterables such as lists to generate
57 | features related to heart rate variability (HRV). GaitPy [@Czech2019] accepts input data in a
58 | customizable format and is mainly used to extract features for gait analysis.
59 | Therefore, packages such as SleepPy, HRV and GaitPy could also be used in a following step to extract features from the output generated by **devicely**.
60 |
61 | FLIRT [@Foell2020] and wearablecompute [@Bent2020] are packages that provide ways for
62 | reading data from specific wearables such as Empatica E4. They also
63 | include functionalities to extract features from electrodermal activity (EDA),
64 | acceleration and HRV. The main difference from FLIRT and wearablecompute to **devicely** is the
65 | focus on feature extraction versus privacy and data sharing.
66 | FLIRT and wearablecompute read the data for extracting features, while **devicely** aims to provide users with
67 | a way to read the data, de-identify them as necessary and write them back in a specified format. In
68 | this way, researchers can ensure even more data privacy and use the data easily
69 | for further analysis and sharing.
70 |
71 | # Statement of need
72 |
73 | Every wearable vendor has a different data format and reading them is
74 | usually a challenge for scientists. Therefore, in order for researchers to be able to use different sensor data in an easy and
75 | friendly way we developed the **devicely** package. The package also contains two methods to help with data de-identification, one
76 | is called timeshift and the other is a write method. The idea behind them is
77 | that researchers can timeshift all their time series data to a different time from the one the
78 | actual experiments occurred and then write this new de-identified dataset back to
79 | the original or a similar data format. This will empower scientists to maintain user privacy
80 | and hopefully share more data to increase research reproducibility.
81 |
82 | # Design
83 |
84 | Different wearables provide incompatible data formats which require specific preprocessing steps.
85 | However, it should be easy for scientists to add data from a new wearable to an
86 | existing pipeline and easy for developers to add a new wearable to the
87 | **devicely** package. To achieve both, **devicely** encapsulates data preparation for
88 | each wearable behind three common methods: _read_, _timeshift_ and _write_.
89 |
90 | After reading, the data is accessible through the reader in common formats such as Pandas DataFrames.
91 | De-identification is achieved by timeshifting the data, either by providing a shifting interval or randomly.
92 | For writing back de-identified data, **devicely** focuses on keeping a format that can be read again using the same reader class.
93 | In almost all cases, this is the same format as the one the wearable originaly provides.
94 | This enables sharing data with the community while maintaining user privacy.
95 |
96 | # Functionalities
97 |
98 | All reader classes support three core functions: reading data created by a wearable, timeshifting them and writing them back.
99 | To _read_ data the corresponding reader class can be initialized using as a
100 | parameter a path to the data created by the wearable. After reading, data can
101 | be accessed through the reader in convenient formats such as dictionaries
102 | and Pandas DataFrames.
103 |
104 | After creating a reader object the method _timeshift_ can be applied upon it. This assures de-identification by shifting all time-related data points.
105 | To control the shifting interval, a parameter can be provided to _timeshift_.
106 | If no parameter is provided, the data is shifted by a random time interval to the past.
107 | The timeshifted data can be written back using the _write_ method.
108 |
109 | For all wearables, the written data can be read again using the same reader
110 | class. \autoref{fig:devicely_structure} depicts the class structure of the **devicely**
111 | package and serves as a guide for future implementation.
112 |
113 | 
117 |
118 | # Availability
119 |
120 | The software can be obtained through the [Python Package Index
121 | (PyPI)](https://pypi.org/project/devicely),
122 | [Conda-forge](https://github.com/conda-forge/devicely-feedstock), [Zenodo](https://zenodo.org/record/5243494) and on
123 | [GitHub](https://github.com/hpi-dhc/devicely) under the MIT License.
124 |
125 | # Mention
126 |
127 | This package was used in the following paper:
128 |
129 | Morassi Sasso A. et al. (2020) HYPE: Predicting Blood Pressure from
130 | Photoplethysmograms in a Hypertensive Population. In: Michalowski M., Moskovitch
131 | R. (eds) Artificial Intelligence in Medicine. AIME 2020. Lecture Notes in
132 | Computer Science, vol 12299. Springer, Cham.
133 | https://doi.org/10.1007/978-3-030-59137-3_29
134 |
135 | [GitHub Repository](https://github.com/arianesasso/aime-2020)
136 |
137 | # Acknowledgements
138 |
139 | We acknowledge contributions from Arpita Kappattanavar, Bjarne Pfitzner, Harry Freitas da Cruz, Lin
140 | Zhou, Pascal Hecker, Philipp Hildebrandt and Sidratul Moontaha during the genesis and testing of this package.
141 |
142 | # References
143 |
--------------------------------------------------------------------------------
/tests/test_shimmer.py:
--------------------------------------------------------------------------------
1 | """
2 | Tests for the ShimmerPlus module
3 | """
4 | import os
5 | import sys
6 | import unittest
7 |
8 | import pandas as pd
9 | import devicely
10 |
11 |
12 | class ShimmerPlusTestCase(unittest.TestCase):
13 | READ_PATH = 'tests/Shimmer_test_data/read_test_data.csv'
14 | WRITE_PATH = 'tests/Shimmer_test_data/write_test_data.csv'
15 |
16 | def setUp(self):
17 | self.expected_delimiter = ';'
18 |
19 | self.expected_data_head = pd.DataFrame(
20 | {'Shimmer_40AC_Timestamp_Unix_CAL': pd.to_datetime(['2020-07-28 10:56:50.034000', '2020-07-28 10:56:50.057000', '2020-07-28 10:56:50.074000', '2020-07-28 10:56:50.099000', '2020-07-28 10:56:50.111000']),
21 | 'Shimmer_40AC_Accel_LN_X_CAL': [-1.434782608695652, -1.4021739130434783, -1.434782608695652, -1.4130434782608696, -1.4456521739130435],
22 | 'Shimmer_40AC_Accel_LN_Y_CAL': [10.0, 10.0, 10.0, 10.0, 10.0],
23 | 'Shimmer_40AC_Accel_LN_Z_CAL': [0.5543478260869565, 0.5543478260869565, 0.5543478260869565, 0.5217391304347826, 0.5108695652173912],
24 | 'Shimmer_40AC_Accel_WR_X_CAL': [-3.9305804907241173, -3.9233991621783364, -3.897067624177139, -3.932974266906044, -3.944943147815679],
25 | 'Shimmer_40AC_Accel_WR_Y_CAL': [8.42130460801915, 8.442848593656493, 8.42848593656493, 8.42130460801915, 8.42848593656493],
26 | 'Shimmer_40AC_Accel_WR_Z_CAL': [-1.620586475164572, -1.5990424895272293, -1.6038300418910831, -1.589467384799521, -1.6612806702573308],
27 | 'Shimmer_40AC_Battery_CAL': [4139.194139194139, 4137.728937728937, 4111.355311355311, 4140.65934065934, 4134.798534798534],
28 | 'Shimmer_40AC_Ext_Exp_A15_CAL': [1684.9816849816848, 1673.260073260073, 1901.098901098901, 1722.3443223443223, 1678.3882783882782],
29 | 'Shimmer_40AC_GSR_Range_CAL': [2.0, 2.0, 2.0, 2.0, 2.0],
30 | 'Shimmer_40AC_GSR_Skin_Conductance_CAL': [2.493040293040293, 2.4915750915750916, 2.49010989010989, 2.4886446886446887, 2.4886446886446887],
31 | 'Shimmer_40AC_GSR_Skin_Resistance_CAL': [401.1166617690273, 401.3525433695972, 401.58870255957635, 401.8251398292611, 401.8251398292611],
32 | 'Shimmer_40AC_Gyro_X_CAL': [0.13740458015267176, 0.183206106870229, 0.2748091603053435, 0.22900763358778625, 0.13740458015267176],
33 | 'Shimmer_40AC_Gyro_Y_CAL': [1.8778625954198473, 1.3282442748091603, 1.282442748091603, 1.450381679389313, 1.5114503816793892],
34 | 'Shimmer_40AC_Gyro_Z_CAL': [-0.183206106870229, -0.41221374045801523, -0.1984732824427481, -0.12213740458015267, -0.47328244274809156],
35 | 'Shimmer_40AC_Int_Exp_A12_CAL': [1680.5860805860805, 1703.296703296703, 1687.179487179487, 1650.5494505494505, 1701.8315018315018],
36 | 'Shimmer_40AC_Mag_X_CAL': [-0.11244377811094453, -0.10944527736131934, -0.10794602698650674, -0.10644677661169415, -0.11394302848575712],
37 | 'Shimmer_40AC_Mag_Y_CAL': [-0.9160419790104948, -0.9130434782608695, -0.9100449775112444, -0.9010494752623688, -0.9040479760119939],
38 | 'Shimmer_40AC_Mag_Z_CAL': [-0.047976011994003, -0.047976011994003, -0.04947526236881559, -0.050974512743628186, -0.037481259370314844],
39 | 'Shimmer_40AC_Pressure_BMP280_CAL': [100.43537920858897, 100.4297311006671, 100.44102732749235, 100.44102732749235, 100.43820326666797],
40 | 'Shimmer_40AC_Temperature_BMP280_CAL': [33.365877509029815, 33.365877509029815, 33.365877509029815, 33.365877509029815, 33.365877509029815],
41 | 'Shimmer_40AC_Accel_LN_mag': [10.1176036 , 10.11303086, 10.1176036 , 10.11280889, 10.11686206]},
42 | )
43 |
44 | self.expected_units = {
45 | 'Shimmer_40AC_Timestamp_Unix_CAL': 'ms',
46 | 'Shimmer_40AC_Accel_LN_X_CAL': 'm/(s^2)',
47 | 'Shimmer_40AC_Accel_LN_Y_CAL': 'm/(s^2)',
48 | 'Shimmer_40AC_Accel_LN_Z_CAL': 'm/(s^2)',
49 | 'Shimmer_40AC_Accel_WR_X_CAL': 'm/(s^2)',
50 | 'Shimmer_40AC_Accel_WR_Y_CAL': 'm/(s^2)',
51 | 'Shimmer_40AC_Accel_WR_Z_CAL': 'm/(s^2)',
52 | 'Shimmer_40AC_Battery_CAL': 'mV',
53 | 'Shimmer_40AC_Ext_Exp_A15_CAL': 'mV',
54 | 'Shimmer_40AC_GSR_Range_CAL': 'no_units',
55 | 'Shimmer_40AC_GSR_Skin_Conductance_CAL': 'uS',
56 | 'Shimmer_40AC_GSR_Skin_Resistance_CAL': 'kOhms',
57 | 'Shimmer_40AC_Gyro_X_CAL': 'deg/s',
58 | 'Shimmer_40AC_Gyro_Y_CAL': 'deg/s',
59 | 'Shimmer_40AC_Gyro_Z_CAL': 'deg/s',
60 | 'Shimmer_40AC_Int_Exp_A12_CAL': 'mV',
61 | 'Shimmer_40AC_Mag_X_CAL': 'local_flux',
62 | 'Shimmer_40AC_Mag_Y_CAL': 'local_flux',
63 | 'Shimmer_40AC_Mag_Z_CAL': 'local_flux',
64 | 'Shimmer_40AC_Pressure_BMP280_CAL': 'kPa',
65 | 'Shimmer_40AC_Temperature_BMP280_CAL': 'Degrees Celsius'
66 | }
67 |
68 | self.reader = devicely.ShimmerPlusReader(self.READ_PATH)
69 |
70 | def _compare_reader_and_expected_values(self, reader, expected_delimiter, expected_units, expected_data_head):
71 | self.assertEqual(reader.delimiter, expected_delimiter)
72 | self.assertEqual(reader.units, expected_units)
73 | pd.testing.assert_frame_equal(reader.data.head(), expected_data_head)
74 |
75 | def test_basic_read(self):
76 | self._compare_reader_and_expected_values(self.reader,
77 | self.expected_delimiter,
78 | self.expected_units,
79 | self.expected_data_head)
80 |
81 | def test_write(self):
82 | self.reader.write(self.WRITE_PATH)
83 | new_reader = devicely.ShimmerPlusReader(self.WRITE_PATH)
84 | self._compare_reader_and_expected_values(new_reader,
85 | self.reader.delimiter,
86 | self.reader.units,
87 | self.reader.data.head())
88 | os.remove(self.WRITE_PATH)
89 |
90 | def test_timeshift_by_timedelta(self):
91 | shift = pd.Timedelta('1 days, 2 hours, 3 minutes, 4 seconds, 5 milliseconds')
92 | self.reader.timeshift(shift)
93 | expected_shifted_time_col = pd.Series(pd.to_datetime(['2020-07-29 12:59:54.039000', '2020-07-29 12:59:54.062000',
94 | '2020-07-29 12:59:54.079000', '2020-07-29 12:59:54.104000',
95 | '2020-07-29 12:59:54.116000']))
96 | pd.testing.assert_series_equal(self.reader.data.loc[:4, 'Shimmer_40AC_Timestamp_Unix_CAL'], expected_shifted_time_col, check_names=False)
97 |
98 | def test_timeshift_to_timestamp(self):
99 | shift = pd.Timestamp(day=23, month=4, year=2009, hour=6, minute=42, second=21, microsecond=int(593.32e3))
100 | self.reader.timeshift(shift)
101 | expected_shifted_time_col = pd.Series(pd.to_datetime(['2009-04-23 06:42:21.593000', '2009-04-23 06:42:21.616000',
102 | '2009-04-23 06:42:21.633000', '2009-04-23 06:42:21.658000',
103 | '2009-04-23 06:42:21.670000']))
104 | pd.testing.assert_series_equal(self.reader.data.loc[:4, 'Shimmer_40AC_Timestamp_Unix_CAL'], expected_shifted_time_col, check_names=False)
105 |
106 | def test_random_timeshift(self):
107 | earliest_possible_time_col = pd.Series(pd.to_datetime(['2018-07-29 10:56:50.034000', '2018-07-29 10:56:50.057000',
108 | '2018-07-29 10:56:50.074000', '2018-07-29 10:56:50.099000',
109 | '2018-07-29 10:56:50.111000']))
110 | latest_possible_time_col = pd.Series(pd.to_datetime(['2020-06-28 10:56:50.034000', '2020-06-28 10:56:50.057000',
111 | '2020-06-28 10:56:50.074000', '2020-06-28 10:56:50.099000',
112 | '2020-06-28 10:56:50.111000']))
113 |
114 | self.reader.timeshift()
115 | self.assertTrue((earliest_possible_time_col <= self.reader.data.loc[:4, 'Shimmer_40AC_Timestamp_Unix_CAL']).all())
116 | self.assertTrue((self.reader.data.loc[:4, 'Shimmer_40AC_Timestamp_Unix_CAL'] <= latest_possible_time_col).all())
117 |
118 |
119 | if __name__ == '__main__':
120 | unittest.main()
121 |
--------------------------------------------------------------------------------
/tests/test_spacelabs.py:
--------------------------------------------------------------------------------
1 | """
2 | Tests for the Spacelabs module
3 | """
4 | import datetime as dt
5 | import os
6 | import unittest
7 |
8 | import numpy as np
9 | import pandas as pd
10 |
11 | import devicely
12 |
13 |
14 | class SpacelabsTestCase(unittest.TestCase):
15 | READ_PATH = "tests/SpaceLabs_test_data/spacelabs.abp"
16 | READ_PATH_REPEATED = "tests/SpaceLabs_test_data/spacelabs_repeated.abp"
17 | WRITE_PATH = "tests/SpaceLabs_test_data/spacelabs_written.abp"
18 |
19 | def __init__(self, *args, **kwargs):
20 | super().__init__(*args, **kwargs)
21 |
22 | self.expected_subject = "000002"
23 | timestamps = pd.to_datetime([
24 | "1.1.99 17:03",
25 | "1.1.99 17:05",
26 | "1.1.99 17:07",
27 | "1.1.99 17:09",
28 | "1.1.99 17:11",
29 | "1.1.99 17:13",
30 | "1.1.99 17:25",
31 | "1.1.99 17:28",
32 | "1.1.99 17:31",
33 | "1.1.99 17:34",
34 | "1.1.99 17:36",
35 | "1.1.99 17:39",
36 | "1.1.99 23:42",
37 | "1.1.99 23:59",
38 | "1.2.99 00:01",
39 | ])
40 |
41 | timestamps_repeated = pd.to_datetime([
42 | "1.1.99 17:05",
43 | "1.1.99 17:05",
44 | "1.1.99 17:07",
45 | "1.1.99 17:09",
46 | "1.1.99 17:11",
47 | "1.1.99 17:13",
48 | "1.1.99 17:25",
49 | "1.1.99 17:28",
50 | "1.1.99 17:31",
51 | "1.1.99 17:34",
52 | "1.1.99 17:36",
53 | "1.1.99 17:39",
54 | "1.1.99 23:42",
55 | "1.1.99 23:59",
56 | "1.2.99 00:01",
57 | ])
58 |
59 | self.expected_data = pd.DataFrame({
60 | "timestamp": timestamps,
61 | "date": timestamps.map(lambda timestamp: timestamp.date()),
62 | "time": timestamps.map(lambda timestamp: timestamp.time()),
63 | "SYS(mmHg)": [11, 142, 152, 151, 145, 3, 4, 164, 154, 149, 153, 148, 148, 148, 148],
64 | "DIA(mmHg)": [0, 118, 112, 115, 110, 0, 0, 119, 116, 119, 118, 114, 114, 114, 114],
65 | "UNKNOW_1": [0, 99, 95, 96, 91, 0, 0, 95, 95, 98, 96, 93, 93, 93, 93],
66 | "UNKNOW_2": [0, 61, 61, 61, 59, 0, 0, 63, 63, 63, 60, 62, 62, 62, 62],
67 | "UNKNOW_3": 15 * [np.nan],
68 | "CODE": ["EB", np.nan, np.nan, np.nan, np.nan, "EB", "EB", np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan],
69 | })
70 |
71 | self.expected_data.set_index('timestamp', inplace=True)
72 |
73 | self.expected_data_repeated = pd.DataFrame({
74 | "timestamp": timestamps_repeated,
75 | "date": timestamps_repeated.map(lambda timestamps_repeated: timestamps_repeated.date()),
76 | "time": timestamps_repeated.map(lambda timestamps_repeated: timestamps_repeated.time()),
77 | "SYS(mmHg)": [11, 142, 152, 151, 145, 3, 4, 164, 154, 149, 153, 148, 148, 148, 148],
78 | "DIA(mmHg)": [0, 118, 112, 115, 110, 0, 0, 119, 116, 119, 118, 114, 114, 114, 114],
79 | "UNKNOW_1": [0, 99, 95, 96, 91, 0, 0, 95, 95, 98, 96, 93, 93, 93, 93],
80 | "UNKNOW_2": [0, 61, 61, 61, 59, 0, 0, 63, 63, 63, 60, 62, 62, 62, 62],
81 | "UNKNOW_3": 15 * [np.nan],
82 | "CODE": ["EB", np.nan, np.nan, np.nan, np.nan, "EB", "EB", np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan],
83 | })
84 |
85 | self.expected_data_repeated.set_index('timestamp', inplace=True)
86 |
87 | self.expected_metadata = {
88 | "PATIENTINFO": {
89 | "DOB": "16.09.1966",
90 | "RACE": "native american"
91 | },
92 | "REPORTINFO": {
93 | "PHYSICIAN": "Dr. Hannibal Lecter",
94 | "NURSETECH": "admin",
95 | "STATUS": "NOTCONFIRMED",
96 | "CALIPERSUMMARY": {
97 | "COUNT": "0"
98 | },
99 | },
100 | }
101 |
102 | def setUp(self):
103 | self.spacelabs_reader = devicely.SpacelabsReader(self.READ_PATH)
104 | self.spacelabs_reader_repertion = devicely.SpacelabsReader(self.READ_PATH_REPEATED)
105 |
106 | def test_read(self):
107 | # Tests a basic reading operation.
108 | pd.testing.assert_frame_equal(self.spacelabs_reader.data,
109 | self.expected_data)
110 |
111 | # Test reading a file with repeated timestamp values.
112 | pd.testing.assert_frame_equal(self.spacelabs_reader_repertion.data,
113 | self.expected_data_repeated)
114 |
115 | self.assertEqual(self.spacelabs_reader.subject, self.expected_subject)
116 | self.assertEqual(self.spacelabs_reader.metadata,
117 | self.expected_metadata)
118 |
119 | def test_deidentify(self):
120 | # Tests if the SpacelabsReader.deidentify method removes all patient metadata.
121 |
122 | self.spacelabs_reader.deidentify()
123 |
124 | self.assertEqual(self.spacelabs_reader.subject, "")
125 | self.assertEqual(
126 | self.spacelabs_reader.metadata,
127 | {
128 | "PATIENTINFO": {
129 | "DOB": "",
130 | "RACE": ""
131 | },
132 | "REPORTINFO": {
133 | "PHYSICIAN": "",
134 | "NURSETECH": "",
135 | "STATUS": "",
136 | "CALIPERSUMMARY": {
137 | "COUNT": ""
138 | },
139 | },
140 | },
141 | )
142 |
143 | def test_write(self):
144 | # Tests the SpacelabsReader.write operation by writing, reading again and comparing the old and new signals.
145 |
146 | self.spacelabs_reader.write(self.WRITE_PATH)
147 | new_reader = devicely.SpacelabsReader(self.WRITE_PATH)
148 |
149 | pd.testing.assert_frame_equal(new_reader.data,
150 | self.spacelabs_reader.data)
151 | self.assertEqual(new_reader.metadata, self.spacelabs_reader.metadata)
152 | self.assertEqual(new_reader.subject, self.spacelabs_reader.subject)
153 |
154 | os.remove(self.WRITE_PATH)
155 |
156 | def test_random_timeshift(self):
157 | earliest_possible_shifted_time_col = pd.to_datetime([
158 | '1997-01-01 17:03:00',
159 | '1997-01-01 17:05:00',
160 | '1997-01-01 17:07:00',
161 | '1997-01-01 17:09:00',
162 | '1997-01-01 17:11:00',
163 | '1997-01-01 17:13:00',
164 | '1997-01-01 17:25:00',
165 | '1997-01-01 17:28:00',
166 | '1997-01-01 17:31:00',
167 | '1997-01-01 17:34:00',
168 | '1997-01-01 17:36:00',
169 | '1997-01-01 17:39:00',
170 | '1997-01-01 23:42:00',
171 | '1997-01-01 23:59:00',
172 | '1997-01-02 00:01:00'
173 | ])
174 | latest_possible_shifted_time_col = pd.to_datetime([
175 | '1998-12-02 17:03:00',
176 | '1998-12-02 17:05:00',
177 | '1998-12-02 17:07:00',
178 | '1998-12-02 17:09:00',
179 | '1998-12-02 17:11:00',
180 | '1998-12-02 17:13:00',
181 | '1998-12-02 17:25:00',
182 | '1998-12-02 17:28:00',
183 | '1998-12-02 17:31:00',
184 | '1998-12-02 17:34:00',
185 | '1998-12-02 17:36:00',
186 | '1998-12-02 17:39:00',
187 | '1998-12-02 23:42:00',
188 | '1998-12-02 23:59:00',
189 | '1998-12-03 00:01:00'
190 | ])
191 |
192 | self.spacelabs_reader.timeshift()
193 | new_timestamp_column = pd.Series(self.spacelabs_reader.data.index)
194 |
195 | self.assertTrue((earliest_possible_shifted_time_col <= new_timestamp_column).all())
196 | self.assertTrue((new_timestamp_column <= latest_possible_shifted_time_col).all())
197 |
198 | new_date_column = self.spacelabs_reader.data["date"]
199 | new_time_column = self.spacelabs_reader.data["time"]
200 | testing_timestamp_column = pd.Series([
201 | dt.datetime.combine(new_date_column[i], new_time_column[i])
202 | for i in range(len(self.spacelabs_reader.data))
203 | ])
204 |
205 | pd.testing.assert_series_equal(new_timestamp_column, testing_timestamp_column, check_names=False)
206 |
207 | if __name__ == "__main__":
208 | unittest.main()
209 |
--------------------------------------------------------------------------------
/devicely/spacelabs.py:
--------------------------------------------------------------------------------
1 | """
2 | Spacelabs (SL 90217) is an oscillometric blood pressure (BP) monitor which can be used to
3 | automatically track a person's BP in specificed time intervals.
4 |
5 | """
6 | import csv
7 | import datetime as dt
8 | import random
9 | from xml.etree import ElementTree as ET
10 |
11 | import pandas as pd
12 |
13 |
14 | class SpacelabsReader:
15 | """
16 | Read, timeshift, deidentify and write data
17 | generated by Spacelabs(SL90217).
18 |
19 | Attributes
20 | ----------
21 | data : DataFrame
22 | DataFrame with the values that were read from the abp file.
23 |
24 | subject : str
25 | Contain the sujbect's id. Can be changed for
26 | deidentification.
27 |
28 | valid_measurements : str
29 | Contain the number of valid measurements in the
30 | abp file.
31 |
32 | metadata : dict
33 | The measurements' metadata. Read from the xml at the bottom
34 | of the abp file. Can be erased for deidentification.
35 | """
36 |
37 | def __init__(self, path):
38 | """
39 | Reads the abp file generated by the Spacelabs device and saves the
40 | parsed DataFrame.
41 |
42 | Parameters
43 | ----------
44 | path : str
45 | Path of the abp file.
46 | """
47 |
48 | # Metadata Definition
49 | metadata = pd.read_csv(path, nrows=5, header=None)
50 | self.subject = str(metadata.loc[0, 0])
51 | base_date = dt.datetime.strptime(metadata.loc[2, 0], '%d.%m.%Y').date()
52 | if metadata.loc[4, 0] != 'Unknown Line':
53 | self.valid_measurements = str(metadata.loc[4, 0])
54 | else:
55 | metadata = pd.read_csv(path, nrows=6, header=None)
56 | self.valid_measurements = str(metadata.loc[5, 0])
57 |
58 | column_names = ['hour', 'minutes', 'SYS(mmHg)', 'DIA(mmHg)', 'UNKNOW_1', 'UNKNOW_2', 'CODE', 'UNKNOW_3']
59 | self.data = pd.read_csv(path, sep=',', skiprows=51, skipfooter=1, header=None, names=column_names, engine='python')
60 |
61 | # Adjusting Date
62 | dates = [base_date]
63 | times = [dt.time(hour=self.data.loc[i, 'hour'], minute=self.data.loc[i, 'minutes']) for i in range(len(self.data))]
64 | current_date = base_date
65 | for i in range(1, len(times)):
66 | if times[i] < times[i-1]:
67 | current_date += dt.timedelta(days=1)
68 | dates.append(current_date)
69 |
70 | self.data.reset_index(inplace=True)
71 | self.data['timestamp'] = pd.to_datetime([dt.datetime.combine(dates[i], times[i]) for i in range(len(dates))])
72 | self.data['date'] = dates
73 | self.data['time'] = times
74 |
75 | order = ['timestamp', 'date', 'time', 'SYS(mmHg)', 'DIA(mmHg)', 'UNKNOW_1', 'UNKNOW_2', 'UNKNOW_3', 'CODE']
76 | self.data = self.data[order]
77 |
78 | try:
79 | self.data.set_index('timestamp', inplace=True)
80 | except KeyError:
81 | print('Timestamp can not be set as an index:')
82 | print(KeyError)
83 |
84 | xml_line = open(path, 'r').readlines()[-1]
85 | xml_root = ET.fromstring(xml_line)
86 | self.metadata = self._etree_to_dict(xml_root)['XML']
87 |
88 |
89 | def deidentify(self, subject_id=None):
90 | """
91 | Deidentify the data by removing the original XML metadata and
92 | subject id.
93 |
94 | Parameters
95 | ----------
96 | subject_id : str, optional
97 | New subject id to be written in the
98 | deidentified file, by default None.
99 | """
100 | # Changing subject id
101 | if subject_id:
102 | self.subject = subject_id
103 | else:
104 | self.subject = ''
105 |
106 | self.metadata = {
107 | 'PATIENTINFO' : {'DOB' : '',
108 | 'RACE' : ''},
109 | 'REPORTINFO' : {'PHYSICIAN' : '',
110 | 'NURSETECH' : '',
111 | 'STATUS' : '',
112 | 'CALIPERSUMMARY' : {'COUNT' : ''}}
113 | }
114 |
115 | def write(self, path):
116 | """
117 | Write the signals and metadata to the
118 | writing path in the same format as it was read.
119 |
120 | Parameters
121 | ----------
122 | path : str
123 | Path to writing file. Writing mode: 'w'.
124 | Use the file extension 'abp' to keep the SpaceLabs standard.
125 | """
126 |
127 | with open(path, 'w') as file:
128 | file.write(f"\n{self.subject}")
129 | file.write(8 * '\n')
130 | file.write("0")
131 | file.write(8 * '\n')
132 | file.write(self.data.date[0].strftime("%d.%m.%Y"))
133 | file.write(7 * '\n')
134 | file.write("Unknown Line")
135 | file.write(26 * '\n')
136 | file.write(self.valid_measurements + "\n")
137 | printing_df = self.data.drop(columns=['date', 'time'])
138 | printing_df['hours'] = self.data.time.map(lambda x: x.strftime("%H"))
139 | printing_df['minutes'] = self.data.time.map(lambda x: x.strftime("%M"))
140 | order = ['hours', 'minutes', 'SYS(mmHg)', 'DIA(mmHg)', 'UNKNOW_1', 'UNKNOW_2', 'CODE', 'UNKNOW_3']
141 | printing_df = printing_df[order]
142 | printing_df.fillna(-9999, inplace=True)
143 | printing_df.replace('EB', -9998, inplace=True)
144 | printing_df.replace('AB', -9997, inplace=True)
145 | printing_df[['SYS(mmHg)', 'DIA(mmHg)', 'UNKNOW_1', 'UNKNOW_2', 'CODE', 'UNKNOW_3']] = printing_df[
146 | ['SYS(mmHg)', 'DIA(mmHg)', 'UNKNOW_1', 'UNKNOW_2', 'CODE', 'UNKNOW_3']].astype(int).astype(str)
147 | printing_df.replace('-9999', '""', inplace=True)
148 | printing_df.replace('-9998', '"EB"', inplace=True)
149 | printing_df.replace('-9997', '"AB"', inplace=True)
150 | printing_df.to_csv(file, header=None, index=None, quoting=csv.QUOTE_NONE, line_terminator='\n')
151 |
152 | xml_node = ET.Element('XML')
153 | xml_node.extend(self._dict_to_etree(self.metadata))
154 | xml_line = ET.tostring(xml_node, encoding="unicode")
155 | file.write(xml_line)
156 |
157 | def _etree_to_dict(self, etree_node):
158 | children = list(iter(etree_node))
159 | if len(children) == 0:
160 | return {etree_node.tag: etree_node.text}
161 | else:
162 | dict_ = dict()
163 | for child in children:
164 | dict_ = {**dict_, **self._etree_to_dict(child)}
165 | return {etree_node.tag: dict_}
166 |
167 | def _dict_to_etree(self, dict_):
168 | def rec(key, value):
169 | node = ET.Element(key)
170 | if isinstance(value, dict):
171 | for child_key in value:
172 | node.append(rec(child_key, value[child_key]))
173 | else:
174 | node.text = str(value) if value else ''
175 | return node
176 |
177 | return [rec(k, v) for k, v in dict_.items()]
178 |
179 | def timeshift(self, shift='random'):
180 | """
181 | Timeshift the data by shifting all time related columns.
182 |
183 | Parameters
184 | ----------
185 | shift : None/'random', pd.Timestamp or pd.Timedelta
186 | If shift is not specified, shifts the data by a random time interval
187 | between one month and two years to the past.
188 |
189 | If shift is a timdelta, shifts the data by that timedelta.
190 |
191 | If shift is a timestamp, shifts the data such that the earliest entry
192 | has that timestamp. The remaining values will mantain the same
193 | time difference to the first entry.
194 | """
195 |
196 | if shift == 'random':
197 | one_month = pd.Timedelta('30 days').value
198 | two_years = pd.Timedelta('730 days').value
199 | random_timedelta = - pd.Timedelta(random.uniform(one_month, two_years)).round('min')
200 | self.timeshift(random_timedelta)
201 |
202 | if not self.data.index.empty:
203 | if isinstance(shift, pd.Timestamp):
204 | timedeltas = self.data.index - self.data.index[0]
205 | self.data.index = shift.round('min') + timedeltas
206 | if isinstance(shift, pd.Timedelta):
207 | self.data.index += shift.round('min')
208 | self.data['date'] = self.data.index.map(lambda timestamp: timestamp.date())
209 | self.data['time'] = self.data.index.map(lambda timestamp: timestamp.time())
210 | else:
211 | if isinstance(shift, pd.Timestamp):
212 | timedeltas = self.data['timestamp'] - self.data['timestamp'].min()
213 | self.data['timestamp'] = shift.round('min') + timedeltas
214 | if isinstance(shift, pd.Timedelta):
215 | self.data['timestamp'] += shift.round('min')
216 | self.data['date'] = self.data['timestamp'].map(lambda timestamp: timestamp.date())
217 | self.data['time'] = self.data['timestamp'].map(lambda timestamp: timestamp.time())
218 |
--------------------------------------------------------------------------------
/docs/_static/doctools.js:
--------------------------------------------------------------------------------
1 | /*
2 | * doctools.js
3 | * ~~~~~~~~~~~
4 | *
5 | * Sphinx JavaScript utilities for all documentation.
6 | *
7 | * :copyright: Copyright 2007-2021 by the Sphinx team, see AUTHORS.
8 | * :license: BSD, see LICENSE for details.
9 | *
10 | */
11 |
12 | /**
13 | * select a different prefix for underscore
14 | */
15 | $u = _.noConflict();
16 |
17 | /**
18 | * make the code below compatible with browsers without
19 | * an installed firebug like debugger
20 | if (!window.console || !console.firebug) {
21 | var names = ["log", "debug", "info", "warn", "error", "assert", "dir",
22 | "dirxml", "group", "groupEnd", "time", "timeEnd", "count", "trace",
23 | "profile", "profileEnd"];
24 | window.console = {};
25 | for (var i = 0; i < names.length; ++i)
26 | window.console[names[i]] = function() {};
27 | }
28 | */
29 |
30 | /**
31 | * small helper function to urldecode strings
32 | *
33 | * See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/decodeURIComponent#Decoding_query_parameters_from_a_URL
34 | */
35 | jQuery.urldecode = function(x) {
36 | if (!x) {
37 | return x
38 | }
39 | return decodeURIComponent(x.replace(/\+/g, ' '));
40 | };
41 |
42 | /**
43 | * small helper function to urlencode strings
44 | */
45 | jQuery.urlencode = encodeURIComponent;
46 |
47 | /**
48 | * This function returns the parsed url parameters of the
49 | * current request. Multiple values per key are supported,
50 | * it will always return arrays of strings for the value parts.
51 | */
52 | jQuery.getQueryParameters = function(s) {
53 | if (typeof s === 'undefined')
54 | s = document.location.search;
55 | var parts = s.substr(s.indexOf('?') + 1).split('&');
56 | var result = {};
57 | for (var i = 0; i < parts.length; i++) {
58 | var tmp = parts[i].split('=', 2);
59 | var key = jQuery.urldecode(tmp[0]);
60 | var value = jQuery.urldecode(tmp[1]);
61 | if (key in result)
62 | result[key].push(value);
63 | else
64 | result[key] = [value];
65 | }
66 | return result;
67 | };
68 |
69 | /**
70 | * highlight a given string on a jquery object by wrapping it in
71 | * span elements with the given class name.
72 | */
73 | jQuery.fn.highlightText = function(text, className) {
74 | function highlight(node, addItems) {
75 | if (node.nodeType === 3) {
76 | var val = node.nodeValue;
77 | var pos = val.toLowerCase().indexOf(text);
78 | if (pos >= 0 &&
79 | !jQuery(node.parentNode).hasClass(className) &&
80 | !jQuery(node.parentNode).hasClass("nohighlight")) {
81 | var span;
82 | var isInSVG = jQuery(node).closest("body, svg, foreignObject").is("svg");
83 | if (isInSVG) {
84 | span = document.createElementNS("http://www.w3.org/2000/svg", "tspan");
85 | } else {
86 | span = document.createElement("span");
87 | span.className = className;
88 | }
89 | span.appendChild(document.createTextNode(val.substr(pos, text.length)));
90 | node.parentNode.insertBefore(span, node.parentNode.insertBefore(
91 | document.createTextNode(val.substr(pos + text.length)),
92 | node.nextSibling));
93 | node.nodeValue = val.substr(0, pos);
94 | if (isInSVG) {
95 | var rect = document.createElementNS("http://www.w3.org/2000/svg", "rect");
96 | var bbox = node.parentElement.getBBox();
97 | rect.x.baseVal.value = bbox.x;
98 | rect.y.baseVal.value = bbox.y;
99 | rect.width.baseVal.value = bbox.width;
100 | rect.height.baseVal.value = bbox.height;
101 | rect.setAttribute('class', className);
102 | addItems.push({
103 | "parent": node.parentNode,
104 | "target": rect});
105 | }
106 | }
107 | }
108 | else if (!jQuery(node).is("button, select, textarea")) {
109 | jQuery.each(node.childNodes, function() {
110 | highlight(this, addItems);
111 | });
112 | }
113 | }
114 | var addItems = [];
115 | var result = this.each(function() {
116 | highlight(this, addItems);
117 | });
118 | for (var i = 0; i < addItems.length; ++i) {
119 | jQuery(addItems[i].parent).before(addItems[i].target);
120 | }
121 | return result;
122 | };
123 |
124 | /*
125 | * backward compatibility for jQuery.browser
126 | * This will be supported until firefox bug is fixed.
127 | */
128 | if (!jQuery.browser) {
129 | jQuery.uaMatch = function(ua) {
130 | ua = ua.toLowerCase();
131 |
132 | var match = /(chrome)[ \/]([\w.]+)/.exec(ua) ||
133 | /(webkit)[ \/]([\w.]+)/.exec(ua) ||
134 | /(opera)(?:.*version|)[ \/]([\w.]+)/.exec(ua) ||
135 | /(msie) ([\w.]+)/.exec(ua) ||
136 | ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec(ua) ||
137 | [];
138 |
139 | return {
140 | browser: match[ 1 ] || "",
141 | version: match[ 2 ] || "0"
142 | };
143 | };
144 | jQuery.browser = {};
145 | jQuery.browser[jQuery.uaMatch(navigator.userAgent).browser] = true;
146 | }
147 |
148 | /**
149 | * Small JavaScript module for the documentation.
150 | */
151 | var Documentation = {
152 |
153 | init : function() {
154 | this.fixFirefoxAnchorBug();
155 | this.highlightSearchWords();
156 | this.initIndexTable();
157 | if (DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS) {
158 | this.initOnKeyListeners();
159 | }
160 | },
161 |
162 | /**
163 | * i18n support
164 | */
165 | TRANSLATIONS : {},
166 | PLURAL_EXPR : function(n) { return n === 1 ? 0 : 1; },
167 | LOCALE : 'unknown',
168 |
169 | // gettext and ngettext don't access this so that the functions
170 | // can safely bound to a different name (_ = Documentation.gettext)
171 | gettext : function(string) {
172 | var translated = Documentation.TRANSLATIONS[string];
173 | if (typeof translated === 'undefined')
174 | return string;
175 | return (typeof translated === 'string') ? translated : translated[0];
176 | },
177 |
178 | ngettext : function(singular, plural, n) {
179 | var translated = Documentation.TRANSLATIONS[singular];
180 | if (typeof translated === 'undefined')
181 | return (n == 1) ? singular : plural;
182 | return translated[Documentation.PLURALEXPR(n)];
183 | },
184 |
185 | addTranslations : function(catalog) {
186 | for (var key in catalog.messages)
187 | this.TRANSLATIONS[key] = catalog.messages[key];
188 | this.PLURAL_EXPR = new Function('n', 'return +(' + catalog.plural_expr + ')');
189 | this.LOCALE = catalog.locale;
190 | },
191 |
192 | /**
193 | * add context elements like header anchor links
194 | */
195 | addContextElements : function() {
196 | $('div[id] > :header:first').each(function() {
197 | $('\u00B6').
198 | attr('href', '#' + this.id).
199 | attr('title', _('Permalink to this headline')).
200 | appendTo(this);
201 | });
202 | $('dt[id]').each(function() {
203 | $('\u00B6').
204 | attr('href', '#' + this.id).
205 | attr('title', _('Permalink to this definition')).
206 | appendTo(this);
207 | });
208 | },
209 |
210 | /**
211 | * workaround a firefox stupidity
212 | * see: https://bugzilla.mozilla.org/show_bug.cgi?id=645075
213 | */
214 | fixFirefoxAnchorBug : function() {
215 | if (document.location.hash && $.browser.mozilla)
216 | window.setTimeout(function() {
217 | document.location.href += '';
218 | }, 10);
219 | },
220 |
221 | /**
222 | * highlight the search words provided in the url in the text
223 | */
224 | highlightSearchWords : function() {
225 | var params = $.getQueryParameters();
226 | var terms = (params.highlight) ? params.highlight[0].split(/\s+/) : [];
227 | if (terms.length) {
228 | var body = $('div.body');
229 | if (!body.length) {
230 | body = $('body');
231 | }
232 | window.setTimeout(function() {
233 | $.each(terms, function() {
234 | body.highlightText(this.toLowerCase(), 'highlighted');
235 | });
236 | }, 10);
237 | $('
123 |
124 |
245 |
246 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
162 |
244 |
163 |
223 |
243 |
164 |
165 |
166 |
173 |
218 |
219 |
220 |
221 |
222 | Python Module Index
167 | 168 |
169 | d
170 |
171 |
172 | | 175 | d | ||
| 179 | |
180 | devicely | 181 | |
| 184 | |
185 | devicely.empatica | 186 | |
| 189 | |
190 | devicely.everion | 191 | |
| 194 | |
195 | devicely.faros | 196 | |
| 199 | |
200 | devicely.muse | 201 | |
| 204 | |
205 | devicely.shimmer_plus | 206 | |
| 209 | |
210 | devicely.spacelabs | 211 | |
| 214 | |
215 | devicely.time_stamp | 216 | |
' + _('Hide Search Matches') + '
') 239 | .appendTo($('#searchbox')); 240 | } 241 | }, 242 | 243 | /** 244 | * init the domain index toggle buttons 245 | */ 246 | initIndexTable : function() { 247 | var togglers = $('img.toggler').click(function() { 248 | var src = $(this).attr('src'); 249 | var idnum = $(this).attr('id').substr(7); 250 | $('tr.cg-' + idnum).toggle(); 251 | if (src.substr(-9) === 'minus.png') 252 | $(this).attr('src', src.substr(0, src.length-9) + 'plus.png'); 253 | else 254 | $(this).attr('src', src.substr(0, src.length-8) + 'minus.png'); 255 | }).css('display', ''); 256 | if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) { 257 | togglers.click(); 258 | } 259 | }, 260 | 261 | /** 262 | * helper function to hide the search marks again 263 | */ 264 | hideSearchWords : function() { 265 | $('#searchbox .highlight-link').fadeOut(300); 266 | $('span.highlighted').removeClass('highlighted'); 267 | }, 268 | 269 | /** 270 | * make the url absolute 271 | */ 272 | makeURL : function(relativeURL) { 273 | return DOCUMENTATION_OPTIONS.URL_ROOT + '/' + relativeURL; 274 | }, 275 | 276 | /** 277 | * get the current relative url 278 | */ 279 | getCurrentURL : function() { 280 | var path = document.location.pathname; 281 | var parts = path.split(/\//); 282 | $.each(DOCUMENTATION_OPTIONS.URL_ROOT.split(/\//), function() { 283 | if (this === '..') 284 | parts.pop(); 285 | }); 286 | var url = parts.join('/'); 287 | return path.substring(url.lastIndexOf('/') + 1, path.length - 1); 288 | }, 289 | 290 | initOnKeyListeners: function() { 291 | $(document).keydown(function(event) { 292 | var activeElementType = document.activeElement.tagName; 293 | // don't navigate when in search box, textarea, dropdown or button 294 | if (activeElementType !== 'TEXTAREA' && activeElementType !== 'INPUT' && activeElementType !== 'SELECT' 295 | && activeElementType !== 'BUTTON' && !event.altKey && !event.ctrlKey && !event.metaKey 296 | && !event.shiftKey) { 297 | switch (event.keyCode) { 298 | case 37: // left 299 | var prevHref = $('link[rel="prev"]').prop('href'); 300 | if (prevHref) { 301 | window.location.href = prevHref; 302 | return false; 303 | } 304 | break; 305 | case 39: // right 306 | var nextHref = $('link[rel="next"]').prop('href'); 307 | if (nextHref) { 308 | window.location.href = nextHref; 309 | return false; 310 | } 311 | break; 312 | } 313 | } 314 | }); 315 | } 316 | }; 317 | 318 | // quick alias for translations 319 | _ = Documentation.gettext; 320 | 321 | $(document).ready(function() { 322 | Documentation.init(); 323 | }); 324 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 |
54 |
55 |
108 |
109 |
110 |
111 |
112 |
118 |
119 |
120 |
260 |
261 |
262 |
263 |
264 |
269 |
270 |
271 |
272 |
273 |
274 |
275 |
276 |
--------------------------------------------------------------------------------
/devicely/faros.py:
--------------------------------------------------------------------------------
1 | """
2 | 1-lead ECG monitor FarosTM 180 from Bittium is a one channel ECG monitor with
3 | sampling frequency up to 1000 Hz and a 3D acceleration sampling up to 100Hz.
4 | """
5 |
6 | import json
7 | import os
8 | import random
9 |
10 | import numpy as np
11 | import pandas as pd
12 | import pyedflib as edf
13 |
14 |
15 | class FarosReader:
16 | """
17 | Read, timeshift and write data
18 | generated by Bittium Faros devices.
19 |
20 | Attributes
21 | ----------
22 | start_time : pandas.Timestamp
23 | Start time of all measurements.
24 |
25 | sample_freqs : dict
26 | Sampling frequencies of all signals in Hz.
27 |
28 | units : dict
29 | Units of all signals
30 |
31 | ECG : pandas.Series
32 | ECG signal, indexed by timestamp.
33 |
34 | ACC : pandas.DataFrame
35 | Three ACC axes, indexed by timestamp.
36 |
37 | Marker : pandas.Series
38 | Markers, indexed by timestamp.
39 |
40 | HRV : pandas.Series
41 | HRV signal, indexed by timestamp.
42 |
43 | data : DataFrame
44 | Contain all signals (ECG, ACC, Marker, HRV) indexed by timestamp.
45 | Since the signals have different sampling frequencies, many values will be NaN.
46 | """
47 |
48 | def __init__(self, path):
49 | """
50 | Read a Faros-generated EDF file or a directory created by a FarosReader.
51 |
52 | Parameters
53 | ----------
54 | path : str
55 | Can either be a Faros-generated EDF file or a directory created by the FarosReader.write() method.
56 | """
57 |
58 | self.start_time = None
59 | self.sample_freqs = None
60 | self.units = None
61 | self._edf_metadata = None
62 | self.ECG = None
63 | self.ACC = None
64 | self.Marker = None
65 | self.HRV = None
66 | self.data = None
67 |
68 | if os.path.isfile(path):
69 | self._read_from_edf_file(path)
70 | if os.path.isdir(path):
71 | self._read_from_directory(path)
72 |
73 | def _read_from_edf_file(self, path):
74 | reader = edf.EdfReader(path)
75 | self.start_time = pd.Timestamp(reader.getStartdatetime())
76 | self.sample_freqs = {
77 | 'ECG': reader.getSampleFrequency(0),
78 | 'ACC': reader.getSampleFrequency(1),
79 | 'Marker': reader.getSampleFrequency(4),
80 | 'HRV': reader.getSampleFrequency(5)
81 | }
82 | self.units = {
83 | 'ECG': reader.getSignalHeader(0)['dimension'],
84 | 'ACC': reader.getSignalHeader(1)['dimension'],
85 | 'HRV': reader.getSignalHeader(5)['dimension'],
86 | }
87 | self._edf_metadata = reader.getSignalHeaders()
88 |
89 | self._n_samples = reader.getNSamples()
90 | self._n_datarecords = reader.datarecords_in_file
91 |
92 | ecg = reader.readSignal(0)
93 | self.ECG = pd.Series(ecg, name='ECG',
94 | index=pd.date_range(start=self.start_time,
95 | periods=len(ecg),
96 | freq=f"{1/self.sample_freqs['ECG']}S")
97 | )
98 |
99 | acc = np.array([reader.readSignal(i) for i in range(1, 4)]).T
100 | self.ACC = pd.DataFrame(acc, columns=['X', 'Y', 'Z'],
101 | index=pd.date_range(start=self.start_time,
102 | periods=len(acc),
103 | freq=f"{1/self.sample_freqs['ACC']}S")
104 | )
105 |
106 | marker = reader.readSignal(4)
107 | self.Marker = pd.Series(marker, name='Marker',
108 | index=pd.date_range(start=self.start_time,
109 | periods=len(marker),
110 | freq=f"{1/self.sample_freqs['Marker']}S")
111 | )
112 |
113 | hrv = reader.readSignal(5)
114 | self.HRV = pd.Series(
115 | hrv, name='HRV',
116 | index=pd.date_range(start=self.start_time,
117 | periods=len(hrv),
118 | freq=f"{1/self.sample_freqs['HRV']}S")
119 | )
120 |
121 | reader.close()
122 |
123 | def _read_from_directory(self, path):
124 | with open(os.path.join(path, 'meta.json'), 'r') as meta_file:
125 | meta = json.load(meta_file)
126 | self.start_time = pd.Timestamp(meta['start_time'])
127 | self.sample_freqs = meta['sample_freqs']
128 | self.units = meta['units']
129 |
130 | self.ECG = self._read_dir_csv(os.path.join(path, 'ECG.csv'), self.start_time, self.sample_freqs['ECG'])
131 | self.ACC = self._read_dir_csv(os.path.join(path, 'ACC.csv'), self.start_time, self.sample_freqs['ACC'])
132 | self.Marker = self._read_dir_csv(os.path.join(path, 'Marker.csv'), self.start_time, self.sample_freqs['Marker'])
133 | self.HRV = self._read_dir_csv(os.path.join(path, 'HRV.csv'), self.start_time, self.sample_freqs['HRV'])
134 |
135 | def _read_dir_csv(self, path, start_time, sample_freq):
136 | dataframe = pd.read_csv(path)
137 | idx = pd.date_range(start=start_time, periods=len(dataframe), freq=f"{1/sample_freq}S")
138 | dataframe.index = idx
139 |
140 | return dataframe.squeeze()
141 |
142 | def join_dataframes(self):
143 | """
144 | Join the individual signal dataframes by timestamp.
145 | The resulting dataframe is saved in the attribute reader.data.
146 | """
147 | # get index for joined dataframe
148 | joined_idx = pd.concat(map(pd.Series, [self.ECG.index, self.ACC.index, self.Marker.index, self.HRV.index]))
149 | joined_idx = pd.Index(joined_idx.drop_duplicates().sort_values())
150 |
151 | # create joined dataframe
152 | col_names = ['ECG', 'ACC_X', 'ACC_Y', 'ACC_Z', 'ACC_mag', 'Marker', 'HRV']
153 | joined_df = pd.DataFrame(index=joined_idx, columns=col_names)
154 |
155 | # set non-nan values of joined dataframe
156 | joined_df.loc[self.ECG.index, 'ECG'] = self.ECG
157 | joined_df.loc[self.ACC.index, 'ACC_X'] = self.ACC['X']
158 | joined_df.loc[self.ACC.index, 'ACC_Y'] = self.ACC['Y']
159 | joined_df.loc[self.ACC.index, 'ACC_Z'] = self.ACC['Z']
160 | joined_df.loc[self.Marker.index, 'Marker'] = self.Marker
161 | joined_df.loc[self.HRV.index, 'HRV'] = self.HRV
162 |
163 | self.data = joined_df
164 |
165 | def write(self, path, file_format='directory'):
166 | """
167 | Write the data either to an EDF file or to several files into a new directory.
168 | Because of the `special structure of EDF files
121 |
122 |
258 |
259 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
164 |
257 |
165 |
233 |
256 |
166 |
167 |
168 |
191 |
231 |
232 |
192 |
228 |
229 |
230 | devicely, a python package for reading, timeshifting and writing sensor data¶
193 |The devicely package is made for reading, writing and de-identifying health sensor data from several sensors:
194 |-
195 |
Empatica E4 is a wearable device that offers real-time physiological data acquisition such as blood volume pulse, electrodermal activity (EDA), heart rate, interbeat intervals, 3-axis acceleration and skin temperature.
196 | Biovotion Everion is a wearable device used for the continuous monitoring of vital signs. Currently, it measures the following vital signs: heart rate, blood pulse wave, heart rate variability, activity, SPO2, blood perfusion, respiration rate, steps, energy expenditure, skin temperature, EDA / galvanic skin response (GSR), barometric pressure and sleep.
197 | 1-lead ECG monitor Faros 180 from Bittium is a one channel ECG monitor with sampling frequency up to 1000 Hz and a 3D acceleration sampling up to 100Hz.
198 | Spacelabs (SL 90217) is an oscillometric blood pressure (BP) monitor which can be used to automatically track a person’s BP in specificed time intervals.
199 | TimeStamp for Android allows you to record the timestamp of an event at the time it occurs. It also allows you to create specific tags such as “Running” or “Walking” and timestamp those specific activities.
200 | Shimmer Consensys GSR is a device that is used to collect sensor data in real time and it contains sensors such as GSR / EDA, photoplethysmography (PPG), 3-axis accelerometer, 3-axis gyroscope, 3-axis magnetometer & integrated altimeter.
201 | Muse S is a consumer grade headband consisting of 4 electrodes electroencephalography (EEG) sensors, 3-axis accelerometer (ACC), gyroscope, and photoplethysmography (PPG) sensors.
202 |
devicely makes it easy to read sensor data and access it in common formats such as dataframes.
204 |We provide a custom reader class for each sensor which has three core methods: read, timeshift and write. 205 | The timeshift method changes the time of measurement, thereby automatically 206 | helping to de-identify the data.
207 |After timeshifting, you can write the data in the original data format. 208 | Now you have one more safety measure in place that will facilitate data sharing.
209 |
210 |
217 |
227 | Usage¶
211 |-
212 |
- 213 |
- 214 |
- 215 |