├── models ├── __init__.py ├── baseline_models.py ├── removed_last_maxpool.py ├── bilinear_model.py ├── build_2dmodel.py └── stacked_attention_network.py ├── utilities ├── __init__.py ├── count_apple_sleep_stage_distribution.py └── cv_folder_builder.py ├── applewatch_dataprocessing ├── test │ ├── __init__.py │ ├── mesa │ │ ├── __init__.py │ │ ├── test_mesa_data_service.py │ │ ├── test_metadata_service.py │ │ ├── test_mesa_heart_rate_service.py │ │ ├── test_mesa_actigraphy_service.py │ │ ├── test_mesa_psg_service.py │ │ └── test_mesa_subject_builder.py │ ├── analysis │ │ ├── __init__.py │ │ ├── setup │ │ │ ├── __init__.py │ │ │ ├── test_data_split.py │ │ │ ├── test_sleep_label.py │ │ │ ├── test_feature_type.py │ │ │ ├── test_attributed_classifier.py │ │ │ ├── test_subject.py │ │ │ ├── test_train_test_splitter.py │ │ │ ├── test_subject_builder.py │ │ │ └── test_feature_set_service.py │ │ ├── tables │ │ │ └── __init__.py │ │ ├── figures │ │ │ └── __init__.py │ │ ├── performance │ │ │ ├── __init__.py │ │ │ ├── test_raw_performance.py │ │ │ ├── test_sleep_labeler.py │ │ │ ├── test_sleep_metrics.py │ │ │ ├── test_curve_performance_objects.py │ │ │ ├── test_sleep_wake_performance.py │ │ │ ├── test_sleep_metrics_calculator.py │ │ │ ├── test_performance_builder.py │ │ │ └── test_performance_summarizer.py │ │ └── classification │ │ │ ├── __init__.py │ │ │ ├── test_classifier_summary.py │ │ │ ├── test_parameter_search.py │ │ │ └── test_classifier_input_builder.py │ ├── preprocessing │ │ ├── __init__.py │ │ ├── motion │ │ │ ├── __init__.py │ │ │ ├── test_motion_collection.py │ │ │ └── test_motion_feature_service.py │ │ ├── psg │ │ │ ├── __init__.py │ │ │ ├── test_psg_report_processor.py │ │ │ ├── test_psg_file_type.py │ │ │ ├── test_stage_item.py │ │ │ ├── test_report_summary.py │ │ │ ├── test_psg_raw_data_collection.py │ │ │ ├── test_psg_converter.py │ │ │ ├── test_psg_label_service.py │ │ │ ├── test_vitaport_processor.py │ │ │ ├── test_compumedics_processor.py │ │ │ └── test_psg_service.py │ │ ├── time │ │ │ └── __init__.py │ │ ├── heart_rate │ │ │ ├── __init__.py │ │ │ ├── test_heart_rate_collection.py │ │ │ └── test_heart_rate_feature_service.py │ │ ├── activity_count │ │ │ ├── __init__.py │ │ │ ├── test_activity_count_collection.py │ │ │ ├── test_activity_count_service.py │ │ │ └── test_activity_count_feature_service.py │ │ ├── test_time_service.py │ │ └── test_feature_builder.py │ ├── test_sleep_stage.py │ └── test_helper.py ├── source │ ├── __init__.py │ ├── analysis │ │ ├── __init__.py │ │ ├── figures │ │ │ └── __init__.py │ │ ├── setup │ │ │ ├── __init__.py │ │ │ ├── attributed_classifier.py │ │ │ ├── data_split.py │ │ │ ├── sleep_label.py │ │ │ ├── subject.py │ │ │ ├── feature_type.py │ │ │ ├── train_test_splitter.py │ │ │ ├── feature_set_service.py │ │ │ ├── sleep_labeler.py │ │ │ └── subject_builder.py │ │ ├── tables │ │ │ └── __init__.py │ │ ├── classification │ │ │ ├── __init__.py │ │ │ ├── classifier_summary.py │ │ │ ├── parameter_search.py │ │ │ └── classifier_input_builder.py │ │ └── performance │ │ │ ├── __init__.py │ │ │ ├── raw_performance.py │ │ │ ├── sleep_metrics.py │ │ │ ├── curve_performance.py │ │ │ ├── epoch_performance.py │ │ │ └── sleep_metrics_calculator.py │ ├── mesa │ │ ├── __init__.py │ │ ├── mesa_data_service.py │ │ ├── mesa_time_based_service.py │ │ ├── mesa_heart_rate_service.py │ │ ├── mesa_actigraphy_service.py │ │ └── mesa_psg_service.py │ ├── preprocessing │ │ ├── __init__.py │ │ ├── psg │ │ │ ├── __init__.py │ │ │ ├── psg_file_type.py │ │ │ ├── stage_item.py │ │ │ ├── report_summary.py │ │ │ ├── psg_converter.py │ │ │ ├── psg_raw_data_collection.py │ │ │ ├── compumedics_processor.py │ │ │ ├── psg_label_service.py │ │ │ ├── vitaport_processor.py │ │ │ └── psg_report_processor.py │ │ ├── time │ │ │ ├── __init__.py │ │ │ ├── .DS_Store │ │ │ ├── clock_proxy │ │ │ │ ├── lux2alpha.m │ │ │ │ ├── cleanStepsData.m │ │ │ │ ├── counts2light.m │ │ │ │ ├── steps2light.m │ │ │ │ ├── ode4.m │ │ │ │ └── circadianModel.m │ │ │ └── circadian_service.py │ │ ├── motion │ │ │ ├── __init__.py │ │ │ ├── motion_collection.py │ │ │ ├── motion_feature_service.py │ │ │ └── motion_service.py │ │ ├── activity_count │ │ │ ├── __init__.py │ │ │ └── activity_count_collection.py │ │ ├── heart_rate │ │ │ ├── __init__.py │ │ │ ├── heart_rate_collection.py │ │ │ └── heart_rate_service.py │ │ ├── interval.py │ │ ├── epoch.py │ │ ├── time_service.py │ │ ├── preprocessing_runner.py │ │ └── fusion_overlapping_service.py │ ├── sleep_stage.py │ └── constants.py ├── outputs │ ├── features │ │ ├── .gitkeep │ │ ├── 3509524_circadian_feature.out │ │ ├── 7749105_hr_feature.out │ │ ├── 7749105_psg_labels.out │ │ ├── 7749105_time_feature.out │ │ ├── 7749105_circadian_feature.out │ │ ├── 7749105_count_feature.out │ │ └── 7749105_cosine_feature.out │ └── figures │ │ └── .gitkeep ├── run_many_batch.sh ├── archive │ └── scripts │ │ ├── clock_modeling │ │ ├── counts2light.m │ │ ├── lux2alpha.m │ │ ├── steps2light.m │ │ ├── cleanStepsData.m │ │ └── circadianModel.m │ │ └── get_parameters.py └── README.md ├── assets ├── apple_watch_test.txt ├── mesa_acc_hr_statistic_feature_list.csv ├── apple_watch_train.txt └── apple_hold_out_split.csv ├── sp_exp_results.csv ├── exp_results.csv ├── rp_exp_results.csv ├── LICENSE ├── data_loader ├── mesa_fold_builder.py ├── apple_fold_builder.py ├── sliding_window.py └── MESA_Acc_HRStatistic_build_h5_file.py ├── tests ├── test_data_loader.py └── test_utilis.py ├── requirements.txt ├── model_settings.yaml ├── .gitignore ├── sleep_stage_config.py ├── README.md └── evaluation_metrics.py /models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utilities/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/mesa/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/outputs/features/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/outputs/figures/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/mesa/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/setup/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/tables/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/figures/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/setup/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/tables/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/psg/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/time/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/figures/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/performance/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/motion/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/psg/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/time/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/classification/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/performance/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/motion/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/classification/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/heart_rate/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/outputs/features/3509524_circadian_feature.out: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/activity_count/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/heart_rate/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/activity_count/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/run_many_batch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## declare an array variable 3 | 4 | -------------------------------------------------------------------------------- /assets/apple_watch_test.txt: -------------------------------------------------------------------------------- 1 | 9961348 2 | 2638030 3 | 759667 4 | 1455390 5 | 2598705 6 | 5498603 7 | -------------------------------------------------------------------------------- /assets/mesa_acc_hr_statistic_feature_list.csv: -------------------------------------------------------------------------------- 1 | feature_list 2 | activity 3 | min_hr 4 | max_hr 5 | mean_hr 6 | skw_hr 7 | kurt_hr 8 | std_hr 9 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/archive/scripts/clock_modeling/counts2light.m: -------------------------------------------------------------------------------- 1 | function light = counts2light(counts) 2 | light = 850*tanh(0.015*counts); 3 | end -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/psg/test_psg_report_processor.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | 4 | class TestPSGReportProcessor(TestCase): 5 | pass -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/psg/psg_file_type.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class PSGFileType(Enum): 5 | Vitaport = 0 6 | Compumedics = 1 7 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/time/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bzhai/Ubi-SleepNet/HEAD/applewatch_dataprocessing/source/preprocessing/time/.DS_Store -------------------------------------------------------------------------------- /applewatch_dataprocessing/archive/scripts/clock_modeling/lux2alpha.m: -------------------------------------------------------------------------------- 1 | function alpha = lux2alpha(I) 2 | 3 | I0 = 9500; 4 | p = .6; 5 | a0 = .16; 6 | alpha = a0*(I.^p/I0.^p); 7 | 8 | end 9 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/archive/scripts/clock_modeling/steps2light.m: -------------------------------------------------------------------------------- 1 | function light = steps2light(steps) 2 | 3 | light = steps; % First-order approximation lacking light sensor validation 4 | 5 | end -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/time/clock_proxy/lux2alpha.m: -------------------------------------------------------------------------------- 1 | function alpha = lux2alpha(I) 2 | 3 | I0 = 9500; 4 | p = .6; 5 | a0 = .16; 6 | alpha = a0*(I.^p/I0.^p); 7 | 8 | end 9 | 10 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/interval.py: -------------------------------------------------------------------------------- 1 | class Interval(object): 2 | def __init__(self, start_time, end_time): 3 | self.start_time = start_time 4 | self.end_time = end_time 5 | 6 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/setup/attributed_classifier.py: -------------------------------------------------------------------------------- 1 | class AttributedClassifier(object): 2 | def __init__(self, name, classifier): 3 | self.name = name 4 | self.classifier = classifier 5 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/setup/data_split.py: -------------------------------------------------------------------------------- 1 | class DataSplit(object): 2 | def __init__(self, training_set, testing_set): 3 | self.training_set = training_set 4 | self.testing_set = testing_set 5 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/sleep_stage.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class SleepStage(Enum): 5 | wake = 0 6 | n1 = 1 7 | n2 = 2 8 | n3 = 3 9 | n4 = 4 10 | rem = 5 11 | unscored = -1 12 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/psg/stage_item.py: -------------------------------------------------------------------------------- 1 | from source.sleep_stage import SleepStage 2 | 3 | 4 | class StageItem(object): 5 | def __init__(self, epoch, stage: SleepStage): 6 | self.epoch = epoch 7 | self.stage = stage 8 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/performance/raw_performance.py: -------------------------------------------------------------------------------- 1 | class RawPerformance(object): 2 | def __init__(self, true_labels, class_probabilities): 3 | self.true_labels = true_labels 4 | self.class_probabilities = class_probabilities 5 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/setup/sleep_label.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class SleepWakeLabel(Enum): 5 | wake = 0 6 | sleep = 1 7 | 8 | 9 | class ThreeClassLabel(Enum): 10 | wake = 0 11 | nrem = 1 12 | rem = 2 13 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/setup/subject.py: -------------------------------------------------------------------------------- 1 | class Subject(object): 2 | 3 | def __init__(self, subject_id, labeled_sleep, feature_dictionary): 4 | self.subject_id = subject_id 5 | self.labeled_sleep = labeled_sleep 6 | self.feature_dictionary = feature_dictionary 7 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/setup/feature_type.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class FeatureType(Enum): 5 | count = "count" 6 | motion = "motion" 7 | heart_rate = "heart rate" 8 | cosine = "cosine" 9 | circadian_model = "circadian model" 10 | time = "time" 11 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/psg/report_summary.py: -------------------------------------------------------------------------------- 1 | class ReportSummary(object): 2 | def __init__(self, study_date, start_epoch, start_time, file_type): 3 | self.study_date = study_date 4 | self.start_epoch = start_epoch 5 | self.start_time = start_time 6 | self.file_type = file_type 7 | -------------------------------------------------------------------------------- /assets/apple_watch_train.txt: -------------------------------------------------------------------------------- 1 | 1818471 2 | 4426783 3 | 8258170 4 | 5383425 5 | 1066528 6 | 8530312 7 | 781756 8 | 3997827 9 | 9106476 10 | 46343 11 | 1449548 12 | 8686948 13 | 3509524 14 | 844359 15 | 4314139 16 | 4018081 17 | 8173033 18 | 6220552 19 | 5132496 20 | 9618981 21 | 8000685 22 | 5797046 23 | 7749105 24 | 1360686 25 | 8692923 26 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/archive/scripts/clock_modeling/cleanStepsData.m: -------------------------------------------------------------------------------- 1 | function data = cleanStepsData(data) 2 | 3 | % Remove data where timestamps = 0 4 | data(data(:,1) == 0,:) = []; 5 | 6 | % If timestamps in milliseconds instead of seconds, convert to seconds 7 | if(max(data(:,1)) > 1.5e10) 8 | data(:,1) = data(:,1)/1000; 9 | end 10 | 11 | end -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/time/clock_proxy/cleanStepsData.m: -------------------------------------------------------------------------------- 1 | function data = cleanStepsData(data) 2 | 3 | % Remove data where timestamps = 0 4 | data(data(:,1) == 0,:) = []; 5 | 6 | % If timestamps in milliseconds instead of seconds, convert to seconds 7 | if(max(data(:,1)) > 1.5e10) 8 | data(:,1) = data(:,1)/1000; 9 | end 10 | 11 | end -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/psg/test_psg_file_type.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.preprocessing.psg.psg_file_type import PSGFileType 4 | 5 | 6 | class TestPSGFileType(TestCase): 7 | def test_file_types_exists(self): 8 | self.assertEqual(PSGFileType.Vitaport.value, 0) 9 | self.assertEqual(PSGFileType.Compumedics.value, 1) 10 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/performance/sleep_metrics.py: -------------------------------------------------------------------------------- 1 | class SleepMetrics(object): 2 | def __init__(self, tst, sol, waso, sleep_efficiency, time_in_rem, time_in_nrem): 3 | self.tst = tst 4 | self.sol = sol 5 | self.waso = waso 6 | self.sleep_efficiency = sleep_efficiency 7 | self.time_in_rem = time_in_rem 8 | self.time_in_nrem = time_in_nrem -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/classification/classifier_summary.py: -------------------------------------------------------------------------------- 1 | from source.analysis.setup.attributed_classifier import AttributedClassifier 2 | 3 | 4 | class ClassifierSummary(object): 5 | def __init__(self, attributed_classifier: AttributedClassifier, performance_dictionary): 6 | self.attributed_classifier = attributed_classifier 7 | self.performance_dictionary = performance_dictionary 8 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/epoch.py: -------------------------------------------------------------------------------- 1 | class Epoch(object): 2 | DURATION = 30 # seconds 3 | 4 | def __init__(self, timestamp, index): 5 | self.timestamp = timestamp 6 | self.index = index 7 | 8 | def to_dict(self): 9 | return { 10 | 'timestamp': self.timestamp, 11 | 'index': self.index, 12 | 'duration': self.DURATION 13 | } -------------------------------------------------------------------------------- /assets/apple_hold_out_split.csv: -------------------------------------------------------------------------------- 1 | uids,segment 3509524,train 5132496,train 1066528,train 5498603,train 2638030,train 2598705,train 5383425,train 1455390,train 4018081,train 9961348,train 1449548,train 8258170,train 781756,train 9106476,train 8686948,train 8530312,train 3997827,test 4314139,test 1818471,test 4426783,test 8173033,test 7749105,test 5797046,test 759667,test 8000685,test 6220552,test 844359,test 9618981,test 1360686,test 46343,test 8692923,test 2 | -------------------------------------------------------------------------------- /utilities/count_apple_sleep_stage_distribution.py: -------------------------------------------------------------------------------- 1 | import h5py as h5py 2 | import h5py as h5py 3 | from sleep_stage_config import Config 4 | from collections import Counter 5 | 6 | cfg =Config() 7 | cache_path = cfg.APPLE_LOOCV_ALL_WINDOWED % 100 8 | with h5py.File(cache_path, 'r') as data: 9 | df_data = data["df_values"][:] 10 | x = data["x"][:] 11 | y = data["y"][:] 12 | columns = data["columns"][:].astype(str).tolist() 13 | data.close() 14 | y = list(y) 15 | 16 | print(Counter(y)) -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/setup/test_data_split.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.analysis.setup.data_split import DataSplit 4 | 5 | 6 | class TestDataSplit(TestCase): 7 | def test_properties(self): 8 | training_set = ["3", "25", "35"] 9 | testing_set = ["2", "A32", "543"] 10 | split = DataSplit(training_set=training_set, testing_set=testing_set) 11 | 12 | self.assertListEqual(split.training_set, training_set) 13 | self.assertListEqual(split.testing_set, testing_set) 14 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/test_sleep_stage.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.sleep_stage import SleepStage 4 | 5 | 6 | class TestSleepStage(TestCase): 7 | 8 | def test_stages(self): 9 | 10 | self.assertEqual(SleepStage.wake.value, 0) 11 | self.assertEqual(SleepStage.n1.value, 1) 12 | self.assertEqual(SleepStage.n2.value, 2) 13 | self.assertEqual(SleepStage.n3.value, 3) 14 | self.assertEqual(SleepStage.n4.value, 4) 15 | self.assertEqual(SleepStage.rem.value, 5) 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/motion/motion_collection.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from source.preprocessing.interval import Interval 4 | 5 | 6 | class MotionCollection(object): 7 | def __init__(self, subject_id, data): 8 | self.subject_id = subject_id 9 | self.data = data 10 | self.timestamps = data[:, 0] 11 | self.values = data[:, 1:] 12 | 13 | def get_interval(self): 14 | return Interval(start_time=np.amin(self.data[:, 0]), 15 | end_time=np.amax(self.data[:, 0])) 16 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/heart_rate/heart_rate_collection.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from source.preprocessing.interval import Interval 4 | 5 | 6 | class HeartRateCollection(object): 7 | def __init__(self, subject_id, data): 8 | self.subject_id = subject_id 9 | self.data = data 10 | self.timestamps = data[:, 0] 11 | self.values = data[:, 1:] 12 | 13 | def get_interval(self): 14 | return Interval(start_time=np.amin(self.data[:, 0]), 15 | end_time=np.amax(self.data[:, 0])) 16 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/performance/curve_performance.py: -------------------------------------------------------------------------------- 1 | from numpy.core.multiarray import ndarray 2 | 3 | 4 | class ROCPerformance(object): 5 | 6 | def __init__(self, false_positive_rates: ndarray, true_positive_rates: ndarray): 7 | self.false_positive_rates = false_positive_rates 8 | self.true_positive_rates = true_positive_rates 9 | 10 | 11 | class PrecisionRecallPerformance(object): 12 | 13 | def __init__(self, recalls: ndarray, precisions: ndarray): 14 | self.recalls = recalls 15 | self.precisions = precisions 16 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/setup/test_sleep_label.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.analysis.setup.sleep_label import SleepWakeLabel, ThreeClassLabel 4 | 5 | 6 | class TestSleepLabel(TestCase): 7 | 8 | def test_label_enums(self): 9 | self.assertEqual(SleepWakeLabel.wake.value, 0) 10 | self.assertEqual(SleepWakeLabel.sleep.value, 1) 11 | self.assertEqual(ThreeClassLabel.wake.value, 0) 12 | self.assertEqual(ThreeClassLabel.nrem.value, 1) 13 | self.assertEqual(ThreeClassLabel.rem.value, 2) 14 | 15 | 16 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/activity_count/activity_count_collection.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from source.preprocessing.interval import Interval 4 | 5 | 6 | class ActivityCountCollection(object): 7 | def __init__(self, subject_id, data): 8 | self.subject_id = subject_id 9 | self.data = data 10 | self.timestamps = data[:, 0] 11 | self.values = data[:, 1:] 12 | 13 | def get_interval(self): 14 | return Interval(start_time=np.amin(self.data[:, 0]), 15 | end_time=np.amax(self.data[:, 0])) 16 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/mesa/mesa_data_service.py: -------------------------------------------------------------------------------- 1 | from source.mesa.mesa_subject_builder import MesaSubjectBuilder 2 | from source.mesa.metadata_service import MetadataService 3 | 4 | 5 | class MesaDataService(object): 6 | 7 | @staticmethod 8 | def get_all_subjects(): 9 | all_files = MetadataService.get_all_files() 10 | all_subjects = [] 11 | for file in all_files: 12 | file_id = file[-8:-4] 13 | subject = MesaSubjectBuilder.build(file_id) 14 | if subject is not None: 15 | all_subjects.append(subject) 16 | 17 | return all_subjects 18 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/setup/test_feature_type.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.analysis.setup.feature_type import FeatureType 4 | 5 | 6 | class TestFeatureType(TestCase): 7 | 8 | def test_types(self): 9 | self.assertEqual(FeatureType.count.value, "count") 10 | self.assertEqual(FeatureType.motion.value, "motion") 11 | self.assertEqual(FeatureType.heart_rate.value, "heart rate") 12 | self.assertEqual(FeatureType.cosine.value, "cosine") 13 | self.assertEqual(FeatureType.circadian_model.value, "circadian model") 14 | self.assertEqual(FeatureType.time.value, "time") 15 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/setup/test_attributed_classifier.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from sklearn.neighbors import KNeighborsClassifier 4 | 5 | from source.analysis.setup.attributed_classifier import AttributedClassifier 6 | 7 | 8 | class TestAttributedClassifier(TestCase): 9 | def test_properties(self): 10 | classifier = KNeighborsClassifier() 11 | name = "k-Nearest Neighbors" 12 | attributed_classifier = AttributedClassifier(name=name, classifier=classifier) 13 | self.assertEqual(name, attributed_classifier.name) 14 | self.assertEqual(classifier, attributed_classifier.classifier) 15 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/mesa/mesa_time_based_service.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | from source import utils 5 | 6 | 7 | class MesaTimeBasedService(object): 8 | 9 | @staticmethod 10 | def load_circadian_model(file_id): 11 | 12 | path = utils.get_project_root().joinpath('data/mesa/clock_proxy/' + file_id + '_clock_proxy.out') 13 | 14 | if path.is_file(): 15 | array = pd.read_csv(str(path), delimiter=',').values 16 | if np.shape(array)[0] > 0: 17 | array = utils.remove_nans(array) 18 | if np.shape(array)[0] > 0: 19 | return array 20 | 21 | return None 22 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/psg/test_stage_item.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.preprocessing.epoch import Epoch 4 | from source.preprocessing.psg.stage_item import StageItem 5 | from source.sleep_stage import SleepStage 6 | 7 | 8 | class TestStageItem(TestCase): 9 | 10 | def test_constructor(self): 11 | stage = SleepStage.rem 12 | epoch_time = 3 13 | epoch_index = 4 14 | stage_item = StageItem(Epoch(timestamp=epoch_time, index=epoch_index), stage=stage) 15 | self.assertEqual(stage, stage_item.stage) 16 | self.assertEqual(epoch_time, stage_item.epoch.timestamp) 17 | self.assertEqual(epoch_index, stage_item.epoch.index) 18 | -------------------------------------------------------------------------------- /sp_exp_results.csv: -------------------------------------------------------------------------------- 1 | machine,period,tf,macro_accuracy,macro_specificity,macro_precision,macro_f1,macro_cohen,macro_recall,nn_type,att_on_modality,optim,log_interval,num_classes,momentum,epochs,batch_size,lr,debug,dataset,feature_type,seq_len,comments,save_eval,seed,Non-REM sleep,REM sleep,Wake,accuracy,cohen-kappa,f1-score,precision,recall,specificity,best epochs 2 | BB-WIN8,s,20220213-091006,75.52722201667757,78.7992818569346,76.63649828618398,59.82468089990225,45.24931042480374,56.020458730603316,ResPlusSplitModal_SANTiDimMatAttMod1NLayer1Con,car,ADAM,100,3,0.9,1,1024,0.0001,0,mesa_hr_statistic,all,100,,0,42,76.9 $\pm$ 5.6,-48.4 $\pm$ 2.9,-28.5 $\pm$ 4.8,75.7 $\pm$ 0.8,40.6 $\pm$ 1.9,56.9 $\pm$ 1.2,72.2 $\pm$ 1.4,56.4 $\pm$ 1.1,78.5 $\pm$ 0.6,0 3 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/time/clock_proxy/counts2light.m: -------------------------------------------------------------------------------- 1 | function light = counts2light(counts, startTimeForCountVector, dtPSG) 2 | count_threshold = 100; 3 | 4 | for i = 1:length(counts) 5 | value = 500; 6 | if mod(startTimeForCountVector + i*dtPSG/3600,24) > 10 && mod(startTimeForCountVector + i*dtPSG/3600,24) < 16 7 | value = 1000; 8 | end 9 | if mod(startTimeForCountVector + i*dtPSG/3600,24) < 7 || mod(startTimeForCountVector + i*dtPSG/3600,24) > 22 10 | value = 50; 11 | 12 | end 13 | 14 | if counts(i) < count_threshold 15 | counts(i) = 0; 16 | end 17 | 18 | light(i) = value*sign(counts(i)); 19 | 20 | end 21 | 22 | 23 | 24 | end 25 | -------------------------------------------------------------------------------- /exp_results.csv: -------------------------------------------------------------------------------- 1 | tf,seq_len,seed,save_eval,optim,num_classes,nn_type,momentum,macro_specificity,macro_recall,macro_precision,macro_f1,macro_cohen,macro_accuracy,machine,lr,log_interval,feature_type,epochs,debug,dataset,comments,best epochs,batch_size,att_on_modality 2 | 20220213-091006,100,42,0,ADAM,3,ResPlusSplitModal_SANTiDimMatAttMod1NLayer1Con,0.9,83.35970462537459,60.67345680087908,79.38325967751382,63.724707479022655,57.780357946414604,75.71421512406603,BB-WIN8,0.0001,100,all,1,0,mesa_hr_statistic,,0,1024,car 3 | 20220212-192655,100,42,0,ADAM,3,ResPlusSplitModal_SANTiDimMatAttMod1NLayer1Con,0.9,87.81006636776577,73.89178157246693,73.58319574991444,73.73498265797087,65.62108186949813,78.57575686647785,BB-WIN8,0.0001,100,all,20,0,mesa,,4,4096,car 4 | -------------------------------------------------------------------------------- /rp_exp_results.csv: -------------------------------------------------------------------------------- 1 | machine,period,tf,macro_accuracy,macro_specificity,macro_precision,macro_f1,macro_cohen,macro_recall,nn_type,att_on_modality,optim,log_interval,num_classes,momentum,epochs,batch_size,lr,debug,dataset,feature_type,seq_len,comments,save_eval,seed,Non-REM sleep,REM sleep,Wake,accuracy,cohen-kappa,f1-score,precision,recall,specificity,best epochs 2 | BB-WIN8,r,20220213-091006,75.71421512406603,83.35970462537459,79.38325967751382,63.724707479022655,57.780357946414604,60.67345680087908,ResPlusSplitModal_SANTiDimMatAttMod1NLayer1Con,car,ADAM,100,3,0.9,1,1024,0.0001,0,mesa_hr_statistic,all,100,,0,42,99.4 $\pm$ 6.9,-47.9 $\pm$ 3.0,-51.5 $\pm$ 6.6,75.9 $\pm$ 0.9,55.3 $\pm$ 1.6,61.5 $\pm$ 1.2,75.5 $\pm$ 1.4,61.1 $\pm$ 1.0,83.3 $\pm$ 0.5,0 3 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/time/clock_proxy/steps2light.m: -------------------------------------------------------------------------------- 1 | function light = steps2light(steps, startTimeForAverageDay, dtSeconds) 2 | light = []; 3 | step_threshold = 20; 4 | 5 | for i = 1:length(steps) 6 | value = 500; 7 | 8 | if mod(startTimeForAverageDay + i*dtSeconds/3600,24) > 10 && mod(startTimeForAverageDay + i*dtSeconds/3600,24) < 16 9 | value = 1000; 10 | end 11 | if mod(startTimeForAverageDay + i*dtSeconds/3600,24) < 7 || mod(startTimeForAverageDay + i*dtSeconds/3600,24) > 22 12 | value = 50; 13 | 14 | end 15 | 16 | if steps(i) < step_threshold 17 | steps(i) = 0; 18 | end 19 | 20 | light(i) = value*sign(steps(i)); 21 | 22 | end 23 | 24 | end -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/performance/test_raw_performance.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | import numpy as np 3 | 4 | from source.analysis.performance.raw_performance import RawPerformance 5 | 6 | 7 | class TestRawPerformance(TestCase): 8 | 9 | def test_properties(self): 10 | 11 | true_labels = np.array([0, 1, 2]) 12 | class_probabilities = np.array([[0.1, 0.9], [0, 1]]) 13 | raw_performance = RawPerformance(true_labels=true_labels, 14 | class_probabilities=class_probabilities) 15 | self.assertEqual(raw_performance.true_labels.tolist(), true_labels.tolist()) 16 | self.assertEqual(raw_performance.class_probabilities.tolist(), class_probabilities.tolist()) 17 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/time/circadian_service.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from source import utils 4 | from source.constants import Constants 5 | 6 | 7 | class CircadianService(object): 8 | 9 | @staticmethod 10 | def build_circadian_model(): 11 | os.system(Constants.MATLAB_PATH + ' -nodisplay -nosplash -nodesktop -r \"run(\'' + str( 12 | utils.get_project_root()) + '/source/preprocessing/time/clock_proxy/runCircadianModel.m\'); exit;\"') 13 | 14 | @staticmethod 15 | def build_circadian_mesa(): 16 | os.system(Constants.MATLAB_PATH + ' -nodisplay -nosplash -nodesktop -r \"run(\'' + str( 17 | utils.get_project_root()) + '/source/preprocessing/time/clock_proxy/runCircadianMESA.m\'); exit;\"') 18 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/motion/motion_feature_service.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | from source.constants import Constants 5 | 6 | 7 | class MotionFeatureService(object): 8 | 9 | @staticmethod 10 | def load(subject_id): 11 | motion_feature_path = MotionFeatureService.get_path(subject_id) 12 | feature = pd.read_csv(str(motion_feature_path)).values 13 | return feature 14 | 15 | @staticmethod 16 | def get_path(subject_id): 17 | return Constants.FEATURE_FILE_PATH.joinpath(subject_id + '_motion_feature.out') 18 | 19 | @staticmethod 20 | def write(subject_id, feature): 21 | motion_feature_path = MotionFeatureService.get_path(subject_id) 22 | np.savetxt(motion_feature_path, feature, fmt='%f') 23 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/mesa/test_mesa_data_service.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | from unittest.mock import call 3 | 4 | from source.mesa.mesa_data_service import MesaDataService 5 | 6 | 7 | class TestMesaDataService(TestCase): 8 | 9 | @mock.patch('source.mesa.mesa_data_service.MesaSubjectBuilder') 10 | @mock.patch('source.mesa.mesa_data_service.MetadataService') 11 | def test_get_all_subjects(self, mock_metadata_service, mock_subject_builder): 12 | mock_metadata_service.get_all_files.return_value = ["file1-3243.edf", "file2-1234.edf"] 13 | 14 | mock_subject_builder.build.side_effect = expected_subjects = ["A", "B"] 15 | 16 | subjects = MesaDataService.get_all_subjects() 17 | 18 | mock_subject_builder.build.assert_has_calls([call("3243"), call("1234")]) 19 | self.assertListEqual(expected_subjects, subjects) 20 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/performance/epoch_performance.py: -------------------------------------------------------------------------------- 1 | class SleepWakePerformance(object): 2 | def __init__(self, accuracy, wake_correct, sleep_correct, kappa, auc, sleep_predictive_value, 3 | wake_predictive_value): 4 | self.accuracy = accuracy 5 | self.wake_correct = wake_correct 6 | self.sleep_correct = sleep_correct 7 | self.kappa = kappa 8 | self.auc = auc 9 | self.wake_predictive_value = wake_predictive_value 10 | self.sleep_predictive_value = sleep_predictive_value 11 | 12 | 13 | class ThreeClassPerformance(object): 14 | def __init__(self, accuracy, wake_correct, rem_correct, nrem_correct, kappa): 15 | self.accuracy = accuracy 16 | self.wake_correct = wake_correct 17 | self.rem_correct = rem_correct 18 | self.nrem_correct = nrem_correct 19 | self.kappa = kappa 20 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/performance/test_sleep_labeler.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.analysis.setup.sleep_labeler import SleepLabeler 4 | import numpy as np 5 | 6 | 7 | class TestSleepLabeler(TestCase): 8 | 9 | def test_label_sleep_wake(self): 10 | sleep_wake_array = np.array([0, 1, 2, 3, 4, 5, 0, 4]) 11 | labeled_sleep = SleepLabeler.label_sleep_wake(sleep_wake_array) 12 | 13 | self.assertEqual(len(sleep_wake_array), len(labeled_sleep)) 14 | self.assertEqual(0, labeled_sleep[0]) 15 | self.assertEqual(1, labeled_sleep[1]) 16 | self.assertEqual(1, labeled_sleep[2]) 17 | self.assertEqual(1, labeled_sleep[3]) 18 | self.assertEqual(1, labeled_sleep[4]) 19 | self.assertEqual(1, labeled_sleep[5]) 20 | self.assertEqual(0, labeled_sleep[6]) 21 | self.assertEqual(1, labeled_sleep[7]) 22 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/classification/parameter_search.py: -------------------------------------------------------------------------------- 1 | from sklearn.model_selection import GridSearchCV 2 | 3 | 4 | class ParameterSearch(object): 5 | parameter_dictionary = { 6 | 'Logistic Regression': {'C': [0.001, 0.01, 0.1, 1, 10, 100], 'penalty': ['l1', 'l2']}, 7 | 'Random Forest': {'max_depth': [10, 50, 100]}, 8 | 'k-Nearest Neighbors': {'n_neighbors': [500, 1000]}, 9 | 'Neural Net': {'alpha': [0.1, 0.01, 0.001, 0.0001, 0.00001]} 10 | } 11 | 12 | @staticmethod 13 | def run_search(attributed_classifier, training_x, training_y, scoring): 14 | parameter_range = ParameterSearch.parameter_dictionary[attributed_classifier.name] 15 | grid_search = GridSearchCV(attributed_classifier.classifier, parameter_range, scoring=scoring, cv=3, iid=False) 16 | grid_search.fit(training_x, training_y) 17 | return grid_search.best_params_ 18 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/psg/test_report_summary.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.preprocessing.psg.psg_file_type import PSGFileType 4 | from source.preprocessing.psg.report_summary import ReportSummary 5 | 6 | 7 | class TestReportSummary(TestCase): 8 | def test_constructor(self): 9 | start_time = "12:00:02 AM" 10 | start_epoch = 4 11 | study_date = "11/2/2019" 12 | file_type = PSGFileType.Vitaport 13 | report_summary = ReportSummary(study_date=study_date, start_epoch=start_epoch, start_time=start_time, 14 | file_type=file_type) 15 | 16 | self.assertEqual(report_summary.study_date, study_date) 17 | self.assertEqual(report_summary.start_epoch, start_epoch) 18 | self.assertEqual(report_summary.start_time, start_time) 19 | self.assertEqual(report_summary.file_type, file_type) 20 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/mesa/mesa_heart_rate_service.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pyedflib as pyedflib 3 | 4 | from source import utils 5 | from source.preprocessing.heart_rate.heart_rate_collection import HeartRateCollection 6 | 7 | 8 | class MesaHeartRateService(object): 9 | @staticmethod 10 | def load_raw(file_id): 11 | project_root = str(utils.get_project_root()) 12 | 13 | edf_file = pyedflib.EdfReader(project_root + '/data/mesa/polysomnography/edfs/mesa-sleep-' + file_id + '.edf') 14 | signal_labels = edf_file.getSignalLabels() 15 | 16 | hr_column = len(signal_labels) - 2 17 | 18 | sample_frequencies = edf_file.getSampleFrequencies() 19 | 20 | heart_rate = edf_file.readSignal(hr_column) 21 | sf = sample_frequencies[hr_column] 22 | 23 | time_hr = np.array(range(0, len(heart_rate))) # Get timestamps for heart rate data 24 | time_hr = time_hr / sf 25 | 26 | data = np.transpose(np.vstack((time_hr, heart_rate))) 27 | data = utils.remove_nans(data) 28 | return HeartRateCollection(subject_id=file_id, data=data) 29 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/README.md: -------------------------------------------------------------------------------- 1 | # sleep_classifiers 2 | 3 | This code is adapted from [link](https://github.com/ojwalch/sleep_classifiers.git). 4 | 5 | ## Getting Started 6 | 7 | This code uses Python 3.7. 8 | 9 | ## Data 10 | 11 | Data collected using the Apple Watch is available on PhysioNet: [link](https://alpha.physionet.org/content/sleep-accel/1.0.0/) 12 | 13 | The MESA dataset is available for download at the [National Sleep Research Resource](https://sleepdata.org). You will have to request access from NSRR for the data. 14 | 15 | Once the data has been download, please put all files along with the directories under the **data** folder. 16 | ## Features + figures 17 | 18 | * All raw data are cleaned and features are generated in ```preprocessing_runner.py.```. 19 | * For the first time run, please make sure you change the value as:``first_time_build=True``. 20 | 21 | * The file ```analysis_runner.py``` can be used to generate figures showing classifier performance. You can comment and uncomment the figures you want to run. 22 | 23 | 24 | ## License 25 | 26 | This software is open source and under an MIT license. 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 bzhai 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/time_service.py: -------------------------------------------------------------------------------- 1 | import datetime as dt 2 | 3 | from source.preprocessing.psg.psg_file_type import PSGFileType 4 | from source.preprocessing.psg.report_summary import ReportSummary 5 | 6 | 7 | class TimeService(object): 8 | @staticmethod 9 | def get_start_epoch_timestamp(report_summary: ReportSummary): 10 | if report_summary.file_type == PSGFileType.Compumedics: 11 | study_date = dt.datetime.strptime(report_summary.study_date + ' ' + report_summary.start_time, 12 | '%m/%d/%Y %I:%M:%S %p') 13 | if study_date.strftime('%p') == 'AM': 14 | study_date += dt.timedelta(days=1) 15 | return study_date.timestamp() 16 | 17 | if report_summary.file_type == PSGFileType.Vitaport: 18 | study_date = dt.datetime.strptime(report_summary.study_date + ' ' + report_summary.start_time, 19 | '%m/%d/%y %H:%M:%S') 20 | if int(study_date.strftime('%H')) < 12: 21 | study_date += dt.timedelta(days=1) 22 | return study_date.timestamp() 23 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/performance/test_sleep_metrics.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.analysis.performance.sleep_metrics import SleepMetrics 4 | 5 | 6 | class TestSleepMetrics(TestCase): 7 | 8 | def test_properties(self): 9 | tst = 600 10 | sol = 45 11 | waso = 30 12 | sleep_efficiency = 0.3 13 | time_in_rem = 150 14 | time_in_nrem = 250 15 | sleep_metrics = SleepMetrics(tst=tst, 16 | sol=sol, 17 | waso=waso, 18 | sleep_efficiency=sleep_efficiency, 19 | time_in_rem=time_in_rem, 20 | time_in_nrem=time_in_nrem) 21 | 22 | self.assertEqual(tst, sleep_metrics.tst) 23 | self.assertEqual(sol, sleep_metrics.sol) 24 | self.assertEqual(waso, sleep_metrics.waso) 25 | self.assertEqual(sleep_efficiency, sleep_metrics.sleep_efficiency) 26 | self.assertEqual(time_in_rem, sleep_metrics.time_in_rem) 27 | self.assertEqual(time_in_nrem, sleep_metrics.time_in_nrem) 28 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/constants.py: -------------------------------------------------------------------------------- 1 | from source import utils 2 | import os 3 | 4 | class Constants(object): 5 | # WAKE_THRESHOLD = 0.3 # These values were used for scikit-learn 0.20.3, See: 6 | # REM_THRESHOLD = 0.35 # https://scikit-learn.org/stable/whats_new.html#version-0-21-0 7 | WAKE_THRESHOLD = 0.5 # 8 | REM_THRESHOLD = 0.35 9 | 10 | EPOCH_DURATION_IN_SECONDS = 30 11 | SECONDS_PER_MINUTE = 60 12 | SECONDS_PER_DAY = 3600 * 24 13 | SECONDS_PER_HOUR = 3600 14 | VERBOSE = True 15 | RAW_PATH = r"" # for UbiSleepNet, you don't need this path. 16 | CROPPED_FILE_PATH = utils.get_project_root().joinpath('outputs/cropped/') 17 | FEATURE_FILE_PATH = utils.get_project_root().joinpath('outputs/features/') 18 | FIGURE_FILE_PATH = utils.get_project_root().joinpath('outputs/figures/') 19 | # CROPPED_FILE_PATH = os.path.join(RAW_PATH, 'outputs', 'cropped') 20 | # FEATURE_FILE_PATH = os.path.join(RAW_PATH, 'outputs', 'features') 21 | # FIGURE_FILE_PATH = os.path.join(RAW_PATH, 'outputs', 'figures') 22 | LOWER_BOUND = -0.2 23 | MATLAB_PATH = '/Applications/MATLAB_R2019a.app/bin/matlab' # Replace with your MATLAB path 24 | 25 | -------------------------------------------------------------------------------- /utilities/cv_folder_builder.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pandas as pd 3 | import numpy as np 4 | from sklearn.model_selection import KFold 5 | 6 | 7 | def build_fold(total_fold, pids, target_dir): 8 | train_set = [] 9 | test_set = [] 10 | new_train_set = [] 11 | fold_idx = 0 12 | k_fold = KFold(n_splits=total_fold) 13 | fold_df = [] 14 | for fold_idx, (train_set, test_set) in enumerate(k_fold.split(pids)): 15 | new_train_set = train_set[:int(np.floor(len(train_set)*0.8))] 16 | val_set = train_set[int(np.floor(len(train_set)*0.8)):] 17 | tmp_df_list = [] 18 | for set_type, pid_set in {"train": new_train_set, "val": val_set, "test": test_set}.items(): 19 | tmp_df = pd.DataFrame({"fold_num": [fold_idx]*len(pid_set), 20 | "set_type": [set_type]*len(pid_set), 21 | "pid": pids[pid_set]}) 22 | tmp_df_list.append(tmp_df) 23 | tmp_df_list = pd.concat(tmp_df_list, axis=0, ignore_index=True) 24 | fold_df.append(tmp_df_list) 25 | fold_df = pd.concat(fold_df, axis=0, ignore_index=True) 26 | fold_df.to_csv(os.path.join(target_dir, r"%s_fold.csv" % total_fold), 27 | index=False) 28 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/setup/train_test_splitter.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | import numpy as np 4 | 5 | from source.analysis.setup.data_split import DataSplit 6 | 7 | 8 | class TrainTestSplitter(object): 9 | 10 | @staticmethod 11 | def leave_one_out(subject_ids): 12 | splits = [] 13 | 14 | for index in range(len(subject_ids)): 15 | training_set = subject_ids.copy() 16 | testing_set = [training_set.pop(index)] 17 | 18 | splits.append(DataSplit(training_set=training_set, testing_set=testing_set)) 19 | 20 | return splits 21 | 22 | @staticmethod 23 | def by_fraction(subject_ids, test_fraction, number_of_splits): 24 | 25 | test_index = int(np.round(test_fraction * len(subject_ids))) 26 | 27 | splits = [] 28 | for trial in range(number_of_splits): 29 | random.shuffle(subject_ids) 30 | 31 | training_set = subject_ids.copy() 32 | testing_set = [] 33 | for index in range(test_index): 34 | testing_set.append(training_set.pop(0)) 35 | 36 | splits.append(DataSplit(training_set=training_set, testing_set=testing_set)) 37 | 38 | return splits 39 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/setup/test_subject.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | import numpy as np 3 | 4 | from source.analysis.setup.feature_type import FeatureType 5 | from source.analysis.setup.subject import Subject 6 | 7 | 8 | class TestSubject(TestCase): 9 | 10 | def test_subject(self): 11 | subject_id = "subjectA" 12 | labeled_sleep = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1]) 13 | feature_dictionary = {FeatureType.count: np.array([]), 14 | FeatureType.motion: np.array([]), 15 | FeatureType.heart_rate: np.array([]), 16 | FeatureType.cosine: np.array([]), 17 | FeatureType.circadian_model: np.array([]), 18 | FeatureType.time: np.array([])} 19 | subject = Subject(subject_id=subject_id, 20 | labeled_sleep=labeled_sleep, 21 | feature_dictionary=feature_dictionary) 22 | 23 | self.assertEqual(subject.subject_id, subject_id) 24 | self.assertListEqual(subject.labeled_sleep.tolist(), labeled_sleep.tolist()) 25 | self.assertDictEqual(subject.feature_dictionary, feature_dictionary) 26 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/motion/test_motion_collection.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | import numpy as np 3 | 4 | from source.preprocessing.motion.motion_collection import MotionCollection 5 | from source.preprocessing.interval import Interval 6 | 7 | 8 | class TestMotionCollection(TestCase): 9 | def test_properties(self): 10 | subject_id = "subjectA" 11 | motion_collection = MotionCollection(subject_id=subject_id, 12 | data=np.array([[1, 2, 3], [4, 5, 6]])) 13 | 14 | self.assertEqual(subject_id, motion_collection.subject_id) 15 | self.assertEqual(np.array([1, 4]).tolist(), motion_collection.timestamps.tolist()) 16 | self.assertEqual(np.array([[2, 3], [5, 6]]).tolist(), motion_collection.values.tolist()) 17 | 18 | def test_get_interval(self): 19 | motion_collection = MotionCollection(subject_id="subjectA", 20 | data=np.array([[1, 2, 3], [4, 5, 6]])) 21 | interval = Interval(start_time=1, end_time=4) 22 | self.assertEqual(interval.start_time, motion_collection.get_interval().start_time) 23 | self.assertEqual(interval.end_time, motion_collection.get_interval().end_time) 24 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/psg/psg_converter.py: -------------------------------------------------------------------------------- 1 | from source.sleep_stage import SleepStage 2 | 3 | 4 | class PSGConverter(object): 5 | strings_to_labels = { 6 | "?": SleepStage.unscored, 7 | "W": SleepStage.wake, 8 | "1": SleepStage.n1, 9 | "N1": SleepStage.n1, 10 | "2": SleepStage.n2, 11 | "N2": SleepStage.n2, 12 | "3": SleepStage.n3, 13 | "N3": SleepStage.n3, 14 | "4": SleepStage.n4, 15 | "N4": SleepStage.n4, 16 | "R": SleepStage.rem, 17 | "M": SleepStage.wake} 18 | 19 | ints_to_labels = { 20 | -1: SleepStage.unscored, 21 | 0: SleepStage.wake, 22 | 1: SleepStage.n1, 23 | 2: SleepStage.n2, 24 | 3: SleepStage.n3, 25 | 4: SleepStage.n4, 26 | 5: SleepStage.rem, 27 | 6: SleepStage.unscored} 28 | 29 | @staticmethod 30 | def get_label_from_string(stage_string): 31 | if stage_string in PSGConverter.strings_to_labels: 32 | return PSGConverter.strings_to_labels[stage_string] 33 | 34 | @staticmethod 35 | def get_label_from_int(stage_int): 36 | if stage_int in PSGConverter.ints_to_labels: 37 | return PSGConverter.ints_to_labels[stage_int] 38 | -------------------------------------------------------------------------------- /data_loader/mesa_fold_builder.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | from sklearn.model_selection import KFold 4 | import csv 5 | total_fold = 16 6 | k_fold = KFold(n_splits=total_fold) 7 | pids = pd.read_csv("../assets/train_test_pid_split.csv") 8 | pids = pids.uids.tolist() 9 | np.random.shuffle(pids) 10 | pids = np.asarray(pids[:31]) 11 | 12 | fold_df = [] # pd.DataFrame(columns=["fold_num", "set_name", "pid"]) 13 | for fold_idx, (train_set, test_set) in enumerate(k_fold.split(pids)): 14 | np.random.shuffle(train_set) 15 | new_train_set = train_set[:int(np.floor(len(train_set)*0.8))] 16 | val_set = train_set[int(np.floor(len(train_set)*0.8)):] 17 | tmp_df_list = [] 18 | for set_type, pid_set in {"train": new_train_set, "val": val_set, "test": test_set}.items(): 19 | tmp_df = pd.DataFrame({"fold_num": [fold_idx]*len(pid_set), 20 | "set_type": [set_type]*len(pid_set), 21 | "pid": pids[pid_set]}) 22 | tmp_df_list.append(tmp_df) 23 | tmp_df_list = pd.concat(tmp_df_list, axis=0, ignore_index=True) 24 | fold_df.append(tmp_df_list) 25 | fold_df = pd.concat(fold_df, axis=0, ignore_index=True) 26 | fold_df.to_csv(r"../assets/mesa_%s_fold.csv" % total_fold, 27 | index=False) 28 | -------------------------------------------------------------------------------- /data_loader/apple_fold_builder.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | from sklearn.model_selection import KFold 4 | import csv 5 | total_fold = 16 6 | k_fold = KFold(n_splits=total_fold) 7 | pids_1 = pd.read_csv(r"../assets/apple_watch_test.txt", header=None) 8 | pids_2 = pd.read_csv(r"../assets/apple_watch_train.txt", header=None) 9 | pids = pd.concat([pids_1, pids_2], axis=0, ignore_index=True)[0].values 10 | fold_df = [] # pd.DataFrame(columns=["fold_num", "set_name", "pid"]) 11 | for fold_idx, (train_set, test_set) in enumerate(k_fold.split(pids)): 12 | new_train_set = train_set[:int(np.floor(len(train_set)*0.8))] 13 | val_set = train_set[int(np.floor(len(train_set)*0.8)):] 14 | tmp_df_list = [] 15 | for set_type, pid_set in {"train": new_train_set, "val": val_set, "test": test_set}.items(): 16 | tmp_df = pd.DataFrame({"fold_num": [fold_idx]*len(pid_set), 17 | "set_type": [set_type]*len(pid_set), 18 | "pid": pids[pid_set]}) 19 | tmp_df_list.append(tmp_df) 20 | tmp_df_list = pd.concat(tmp_df_list, axis=0, ignore_index=True) 21 | fold_df.append(tmp_df_list) 22 | fold_df = pd.concat(fold_df, axis=0, ignore_index=True) 23 | fold_df.to_csv(r"../assets/apple_%s_fold.csv" % total_fold, 24 | index=False) 25 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/heart_rate/test_heart_rate_collection.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | import numpy as np 3 | 4 | from source.preprocessing.heart_rate.heart_rate_collection import HeartRateCollection 5 | from source.preprocessing.interval import Interval 6 | 7 | 8 | class TestHeartRateCollection(TestCase): 9 | def test_properties(self): 10 | subject_id = "subjectA" 11 | heart_rate_collection = HeartRateCollection(subject_id=subject_id, 12 | data=np.array([[1, 2, 3], [4, 5, 6]])) 13 | 14 | self.assertEqual(subject_id, heart_rate_collection.subject_id) 15 | self.assertEqual(np.array([1, 4]).tolist(), heart_rate_collection.timestamps.tolist()) 16 | self.assertEqual(np.array([[2, 3], [5, 6]]).tolist(), heart_rate_collection.values.tolist()) 17 | 18 | def test_get_interval(self): 19 | heart_rate_collection = HeartRateCollection(subject_id="subjectA", 20 | data=np.array([[1, 2, 3], [4, 5, 6]])) 21 | interval = Interval(start_time=1, end_time=4) 22 | self.assertEqual(interval.start_time, heart_rate_collection.get_interval().start_time) 23 | self.assertEqual(interval.end_time, heart_rate_collection.get_interval().end_time) 24 | -------------------------------------------------------------------------------- /tests/test_data_loader.py: -------------------------------------------------------------------------------- 1 | from data_loader.TorchFrameDataLoader import get_apple_loocv_ids 2 | from data_loader.raw_data_loader import * 3 | 4 | 5 | def test_raw_apple_df_loader(): 6 | cfg = Config() 7 | results = get_raw_test_df(3509524, cfg, 'apple_raw', 3, 100) 8 | assert results.shape[0] == 415 9 | 10 | 11 | def test_get_raw_apple_dataset_by_id(): 12 | cfg = Config() 13 | data_loader = get_raw_dataloader_by_id(46343, cfg, False, 100,'apple_raw', 100, 1) 14 | total_samples = 0 15 | for (acc, hrv, y, idx) in data_loader: 16 | total_samples += len(idx) 17 | print("total num of samples is:") 18 | assert True 19 | 20 | 21 | def test_get_apple_loocv_ids(): 22 | cfg = Config() 23 | for fold in np.arange(16): 24 | train_id, val_id, test_id = get_apple_loocv_ids(cfg, fold) 25 | if fold != 15: 26 | assert len(train_id) + len(val_id) == 15*2-1 27 | else: 28 | assert len(train_id) + len(val_id) == 15*2 29 | 30 | def test_get_dis_dataloader(): 31 | cfg = Config() 32 | tr, va, te = get_win_train_test_val_dis_loader(cfg, 64, 100, 3, "mesa", 0, 50, 'sleepage5c') 33 | assert tr.batch_size == 64 34 | 35 | 36 | def test_get_dis_id_testdf(): 37 | cfg = Config() 38 | df = get_dis_test_df(cfg, "mesa", 3) 39 | print(df.shape) 40 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/psg/psg_raw_data_collection.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from source.preprocessing.interval import Interval 4 | from source.sleep_stage import SleepStage 5 | 6 | 7 | class PSGRawDataCollection(object): 8 | def __init__(self, subject_id, data: [SleepStage]): 9 | self.subject_id = subject_id 10 | self.data = data 11 | 12 | def get_np_array(self): 13 | number_of_epochs = len(self.data) 14 | array = np.zeros((number_of_epochs, 2)) 15 | 16 | for index in range(number_of_epochs): 17 | stage_item = self.data[index] 18 | array[index, 0] = stage_item.epoch.timestamp 19 | array[index, 1] = stage_item.stage.value 20 | 21 | return array 22 | 23 | def get_interval(self): 24 | number_of_epochs = len(self.data) 25 | min_timestamp = 1e15 26 | max_timestamp = -1 27 | 28 | for index in range(number_of_epochs): 29 | stage_item = self.data[index] 30 | if stage_item.epoch.timestamp < min_timestamp: 31 | min_timestamp = stage_item.epoch.timestamp 32 | if stage_item.epoch.timestamp > max_timestamp: 33 | max_timestamp = stage_item.epoch.timestamp 34 | 35 | return Interval(start_time=min_timestamp, end_time=max_timestamp) 36 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/psg/compumedics_processor.py: -------------------------------------------------------------------------------- 1 | import csv 2 | 3 | from source.preprocessing.epoch import Epoch 4 | from source.preprocessing.psg.psg_converter import PSGConverter 5 | from source.preprocessing.psg.stage_item import StageItem 6 | from source.preprocessing.time_service import TimeService 7 | 8 | 9 | class CompumedicsProcessor(object): 10 | DT_COMPUMEDICS_PSG = 30 11 | 12 | @staticmethod 13 | def parse(report_summary, psg_stage_path): 14 | data = [] 15 | score_strings = [] 16 | with open(psg_stage_path, 'rt') as csv_file: 17 | file_reader = csv.reader(csv_file, delimiter=',', quotechar='|') 18 | 19 | for row in file_reader: 20 | score_strings.append(row[0]) 21 | 22 | start_epoch = report_summary.start_epoch 23 | start_time_seconds = TimeService.get_start_epoch_timestamp(report_summary) 24 | 25 | for epoch_index in range(start_epoch - 1, len(score_strings)): 26 | timestamp = start_time_seconds + (epoch_index - start_epoch + 1) * CompumedicsProcessor.DT_COMPUMEDICS_PSG 27 | epoch = Epoch(timestamp=timestamp, index=epoch_index + 1) 28 | 29 | stage = PSGConverter.get_label_from_string(score_strings[epoch_index]) 30 | data.append(StageItem(epoch=epoch, stage=stage)) 31 | 32 | return data 33 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/psg/test_psg_raw_data_collection.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.preprocessing.epoch import Epoch 4 | from source.preprocessing.interval import Interval 5 | from source.preprocessing.psg.psg_raw_data_collection import PSGRawDataCollection 6 | from source.preprocessing.psg.stage_item import StageItem 7 | from source.sleep_stage import SleepStage 8 | from test.test_helper import TestHelper 9 | import numpy as np 10 | 11 | 12 | class TestPSGRawDataCollection(TestCase): 13 | 14 | def test_constructor_and_get_functions(self): 15 | subject_id = "subject9000" 16 | data = [StageItem(Epoch(timestamp=2, index=2), stage=SleepStage.rem), 17 | StageItem(Epoch(timestamp=200, index=4), stage=SleepStage.n1)] 18 | expected_interval = Interval(start_time=2, end_time=200) 19 | expected_np_array = np.array([[2, 5], [200, 1]]) 20 | 21 | psg_raw_data_collection = PSGRawDataCollection(subject_id=subject_id, data=data) 22 | 23 | self.assertListEqual(data, psg_raw_data_collection.data) 24 | self.assertEqual(subject_id, psg_raw_data_collection.subject_id) 25 | 26 | TestHelper.assert_models_equal(self, expected_interval, psg_raw_data_collection.get_interval()) 27 | self.assertListEqual(expected_np_array.tolist(), psg_raw_data_collection.get_np_array().tolist()) 28 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/performance/test_curve_performance_objects.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | 5 | from source.analysis.performance.curve_performance import ROCPerformance, PrecisionRecallPerformance 6 | 7 | 8 | class TestROCPerformance(TestCase): 9 | 10 | def test_properties(self): 11 | true_positive_rates = np.array([1, 2]) 12 | false_positive_rates = np.array([3, 4]) 13 | roc_performance = ROCPerformance(true_positive_rates=true_positive_rates, 14 | false_positive_rates=false_positive_rates) 15 | 16 | self.assertListEqual(true_positive_rates.tolist(), roc_performance.true_positive_rates.tolist()) 17 | self.assertListEqual(false_positive_rates.tolist(), roc_performance.false_positive_rates.tolist()) 18 | 19 | 20 | class TestPRPerformance(TestCase): 21 | 22 | def test_properties(self): 23 | precisions = np.array([1, 2]) 24 | recalls = np.array([3, 4]) 25 | precision_recall_performance = PrecisionRecallPerformance(precisions=precisions, 26 | recalls=recalls) 27 | 28 | self.assertListEqual(precisions.tolist(), precision_recall_performance.precisions.tolist()) 29 | self.assertListEqual(recalls.tolist(), precision_recall_performance.recalls.tolist()) 30 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/test_time_service.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.preprocessing.psg.psg_file_type import PSGFileType 4 | from source.preprocessing.psg.report_summary import ReportSummary 5 | from source.preprocessing.time_service import TimeService 6 | 7 | 8 | class TestTimeService(TestCase): 9 | 10 | def test_build_from_report_summary_after_midnight(self): 11 | start_time = "2:43:31" 12 | start_epoch = 1235644 13 | study_date = "4/10/19" 14 | report_summary = ReportSummary(study_date=study_date, start_epoch=start_epoch, start_time=start_time, 15 | file_type=PSGFileType.Vitaport) 16 | expected_epoch = 1554965011 17 | epoch = TimeService.get_start_epoch_timestamp(report_summary) 18 | self.assertEqual(expected_epoch, epoch) 19 | 20 | def test_build_from_report_summary_before_midnight(self): 21 | start_time = "11:43:31 PM" 22 | start_epoch = 1235644 23 | study_date = "4/10/2019" 24 | report_summary = ReportSummary(study_date=study_date, start_epoch=start_epoch, start_time=start_time, 25 | file_type=PSGFileType.Compumedics) 26 | expected_epoch = 1554954211 27 | epoch = TimeService.get_start_epoch_timestamp(report_summary) 28 | self.assertEqual(expected_epoch, epoch) 29 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/activity_count/test_activity_count_collection.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import numpy as np 4 | from source.preprocessing.activity_count.activity_count_collection import ActivityCountCollection 5 | from source.preprocessing.interval import Interval 6 | 7 | 8 | class TestActivityCountCollection(TestCase): 9 | 10 | def test_properties(self): 11 | subject_id = "subjectA" 12 | activity_count_collection = ActivityCountCollection(subject_id=subject_id, 13 | data=np.array([[1, 2, 3], [4, 5, 6]])) 14 | 15 | self.assertEqual(subject_id, activity_count_collection.subject_id) 16 | self.assertEqual(np.array([1, 4]).tolist(), activity_count_collection.timestamps.tolist()) 17 | self.assertEqual(np.array([[2, 3], [5, 6]]).tolist(), activity_count_collection.values.tolist()) 18 | 19 | def test_get_interval(self): 20 | activity_count_collection = ActivityCountCollection(subject_id="subjectA", 21 | data=np.array([[1, 2, 3], [4, 5, 6]])) 22 | interval = Interval(start_time=1, end_time=4) 23 | self.assertEqual(interval.start_time, activity_count_collection.get_interval().start_time) 24 | self.assertEqual(interval.end_time, activity_count_collection.get_interval().end_time) 25 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/test_feature_builder.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | 3 | from source import utils 4 | from source.preprocessing.epoch import Epoch 5 | from source.preprocessing.feature_builder import FeatureBuilder 6 | from source.preprocessing.psg.psg_service import PSGService 7 | from source.preprocessing.raw_data_processor import RawDataProcessor 8 | from test.test_helper import TestHelper 9 | 10 | 11 | class TestFeatureBuilder(TestCase): 12 | 13 | @mock.patch.object(FeatureBuilder, 'build_from_time') 14 | @mock.patch.object(FeatureBuilder, 'build_from_wearables') 15 | @mock.patch.object(FeatureBuilder, 'build_labels') 16 | @mock.patch.object(RawDataProcessor, 'get_valid_epochs') 17 | def test_builds_features(self, mock_get_valid_epochs, mock_build_labels, mock_build_from_wearables, 18 | mock_build_from_time): 19 | subject_id = "subjectA" 20 | mock_get_valid_epochs.return_value = valid_epochs = [Epoch(timestamp=1, index=1000)] 21 | 22 | FeatureBuilder.build(subject_id) 23 | 24 | mock_get_valid_epochs.assert_called_once_with(subject_id) 25 | mock_build_labels.assert_called_once_with(subject_id, valid_epochs) 26 | mock_build_from_wearables.assert_called_once_with(subject_id, valid_epochs) 27 | mock_build_from_time.assert_called_once_with(subject_id, valid_epochs) 28 | -------------------------------------------------------------------------------- /models/baseline_models.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | import torch 3 | 4 | def num_flat_features(x): 5 | size = x.size()[1:] 6 | num_features = 1 7 | for s in size: 8 | num_features *= s 9 | return num_features 10 | 11 | 12 | class CNN(nn.Module): 13 | def __init__(self, in_channels, num_classes, init_weights: bool = True): 14 | super(CNN, self).__init__() 15 | self.features = nn.Sequential( 16 | nn.Conv1d(in_channels, 64, kernel_size=2), 17 | nn.ReLU(), 18 | ) 19 | 20 | self.classifier = nn.Sequential( 21 | nn.Linear(6400, num_classes), 22 | # nn.ReLU(), 23 | ) 24 | 25 | def forward(self, x): 26 | x = self.features(x) 27 | x = torch.flatten(x, 1) 28 | x = self.classifier(x) 29 | return x 30 | 31 | def _initialize_weights(self) -> None: 32 | for m in self.modules(): 33 | if isinstance(m, nn.Conv2d): 34 | nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') 35 | if m.bias is not None: 36 | nn.init.constant_(m.bias, 0) 37 | elif isinstance(m, nn.BatchNorm2d): 38 | nn.init.constant_(m.weight, 1) 39 | nn.init.constant_(m.bias, 0) 40 | elif isinstance(m, nn.Linear): 41 | nn.init.normal_(m.weight, 0, 0.01) 42 | nn.init.constant_(m.bias, 0) 43 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/motion/test_motion_feature_service.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | from unittest.mock import MagicMock 3 | import numpy as np 4 | from source.constants import Constants 5 | from source.preprocessing.motion.motion_feature_service import MotionFeatureService 6 | 7 | 8 | class TestMotionFeatureService(TestCase): 9 | 10 | @mock.patch('source.preprocessing.motion.motion_feature_service.pd') 11 | def test_load(self, mock_pd): 12 | mock_pd.read_csv.return_value = mock_return = MagicMock() 13 | mock_return.values = expected_return = np.array([1, 2, 3, 4, 5]) 14 | actual_returned_value = MotionFeatureService.load("subjectA") 15 | 16 | self.assertListEqual(expected_return.tolist(), actual_returned_value.tolist()) 17 | mock_pd.read_csv.assert_called_once_with(str(MotionFeatureService.get_path("subjectA"))) 18 | 19 | def test_get_path(self): 20 | expected_path = Constants.FEATURE_FILE_PATH.joinpath("subjectA" + '_motion_feature.out') 21 | 22 | self.assertEqual(expected_path, MotionFeatureService.get_path("subjectA")) 23 | 24 | @mock.patch('source.preprocessing.motion.motion_feature_service.np') 25 | def test_write(self, mock_np): 26 | feature_to_write = np.array([1, 2, 3, 4]) 27 | subject_id = "subjectA" 28 | MotionFeatureService.write(subject_id, feature_to_write) 29 | 30 | mock_np.savetxt.assert_called_once_with(MotionFeatureService.get_path(subject_id), feature_to_write, 31 | fmt='%f') 32 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/performance/test_sleep_wake_performance.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.analysis.performance.epoch_performance import SleepWakePerformance 4 | 5 | 6 | class TestSleepWakePerformance(TestCase): 7 | 8 | def test_has_properties(self): 9 | accuracy = 0.9 10 | wake_correct = 0.2 11 | sleep_correct = 0.99 12 | kappa = 0.5 13 | auc = 0.8 14 | sleep_predictive_value = 0.3 15 | wake_predictive_value = 0.2 16 | sleep_wake_performance = SleepWakePerformance(accuracy=accuracy, 17 | wake_correct=wake_correct, 18 | sleep_correct=sleep_correct, 19 | kappa=kappa, 20 | auc=auc, 21 | sleep_predictive_value=sleep_predictive_value, 22 | wake_predictive_value=wake_predictive_value) 23 | self.assertEqual(sleep_wake_performance.accuracy, accuracy) 24 | self.assertEqual(sleep_wake_performance.wake_correct, wake_correct) 25 | self.assertEqual(sleep_wake_performance.sleep_correct, sleep_correct) 26 | self.assertEqual(sleep_wake_performance.kappa, kappa) 27 | self.assertEqual(sleep_wake_performance.auc, auc) 28 | self.assertEqual(sleep_wake_performance.sleep_predictive_value, sleep_predictive_value) 29 | self.assertEqual(sleep_wake_performance.wake_predictive_value, wake_predictive_value) 30 | 31 | 32 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/test_helper.py: -------------------------------------------------------------------------------- 1 | import os 2 | from builtins import FileNotFoundError 3 | from unittest import TestCase 4 | 5 | from numpy.core.multiarray import ndarray 6 | 7 | 8 | class TestHelper(TestCase): 9 | @staticmethod 10 | def remove_file(path): 11 | try: 12 | os.remove(str(path.resolve())) 13 | except OSError or FileNotFoundError: 14 | pass 15 | 16 | @staticmethod 17 | def assert_models_equal(test_case, object1, object2): 18 | 19 | for key in object1.__dict__: 20 | if not type(object1.__dict__[key]) == (type(object2.__dict__[key])): 21 | test_case.fail('Types do not match') 22 | 23 | if key not in object2.__dict__: 24 | test_case.fail("Missing Key") 25 | try: 26 | value1_as_float = float(object1.__dict__[key]) 27 | value2_as_float = float(object2.__dict__[key]) 28 | if abs(value1_as_float - value2_as_float) > 0.0000000001: 29 | test_case.fail("Float values do not match. " + str(value1_as_float) + " does not equal " 30 | + str(value2_as_float)) 31 | 32 | except ValueError: 33 | test_case.assertEqual(object1.__dict__[key], object2.__dict__[key]) 34 | except TypeError: 35 | 36 | if type(object1.__dict__[key]) == ndarray: 37 | test_case.assertListEqual(object1.__dict__[key].tolist(), object2.__dict__[key].tolist()) 38 | 39 | if type(object1.__dict__[key]) == list: 40 | test_case.assertListEqual(object1.__dict__[key], object2.__dict__[key]) 41 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/setup/test_train_test_splitter.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.analysis.setup.data_split import DataSplit 4 | from source.analysis.setup.train_test_splitter import TrainTestSplitter 5 | from test_helper import TestHelper 6 | 7 | 8 | class TestTrainTestSplitter(TestCase): 9 | 10 | def test_leave_one_out(self): 11 | subject_ids = ["subjectA", "subjectB", "subjectC"] 12 | results = TrainTestSplitter.leave_one_out(subject_ids) 13 | 14 | TestHelper.assert_models_equal(self, DataSplit(training_set=["subjectB", "subjectC"], testing_set=["subjectA"]), 15 | results[0]) 16 | TestHelper.assert_models_equal(self, DataSplit(training_set=["subjectA", "subjectC"], testing_set=["subjectB"]), 17 | results[1]) 18 | TestHelper.assert_models_equal(self, DataSplit(training_set=["subjectA", "subjectB"], testing_set=["subjectC"]), 19 | results[2]) 20 | 21 | def test_by_fraction(self): 22 | subject_ids = ["subjectA", "subjectB", "subjectC", "subjectD", "subjectE", "subjectF", "subjectG", "subjectH"] 23 | results = TrainTestSplitter.by_fraction(subject_ids=subject_ids, test_fraction=0.25, number_of_splits=5) 24 | 25 | self.assertEqual(5, len(results)) 26 | training_set = results[3].training_set 27 | testing_set = results[3].testing_set 28 | 29 | self.assertEqual(len(training_set), 6) 30 | self.assertEqual(len(testing_set), 2) 31 | 32 | unique_elements = list(set(training_set) & set(testing_set)) 33 | 34 | self.assertEqual(len(unique_elements), 0) 35 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/classification/test_classifier_summary.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from sklearn.linear_model import LogisticRegression 4 | 5 | from source.analysis.classification.classifier_summary import ClassifierSummary 6 | from source.analysis.performance.raw_performance import RawPerformance 7 | from source.analysis.setup.attributed_classifier import AttributedClassifier 8 | from source.analysis.setup.feature_type import FeatureType 9 | 10 | import numpy as np 11 | 12 | from test.test_helper import TestHelper 13 | 14 | 15 | class TestClassifierSummary(TestCase): 16 | def test_properties(self): 17 | attributed_classifier = AttributedClassifier(name="Logistic Regression", classifier=LogisticRegression()) 18 | 19 | performance_dictionary = {(FeatureType.count, FeatureType.cosine): 20 | [RawPerformance(true_labels=np.array([1, 0, 1, 0]), 21 | class_probabilities=np.array([0.1, 0.9]))], 22 | FeatureType.count: 23 | [RawPerformance(true_labels=np.array([0, 0, 1, 0]), 24 | class_probabilities=np.array([0.9, 0.1]))] 25 | } 26 | 27 | classifier_summary = ClassifierSummary(attributed_classifier=attributed_classifier, 28 | performance_dictionary=performance_dictionary) 29 | 30 | TestHelper.assert_models_equal(self, attributed_classifier, classifier_summary.attributed_classifier) 31 | self.assertDictEqual(performance_dictionary, classifier_summary.performance_dictionary) -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/performance/test_sleep_metrics_calculator.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | import numpy as np 3 | 4 | from source.analysis.performance.sleep_metrics_calculator import SleepMetricsCalculator 5 | 6 | 7 | class TestSleepMetricsCalculator(TestCase): 8 | 9 | def test_get_tst(self): 10 | labeled_night = np.array([0, 0, 1, 1, 1]) 11 | tst = SleepMetricsCalculator.get_tst(labeled_night) 12 | 13 | self.assertEqual(1.5, tst) 14 | 15 | def test_get_waso(self): 16 | labeled_night = np.array([0, 0, 0, 0, 1, 2, 1, 0, 0, 0]) 17 | waso = SleepMetricsCalculator.get_wake_after_sleep_onset(labeled_night) 18 | 19 | self.assertEqual(1.5, waso) 20 | 21 | def test_get_sleep_efficiency(self): 22 | labeled_night = np.array([0, 0, 1, 1, 2, 1, 2, 0, 1, 1]) 23 | sleep_efficiency = SleepMetricsCalculator.get_sleep_efficiency(labeled_night) 24 | self.assertEqual(0.7, sleep_efficiency) 25 | 26 | def test_get_sleep_onset_latency(self): 27 | labeled_night = np.array([0, 0, 0, 1, 2, 2, 1, 0, 1, 1]) 28 | sleep_onset_latency = SleepMetricsCalculator.get_sleep_onset_latency(labeled_night) 29 | self.assertEqual(1.5, sleep_onset_latency) 30 | 31 | def test_time_in_rem(self): 32 | labeled_night = np.array([0, 0, 0, 1, 2, 2, 1, 0, 1, 1]) 33 | time_in_rem = SleepMetricsCalculator.get_time_in_rem(labeled_night) 34 | self.assertEqual(1, time_in_rem) 35 | 36 | def test_time_in_nrem(self): 37 | labeled_night = np.array([0, 0, 0, 1, 2, 2, 1, 0, 0, 1]) 38 | time_in_nrem = SleepMetricsCalculator.get_time_in_nrem(labeled_night) 39 | self.assertEqual(1.5, time_in_nrem) 40 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/outputs/features/7749105_hr_feature.out: -------------------------------------------------------------------------------- 1 | 0.754734 2 | 0.752762 3 | 0.736219 4 | 0.670241 5 | 0.617347 6 | 0.599676 7 | 0.592784 8 | 0.567579 9 | 0.455140 10 | 0.319274 11 | 0.213509 12 | 0.202712 13 | 0.220264 14 | 0.290018 15 | 0.323068 16 | 0.323042 17 | 0.319111 18 | 0.320566 19 | 0.327498 20 | 0.332809 21 | 0.334540 22 | 0.336352 23 | 0.341850 24 | 0.345404 25 | 0.351081 26 | 0.365114 27 | 0.378766 28 | 0.379213 29 | 0.385107 30 | 0.384208 31 | 0.379085 32 | 0.383141 33 | 0.391661 34 | 0.388961 35 | 0.390043 36 | 0.396801 37 | 0.383723 38 | 0.354812 39 | 0.304333 40 | 0.245155 41 | 0.187943 42 | 0.156395 43 | 0.150357 44 | 0.137044 45 | 0.135964 46 | 0.146478 47 | 0.156654 48 | 0.158207 49 | 0.159819 50 | 0.168426 51 | 0.167746 52 | 0.224693 53 | 0.330479 54 | 0.408527 55 | 0.446124 56 | 0.453839 57 | 0.461365 58 | 0.504165 59 | 0.556505 60 | 0.559009 61 | 0.563768 62 | 0.580118 63 | 0.597282 64 | 0.611965 65 | 0.606691 66 | 0.594405 67 | 0.581210 68 | 0.572799 69 | 0.568049 70 | 0.552985 71 | 0.547874 72 | 0.552961 73 | 0.532989 74 | 0.498974 75 | 0.468103 76 | 0.458022 77 | 0.451733 78 | 0.399948 79 | 0.296742 80 | 0.289329 81 | 0.295525 82 | 0.277437 83 | 0.247373 84 | 0.205021 85 | 0.218154 86 | 0.259075 87 | 0.275187 88 | 0.276053 89 | 0.289018 90 | 0.312291 91 | 0.341185 92 | 0.353819 93 | 0.351333 94 | 0.344484 95 | 0.346470 96 | 0.347388 97 | 0.350168 98 | 0.351372 99 | 0.352055 100 | 0.352295 101 | 0.351010 102 | 0.351528 103 | 0.350972 104 | 0.347860 105 | 0.330910 106 | 0.301787 107 | 0.286354 108 | 0.286295 109 | 0.278762 110 | 0.255961 111 | 0.213706 112 | 0.175966 113 | 0.164336 114 | 0.176073 115 | 0.185922 116 | 0.196421 117 | 0.205585 118 | 0.215495 119 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/outputs/features/7749105_psg_labels.out: -------------------------------------------------------------------------------- 1 | 0.000000 2 | 0.000000 3 | 0.000000 4 | 0.000000 5 | 0.000000 6 | 0.000000 7 | 0.000000 8 | 0.000000 9 | 0.000000 10 | 0.000000 11 | 0.000000 12 | 0.000000 13 | 0.000000 14 | 0.000000 15 | 0.000000 16 | 0.000000 17 | 0.000000 18 | 0.000000 19 | 0.000000 20 | 0.000000 21 | 0.000000 22 | 0.000000 23 | 0.000000 24 | 0.000000 25 | 0.000000 26 | 0.000000 27 | 0.000000 28 | 0.000000 29 | 0.000000 30 | 0.000000 31 | 0.000000 32 | 0.000000 33 | 0.000000 34 | 0.000000 35 | 0.000000 36 | 1.000000 37 | 1.000000 38 | 1.000000 39 | 1.000000 40 | 1.000000 41 | 2.000000 42 | 2.000000 43 | 2.000000 44 | 2.000000 45 | 2.000000 46 | 2.000000 47 | 2.000000 48 | 2.000000 49 | 2.000000 50 | 2.000000 51 | 2.000000 52 | 2.000000 53 | 2.000000 54 | 1.000000 55 | 2.000000 56 | 2.000000 57 | 2.000000 58 | 2.000000 59 | 2.000000 60 | 2.000000 61 | 2.000000 62 | 2.000000 63 | 2.000000 64 | 0.000000 65 | 0.000000 66 | 0.000000 67 | 1.000000 68 | 1.000000 69 | 2.000000 70 | 2.000000 71 | 2.000000 72 | 2.000000 73 | 2.000000 74 | 2.000000 75 | 2.000000 76 | 2.000000 77 | 2.000000 78 | 2.000000 79 | 2.000000 80 | 2.000000 81 | 2.000000 82 | 2.000000 83 | 2.000000 84 | 2.000000 85 | 2.000000 86 | 2.000000 87 | 2.000000 88 | 2.000000 89 | 3.000000 90 | 3.000000 91 | 3.000000 92 | 3.000000 93 | 3.000000 94 | 3.000000 95 | 3.000000 96 | 3.000000 97 | 3.000000 98 | 2.000000 99 | 2.000000 100 | 2.000000 101 | 2.000000 102 | 2.000000 103 | 2.000000 104 | 2.000000 105 | 2.000000 106 | 2.000000 107 | 2.000000 108 | 3.000000 109 | 3.000000 110 | 2.000000 111 | 2.000000 112 | 2.000000 113 | 2.000000 114 | 2.000000 115 | 2.000000 116 | 2.000000 117 | 2.000000 118 | 3.000000 119 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/outputs/features/7749105_time_feature.out: -------------------------------------------------------------------------------- 1 | 0.000000 2 | 0.008333 3 | 0.016667 4 | 0.025000 5 | 0.033333 6 | 0.041667 7 | 0.050000 8 | 0.058333 9 | 0.066667 10 | 0.075000 11 | 0.083333 12 | 0.091667 13 | 0.100000 14 | 0.108333 15 | 0.116667 16 | 0.125000 17 | 0.133333 18 | 0.141667 19 | 0.150000 20 | 0.158333 21 | 0.166667 22 | 0.175000 23 | 0.183333 24 | 0.191667 25 | 0.200000 26 | 0.208333 27 | 0.216667 28 | 0.225000 29 | 0.233333 30 | 0.241667 31 | 0.250000 32 | 0.258333 33 | 0.266667 34 | 0.275000 35 | 0.283333 36 | 0.291667 37 | 0.300000 38 | 0.308333 39 | 0.316667 40 | 0.325000 41 | 0.333333 42 | 0.341667 43 | 0.350000 44 | 0.358333 45 | 0.366667 46 | 0.375000 47 | 0.383333 48 | 0.391667 49 | 0.400000 50 | 0.408333 51 | 0.416667 52 | 0.425000 53 | 0.433333 54 | 0.441667 55 | 0.450000 56 | 0.458333 57 | 0.466667 58 | 0.475000 59 | 0.483333 60 | 0.491667 61 | 0.500000 62 | 0.508333 63 | 0.516667 64 | 0.525000 65 | 0.533333 66 | 0.541667 67 | 0.550000 68 | 0.558333 69 | 0.566667 70 | 0.575000 71 | 0.583333 72 | 0.591667 73 | 0.600000 74 | 0.608333 75 | 0.616667 76 | 0.625000 77 | 0.633333 78 | 0.641667 79 | 0.650000 80 | 0.658333 81 | 0.666667 82 | 0.675000 83 | 0.683333 84 | 0.691667 85 | 0.700000 86 | 0.708333 87 | 0.716667 88 | 0.725000 89 | 0.733333 90 | 0.741667 91 | 0.750000 92 | 0.758333 93 | 0.766667 94 | 0.775000 95 | 0.783333 96 | 0.791667 97 | 0.800000 98 | 0.808333 99 | 0.816667 100 | 0.825000 101 | 0.833333 102 | 0.841667 103 | 0.850000 104 | 0.858333 105 | 0.866667 106 | 0.875000 107 | 0.883333 108 | 0.891667 109 | 0.900000 110 | 0.908333 111 | 0.916667 112 | 0.925000 113 | 0.933333 114 | 0.941667 115 | 0.950000 116 | 0.958333 117 | 0.966667 118 | 0.975000 119 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/psg/psg_label_service.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | from source.constants import Constants 5 | from source.preprocessing.psg.psg_service import PSGService 6 | 7 | 8 | class PSGLabelService(object): 9 | @staticmethod 10 | def load(subject_id): 11 | psg_label_path = PSGLabelService.get_path(subject_id) 12 | feature = pd.read_csv(str(psg_label_path)).values 13 | return feature 14 | 15 | @staticmethod 16 | def get_path(subject_id): 17 | return Constants.FEATURE_FILE_PATH.joinpath(subject_id + '_psg_labels.out') 18 | 19 | @staticmethod 20 | def get_path_mesa_protocol(subject_id): 21 | return Constants.FEATURE_FILE_PATH.joinpath("mesa", subject_id + '_psg_labels.out') 22 | 23 | @staticmethod 24 | def build(subject_id, valid_epochs): 25 | psg_array = PSGService.load_cropped_array(subject_id) 26 | labels = [] # the following code interpolate the missing data by learning pattern from the x_given and y_given 27 | epoch_timestamp = [] 28 | for epoch in valid_epochs: 29 | value = np.interp(epoch.timestamp, psg_array[:, 0], psg_array[:, 1]) 30 | labels.append(value) 31 | epoch_timestamp.append(epoch.timestamp) 32 | return np.array(np.stack([epoch_timestamp, labels], axis=1)) 33 | 34 | @staticmethod 35 | def write(subject_id, labels): 36 | psg_labels_path = PSGLabelService.get_path(subject_id) 37 | np.savetxt(psg_labels_path, labels, fmt='%f') 38 | 39 | @staticmethod 40 | def write_mesa_protocol(subject_id, labels): 41 | psg_labels_path = PSGLabelService.get_path_mesa_protocol(subject_id) 42 | np.savetxt(psg_labels_path, labels, fmt='%f') 43 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/outputs/features/7749105_circadian_feature.out: -------------------------------------------------------------------------------- 1 | -0.000000 2 | 0.003165 3 | 0.006329 4 | 0.009494 5 | 0.012659 6 | 0.015824 7 | 0.018988 8 | 0.022111 9 | 0.025234 10 | 0.028357 11 | 0.031480 12 | 0.034603 13 | 0.037726 14 | 0.040848 15 | 0.043971 16 | 0.047094 17 | 0.050217 18 | 0.053340 19 | 0.056463 20 | 0.059543 21 | 0.062623 22 | 0.065702 23 | 0.068782 24 | 0.071862 25 | 0.074942 26 | 0.078022 27 | 0.081101 28 | 0.084181 29 | 0.087261 30 | 0.090341 31 | 0.093421 32 | 0.096456 33 | 0.099491 34 | 0.102527 35 | 0.105562 36 | 0.108598 37 | 0.111633 38 | 0.114669 39 | 0.117704 40 | 0.120739 41 | 0.123775 42 | 0.126810 43 | 0.129846 44 | 0.132835 45 | 0.135825 46 | 0.138815 47 | 0.141805 48 | 0.144795 49 | 0.147784 50 | 0.150774 51 | 0.153764 52 | 0.156754 53 | 0.159744 54 | 0.162733 55 | 0.165723 56 | 0.168666 57 | 0.171609 58 | 0.174552 59 | 0.177495 60 | 0.180438 61 | 0.183381 62 | 0.186324 63 | 0.189267 64 | 0.192210 65 | 0.195153 66 | 0.198096 67 | 0.201039 68 | 0.203933 69 | 0.206828 70 | 0.209723 71 | 0.212618 72 | 0.215513 73 | 0.218408 74 | 0.221303 75 | 0.224198 76 | 0.227092 77 | 0.229987 78 | 0.232882 79 | 0.235777 80 | 0.238623 81 | 0.241468 82 | 0.244314 83 | 0.247159 84 | 0.250005 85 | 0.252851 86 | 0.255696 87 | 0.258542 88 | 0.261387 89 | 0.264233 90 | 0.267079 91 | 0.269924 92 | 0.272719 93 | 0.275515 94 | 0.278310 95 | 0.281105 96 | 0.283900 97 | 0.286695 98 | 0.289490 99 | 0.292285 100 | 0.295081 101 | 0.297876 102 | 0.300671 103 | 0.303466 104 | 0.306210 105 | 0.308953 106 | 0.311697 107 | 0.314440 108 | 0.317184 109 | 0.319927 110 | 0.322671 111 | 0.325414 112 | 0.328158 113 | 0.330901 114 | 0.333645 115 | 0.336389 116 | 0.339079 117 | 0.341770 118 | 0.344461 119 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/outputs/features/7749105_count_feature.out: -------------------------------------------------------------------------------- 1 | 40.949911 2 | 26.398165 3 | 13.450962 4 | 6.288792 5 | 3.896062 6 | 4.261030 7 | 5.368996 8 | 5.375557 9 | 3.869842 10 | 1.963595 11 | 0.699524 12 | 0.174820 13 | 0.030643 14 | 0.003767 15 | 0.000325 16 | 0.000020 17 | 0.000001 18 | 0.000001 19 | 0.000013 20 | 0.000194 21 | 0.002073 22 | 0.015574 23 | 0.082046 24 | 0.303151 25 | 0.785642 26 | 1.428150 27 | 1.821039 28 | 1.628792 29 | 1.021905 30 | 0.449722 31 | 0.138819 32 | 0.030054 33 | 0.004563 34 | 0.000486 35 | 0.000036 36 | 0.000002 37 | 0.000000 38 | 0.000000 39 | 0.000000 40 | 0.000000 41 | 0.000000 42 | 0.000000 43 | 0.000000 44 | 0.000000 45 | 0.000000 46 | 0.000000 47 | 0.000000 48 | 0.000000 49 | 0.000000 50 | 0.000000 51 | 0.000000 52 | 0.000000 53 | 0.000000 54 | 0.000002 55 | 0.000042 56 | 0.000689 57 | 0.007854 58 | 0.062795 59 | 0.352099 60 | 1.384622 61 | 3.819022 62 | 7.388360 63 | 10.026103 64 | 9.543566 65 | 6.372095 66 | 2.984270 67 | 0.980307 68 | 0.225856 69 | 0.036494 70 | 0.004135 71 | 0.000329 72 | 0.000031 73 | 0.000183 74 | 0.001865 75 | 0.013476 76 | 0.068721 77 | 0.247494 78 | 0.629769 79 | 1.132645 80 | 1.440130 81 | 1.294638 82 | 0.822840 83 | 0.369679 84 | 0.117366 85 | 0.026320 86 | 0.004167 87 | 0.000466 88 | 0.000037 89 | 0.000002 90 | 0.000000 91 | 0.000000 92 | 0.000000 93 | 0.000000 94 | 0.000000 95 | 0.000000 96 | 0.000000 97 | 0.000000 98 | 0.000000 99 | 0.000000 100 | 0.000000 101 | 0.000000 102 | 0.000000 103 | 0.000000 104 | 0.000000 105 | 0.000000 106 | 0.000000 107 | 0.000000 108 | 0.000000 109 | 0.000000 110 | 0.000000 111 | 0.000000 112 | 0.000000 113 | 0.000000 114 | 0.000000 115 | 0.000000 116 | 0.000000 117 | 0.000000 118 | 0.000000 119 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/mesa/mesa_actigraphy_service.py: -------------------------------------------------------------------------------- 1 | import csv 2 | 3 | import numpy as np 4 | 5 | from source import utils 6 | from source.preprocessing.activity_count.activity_count_collection import ActivityCountCollection 7 | 8 | 9 | class MesaActigraphyService(object): 10 | @staticmethod 11 | def load_raw(file_id): 12 | line_align = -1 # Find alignment line between PSG and actigraphy 13 | project_root = str(utils.get_project_root()) 14 | 15 | with open(project_root + '/data/mesa/overlap/mesa-actigraphy-psg-overlap.csv') as csv_file: 16 | csv_reader = csv.reader(csv_file, delimiter=',') 17 | next(csv_file) 18 | for row in csv_reader: 19 | if int(row[0]) == int(file_id): 20 | line_align = int(row[1]) 21 | 22 | activity = [] 23 | elapsed_time_counter = 0 24 | 25 | if line_align == -1: # If there was no alignment found 26 | return ActivityCountCollection(subject_id=file_id, data=np.array([[-1], [-1]])) 27 | 28 | with open(project_root + '/data/mesa/actigraphy/mesa-sleep-' + file_id + '.csv') as csv_file: 29 | csv_reader = csv.reader(csv_file, delimiter=',') 30 | next(csv_file) 31 | for row in csv_reader: 32 | if int(row[1]) >= line_align: 33 | if row[4] == '': 34 | activity.append([elapsed_time_counter, np.nan]) 35 | else: 36 | activity.append([elapsed_time_counter, float(row[4])]) 37 | elapsed_time_counter = elapsed_time_counter + 30 38 | 39 | data = np.array(activity) 40 | data = utils.remove_nans(data) 41 | 42 | return ActivityCountCollection(subject_id=file_id, data=data) 43 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/psg/vitaport_processor.py: -------------------------------------------------------------------------------- 1 | import csv 2 | 3 | import numpy as np 4 | 5 | from source.preprocessing.epoch import Epoch 6 | from source.preprocessing.psg.psg_converter import PSGConverter 7 | from source.preprocessing.psg.stage_item import StageItem 8 | from source.preprocessing.time_service import TimeService 9 | 10 | 11 | class VitaportProcessor(object): 12 | DT_TXT_PSG = 10 13 | 14 | @staticmethod 15 | def parse(report_summary, psg_stage_path): 16 | data = [] 17 | with open(psg_stage_path, 'rt') as csv_file: 18 | file_reader = csv.reader(csv_file, delimiter=',', quotechar='|') 19 | count = 0 20 | rows_per_epoch = Epoch.DURATION / VitaportProcessor.DT_TXT_PSG 21 | 22 | for row in file_reader: 23 | if count == 0: 24 | start_time = row[1] 25 | start_score = int(row[0]) 26 | report_summary.start_time = start_time 27 | start_time_seconds = TimeService.get_start_epoch_timestamp(report_summary) 28 | epoch = Epoch(timestamp=start_time_seconds, index=1) 29 | data.append(StageItem(epoch=epoch, stage=PSGConverter.get_label_from_int(start_score))) 30 | 31 | if np.mod(count, rows_per_epoch) == 0 and count != 0: 32 | timestamp = start_time_seconds + count * VitaportProcessor.DT_TXT_PSG 33 | score = int(row[0]) 34 | epoch = Epoch(timestamp=timestamp, 35 | index=(1 + int(np.floor(count / rows_per_epoch)))) 36 | 37 | data.append(StageItem(epoch=epoch, stage=PSGConverter.get_label_from_int(score))) 38 | count = count + 1 39 | 40 | return data 41 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/preprocessing_runner.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from source.analysis.figures.data_plot_builder import DataPlotBuilder 4 | from source.analysis.setup.subject_builder import SubjectBuilder 5 | from source.preprocessing.activity_count.activity_count_service import ActivityCountService 6 | from source.preprocessing.fusion_feature_builder import FusionFeatureBuilder 7 | from source.preprocessing.raw_data_processor import RawDataProcessor 8 | from source.preprocessing.time.circadian_service import CircadianService 9 | from source.preprocessing.feature_builder import FeatureBuilder 10 | 11 | def run_preprocessing(subject_set, clean=False, build=False, method="mesa", hz=1): 12 | start_time = time.time() 13 | 14 | if clean: 15 | for subject in subject_set: 16 | print("Cropping data from subject " + str(subject) + "...") 17 | RawDataProcessor.crop_all(str(subject), hz=hz) 18 | 19 | # ActivityCountService.build_activity_counts() # This uses MATLAB, but has been replaced with a python implementation 20 | # CircadianService.build_circadian_model() # requires MATLAB to run 21 | # CircadianService.build_circadian_mesa() 22 | if build: 23 | for subject in subject_set: 24 | if method == "mesa": 25 | FusionFeatureBuilder.build_mesa_protocol(str(subject), hz) 26 | # FeatureBuilder.build(str(subject)) 27 | else: 28 | FusionFeatureBuilder.build(str(subject)) 29 | end_time = time.time() 30 | print("Execution took " + str((end_time - start_time) / 60) + " minutes") 31 | 32 | 33 | subject_ids = SubjectBuilder.get_all_subject_ids() 34 | run_preprocessing(subject_ids, clean=True, build=True, method="mesa", hz=1) 35 | 36 | # for subject in subject_ids: 37 | # DataPlotBuilder.make_data_demo(subject, True) 38 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/setup/feature_set_service.py: -------------------------------------------------------------------------------- 1 | import seaborn as sns 2 | 3 | from source.analysis.setup.feature_type import FeatureType 4 | 5 | 6 | class FeatureSetService(object): 7 | 8 | @staticmethod 9 | def get_label(feature_set: [FeatureType]): 10 | if set(feature_set) == {FeatureType.count}: 11 | return 'Motion only' 12 | if set(feature_set) == {FeatureType.heart_rate}: 13 | return 'HR only' 14 | if set(feature_set) == {FeatureType.count, FeatureType.heart_rate}: 15 | return 'Motion, HR' 16 | if set(feature_set) == {FeatureType.count, FeatureType.heart_rate, FeatureType.circadian_model}: 17 | return 'Motion, HR, and Clock' 18 | if set(feature_set) == {FeatureType.count, FeatureType.heart_rate, FeatureType.cosine}: 19 | return 'Motion, HR, and Cosine' 20 | if set(feature_set) == {FeatureType.count, FeatureType.heart_rate, FeatureType.time}: 21 | return 'Motion, HR, and Time' 22 | 23 | @staticmethod 24 | def get_color(feature_set: [FeatureType]): 25 | if set(feature_set) == {FeatureType.count}: 26 | return sns.xkcd_rgb["denim blue"] 27 | if set(feature_set) == {FeatureType.heart_rate}: 28 | return sns.xkcd_rgb["yellow orange"] 29 | if set(feature_set) == {FeatureType.count, FeatureType.heart_rate}: 30 | return sns.xkcd_rgb["medium green"] 31 | if set(feature_set) == {FeatureType.count, FeatureType.heart_rate, FeatureType.circadian_model}: 32 | return sns.xkcd_rgb["medium pink"] 33 | if set(feature_set) == {FeatureType.count, FeatureType.heart_rate, FeatureType.cosine}: 34 | return sns.xkcd_rgb["plum"] 35 | if set(feature_set) == {FeatureType.count, FeatureType.heart_rate, FeatureType.time}: 36 | return sns.xkcd_rgb["greyish"] 37 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/outputs/features/7749105_cosine_feature.out: -------------------------------------------------------------------------------- 1 | -0.258819 2 | -0.260926 3 | -0.263031 4 | -0.265135 5 | -0.267238 6 | -0.269340 7 | -0.271440 8 | -0.273540 9 | -0.275637 10 | -0.277734 11 | -0.279829 12 | -0.281923 13 | -0.284015 14 | -0.286106 15 | -0.288196 16 | -0.290285 17 | -0.292372 18 | -0.294457 19 | -0.296542 20 | -0.298624 21 | -0.300706 22 | -0.302786 23 | -0.304864 24 | -0.306941 25 | -0.309017 26 | -0.311091 27 | -0.313164 28 | -0.315235 29 | -0.317305 30 | -0.319373 31 | -0.321439 32 | -0.323505 33 | -0.325568 34 | -0.327630 35 | -0.329691 36 | -0.331750 37 | -0.333807 38 | -0.335863 39 | -0.337917 40 | -0.339969 41 | -0.342020 42 | -0.344069 43 | -0.346117 44 | -0.348163 45 | -0.350207 46 | -0.352250 47 | -0.354291 48 | -0.356330 49 | -0.358368 50 | -0.360404 51 | -0.362438 52 | -0.364470 53 | -0.366501 54 | -0.368530 55 | -0.370557 56 | -0.372583 57 | -0.374607 58 | -0.376629 59 | -0.378649 60 | -0.380667 61 | -0.382683 62 | -0.384698 63 | -0.386711 64 | -0.388722 65 | -0.390731 66 | -0.392738 67 | -0.394744 68 | -0.396747 69 | -0.398749 70 | -0.400749 71 | -0.402747 72 | -0.404743 73 | -0.406737 74 | -0.408729 75 | -0.410719 76 | -0.412707 77 | -0.414693 78 | -0.416677 79 | -0.418660 80 | -0.420640 81 | -0.422618 82 | -0.424595 83 | -0.426569 84 | -0.428541 85 | -0.430511 86 | -0.432479 87 | -0.434445 88 | -0.436409 89 | -0.438371 90 | -0.440331 91 | -0.442289 92 | -0.444244 93 | -0.446198 94 | -0.448149 95 | -0.450098 96 | -0.452046 97 | -0.453990 98 | -0.455933 99 | -0.457874 100 | -0.459812 101 | -0.461749 102 | -0.463683 103 | -0.465615 104 | -0.467544 105 | -0.469472 106 | -0.471397 107 | -0.473320 108 | -0.475240 109 | -0.477159 110 | -0.479075 111 | -0.480989 112 | -0.482900 113 | -0.484810 114 | -0.486717 115 | -0.488621 116 | -0.490524 117 | -0.492424 118 | -0.494321 119 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/mesa/test_metadata_service.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | 3 | from source.mesa.metadata_service import MetadataService 4 | from mock import mock_open 5 | 6 | 7 | class TestMetadataService(TestCase): 8 | 9 | @mock.patch('source.mesa.metadata_service.csv') 10 | @mock.patch("builtins.open", new_callable=mock_open, read_data='') 11 | def test_gets_metadata_dictionary(self, mock_open, mock_csv): 12 | subject_id = 1234567 13 | first_row = ['subject_id', 14 | 'ahiu35', 15 | 'sleepage5c', 16 | 'gender1', 17 | 'slpprdp5', 18 | 'time_bed5', 19 | 'waso5', 20 | 'slp_eff5', 21 | 'timerem5', 22 | 'timest15', 23 | 'timest25', 24 | 'timest345'] 25 | 26 | second_row = [str(subject_id), '0', '10', '20', '30', '40', '50', '60', '70', '80', '90', '100'] 27 | 28 | first_subject_dictionary = {'ahi': 0, 29 | 'age': 10, 30 | 'gender': 20, 31 | 'tst': 30, 32 | 'tib': 40, 33 | 'waso': 50, 34 | 'slp_eff': 60, 35 | 'time_rem': 70, 36 | 'time_n1': 80, 37 | 'time_n2': 90, 38 | 'time_n34': 100} 39 | 40 | mock_csv.reader.return_value = [first_row, 41 | second_row] 42 | 43 | dictionary = MetadataService.get_metadata_dictionary() 44 | 45 | self.assertDictEqual({subject_id: first_subject_dictionary}, dictionary) 46 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==1.0.0 2 | Bottleneck==1.3.2 3 | cachetools==5.0.0 4 | certifi==2021.10.8 5 | charset-normalizer==2.0.12 6 | colorama==0.4.4 7 | cycler==0.11.0 8 | easydict==1.9 9 | fonttools==4.29.1 10 | google-auth==2.6.0 11 | google-auth-oauthlib==0.4.6 12 | grpcio==1.43.0 13 | h5py==2.10.0 14 | idna==3.3 15 | imbalanced-learn==0.9.0 16 | imblearn==0.0 17 | importlib-metadata==4.11.0 18 | joblib==1.1.0 19 | kiwisolver==1.3.2 20 | Markdown==3.3.6 21 | matplotlib==3.5.1 22 | mkl-fft==1.3.1 23 | mkl-random @ file:///C:/ci/mkl_random_1626186184278/work 24 | mkl-service==2.4.0 25 | mlxtend==0.19.0 26 | mock @ file:///tmp/build/80754af9/mock_1607622725907/work 27 | numexpr @ file:///C:/ci/numexpr_1640704408721/work 28 | numpy @ file:///C:/ci_310/numpy_and_numpy_base_1643798589088/work 29 | oauthlib==3.2.0 30 | olefile @ file:///Users/ktietz/demo/mc3/conda-bld/olefile_1629805411829/work 31 | packaging @ file:///tmp/build/80754af9/packaging_1637314298585/work 32 | pandas @ file:///C:/ci/pandas_1641443171711/work 33 | patsy==0.5.2 34 | Pillow==8.4.0 35 | protobuf==3.19.4 36 | pyasn1==0.4.8 37 | pyasn1-modules==0.2.8 38 | pyparsing @ file:///tmp/build/80754af9/pyparsing_1635766073266/work 39 | pyreadline==2.1 40 | python-dateutil @ file:///tmp/build/80754af9/python-dateutil_1626374649649/work 41 | pytz==2021.3 42 | PyYAML==6.0 43 | requests==2.27.1 44 | requests-oauthlib==1.3.1 45 | rsa==4.8 46 | scikit-learn==1.0.2 47 | scipy==1.8.0 48 | six @ file:///tmp/build/80754af9/six_1623709665295/work 49 | statsmodels==0.13.0 50 | tables==3.6.1 51 | tensorboard==2.8.0 52 | tensorboard-data-server==0.6.1 53 | tensorboard-plugin-wit==1.8.1 54 | threadpoolctl==3.1.0 55 | torch==1.10.2 56 | torchaudio==0.10.2 57 | torchsummary==1.5.1 58 | torchvision==0.11.3 59 | tqdm==4.62.3 60 | typing-extensions @ file:///tmp/build/80754af9/typing_extensions_1631814937681/work 61 | urllib3==1.26.8 62 | Werkzeug==2.0.3 63 | wincertstore==0.2 64 | zipp==3.7.0 65 | -------------------------------------------------------------------------------- /models/removed_last_maxpool.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | 4 | def num_flat_features(x): 5 | size = x.size()[1:] 6 | num_features = 1 7 | for s in size: 8 | num_features *= s 9 | return num_features 10 | 11 | 12 | class VggAcc79F174NOFCRM(nn.Module): 13 | def __init__(self, in_channels): 14 | """ 15 | this is the VGG backbone for hybrid fusion strategy, which removed all FC layers 16 | @param in_channels: the number of input channels. e.g., 7, 9 17 | """ 18 | super(VggAcc79F174NOFCRM, self).__init__() 19 | self.con_1 = nn.Conv1d(in_channels, 512, kernel_size=3, padding=1) 20 | self.con_2 = nn.Conv1d(512, 512, kernel_size=3, padding=1) 21 | 22 | self.con_3 = nn.Conv1d(512, 128, kernel_size=3, padding=1) 23 | self.con_4 = nn.Conv1d(128, 128, kernel_size=3, padding=1) 24 | 25 | self.con_5 = nn.Conv1d(128, 512, kernel_size=3, padding=1) 26 | self.con_6 = nn.Conv1d(512, 512, kernel_size=3, padding=1) 27 | self.con_7 = nn.Conv1d(512, 512, kernel_size=3, padding=1) 28 | 29 | self.max_pool = nn.MaxPool1d(kernel_size=2, stride=2) 30 | self.relu = nn.ReLU() 31 | self.drop_out = nn.Dropout(p=0.25) 32 | 33 | def forward(self, x): 34 | output = self.con_1(x) 35 | output = self.relu(output) 36 | output = self.con_2(output) 37 | output = self.relu(output) 38 | output = self.max_pool(output) 39 | 40 | output = self.con_3(output) 41 | output = self.relu(output) 42 | output = self.con_4(output) 43 | output = self.relu(output) 44 | output = self.max_pool(output) 45 | 46 | output = self.con_5(output) 47 | output = self.relu(output) 48 | output = self.con_6(output) 49 | output = self.relu(output) 50 | output = self.con_7(output) 51 | output = self.relu(output) 52 | return output 53 | 54 | 55 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/psg/test_psg_converter.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.preprocessing.psg.psg_converter import PSGConverter 4 | from source.sleep_stage import SleepStage 5 | 6 | 7 | class TestPSGConverter(TestCase): 8 | def test_converts_between_strings_and_labels(self): 9 | self.assertEqual(PSGConverter.get_label_from_string("W"), SleepStage.wake) 10 | self.assertEqual(PSGConverter.get_label_from_string("M"), SleepStage.wake) 11 | self.assertEqual(PSGConverter.get_label_from_string("1"), SleepStage.n1) 12 | self.assertEqual(PSGConverter.get_label_from_string("2"), SleepStage.n2) 13 | self.assertEqual(PSGConverter.get_label_from_string("3"), SleepStage.n3) 14 | self.assertEqual(PSGConverter.get_label_from_string("4"), SleepStage.n4) 15 | self.assertEqual(PSGConverter.get_label_from_string("N1"), SleepStage.n1) 16 | self.assertEqual(PSGConverter.get_label_from_string("N2"), SleepStage.n2) 17 | self.assertEqual(PSGConverter.get_label_from_string("N3"), SleepStage.n3) 18 | self.assertEqual(PSGConverter.get_label_from_string("R"), SleepStage.rem) 19 | self.assertEqual(PSGConverter.get_label_from_string("?"), SleepStage.unscored) 20 | 21 | def test_converts_between_ints_and_labels(self): 22 | self.assertEqual(PSGConverter.get_label_from_int(-1), SleepStage.unscored) 23 | self.assertEqual(PSGConverter.get_label_from_int(0), SleepStage.wake) 24 | self.assertEqual(PSGConverter.get_label_from_int(1), SleepStage.n1) 25 | self.assertEqual(PSGConverter.get_label_from_int(2), SleepStage.n2) 26 | self.assertEqual(PSGConverter.get_label_from_int(3), SleepStage.n3) 27 | self.assertEqual(PSGConverter.get_label_from_int(4), SleepStage.n4) 28 | self.assertEqual(PSGConverter.get_label_from_int(5), SleepStage.rem) 29 | self.assertEqual(PSGConverter.get_label_from_int(6), SleepStage.unscored) 30 | 31 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/classification/test_parameter_search.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | from unittest.mock import MagicMock 3 | 4 | from sklearn.ensemble import RandomForestClassifier 5 | from sklearn.linear_model import LogisticRegression 6 | from sklearn.neighbors import KNeighborsClassifier 7 | from sklearn.neural_network import MLPClassifier 8 | 9 | from source.analysis.classification.parameter_search import ParameterSearch 10 | from source.analysis.setup.attributed_classifier import AttributedClassifier 11 | 12 | import numpy as np 13 | 14 | 15 | class TestParameterSearch(TestCase): 16 | 17 | @mock.patch('source.analysis.classification.parameter_search.GridSearchCV') 18 | def test_run_best_parameter_search(self, mock_grid_search): 19 | attributed_classifier = AttributedClassifier(name="Logistic Regression", classifier=LogisticRegression()) 20 | training_x = np.array([1, 2, 3]) 21 | training_y = np.array([4, 5, 6]) 22 | expected_parameter_range = ParameterSearch.parameter_dictionary[attributed_classifier.name] 23 | scoring = 'roc_auc' 24 | mock_parameter_search_classifier = MagicMock() 25 | mock_grid_search.return_value = mock_parameter_search_classifier 26 | mock_parameter_search_classifier.best_params_ = expected_parameters = {'parameter': 'value'} 27 | 28 | returned_parameters = ParameterSearch.run_search(attributed_classifier, training_x, 29 | training_y, scoring=scoring) 30 | mock_grid_search.assert_called_once_with(attributed_classifier.classifier, expected_parameter_range, 31 | scoring=scoring, 32 | iid=False, 33 | cv=3) 34 | mock_parameter_search_classifier.fit.assert_called_once_with(training_x, training_y) 35 | self.assertDictEqual(expected_parameters, returned_parameters) 36 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/psg/psg_report_processor.py: -------------------------------------------------------------------------------- 1 | import docx2txt as docx2txt 2 | 3 | from source import utils 4 | from source.preprocessing.psg.psg_file_type import PSGFileType 5 | from source.preprocessing.psg.report_summary import ReportSummary 6 | 7 | 8 | class PSGReportProcessor(object): 9 | 10 | @staticmethod 11 | def get_start_epoch_for_subject(subject_id): 12 | if int(subject_id) < 38: 13 | return 1 14 | if int(subject_id) == 40: 15 | return 35 16 | if int(subject_id) == 39: 17 | return 32 18 | if int(subject_id) == 38: 19 | return 37 20 | if int(subject_id) == 42: 21 | return 27 22 | if int(subject_id) == 41: 23 | return 21 24 | 25 | @staticmethod 26 | def get_summary_from_pdf(report_file_path): 27 | report_raw_text = utils.convert_pdf_to_txt(str(report_file_path), True) 28 | raw_text_split_at_epoch = report_raw_text.split('Epoch') 29 | 30 | split_at_study_date = (raw_text_split_at_epoch[0]).split('Study Date: ') 31 | study_date = (split_at_study_date[1]).split(' \n')[0] 32 | 33 | split_at_zero = (raw_text_split_at_epoch[1]).split('\n\n0') 34 | 35 | split_at_newline = (split_at_zero[1]).split('\n') 36 | split_at_colon = (raw_text_split_at_epoch[1]).split(':') 37 | 38 | start_epoch = split_at_newline[1] 39 | start_time = split_at_colon[0][-2:] + ':' + split_at_colon[1] + ':' + split_at_colon[2][:5] 40 | 41 | return ReportSummary(study_date=study_date, start_time=start_time, start_epoch=start_epoch, 42 | file_type=PSGFileType.Compumedics) 43 | 44 | @staticmethod 45 | def get_summary_from_docx(report_file_path): 46 | report_text = docx2txt.process(report_file_path) 47 | report_split = report_text.split('DATE: ') 48 | date = report_split[1].split('\n')[0] 49 | 50 | return ReportSummary(study_date=date, start_time=None, start_epoch=1, file_type=PSGFileType.Vitaport) 51 | -------------------------------------------------------------------------------- /models/bilinear_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class BilinearFusion(torch.nn.Module): 5 | """ 6 | The B-CNN model is illustrated as follows. 7 | conv1^2 (64) -> pool1 -> conv2^2 (128) -> pool2 -> conv3^3 (256) -> pool3 8 | -> conv4^3 (512) -> pool4 -> conv5^3 (512) -> bilinear pooling 9 | -> sqrt-normalize -> L2-normalize -> fc (200). 10 | The network accepts a 3*448*448 input, and the pool5 activation has shape 11 | 512*28*28 since we down-sample 5 times. 12 | Original github: https://github.com/HaoMood/bilinear-cnn/blob/master/src/bilinear_cnn_all.py 13 | Adapted by: Bing Zhai 14 | Attributes: 15 | features, torch.nn.Module: Convolution and pooling layers. 16 | fc, torch.nn.Module: 200. 17 | """ 18 | def __init__(self, feature_map_dim=512, linear_dim=1024): 19 | """Declare all needed layers.""" 20 | super(BilinearFusion, self).__init__() 21 | self.feature_map_dim = feature_map_dim 22 | self.linear_dim = linear_dim 23 | # Linear classifier. 24 | self.fc = torch.nn.Linear(self.feature_map_dim**2, self.linear_dim) 25 | 26 | def forward(self, x_1, x_2): 27 | """Forward pass of the network. 28 | Args: 29 | x_1, the first input matrix shape= [N, feature_map_dim, temporal_steps] , e.g., [32, 512, 10]. 30 | x_2, the second input matrix shape= [N, feature_map_dim, temporal_steps] 31 | Returns: 32 | x, fused bilinear results shape = [N, feature_map_dim, temporal_steps]. 33 | """ 34 | assert x_1.size() == x_2.size() 35 | N = x_1.size()[0] 36 | num_time_step = x_1.size()[2] 37 | x = torch.bmm(x_1, torch.transpose(x_2, 1, 2)) / num_time_step # Bilinear 38 | assert x.size() == (N, self.feature_map_dim, self.feature_map_dim) 39 | x = x.view(N, self.feature_map_dim ** 2) # reduce the dimensions 40 | x = torch.sqrt(x + 1e-5) 41 | x = torch.nn.functional.normalize(x) 42 | x = self.fc(x) 43 | assert x.size() == (N, self.linear_dim) 44 | return x 45 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/mesa/mesa_psg_service.py: -------------------------------------------------------------------------------- 1 | from xml.dom import minidom 2 | 3 | import numpy as np 4 | 5 | from source import utils 6 | 7 | 8 | class MesaPSGService(object): 9 | 10 | @staticmethod 11 | def load_raw(file_id): 12 | stage_to_num = {'Stage 4 sleep|4': 4, 'Stage 3 sleep|3': 3, 'Stage 2 sleep|2': 2, 'Stage 1 sleep|1': 1, 13 | 'Wake|0': 0, 'REM sleep|5': 5} 14 | project_root = str(utils.get_project_root()) 15 | 16 | xml_document = minidom.parse( 17 | project_root + '/data/mesa/polysomnography/annotations-events-nsrr/mesa-sleep-' + file_id + '-nsrr.xml') 18 | list_of_scored_events = xml_document.getElementsByTagName('ScoredEvent') 19 | 20 | stage_data = [] 21 | 22 | for scored_event in list_of_scored_events: # 3 is stage, 5 is start, 7 is duration 23 | duration = scored_event.childNodes[7].childNodes[0].nodeValue 24 | start = scored_event.childNodes[5].childNodes[0].nodeValue 25 | stage = scored_event.childNodes[3].childNodes[0].nodeValue 26 | 27 | if stage in stage_to_num: 28 | # # For debugging: print(file_id + ' ' + str(stage) + ' ' + str(start) + ' ' + str(duration)) 29 | stage_data.append([stage_to_num[stage], float(start), float(duration)]) 30 | 31 | stages = [] 32 | for staged_window in stage_data[:]: # Ignore last PSG overflow entry: it's long & doesn't have valid HR 33 | elapsed_time_counter = 0 34 | stage_value = staged_window[0] 35 | duration = staged_window[2] 36 | 37 | while elapsed_time_counter < duration: 38 | stages.append(stage_value) 39 | elapsed_time_counter = elapsed_time_counter + 1 40 | 41 | return np.array(stages) 42 | 43 | @staticmethod 44 | def crop(psg_labels, valid_epochs): 45 | cropped_psg_labels = [] 46 | 47 | for epoch in valid_epochs: 48 | index = int(epoch.timestamp) 49 | 50 | cropped_psg_labels.append(psg_labels[index]) 51 | return np.array(cropped_psg_labels) 52 | -------------------------------------------------------------------------------- /models/build_2dmodel.py: -------------------------------------------------------------------------------- 1 | from models.mix_2dmodel import * 2 | from models.build_model import get_model_time_step_dim 3 | 4 | 5 | def get_num_in_channel(dataset_name="mesa"): 6 | if dataset_name == "mesa": 7 | in_channel = 9 8 | elif dataset_name == "apple": 9 | in_channel = 7 10 | elif dataset_name == "mesa_hr_statistic": 11 | in_channel = 7 12 | else: 13 | raise ValueError("Sorry, dataset is not recognised should be mesa, mesa_hr_statistic, apple") 14 | return in_channel 15 | 16 | 17 | def get_num_in_channel_1_2(dataset_name="mesa"): 18 | if dataset_name == "mesa": 19 | in_channel_1 = 1 20 | in_channel_2 = 8 21 | elif dataset_name == "apple": 22 | in_channel_1 = 1 23 | in_channel_2 = 6 24 | elif dataset_name == "mesa_hr_statistic": 25 | in_channel_1 = 1 26 | in_channel_2 = 6 27 | else: 28 | raise ValueError("Sorry, dataset is not recognised should be mesa, mesa_hr_statistic, apple") 29 | return in_channel_1, in_channel_2 30 | 31 | 32 | def build_2d_model(nn_type, dataset, num_classes, seq_len): 33 | if nn_type == "VggLateConcat": 34 | sleep_model = VggLateConcat(in_channels=get_num_in_channel(dataset), num_classes=num_classes, 35 | time_step_dim=get_model_time_step_dim(nn_type, seq_len)) 36 | elif nn_type == "VggLateSum": 37 | sleep_model = VggLateSum(in_channels=get_num_in_channel(dataset), num_classes=num_classes, 38 | time_step_dim=get_model_time_step_dim(nn_type, seq_len)) 39 | elif nn_type == "ResLateConcat": 40 | sleep_model = ResLateConcat(in_channels=get_num_in_channel(dataset), num_classes=num_classes, 41 | time_step_dim=get_model_time_step_dim(nn_type, seq_len)) 42 | elif nn_type == "ResLateSum": 43 | sleep_model = ResLateSum(in_channels=get_num_in_channel(dataset), num_classes=num_classes, 44 | time_step_dim=get_model_time_step_dim(nn_type, seq_len)) 45 | else: 46 | raise Exception("model specified not existed!") 47 | return sleep_model 48 | -------------------------------------------------------------------------------- /models/stacked_attention_network.py: -------------------------------------------------------------------------------- 1 | import torch.nn.functional as F 2 | import torch.nn as nn 3 | import torch 4 | import numpy as np 5 | 6 | 7 | class SANTimeDimMatrixAttOnModality1NLayer1Concat(nn.Module): 8 | def __init__(self, input_feature_dim=512, attention_dim=256, time_step_dim=12): 9 | """ 10 | The original paper set the attention latent space dim to be 0.5 of the feature map dimension 11 | """ 12 | super(SANTimeDimMatrixAttOnModality1NLayer1Concat, self).__init__() 13 | 14 | self.input_feature_dim = input_feature_dim, 15 | self.latent_dim = attention_dim 16 | self.modality_1 = nn.Linear(input_feature_dim, attention_dim) # 512 - > 256 17 | self.modality_2 = nn.Linear(input_feature_dim, attention_dim) # 512 -> 256 18 | self.attention = nn.Linear(attention_dim, time_step_dim) # 256 -> 12 19 | 20 | self.tanh = nn.Tanh() 21 | self.softmax = nn.Softmax(dim=-1) 22 | self.__initial_weight() 23 | 24 | def __initial_weight(self): 25 | nn.init.xavier_uniform_(self.modality_2.weight) 26 | nn.init.xavier_uniform_(self.modality_1.weight) 27 | nn.init.xavier_uniform_(self.attention.weight) 28 | 29 | def forward(self, v_modality_1, v_modality_2): 30 | """ 31 | :param v_modality_2: e.g. cardiac CNN final layer output matrix [batch, 512, 15] 32 | :param v_modality_1: e.g. activity count CNN final layer output matrix [batch, 512, 15] 33 | """ 34 | # attention layer 1, car_rep: cardiac representation $$W_{cardiac,A}v_cardiac$$ 35 | # 36 | mod_1_rep_l_1 = self.modality_1(torch.transpose(v_modality_1, 1, 2)) 37 | mod_2_rep_l_1 = self.modality_2(torch.transpose(v_modality_2, 1, 2)) 38 | h_A_l_1 = self.tanh(mod_2_rep_l_1 + mod_1_rep_l_1) # $$h_A$$ at layer one 39 | p_att = self.attention(h_A_l_1) 40 | p_I = torch.transpose(self.softmax(p_att), 1, 2) # here p_I is size of [t_step x t_step] after T column sum = 1 41 | mod_1_rep_hat = torch.matmul(v_modality_1, p_I) 42 | u = torch.cat([mod_1_rep_hat, v_modality_2], dim=1) # refer to original paper's u = v^hat + v_Q if mod_1 is v 43 | return u 44 | 45 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/fusion_overlapping_service.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | from source import utils 5 | from source.constants import Constants 6 | from source.preprocessing.activity_count.activity_count_service import ActivityCountService 7 | from source.preprocessing.heart_rate.heart_rate_feature_service import HeartRateService 8 | from source.preprocessing.epoch import Epoch 9 | 10 | class FusionOverlappingService(object): 11 | 12 | @staticmethod 13 | def get_overlapped_epochs(subject_id, valid_epochs): 14 | activity_count_collection = ActivityCountService.load_cropped(subject_id) 15 | heart_rate_collection = HeartRateService.load_cropped(subject_id) 16 | activity_count_timestamp = np.round(activity_count_collection.timestamps.flatten()) 17 | heart_timestamp = np.round(heart_rate_collection.timestamps.flatten()) 18 | invalid_epochs = [] 19 | 20 | for i, epoch in enumerate(valid_epochs): 21 | act_idx = np.where((activity_count_timestamp >= (epoch.timestamp - Epoch.DURATION)) & 22 | (activity_count_timestamp < epoch.timestamp))[0] 23 | heart_idx = np.where((heart_timestamp > epoch.timestamp - Epoch.DURATION) & 24 | (heart_timestamp < epoch.timestamp))[0] 25 | if (len(act_idx) == 0) | (len(heart_idx) == 0): 26 | invalid_epochs.append(epoch) 27 | # assert len(invalid_epochs) == 0, print("we found invalid_epochs") 28 | new_valid_epochs = [x for x in valid_epochs if x not in invalid_epochs] 29 | FusionOverlappingService.write_mesa_protocol(subject_id, new_valid_epochs) 30 | return new_valid_epochs 31 | 32 | @staticmethod 33 | def get_path_mesa_protocol(subject_id): 34 | return Constants.FEATURE_FILE_PATH.joinpath("mesa", subject_id + '_valid_epochs.out') 35 | 36 | @staticmethod 37 | def write_mesa_protocol(subject_id, feature): 38 | heart_rate_feature_path = FusionOverlappingService.get_path_mesa_protocol(subject_id) 39 | df = pd.DataFrame.from_records([s.to_dict() for s in feature]) 40 | df.to_csv(heart_rate_feature_path, index=False) 41 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/mesa/test_mesa_heart_rate_service.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | 3 | from mock import MagicMock 4 | import numpy as np 5 | 6 | from source.mesa.mesa_heart_rate_service import MesaHeartRateService 7 | 8 | 9 | class TestMesaHeartRateService(TestCase): 10 | @mock.patch('source.mesa.mesa_heart_rate_service.np') 11 | @mock.patch('source.mesa.mesa_heart_rate_service.pyedflib') 12 | @mock.patch('source.mesa.mesa_heart_rate_service.utils') 13 | def test_load_raw(self, mock_utils, mock_pyedflib, mock_np): 14 | file_id = '4' 15 | mock_utils.get_project_root.return_value = project_root = 'project_root' 16 | expected_edf_file_location = project_root + '/data/mesa/polysomnography/edfs/mesa-sleep-' + file_id + '.edf' 17 | 18 | mock_pyedflib.EdfReader.return_value = mock_edf_file = MagicMock() 19 | mock_edf_file.getSignalLabels.return_value = ["A", "B", "C", "D"] 20 | mock_edf_file.getSampleFrequencies.return_value = [1, 2, 3, 4] 21 | mock_edf_file.readSignal.return_value = heart_rate_signal = [10, 20, 30] 22 | mock_np.array.return_value = np.array([0, 1, 2]) 23 | mock_np.transpose.return_value = transposed_value = np.array([[1, 2], [3, 4]]) 24 | mock_np.vstack.return_value = vstack_return_value = np.array([[11, 21, 31], [4, 5, 6]]) 25 | mock_utils.remove_nans.return_value = remove_nan_return_value = np.array([[0, 1], [2, 3]]) 26 | 27 | heart_rate_collection = MesaHeartRateService.load_raw(file_id) 28 | 29 | mock_utils.get_project_root.assert_called_once() 30 | mock_pyedflib.EdfReader.assert_called_once_with(expected_edf_file_location) 31 | 32 | mock_edf_file.getSignalLabels.assert_called_once() 33 | mock_edf_file.getSampleFrequencies.assert_called_once() 34 | mock_edf_file.readSignal.assert_called_once_with(2) 35 | mock_np.array.assert_called_once_with(range(0, len(heart_rate_signal))) 36 | mock_np.transpose.assert_called_once_with(vstack_return_value) 37 | mock_utils.remove_nans.assert_called_once_with(transposed_value) 38 | 39 | self.assertListEqual(remove_nan_return_value.tolist(), heart_rate_collection.data.tolist()) 40 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/mesa/test_mesa_actigraphy_service.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | 3 | from mock import mock_open, call 4 | import numpy as np 5 | from source.mesa.mesa_actigraphy_service import MesaActigraphyService 6 | 7 | 8 | class TestMesaActigraphyService(TestCase): 9 | 10 | @mock.patch('source.mesa.mesa_actigraphy_service.np') 11 | @mock.patch('source.mesa.mesa_actigraphy_service.csv') 12 | @mock.patch("builtins.open", new_callable=mock_open, read_data='') 13 | @mock.patch('source.mesa.mesa_actigraphy_service.utils') 14 | def test_load_raw(self, mock_utils, mock_open, mock_csv, mock_np): 15 | file_id = '3' 16 | align_line_1 = ['1', '244', '20:20:00', '20:21:30'] 17 | align_line_2 = ['3', '104', '14:20:00', '14:21:30'] 18 | 19 | sleep_row_1 = ['3', '103', '20:50:00', '0', '0.00', '0', '2.7500', '0.5790', '0.1650', '0.0206', '0', 'ACTIVE', 20 | '5', '1', '1'] 21 | sleep_row_2 = ['3', '104', '20:50:30', '0', '0.00', '0', '2.7500', '0.5790', '0.1650', '0.0206', '0', 'ACTIVE', 22 | '5', '1', '1'] 23 | sleep_row_3 = ['3', '105', '20:51:00', '0', '200.0', '0', '2.7500', '0.5790', '0.1650', '0.0206', '0', 24 | 'ACTIVE', '5', '1', '1'] 25 | 26 | expected_data = [[0.0, 0.0], [30.0, 200.0]] 27 | 28 | mock_csv.reader.side_effect = [[align_line_1, 29 | align_line_2], 30 | [sleep_row_1, 31 | sleep_row_2, 32 | sleep_row_3]] 33 | 34 | mock_np.array.return_value = array_return_value = 'value_returned_from_array' 35 | mock_utils.remove_nans.return_value = remove_nan_return_value = np.array([[0, 1], [2, 3]]) 36 | 37 | activity_count_collection = MesaActigraphyService.load_raw(file_id) 38 | 39 | mock_utils.get_project_root.assert_called_once() 40 | mock_np.array.assert_called_once_with(expected_data) 41 | mock_utils.remove_nans.assert_called_once_with(array_return_value) 42 | 43 | self.assertEqual(remove_nan_return_value.tolist(), activity_count_collection.data.tolist()) 44 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/psg/test_psg_label_service.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | from unittest.mock import MagicMock 3 | import numpy as np 4 | 5 | from source.constants import Constants 6 | from source.preprocessing.epoch import Epoch 7 | from source.preprocessing.psg.psg_label_service import PSGLabelService 8 | from source.preprocessing.psg.psg_service import PSGService 9 | 10 | 11 | class TestPSGLabelService(TestCase): 12 | 13 | @mock.patch('source.preprocessing.psg.psg_label_service.pd') 14 | def test_load(self, mock_pd): 15 | mock_pd.read_csv.return_value = mock_return = MagicMock() 16 | mock_return.values = expected_return = np.array([1, 2, 3, 4, 5]) 17 | actual_returned_value = PSGLabelService.load("subjectA") 18 | 19 | self.assertListEqual(expected_return.tolist(), actual_returned_value.tolist()) 20 | mock_pd.read_csv.assert_called_once_with(str(PSGLabelService.get_path("subjectA"))) 21 | 22 | def test_get_path(self): 23 | expected_path = Constants.FEATURE_FILE_PATH.joinpath("subjectA" + '_psg_labels.out') 24 | 25 | self.assertEqual(expected_path, PSGLabelService.get_path("subjectA")) 26 | 27 | @mock.patch.object(PSGService, 'load_cropped_array') 28 | def test_build(self, mock_load_cropped_array): 29 | subject_id = 'subjectA' 30 | data = np.array( 31 | [[1, 1], [10, 2], [20, 0], [40, 1], [70, 2], [90, 3], [100, 1], [120, 2]]) 32 | 33 | valid_epochs = [Epoch(timestamp=10, index=1), Epoch(timestamp=40, index=2)] 34 | mock_load_cropped_array.return_value = data 35 | expected_labels = np.array([2, 1]) 36 | 37 | returned_labels = PSGLabelService.build(subject_id, valid_epochs) 38 | 39 | self.assertEqual(expected_labels.tolist(), returned_labels.tolist()) 40 | 41 | @mock.patch.object(PSGLabelService, 'get_path') 42 | @mock.patch('source.preprocessing.psg.psg_label_service.np') 43 | def test_write(self, mock_np, mock_get_path): 44 | labels = np.array([1, 2, 3]) 45 | mock_get_path.return_value = path = 'path/to/return' 46 | PSGLabelService.write("subjectA", labels) 47 | 48 | mock_np.savetxt.assert_called_once_with(path, labels, fmt='%f') 49 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/performance/test_performance_builder.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | import numpy as np 3 | from sklearn.metrics import precision_score, auc, roc_curve, cohen_kappa_score 4 | 5 | from source.analysis.performance.performance_builder import PerformanceBuilder 6 | from source.analysis.performance.raw_performance import RawPerformance 7 | from source.analysis.performance.epoch_performance import SleepWakePerformance 8 | from test.test_helper import TestHelper 9 | 10 | 11 | class TestPerformanceBuilder(TestCase): 12 | 13 | def test_build_from_raw(self): 14 | threshold = 0.2 15 | raw_performance = RawPerformance(true_labels=np.array([1, 0]), 16 | class_probabilities=np.array([[0.1, 0.9], [0.3, 0.7]])) 17 | 18 | predicted_labels = np.array([1, 1]) 19 | kappa = cohen_kappa_score(raw_performance.true_labels, predicted_labels) 20 | 21 | sleep_predictive_value = precision_score(raw_performance.true_labels, predicted_labels, pos_label=1) 22 | wake_predictive_value = precision_score(raw_performance.true_labels, predicted_labels, pos_label=0) 23 | false_positive_rates, true_positive_rates, thresholds = roc_curve(raw_performance.true_labels, 24 | raw_performance.class_probabilities[:, 1], 25 | pos_label=1, 26 | drop_intermediate=False) 27 | auc_value = auc(false_positive_rates, true_positive_rates) 28 | 29 | expected_performance = SleepWakePerformance(accuracy=np.float64(0.5), wake_correct=np.float64(0), 30 | sleep_correct=np.float64(1.0), kappa=kappa, 31 | auc=auc_value, sleep_predictive_value=sleep_predictive_value, 32 | wake_predictive_value=wake_predictive_value) 33 | 34 | performance = PerformanceBuilder.build_with_sleep_threshold(raw_performance, threshold) 35 | TestHelper.assert_models_equal(self, expected_performance, performance) 36 | -------------------------------------------------------------------------------- /model_settings.yaml: -------------------------------------------------------------------------------- 1 | VggEarlyConcat: 2 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 3 | VggHybridConcat: 4 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 5 | VggHybridSum: 6 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 7 | VggHybridAtt: 8 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 9 | VggAcc79F174_RM_SANTimeDimMatrixAttOnMod1NLayer1: 10 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 11 | VggHybridBil: 12 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 13 | VggLateConcat: 14 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 15 | VggLateSum: 16 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 17 | ResLateConcat: 18 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 19 | ResLateSum: 20 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 21 | ResEarlyConcat: 22 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 23 | ResPlusSplitModalEleMul: 24 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 25 | ResHybridBil: 26 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 27 | ResHybridAtt: 28 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 29 | VggRawSANTiDimMatAttMod1NLayer1Con: 30 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 31 | ResHybridConcat: 32 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 33 | ResHybridSum: 34 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 35 | ResRawPlusSplitModal_SANTiDimMatAttMod1NLayer1Con: 36 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 37 | VggAcc79F174_RM_BiLinearV2: 38 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 39 | VggResdPlusCBAM: 40 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 41 | VggRaw2DConcate: 42 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 43 | VggRaw2DSum: 44 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 45 | VggRaw2DResConcate: 46 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 47 | VggRaw2DResSum: 48 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 49 | VggAcc79F174_7_RM_Raw_Appl_1_hr: 50 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 51 | VggRawSANTiDimMatAttMod1NLayer1Con_hr: 52 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 53 | ResPlusRawSplitModalCon_hr: 54 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 55 | ResPlusRawSplitModalPlus_hr: 56 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 57 | ResRawPlusSplitModal_SANTiDimMatAttMod1NLayer1Con_hr: 58 | FC_TEMP_STEPS: {'20': 5, '50': 12, '100': 25} 59 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/performance/test_performance_summarizer.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.analysis.performance.performance_summarizer import PerformanceSummarizer 4 | from source.analysis.performance.epoch_performance import SleepWakePerformance 5 | from test_helper import TestHelper 6 | 7 | 8 | class TestPerformanceSummarizer(TestCase): 9 | 10 | def test_averages(self): 11 | sleep_wake_performance_1 = SleepWakePerformance(accuracy=0, 12 | wake_correct=0.1, 13 | sleep_correct=0.2, 14 | auc=0.3, 15 | kappa=0.4, 16 | wake_predictive_value=0.5, 17 | sleep_predictive_value=0.6) 18 | 19 | sleep_wake_performance_2 = SleepWakePerformance(accuracy=0.2, 20 | wake_correct=0.3, 21 | sleep_correct=0.4, 22 | auc=0.5, 23 | kappa=0.6, 24 | wake_predictive_value=0.7, 25 | sleep_predictive_value=0.8) 26 | 27 | expected_performance = SleepWakePerformance(accuracy=0.1, 28 | wake_correct=0.2, 29 | sleep_correct=0.3, 30 | auc=0.4, 31 | kappa=0.5, 32 | wake_predictive_value=0.6, 33 | sleep_predictive_value=0.7) 34 | 35 | performance_list = [sleep_wake_performance_1, sleep_wake_performance_2] 36 | 37 | actual_averaged_performance = PerformanceSummarizer.average(performance_list) 38 | TestHelper.assert_models_equal(self, expected_performance, actual_averaged_performance) 39 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/psg/test_vitaport_processor.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | from mock import mock_open 3 | 4 | from source.preprocessing.epoch import Epoch 5 | from source.preprocessing.psg.psg_file_type import PSGFileType 6 | from source.preprocessing.psg.report_summary import ReportSummary 7 | from source.preprocessing.psg.stage_item import StageItem 8 | from source.preprocessing.psg.vitaport_processor import VitaportProcessor 9 | from source.sleep_stage import SleepStage 10 | from test.test_helper import TestHelper 11 | 12 | 13 | class TestVitaportProcessor(TestCase): 14 | 15 | @mock.patch("source.preprocessing.psg.vitaport_processor.TimeService") 16 | @mock.patch("builtins.open", new_callable=mock_open, read_data='') 17 | @mock.patch('source.preprocessing.psg.vitaport_processor.csv') 18 | def test_parse(self, mock_csv, mock_open, mock_time_service): 19 | mock_csv.reader.return_value = [['0', '23:20:55'], ['0', '23:21:05'], ['0', '23:21:15'], ['2', '23:21:25'], 20 | ['2', '23:21:35']] 21 | report_summary = ReportSummary(study_date="04/02/2019", 22 | start_epoch=1, 23 | start_time="11:20:55 PM", 24 | file_type=PSGFileType.Vitaport) 25 | psg_stage_path = 'path/to/file' 26 | mock_time_service.get_start_epoch_timestamp.return_value = expected_start_timestamp = 1234567890 27 | 28 | expected_data = [StageItem(epoch=Epoch(timestamp=expected_start_timestamp, 29 | index=1), stage=SleepStage.wake), 30 | StageItem(epoch=Epoch(timestamp=expected_start_timestamp + Epoch.DURATION, 31 | index=2), stage=SleepStage.n2)] 32 | 33 | data = VitaportProcessor.parse(report_summary, psg_stage_path) 34 | 35 | TestHelper.assert_models_equal(self, expected_data[0].epoch, data[0].epoch) 36 | TestHelper.assert_models_equal(self, expected_data[0].stage, data[0].stage) 37 | TestHelper.assert_models_equal(self, expected_data[1].epoch, data[1].epoch) 38 | TestHelper.assert_models_equal(self, expected_data[1].stage, data[1].stage) 39 | 40 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/performance/sleep_metrics_calculator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from source.constants import Constants 4 | 5 | 6 | class SleepMetricsCalculator(object): 7 | 8 | @staticmethod 9 | def get_tst(labels): 10 | sleep_epoch_indices = np.where(labels > 0) 11 | tst = np.shape(sleep_epoch_indices)[1] 12 | return tst * Constants.EPOCH_DURATION_IN_SECONDS / Constants.SECONDS_PER_MINUTE 13 | 14 | @staticmethod 15 | def get_wake_after_sleep_onset(labels): 16 | sleep_indices = np.argwhere(labels > 0) 17 | if np.shape(sleep_indices)[0] > 0: 18 | sol_index = np.amin(sleep_indices) 19 | indices_where_wake_occurred = np.where(labels == 0) 20 | 21 | waso_indices = np.where(indices_where_wake_occurred > sol_index) 22 | waso_indices = waso_indices[1] 23 | number_waso_indices = np.shape(waso_indices)[0] 24 | return number_waso_indices * Constants.EPOCH_DURATION_IN_SECONDS / Constants.SECONDS_PER_MINUTE 25 | else: 26 | return len(labels) * Constants.EPOCH_DURATION_IN_SECONDS / Constants.SECONDS_PER_MINUTE 27 | 28 | @staticmethod 29 | def get_sleep_efficiency(labels): 30 | sleep_indices = np.where(labels > 0) 31 | sleep_efficiency = float(np.shape(sleep_indices)[1]) / float(np.shape(labels)[0]) 32 | return sleep_efficiency 33 | 34 | @staticmethod 35 | def get_sleep_onset_latency(labels): 36 | sleep_indices = np.argwhere(labels > 0) 37 | if np.shape(sleep_indices)[0] > 0: 38 | return np.amin(sleep_indices) * Constants.EPOCH_DURATION_IN_SECONDS / Constants.SECONDS_PER_MINUTE 39 | else: 40 | return len(labels) * Constants.EPOCH_DURATION_IN_SECONDS / Constants.SECONDS_PER_MINUTE 41 | 42 | @staticmethod 43 | def get_time_in_rem(labels): 44 | rem_epoch_indices = np.where(labels == 2) 45 | rem_time = np.shape(rem_epoch_indices)[1] 46 | return rem_time * Constants.EPOCH_DURATION_IN_SECONDS / Constants.SECONDS_PER_MINUTE 47 | 48 | @staticmethod 49 | def get_time_in_nrem(labels): 50 | rem_epoch_indices = np.where(labels == 1) 51 | rem_time = np.shape(rem_epoch_indices)[1] 52 | return rem_time * Constants.EPOCH_DURATION_IN_SECONDS / Constants.SECONDS_PER_MINUTE 53 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/time/clock_proxy/ode4.m: -------------------------------------------------------------------------------- 1 | function Y = ode4(odefun,tspan,y0,varargin) 2 | %ODE4 Solve differential equations with a non-adaptive method of order 4. 3 | % Y = ODE4(ODEFUN,TSPAN,Y0) with TSPAN = [T1, T2, T3, ... TN] integrates 4 | % the system of differential equations y' = f(t,y) by stepping from T0 to 5 | % T1 to TN. Function ODEFUN(T,Y) must return f(t,y) in a column vector. 6 | % The vector Y0 is the initial conditions at T0. Each row in the solution 7 | % array Y corresponds to a time specified in TSPAN. 8 | % 9 | % Y = ODE4(ODEFUN,TSPAN,Y0,P1,P2...) passes the additional parameters 10 | % P1,P2... to the derivative function as ODEFUN(T,Y,P1,P2...). 11 | % 12 | % This is a non-adaptive solver. The step sequence is determined by TSPAN 13 | % but the derivative function ODEFUN is evaluated multiple times per step. 14 | % The solver implements the classical Runge-Kutta method of order 4. 15 | % 16 | % Example 17 | % tspan = 0:0.1:20; 18 | % y = ode4(@vdp1,tspan,[2 0]); 19 | % plot(tspan,y(:,1)); 20 | % solves the system y' = vdp1(t,y) with a constant step size of 0.1, 21 | % and plots the first component of the solution. 22 | % 23 | 24 | if ~isnumeric(tspan) 25 | error('TSPAN should be a vector of integration steps.'); 26 | end 27 | 28 | if ~isnumeric(y0) 29 | error('Y0 should be a vector of initial conditions.'); 30 | end 31 | 32 | h = diff(tspan); 33 | if any(sign(h(1))*h <= 0) 34 | error('Entries of TSPAN are not in order.') 35 | end 36 | 37 | % try 38 | f0 = feval(odefun,tspan(1),y0,varargin{:}); 39 | % catch 40 | % msg = ['Unable to evaluate the ODEFUN at t0,y0. ',lasterr]; 41 | % error(msg); 42 | % end 43 | 44 | y0 = y0(:); % Make a column vector. 45 | if ~isequal(size(y0),size(f0)) 46 | error('Inconsistent sizes of Y0 and f(t0,y0).'); 47 | end 48 | 49 | neq = length(y0); 50 | N = length(tspan); 51 | Y = zeros(neq,N); 52 | F = zeros(neq,4); 53 | 54 | Y(:,1) = y0; 55 | for i = 2:N 56 | ti = tspan(i-1); 57 | hi = h(i-1); 58 | yi = Y(:,i-1); 59 | F(:,1) = feval(odefun,ti,yi,varargin{:}); 60 | F(:,2) = feval(odefun,ti+0.5*hi,yi+0.5*hi*F(:,1),varargin{:}); 61 | F(:,3) = feval(odefun,ti+0.5*hi,yi+0.5*hi*F(:,2),varargin{:}); 62 | F(:,4) = feval(odefun,tspan(i),yi+hi*F(:,3),varargin{:}); 63 | Y(:,i) = yi + (hi/6)*(F(:,1) + 2*F(:,2) + 2*F(:,3) + F(:,4)); 64 | end 65 | Y = Y.'; -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/setup/sleep_labeler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from source.analysis.performance.raw_performance import RawPerformance 4 | from source.analysis.setup.sleep_label import SleepWakeLabel 5 | from source.analysis.setup.sleep_label import ThreeClassLabel 6 | 7 | 8 | class SleepLabeler(object): 9 | 10 | @staticmethod 11 | def label_sleep_wake(raw_sleep_wake): 12 | labeled_sleep = [] 13 | 14 | for value in raw_sleep_wake: 15 | if value > 0: 16 | converted_value = SleepWakeLabel.sleep.value 17 | else: 18 | converted_value = SleepWakeLabel.wake.value 19 | labeled_sleep.append(converted_value) 20 | 21 | return np.array(labeled_sleep) 22 | 23 | @staticmethod 24 | def label_three_class(raw_sleep_wake): 25 | labeled_sleep = [] 26 | 27 | for value in raw_sleep_wake: 28 | if value == 0: 29 | converted_value = ThreeClassLabel.wake.value 30 | elif value == 5: 31 | converted_value = ThreeClassLabel.rem.value 32 | else: 33 | converted_value = ThreeClassLabel.nrem.value 34 | 35 | labeled_sleep.append(converted_value) 36 | 37 | return np.array(labeled_sleep) 38 | 39 | @staticmethod 40 | def label_one_vs_rest(sleep_wake_labels, positive_class): 41 | labeled_sleep = [] 42 | 43 | for value in sleep_wake_labels: 44 | if value == positive_class: 45 | converted_value = 1 46 | else: 47 | converted_value = 0 48 | 49 | labeled_sleep.append(converted_value) 50 | 51 | return np.array(labeled_sleep) 52 | 53 | @staticmethod 54 | def convert_three_class_to_two(raw_performance: RawPerformance): 55 | raw_performance.true_labels = SleepLabeler.label_sleep_wake(raw_performance.true_labels) 56 | number_of_samples = np.shape(raw_performance.class_probabilities)[0] 57 | for index in range(number_of_samples): 58 | raw_performance.class_probabilities[index, 1] = raw_performance.class_probabilities[index, 1] + \ 59 | raw_performance.class_probabilities[index, 2] 60 | raw_performance.class_probabilities = raw_performance.class_probabilities[:, :-1] 61 | 62 | return raw_performance 63 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/classification/classifier_input_builder.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from source.analysis.setup.sleep_labeler import SleepLabeler 4 | 5 | 6 | class ClassifierInputBuilder(object): 7 | 8 | @staticmethod 9 | def get_array(subject_ids, subject_dictionary, feature_set): 10 | 11 | all_subjects_features = np.array([]) 12 | all_subjects_labels = np.array([]) 13 | 14 | for subject_id in subject_ids: 15 | subject_features = np.array([]) 16 | subject = subject_dictionary[subject_id] 17 | feature_dictionary = subject.feature_dictionary 18 | 19 | for feature in feature_set: 20 | feature_data = feature_dictionary[feature] 21 | subject_features = ClassifierInputBuilder.__append_feature(subject_features, feature_data) 22 | 23 | all_subjects_features = ClassifierInputBuilder.__stack(all_subjects_features, subject_features) 24 | all_subjects_labels = ClassifierInputBuilder.__stack(all_subjects_labels, subject.labeled_sleep) 25 | 26 | return all_subjects_features, all_subjects_labels 27 | 28 | @staticmethod 29 | def get_sleep_wake_inputs(subject_ids, subject_dictionary, feature_set): 30 | values, raw_labels = ClassifierInputBuilder.get_array(subject_ids, subject_dictionary, feature_set) 31 | processed_labels = SleepLabeler.label_sleep_wake(raw_labels) 32 | return values, processed_labels 33 | 34 | @staticmethod 35 | def get_three_class_inputs(subject_ids, subject_dictionary, feature_set): 36 | values, raw_labels = ClassifierInputBuilder.get_array(subject_ids, subject_dictionary, feature_set) 37 | processed_labels = SleepLabeler.label_three_class(raw_labels) 38 | return values, processed_labels 39 | 40 | @staticmethod 41 | def __append_feature(array, feature): 42 | if len(np.shape(feature)) < 2: 43 | feature = np.transpose([feature]) 44 | if np.shape(array)[0] == 0: 45 | array = feature 46 | else: 47 | array = np.hstack((array, feature)) 48 | 49 | return array 50 | 51 | @staticmethod 52 | def __stack(combined_array, new_array): 53 | if np.shape(combined_array)[0] == 0: 54 | combined_array = new_array 55 | else: 56 | combined_array = np.vstack((combined_array, new_array)) 57 | return combined_array 58 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/heart_rate/heart_rate_service.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import os 4 | 5 | from source import utils 6 | from source.constants import Constants 7 | from source.preprocessing.heart_rate.heart_rate_collection import HeartRateCollection 8 | 9 | 10 | class HeartRateService(object): 11 | 12 | @staticmethod 13 | def load_raw(subject_id): 14 | raw_hr_path = HeartRateService.get_raw_file_path(subject_id) 15 | heart_rate_array = HeartRateService.load(raw_hr_path, ",") 16 | heart_rate_array = utils.remove_repeats(heart_rate_array) 17 | return HeartRateCollection(subject_id=subject_id, data=heart_rate_array) 18 | 19 | @staticmethod 20 | def load_cropped(subject_id): 21 | cropped_hr_path = HeartRateService.get_cropped_file_path(subject_id) 22 | heart_rate_array = HeartRateService.load(cropped_hr_path) 23 | return HeartRateCollection(subject_id=subject_id, data=heart_rate_array) 24 | 25 | @staticmethod 26 | def load(hr_file, delimiter=" "): 27 | heart_rate_array = pd.read_csv(str(hr_file), delimiter=delimiter).values 28 | return heart_rate_array 29 | 30 | @staticmethod 31 | def write(heart_rate_collection): 32 | hr_output_path = HeartRateService.get_cropped_file_path(heart_rate_collection.subject_id) 33 | np.savetxt(hr_output_path, heart_rate_collection.data, fmt='%f') 34 | 35 | @staticmethod 36 | def crop(heart_rate_collection, interval): 37 | subject_id = heart_rate_collection.subject_id 38 | timestamps = heart_rate_collection.timestamps 39 | valid_indices = ((timestamps >= interval.start_time) 40 | & (timestamps < interval.end_time)).nonzero()[0] 41 | 42 | cropped_data = heart_rate_collection.data[valid_indices, :] 43 | return HeartRateCollection(subject_id=subject_id, data=cropped_data) 44 | 45 | @staticmethod 46 | def get_cropped_file_path(subject_id): 47 | return Constants.CROPPED_FILE_PATH.joinpath(subject_id + "_cleaned_hr.out") 48 | 49 | @staticmethod 50 | def get_raw_file_path(subject_id): 51 | # heart_rate_dir = utils.get_project_root().joinpath('data/heart_rate/') 52 | heart_rate_dir = Constants.RAW_PATH 53 | return os.path.join(heart_rate_dir, "heart_rate", subject_id + '_heartrate.txt') 54 | # return heart_rate_dir.joinpath(subject_id + '_heartrate.txt') 55 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/activity_count/test_activity_count_service.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | 3 | from source import utils 4 | from source.constants import Constants 5 | from source.preprocessing.activity_count.activity_count_collection import ActivityCountCollection 6 | from source.preprocessing.activity_count.activity_count_service import ActivityCountService 7 | import numpy as np 8 | 9 | from test.test_helper import TestHelper 10 | 11 | 12 | class TestActivityCountService(TestCase): 13 | 14 | @mock.patch.object(ActivityCountService, 'load') 15 | @mock.patch.object(ActivityCountService, 'get_cropped_file_path') 16 | def test_load_cropped(self, mock_cropped_file_path, mock_load): 17 | subject_id = 'subjectA' 18 | mock_cropped_file_path.return_value = path = 'path/to/file' 19 | data = np.array([[1, 2, 3], [4, 5, 6]]) 20 | expected_activity_count_collection = ActivityCountCollection(subject_id=subject_id, data=data) 21 | mock_load.return_value = data 22 | returned_activity_count_collection = ActivityCountService.load_cropped(subject_id) 23 | mock_cropped_file_path.assert_called_once_with(subject_id) 24 | mock_load.assert_called_once_with(path) 25 | TestHelper.assert_models_equal(self, expected_activity_count_collection, returned_activity_count_collection) 26 | 27 | @mock.patch('source.preprocessing.activity_count.activity_count_service.pd') 28 | def test_load(self, mock_pd): 29 | counts_file_path = 'path/to/file' 30 | mock_pd.read_csv.return_value.values = expected_data = np.array([1, 2, 3, 4, 5]) 31 | 32 | returned_data = ActivityCountService.load(counts_file_path) 33 | 34 | mock_pd.read_csv.assert_called_once_with(counts_file_path) 35 | self.assertEqual(expected_data.tolist(), returned_data.tolist()) 36 | 37 | def test_get_cropped_file_path(self): 38 | subject_id = 'subject1' 39 | file_path = ActivityCountService.get_cropped_file_path(subject_id) 40 | self.assertEqual(Constants.CROPPED_FILE_PATH.joinpath("subject1_cleaned_counts.out"), file_path) 41 | 42 | @mock.patch('source.preprocessing.activity_count.activity_count_service.os') 43 | def test_build_activity_counts(self, mock_os): 44 | expected_argument = 'matlab -nodisplay -nosplash -nodesktop -r \"run(\'' + str( 45 | utils.get_project_root()) + '/source/make_counts.m\'); exit;\"' 46 | 47 | ActivityCountService.build_activity_counts() 48 | 49 | mock_os.system.assert_called_once_with(expected_argument) 50 | 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # customised folders 2 | tfboard/ 3 | Dataset/ 4 | outputs/ 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | 14 | # Distribution / packaging 15 | .Python 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | 29 | pip-wheel-metadata/ 30 | share/python-wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | MANIFEST 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .nox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *.cover 56 | *.py,cover 57 | .hypothesis/ 58 | .pytest_cache/ 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | target/ 82 | 83 | # Jupyter Notebook 84 | .ipynb_checkpoints 85 | 86 | # IPython 87 | profile_default/ 88 | ipython_config.py 89 | 90 | # pyenv 91 | .python-version 92 | 93 | # pipenv 94 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 95 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 96 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 97 | # install all needed dependencies. 98 | #Pipfile.lock 99 | 100 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 101 | __pypackages__/ 102 | 103 | # Celery stuff 104 | celerybeat-schedule 105 | celerybeat.pid 106 | 107 | # SageMath parsed files 108 | *.sage.py 109 | 110 | # Environments 111 | .env 112 | .venv 113 | env/ 114 | venv/ 115 | ENV/ 116 | env.bak/ 117 | venv.bak/ 118 | 119 | # Spyder project settings 120 | .spyderproject 121 | .spyproject 122 | 123 | # Rope project settings 124 | .ropeproject 125 | 126 | # mkdocs documentation 127 | /site 128 | 129 | # mypy 130 | .mypy_cache/ 131 | .dmypy.json 132 | dmypy.json 133 | 134 | # Pyre type checker 135 | .pyre/ 136 | 137 | # exclude idea settings 138 | .idea/ -------------------------------------------------------------------------------- /tests/test_utilis.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from numpy.testing import assert_almost_equal, assert_array_equal 3 | from utilities.utils import build_windowed_data 4 | from utilities.utils import interpolate_rri_nan_values, up_down_rri_sampling 5 | import numpy as np 6 | 7 | 8 | def test_interpolate_rri_nan_values(): 9 | # Setup 10 | test_data = [np.nan, np.nan, 1.0, 2, 3, np.nan, 4, 5, np.nan, np.nan, 6, np.nan, np.nan, np.nan] 11 | # Exercise 12 | results = interpolate_rri_nan_values(test_data) 13 | # Verify 14 | expected = [1.0, 1.0, 1.0, 2.0, 3.0, 3.5, 4.0, 5.0, 5.33, 5.67, 6.0, 6.0, 6.0, 6.0] 15 | assert_almost_equal(expected, results, decimal=2) 16 | 17 | 18 | def test_up_down_rri_sampling(): 19 | test_data = [714.84375, 785.15625, 781.25, 769.53125, 792.96875, 789.0625, 777.34375, 785.15625] 20 | test_data_idx = np.asarray(pd.Series(test_data).cumsum()) 21 | test_data = np.asarray(test_data) 22 | seconds_to_sample = 30 23 | results = up_down_rri_sampling(test_data_idx, np.asarray(test_data), seconds=seconds_to_sample, target_sampling_rate=1) 24 | assert results[len(test_data):].sum() == test_data.mean() * (seconds_to_sample-len(test_data)) 25 | 26 | 27 | def test_build_windowed_data(): 28 | data = np.arange(1, 121) 29 | # let's make a time series with sampling rate in 1Hz and per 10 seconds as an epoch 30 | data = data.reshape(-1, 10) 31 | sampling_rate = 1 32 | win_len = 4 33 | # we should expect the return data in window length 5 34 | output = build_windowed_data(data, sampling_rate=sampling_rate, epoch_len=10, win_len=win_len) 35 | assert output.shape == (12, 50) 36 | # let's calculate the first 4 rows 37 | expected_result_1 = np.asarray([np.arange(1, 31).sum() + (-1)*20, 38 | np.arange(1, 41).sum() + (-1) * 10, 39 | np.arange(1, 51).sum(), 40 | np.arange(11, 61).sum()]) 41 | assert_array_equal(output[:4, :].sum(axis=-1) == expected_result_1, [True, True, True, True]) 42 | # let's calculate the rows 4-6 43 | expected_result_2 = np.asarray([np.arange(31, 81).sum(), 44 | np.arange(41, 91).sum(), 45 | np.arange(51, 101).sum()]) 46 | assert_array_equal(output[5:8, :].sum(axis=-1) == expected_result_2, [True, True, True]) 47 | # let's calculate the last two rows 48 | expected_result_2 = np.asarray([np.arange(81, 121).sum() + (-1)*10, 49 | np.arange(91, 121).sum() + (-1)*20]) 50 | assert_array_equal(output[-2:, :].sum(axis=-1) == expected_result_2, [True, True]) 51 | 52 | -------------------------------------------------------------------------------- /sleep_stage_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import platform 3 | 4 | 5 | class Config(object): 6 | def __init__(self): 7 | # {"activity": "Activity Counts", 8 | # "mean_nni": "Mean NNI", 9 | # "sdnn": "Standard Deviation of NN Intervals", 10 | # "sdsd": "Successive RR Interval Differences", 11 | # "vlf": "Very-Low-Frequency Band", 12 | # "lf": "Low-Frequency Band", 13 | # "hf": "High-Frequency Band", 14 | # "lf_hf_ratio": "A ratio of Low Frequency to High Frequency", 15 | # "total_power": "The Signal's Power Intensity" 16 | # } 17 | self.MESA_ACC_HR_STATISTIC_FEATURE_LIST = ["activity", "min_hr", "max_hr", "mean_hr", "skw_hr", "kurt_hr", 18 | "std_hr"] 19 | self.SUMMARY_FILE_PATH = r"./exp_results.csv" 20 | self.MESA_LOOCV_PID_PATH = r"./assets/mesa_16_fold.csv" 21 | if platform.uname()[1] == 'BB-WIN8': 22 | self.HR_PATH = "Dataset/annotations-rpoints" 23 | self.ACC_PATH = "Dataset/actigraphy" 24 | self.OVERLAP_PATH = "Dataset/mesa-actigraphy-psg-overlap.csv" 25 | self.HRV30_ACC_STD_PATH = "Dataset/HRV30s_ACC30s_H5/std/hrv30s_acc30s_full_feat_stand.h5" 26 | self.NN_ACC_HRV_STD = "Dataset/HRV30s_ACC30s_H5/std/nn_acc_hrv30s_%d.h5" 27 | self.MESA_ACC_HR_STATISTICS_STD_DATA_PATH = "Dataset/HR_STATISTIC30s_ACC30s_H5/hr_statistic_30s_acc30s_full_feat_stand.h5" 28 | self.MESA_NN_ACC_HR_STATISTIC = "Dataset/HR_STATISTIC30s_ACC30s_H5/nn_acc_hr_statistic_30s_%d.h5" 29 | self.MESA_ACC_HR_STATISTIC_CSV_ALIGNED = "Dataset/Aligned_ACC_HR_STATISTIC_CSV" 30 | self.MESA_ACC_HR_STATISTIC_FEATURE = "Dataset/mesa_acc_hr_statistic_feature_list.csv" 31 | # apple data set settings 32 | self.APPLE_NN_ACC_HRV = r"Dataset/Apple_watch_sleep_dataset/outputs/features/apple_nn_acc_hrv30s_%d.h5" 33 | self.APPLE_LOOCV_ALL_WINDOWED = r"Dataset/Apple_watch_sleep_dataset/outputs/features/apple_loocv_windowed_%d.h5" 34 | self.APPLE_HRV30_ACC_STD_PATH = r"Dataset/Apple_watch_sleep_dataset/outputs/features/apple_hr30s_acc30s_full_feat_stand.h5" 35 | self.APPLE_LOOCV_PID_PATH = r"assets/apple_16_fold.csv" 36 | self.APPLE_CROPPED_RAW_PATH = r"Dataset/Apple_watch_sleep_dataset/outputs/cropped" 37 | self.APPLE_LOOCV_ALL_WINDOWED = r"Dataset/apple_loocv_windowed_%d.h5" 38 | self.TRAIN_TEST_SPLIT = "/assets/train_test_pid_split.csv" 39 | self.SUMMARY_FOLDER_DICT = {"s": r"sp_exp_results.csv", 40 | "r": r"rp_exp_results.csv", } 41 | self.MESA_HRS_LOOCV = r"Dataset/MESA/mesa_hrs_loocv" 42 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/mesa/test_mesa_psg_service.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | 3 | from mock import MagicMock 4 | 5 | from source.mesa.mesa_psg_service import MesaPSGService 6 | 7 | 8 | class TestMesaPSGService(TestCase): 9 | 10 | @mock.patch('source.mesa.mesa_psg_service.minidom') 11 | @mock.patch('source.mesa.mesa_psg_service.np') 12 | @mock.patch('source.mesa.mesa_psg_service.utils') 13 | def test_load_raw(self, mock_utils, mock_np, mock_minidom): 14 | file_id = "2" 15 | mock_utils.get_project_root.return_value = "project/root" 16 | mock_minidom.parse.return_value = mock_xml_document = MagicMock() 17 | first_scored_event = MagicMock() 18 | second_scored_event = MagicMock() 19 | third_scored_event = MagicMock() 20 | 21 | child_nodes = [MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), 22 | MagicMock()] 23 | child_nodes[3].childNodes[0].nodeValue = 'Stage 4 sleep|4' 24 | child_nodes[5].childNodes[0].nodeValue = 0.0 25 | child_nodes[7].childNodes[0].nodeValue = 90.0 26 | 27 | first_scored_event.childNodes = child_nodes 28 | 29 | child_nodes = [MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), 30 | MagicMock()] 31 | child_nodes[3].childNodes[0].nodeValue = 'REM sleep|5' 32 | child_nodes[5].childNodes[0].nodeValue = 90.0 33 | child_nodes[7].childNodes[0].nodeValue = 30.0 34 | 35 | second_scored_event.childNodes = child_nodes 36 | 37 | child_nodes = [MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), MagicMock(), 38 | MagicMock()] 39 | child_nodes[3].childNodes[0].nodeValue = 'Stage 4 sleep|4' 40 | child_nodes[5].childNodes[0].nodeValue = 120.0 41 | child_nodes[7].childNodes[0].nodeValue = 900.0 42 | 43 | third_scored_event.childNodes = child_nodes 44 | 45 | scored_events = [first_scored_event, 46 | second_scored_event, 47 | third_scored_event] 48 | 49 | mock_xml_document.getElementsByTagName.return_value = scored_events 50 | 51 | expected_stages = [] 52 | for i in range(90): 53 | expected_stages.append(4) 54 | for i in range(30): 55 | expected_stages.append(5) 56 | for i in range(900): 57 | expected_stages.append(4) 58 | mock_np.array.return_value = expected_return = [1, 10, 100] 59 | 60 | stages = MesaPSGService.load_raw(file_id) 61 | 62 | mock_utils.get_project_root.assert_called_once() 63 | 64 | mock_np.array.assert_called_once_with(expected_stages) 65 | self.assertListEqual(expected_return, stages) 66 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/psg/test_compumedics_processor.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | 3 | from mock import mock_open, MagicMock 4 | 5 | from source.preprocessing.epoch import Epoch 6 | from source.preprocessing.psg.compumedics_processor import CompumedicsProcessor 7 | from source.preprocessing.psg.psg_file_type import PSGFileType 8 | from source.preprocessing.psg.report_summary import ReportSummary 9 | from source.preprocessing.psg.stage_item import StageItem 10 | from source.sleep_stage import SleepStage 11 | from test.test_helper import TestHelper 12 | 13 | 14 | class TestCompumedicsProcessor(TestCase): 15 | @mock.patch("source.preprocessing.psg.compumedics_processor.TimeService") 16 | @mock.patch("builtins.open", new_callable=mock_open, read_data='') 17 | @mock.patch('source.preprocessing.psg.compumedics_processor.csv') 18 | def test_parse(self, mock_csv, mock_open, mock_time_service): 19 | mock_csv.reader.return_value = ['W', '1', 'W', '2', 'R'] 20 | report_summary = ReportSummary(study_date="04/02/2019", 21 | start_epoch=2, 22 | start_time="10:30:13 PM", 23 | file_type=PSGFileType.Compumedics) 24 | psg_stage_path = 'path/to/file' 25 | mock_time_service.get_start_epoch_timestamp.return_value = expected_start_timestamp = 1234567890 26 | 27 | expected_data = [StageItem(epoch=Epoch(timestamp=expected_start_timestamp, 28 | index=2), stage=SleepStage.n1), 29 | StageItem(epoch=Epoch(timestamp=expected_start_timestamp + Epoch.DURATION, 30 | index=3), stage=SleepStage.wake), 31 | StageItem(epoch=Epoch(timestamp=expected_start_timestamp + Epoch.DURATION*2, 32 | index=4), stage=SleepStage.n2), 33 | StageItem(epoch=Epoch(timestamp=expected_start_timestamp + Epoch.DURATION*3, 34 | index=5), stage=SleepStage.rem)] 35 | 36 | data = CompumedicsProcessor.parse(report_summary, psg_stage_path) 37 | 38 | TestHelper.assert_models_equal(self, expected_data[0].epoch, data[0].epoch) 39 | TestHelper.assert_models_equal(self, expected_data[0].stage, data[0].stage) 40 | TestHelper.assert_models_equal(self, expected_data[1].epoch, data[1].epoch) 41 | TestHelper.assert_models_equal(self, expected_data[1].stage, data[1].stage) 42 | TestHelper.assert_models_equal(self, expected_data[2].epoch, data[2].epoch) 43 | TestHelper.assert_models_equal(self, expected_data[2].stage, data[2].stage) 44 | TestHelper.assert_models_equal(self, expected_data[3].epoch, data[3].epoch) 45 | TestHelper.assert_models_equal(self, expected_data[3].stage, data[3].stage) 46 | 47 | 48 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/setup/test_subject_builder.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | import numpy as np 3 | 4 | from source.analysis.setup.feature_type import FeatureType 5 | from source.analysis.setup.subject import Subject 6 | from source.analysis.setup.subject_builder import SubjectBuilder 7 | from test.test_helper import TestHelper 8 | 9 | 10 | class TestSubjectBuilder(TestCase): 11 | def test_get_all_subject_ids(self): 12 | self.assertListEqual( 13 | ['1', '2', '4', '5', '6', '7', '8', '9', '10', '11', '12', '14', '15', '16', '19', '20', '22', '23', '25', 14 | '27', '28', '29', '30', '32', '33', '34', '35', '38', '39', '41', '42'], 15 | SubjectBuilder.get_all_subject_ids()) 16 | 17 | @mock.patch('source.analysis.setup.subject_builder.PSGLabelService') 18 | @mock.patch('source.analysis.setup.subject_builder.TimeBasedFeatureService') 19 | @mock.patch('source.analysis.setup.subject_builder.HeartRateFeatureService') 20 | @mock.patch('source.analysis.setup.subject_builder.ActivityCountFeatureService') 21 | def test_build(self, mock_activity_count_fs, mock_heart_rate_fs, mock_time_based_fs, mock_psg_label_service): 22 | subject_id = "subjectA" 23 | mock_activity_count_fs.get_path_to_file.return_value = activity_count_feature = np.array([1, 2, 3, 4, 5]) 24 | mock_heart_rate_fs.get_path_to_file.return_value = heart_rate_feature = np.array([6, 7, 8, 9, 10]) 25 | mock_time_based_fs.load_time.return_value = time_feature = np.array([11]) 26 | mock_time_based_fs.load_circadian_model.return_value = circadian_feature = np.array([12]) 27 | mock_time_based_fs.load_cosine.return_value = cosine = np.array([12]) 28 | mock_psg_label_service.load.return_value = labels = np.array([13]) 29 | 30 | feature_dictionary = {FeatureType.count: activity_count_feature, 31 | FeatureType.heart_rate: heart_rate_feature, 32 | FeatureType.time: time_feature, 33 | FeatureType.circadian_model: circadian_feature, 34 | FeatureType.cosine: cosine 35 | } 36 | 37 | expected_subject = Subject(subject_id=subject_id, labeled_sleep=labels, feature_dictionary=feature_dictionary) 38 | returned_subject = SubjectBuilder.build(subject_id) 39 | 40 | TestHelper.assert_models_equal(self, expected_subject, returned_subject) 41 | 42 | @mock.patch.object(SubjectBuilder, 'build') 43 | @mock.patch.object(SubjectBuilder, 'get_all_subject_ids') 44 | def test_build_subject_dictionary(self, mock_subject_ids, mock_build): 45 | mock_subject_ids.return_value = subject_ids = ["subject123"] 46 | mock_build.return_value = placeholder = np.array([1, 2, 3]) 47 | 48 | subject_dictionary = SubjectBuilder.get_subject_dictionary() 49 | 50 | mock_build.assert_called_once_with(subject_ids[0]) 51 | self.assertDictEqual({subject_ids[0]: placeholder}, subject_dictionary) 52 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/motion/motion_service.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import os 4 | 5 | from source import utils 6 | from source.constants import Constants 7 | from source.preprocessing.motion.motion_collection import MotionCollection 8 | 9 | 10 | class MotionService(object): 11 | 12 | @staticmethod 13 | def load_raw(subject_id): 14 | raw_motion_path = MotionService.get_raw_file_path(subject_id) 15 | motion_array = MotionService.load(raw_motion_path) 16 | motion_array = utils.remove_repeats(motion_array) 17 | return MotionCollection(subject_id=subject_id, data=motion_array) 18 | 19 | @staticmethod 20 | def load_cropped(subject_id): 21 | cropped_motion_path = MotionService.get_cropped_file_path(subject_id) 22 | motion_array = MotionService.load(cropped_motion_path) 23 | return MotionCollection(subject_id=subject_id, data=motion_array) 24 | 25 | @staticmethod 26 | def load(motion_file, delimiter=' '): 27 | motion_array = pd.read_csv(str(motion_file), delimiter=delimiter).values 28 | return motion_array 29 | 30 | @staticmethod 31 | def write(motion_collection): 32 | motion_output_path = MotionService.get_cropped_file_path(motion_collection.subject_id) 33 | np.savetxt(motion_output_path, motion_collection.data, fmt='%f') 34 | 35 | @staticmethod 36 | def crop(motion_collection, interval): 37 | subject_id = motion_collection.subject_id 38 | timestamps = motion_collection.timestamps 39 | valid_indices = ((timestamps >= interval.start_time) 40 | & (timestamps < interval.end_time)).nonzero()[0] 41 | 42 | cropped_data = motion_collection.data[valid_indices, :] 43 | return MotionCollection(subject_id=subject_id, data=cropped_data) 44 | 45 | @staticmethod 46 | def get_cropped_file_path(subject_id): 47 | return Constants.CROPPED_FILE_PATH.joinpath(subject_id + "_cleaned_motion.out") 48 | 49 | @staticmethod 50 | def get_raw_file_path(subject_id): 51 | project_root = Constants.RAW_PATH 52 | # project_root = utils.get_project_root() 53 | # return project_root.joinpath('data/motion/' + subject_id + '_acceleration.txt') 54 | return os.path.join(project_root, 'motion', subject_id + '_acceleration.txt') 55 | 56 | @staticmethod 57 | def up_down_sampling_xyz_from_collection_write(motion_collection, interval, hz): 58 | values = motion_collection.values 59 | new_index = np.arange(round(interval.start_time), round(interval.end_time), 1/hz) 60 | data_list = [np.expand_dims(new_index, -1)] 61 | for ch in np.arange(values.shape[1]): 62 | tmp_data = np.interp(new_index, motion_collection.timestamps, values[:, ch]) 63 | data_list.append(np.expand_dims(tmp_data, -1)) 64 | data = np.hstack(data_list) 65 | 66 | motion_output_path = Constants.CROPPED_FILE_PATH.joinpath( 67 | motion_collection.subject_id + r"_cleaned_resampled_%s_hz.out" % hz) 68 | np.savetxt(motion_output_path, data, fmt='%f') -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/heart_rate/test_heart_rate_feature_service.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | from unittest.mock import MagicMock 3 | import numpy as np 4 | from source.constants import Constants 5 | from source.preprocessing.epoch import Epoch 6 | from source.preprocessing.heart_rate.heart_rate_collection import HeartRateCollection 7 | from source.preprocessing.heart_rate.heart_rate_feature_service import HeartRateFeatureService 8 | 9 | 10 | class TestHeartRateFeatureService(TestCase): 11 | 12 | @mock.patch('source.preprocessing.heart_rate.heart_rate_feature_service.pd') 13 | def test_load(self, mock_pd): 14 | mock_pd.read_csv.return_value = mock_return = MagicMock() 15 | mock_return.values = expected_return = np.array([1, 2, 3, 4, 5]) 16 | actual_returned_value = HeartRateFeatureService.load("subjectA") 17 | 18 | self.assertListEqual(expected_return.tolist(), actual_returned_value.tolist()) 19 | mock_pd.read_csv.assert_called_once_with(str(HeartRateFeatureService.get_path("subjectA")), delimiter=' ') 20 | 21 | def test_get_path(self): 22 | expected_path = Constants.FEATURE_FILE_PATH.joinpath("subjectA" + '_hr_feature.out') 23 | 24 | self.assertEqual(expected_path, HeartRateFeatureService.get_path("subjectA")) 25 | 26 | @mock.patch('source.preprocessing.heart_rate.heart_rate_feature_service.np') 27 | def test_write(self, mock_np): 28 | feature_to_write = np.array([1, 2, 3, 4]) 29 | subject_id = "subjectA" 30 | HeartRateFeatureService.write(subject_id, feature_to_write) 31 | 32 | mock_np.savetxt.assert_called_once_with(HeartRateFeatureService.get_path(subject_id), feature_to_write, 33 | fmt='%f') 34 | 35 | def test_get_window(self): 36 | timestamps = np.array([-1000, -500, 32, 50, 60, 800, 1000]) 37 | epoch = Epoch(timestamp=55, index=120) 38 | expected_indices_in_range = np.array([2, 3, 4]) 39 | 40 | actual_indices_in_range = HeartRateFeatureService.get_window(timestamps, epoch) 41 | 42 | self.assertEqual(expected_indices_in_range.tolist(), actual_indices_in_range.tolist()) 43 | 44 | @mock.patch.object(HeartRateFeatureService, 'get_feature') 45 | @mock.patch('source.preprocessing.heart_rate.heart_rate_feature_service.HeartRateService') 46 | def test_build_feature_array(self, mock_heart_rate_service, mock_get_feature): 47 | subject_id = "subjectA" 48 | data = np.array( 49 | [[1, 10], [10, 220], [20, 0], [40, 500], [70, 200], [90, 0], [100, 0], [400, 4]]) 50 | motion_collection = HeartRateCollection(subject_id=subject_id, data=data) 51 | mock_heart_rate_service.load_cropped.return_value = motion_collection 52 | expected_features = [np.array([0.1]), np.array([0.2])] 53 | mock_get_feature.side_effect = expected_features 54 | expected_feature_array = np.array(expected_features) 55 | 56 | valid_epochs = [Epoch(timestamp=4, index=1), Epoch(timestamp=50, index=2)] 57 | 58 | returned_feature_array = HeartRateFeatureService.build(subject_id, valid_epochs) 59 | 60 | self.assertEqual(expected_feature_array.tolist(), returned_feature_array.tolist()) 61 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/archive/scripts/clock_modeling/circadianModel.m: -------------------------------------------------------------------------------- 1 | function [tc,y] = circadianModel(lightStruct,tau) 2 | % CIRCADIANMODEL Simulates the model of the circadian clock given light 3 | % input in a struct and a value for tau; assumes light sufficiently padded 4 | % to remove initial condition effects 5 | % 6 | % [T,Y] = circadianModel(LIGHT,TAU) returns the timesteps in T and output 7 | % variables (x, xc, n) in Y 8 | % 9 | % Example: 10 | % [t,y] = circadianModel(struct('dur',dur,'time',time,'light',light_vec),24.2); 11 | 12 | 13 | % Random initial conditions 14 | ics = rand(1,3); 15 | ics(1:2) = ics(1:2)*2 - 1; 16 | dt = 0.1; % hours 17 | 18 | [tc,y] = ode23s(@simple, 0:dt:lightStruct.dur,ics,[],lightStruct); 19 | 20 | function dydt = simple(t,y,u) 21 | % Forger, 1999 - This is the one we use in the paper 22 | x = y(1); 23 | xc = y(2); 24 | n = y(3); 25 | 26 | alph = interp1(u.time,u.light,t); 27 | tx = tau; % 24.2; 28 | G = 19.875; 29 | k = .55; 30 | mu = .23; 31 | b = 0.013; 32 | 33 | Bh = G*(1-n)*alph; 34 | B = Bh*(1 - .4*x)*(1 - .4*xc); 35 | 36 | dydt(1) = pi/12*(xc + B); 37 | dydt(2) = pi/12*(mu*(xc - 4*xc^3/3) - x*((24/(.99669*tx))^2 + k*B)); 38 | dydt(3) = 60*(alph*(1-n) - b*n); 39 | 40 | dydt = dydt'; 41 | end 42 | 43 | function dydt = nonphotic(t,y,u) 44 | % St. Hilaire Model 2007 45 | x = y(1); 46 | xc = y(2); 47 | n = y(3); 48 | alph = interp1(u.time,u.light,t); 49 | 50 | tx = tau; % 24.2; 51 | 52 | G = 19.875; 53 | k = .55; 54 | b = 0.013; 55 | 56 | mu = .1300; 57 | q = 1/3; 58 | rho = 0.032; 59 | Bh = G*alph*(1-n); 60 | B = Bh*(1-0.4*x)*(1-0.4*xc); 61 | 62 | sw = sign(alph); % Sleep/wake, currently guessed from light exposure 63 | Nsh = rho*(1/3 - sw); 64 | Ns = Nsh*(1 - tanh(10*x)); 65 | 66 | if (x < -0.3 && x > -0.9) 67 | Nsh = rho*(1/3); 68 | Ns = Nsh*(1 - tanh(10*x)); 69 | end 70 | 71 | dydt(1) = pi/12* (xc + mu*(1/3*x+4/3*x^3-256/105*x^7) + B + Ns); 72 | dydt(2) = pi/12* (q*B*xc - x*((24/(0.99729*tx))^2 + k*B)); 73 | dydt(3) = 60*(alph*(1-n) - b*n); 74 | 75 | dydt = dydt'; 76 | end 77 | 78 | function dydt = kronauerJewett(t,y,u) 79 | % Kronauer-Jewett Model 80 | x = y(1); 81 | xc = y(2); 82 | n = y(3); 83 | 84 | alph = interp1(u.time,u.light,t); 85 | tx = tau; % 24.2; 86 | G = 19.875; 87 | k = .55; 88 | b = 0.013; 89 | 90 | mu = .1300; 91 | q = 1/3; 92 | Bh = G*alph*(1-n); 93 | B = Bh*(1-0.4*x)*(1-0.4*xc); 94 | dydt(1) = pi/12* (xc + mu*(1/3*x+4/3*x^3-256/105*x^7) + B); 95 | dydt(2) = pi/12* (q*B*xc - x*((24/(0.99729*tx))^2 + k*B)); 96 | dydt(3) = 60*(alph*(1-n) - b*n); 97 | 98 | dydt = dydt'; 99 | end 100 | 101 | end 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/analysis/setup/subject_builder.py: -------------------------------------------------------------------------------- 1 | from source.analysis.setup.feature_type import FeatureType 2 | from source.analysis.setup.subject import Subject 3 | from source.preprocessing.activity_count.activity_count_feature_service import ActivityCountFeatureService 4 | from source.preprocessing.heart_rate.heart_rate_feature_service import HeartRateFeatureService 5 | from source.preprocessing.psg.psg_label_service import PSGLabelService 6 | from source.preprocessing.time.time_based_feature_service import TimeBasedFeatureService 7 | 8 | 9 | class SubjectBuilder(object): 10 | 11 | @staticmethod 12 | def get_all_subject_ids(): 13 | 14 | subjects_as_ints = [46343, 3509524, 5132496, 1066528, 5498603, 2638030, 2598705, 5383425, 1455390, 4018081, 9961348, 15 | 1449548, 8258170, 781756, 9106476, 8686948, 8530312, 3997827, 4314139, 1818471, 4426783, 16 | 8173033, 7749105, 5797046, 759667, 8000685, 6220552, 844359, 9618981, 1360686, 17 | 8692923] 18 | 19 | subjects_as_strings = [] 20 | 21 | for subject in subjects_as_ints: 22 | subjects_as_strings.append(str(subject)) 23 | return subjects_as_strings 24 | 25 | @staticmethod 26 | def get_subject_dictionary(): 27 | subject_dictionary = {} 28 | all_subject_ids = SubjectBuilder.get_all_subject_ids() 29 | for subject_id in all_subject_ids: 30 | subject_dictionary[subject_id] = SubjectBuilder.build(subject_id) 31 | 32 | return subject_dictionary 33 | 34 | @staticmethod 35 | def build(subject_id): 36 | feature_count = ActivityCountFeatureService.load(subject_id) 37 | feature_hr = HeartRateFeatureService.load(subject_id) 38 | feature_time = TimeBasedFeatureService.load_time(subject_id) 39 | feature_circadian = TimeBasedFeatureService.load_circadian_model(subject_id) 40 | feature_cosine = TimeBasedFeatureService.load_cosine(subject_id) 41 | labeled_sleep = PSGLabelService.load(subject_id) 42 | 43 | feature_dictionary = {FeatureType.count: feature_count, 44 | FeatureType.heart_rate: feature_hr, 45 | FeatureType.time: feature_time, 46 | FeatureType.circadian_model: feature_circadian, 47 | FeatureType.cosine: feature_cosine} 48 | 49 | subject = Subject(subject_id=subject_id, 50 | labeled_sleep=labeled_sleep, 51 | feature_dictionary=feature_dictionary) 52 | 53 | # Uncomment to save plots of every subject's data: 54 | # ax = plt.subplot(5, 1, 1) 55 | # ax.plot(range(len(feature_hr)), feature_hr) 56 | # ax = plt.subplot(5, 1, 2) 57 | # ax.plot(range(len(feature_count)), feature_count) 58 | # ax = plt.subplot(5, 1, 3) 59 | # ax.plot(range(len(feature_cosine)), feature_cosine) 60 | # ax = plt.subplot(5, 1, 4) 61 | # ax.plot(range(len(feature_circadian)), feature_circadian) 62 | # ax = plt.subplot(5, 1, 5) 63 | # ax.plot(range(len(labeled_sleep)), labeled_sleep) 64 | # 65 | # plt.savefig(str(Constants.FIGURE_FILE_PATH.joinpath(subject_id + '_applewatch.png'))) 66 | # plt.close() 67 | return subject 68 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/mesa/test_mesa_subject_builder.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | 3 | from source.mesa.mesa_psg_service import MesaPSGService 4 | from source.mesa.mesa_subject_builder import MesaSubjectBuilder 5 | 6 | import numpy as np 7 | 8 | from source.preprocessing.activity_count.activity_count_collection import ActivityCountCollection 9 | from source.preprocessing.heart_rate.heart_rate_collection import HeartRateCollection 10 | 11 | 12 | class TestMesaSubjectBuilder(TestCase): 13 | 14 | @mock.patch('source.mesa.mesa_subject_builder.MesaTimeBasedService') 15 | @mock.patch('source.mesa.mesa_subject_builder.MesaActigraphyService') 16 | @mock.patch('source.mesa.mesa_subject_builder.MesaHeartRateService') 17 | @mock.patch.object(MesaPSGService, 'load_raw') 18 | def test_build(self, mock_psg_service_load_raw, mock_heart_rate_service, mock_actigraphy_service, 19 | mock_time_service): 20 | file_id = 'subjectA' 21 | 22 | returned_sleep_data = [] 23 | for i in range(30): 24 | returned_sleep_data.append(0) 25 | for i in range(90): 26 | returned_sleep_data.append(1) 27 | for i in range(90): 28 | returned_sleep_data.append(3) 29 | for i in range(30): 30 | returned_sleep_data.append(2) 31 | 32 | heart_rate_data = np.array([[0, 20], 33 | [4, 60], 34 | [25, 95], 35 | [35, 333], 36 | [55, 25], 37 | [100, 233], 38 | [190, 24]]) 39 | 40 | actigraphy_data = np.array([[0, 1], 41 | [15, 2], 42 | [30, 3], 43 | [45, 4], 44 | [60, 5], 45 | [75, 6], 46 | [90, 7]]) 47 | 48 | circadian_data = np.array([[0, 0.3], 49 | [5, 0.4], 50 | [10, 0.1], 51 | [15, 0.6], 52 | [20, 0.7], 53 | [25, 0.2], 54 | [30, 0.1], 55 | [35, 0.3], 56 | [40, 0.4], 57 | [45, 0.1], 58 | [50, 0.6], 59 | [55, 0.7], 60 | [60, 0.2], 61 | [65, 0.1]]) 62 | 63 | mock_psg_service_load_raw.return_value = np.array(returned_sleep_data) 64 | mock_heart_rate_service.load_raw.return_value = HeartRateCollection(subject_id=file_id, data=heart_rate_data) 65 | mock_actigraphy_service.load_raw.return_value = ActivityCountCollection(subject_id=file_id, 66 | data=actigraphy_data) 67 | mock_time_service.load_circadian_model.return_value = circadian_data 68 | 69 | subject = MesaSubjectBuilder.build(file_id) 70 | 71 | self.assertEqual([[0], [1], [1], [1], [3], [3], [3], [2]], subject.labeled_sleep.tolist()) 72 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/setup/test_feature_set_service.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from source.analysis.setup.feature_type import FeatureType 4 | from source.analysis.setup.feature_set_service import FeatureSetService 5 | import seaborn as sns 6 | 7 | 8 | class TestFeatureSetService(TestCase): 9 | 10 | def test_get_feature_set_labels(self): 11 | self.assertEqual("Motion only", FeatureSetService.get_label([FeatureType.count])) 12 | self.assertEqual("HR only", FeatureSetService.get_label([FeatureType.heart_rate])) 13 | self.assertEqual("Motion, HR", FeatureSetService.get_label([FeatureType.count, 14 | FeatureType.heart_rate])) 15 | self.assertEqual("Motion, HR, and Clock", FeatureSetService.get_label([FeatureType.count, 16 | FeatureType.heart_rate, 17 | FeatureType.circadian_model])) 18 | self.assertEqual("Motion, HR, and Time", FeatureSetService.get_label([FeatureType.count, 19 | FeatureType.heart_rate, 20 | FeatureType.time 21 | ])) 22 | self.assertEqual("Motion, HR, and Cosine", FeatureSetService.get_label([FeatureType.count, 23 | FeatureType.heart_rate, 24 | FeatureType.cosine 25 | ])) 26 | 27 | def test_get_feature_set_colors(self): 28 | self.assertEqual(sns.xkcd_rgb["denim blue"], FeatureSetService.get_color([FeatureType.count])) 29 | self.assertEqual(sns.xkcd_rgb["yellow orange"], FeatureSetService.get_color([FeatureType.heart_rate])) 30 | self.assertEqual(sns.xkcd_rgb["medium green"], FeatureSetService.get_color([FeatureType.count, 31 | FeatureType.heart_rate])) 32 | self.assertEqual(sns.xkcd_rgb["medium pink"], FeatureSetService.get_color([FeatureType.count, 33 | FeatureType.heart_rate, 34 | FeatureType.circadian_model])) 35 | self.assertEqual(sns.xkcd_rgb["greyish"], FeatureSetService.get_color([FeatureType.count, 36 | FeatureType.heart_rate, 37 | FeatureType.time 38 | ])) 39 | self.assertEqual(sns.xkcd_rgb["plum"], FeatureSetService.get_color([FeatureType.count, 40 | FeatureType.heart_rate, 41 | FeatureType.cosine 42 | ])) 43 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/source/preprocessing/time/clock_proxy/circadianModel.m: -------------------------------------------------------------------------------- 1 | function [tc,y] = circadianModel(lightStruct,tau) 2 | % CIRCADIANMODEL Simulates the model of the circadian clock given light 3 | % input in a struct and a value for tau; assumes light sufficiently padded 4 | % to remove initial condition effects 5 | % 6 | % [T,Y] = circadianModel(LIGHT,TAU) returns the timesteps in T and output 7 | % variables (x, xc, n) in Y 8 | % 9 | % Example: 10 | % [t,y] = circadianModel(struct('dur',dur,'time',time,'light',light_vec),24.2); 11 | 12 | 13 | % Random initial conditions 14 | ics = rand(1,3); 15 | ics(1:2) = ics(1:2)*2 - 1; 16 | dt = 0.1; % hours 17 | 18 | tc = 0:dt:lightStruct.dur; 19 | y = ode4(@simple,tc,ics,lightStruct); 20 | 21 | %[tc,y] = ode23s(@simple, 0:dt:lightStruct.dur,ics,[],lightStruct); 22 | 23 | function dydt = simple(t,y,u) 24 | % Forger, 1999 - This is the one we use in the paper 25 | x = y(1); 26 | xc = y(2); 27 | n = y(3); 28 | 29 | alph = u.light(find(u.time < t, 1, 'last' )); 30 | 31 | if(isempty(alph)) 32 | alph = 0; 33 | end 34 | 35 | tx = tau; % 24.2; 36 | G = 19.875; 37 | k = .55; 38 | mu = .23; 39 | b = 0.013; 40 | 41 | Bh = G*(1-n)*alph; 42 | B = Bh*(1 - .4*x)*(1 - .4*xc); 43 | 44 | dydt(1) = pi/12*(xc + B); 45 | dydt(2) = pi/12*(mu*(xc - 4*xc^3/3) - x*((24/(.99669*tx))^2 + k*B)); 46 | dydt(3) = 60*(alph*(1-n) - b*n); 47 | 48 | dydt = dydt'; 49 | end 50 | 51 | 52 | function dydt = nonphotic(t,y,u) 53 | % St. Hilaire Model 2007 54 | x = y(1); 55 | xc = y(2); 56 | n = y(3); 57 | alph = interp1(u.time,u.light,t); 58 | 59 | tx = tau; % 24.2; 60 | 61 | G = 19.875; 62 | k = .55; 63 | b = 0.013; 64 | 65 | mu = .1300; 66 | q = 1/3; 67 | rho = 0.032; 68 | Bh = G*alph*(1-n); 69 | B = Bh*(1-0.4*x)*(1-0.4*xc); 70 | 71 | sw = sign(alph); % Sleep/wake, currently guessed from light exposure 72 | Nsh = rho*(1/3 - sw); 73 | Ns = Nsh*(1 - tanh(10*x)); 74 | 75 | if (x < -0.3 && x > -0.9) 76 | Nsh = rho*(1/3); 77 | Ns = Nsh*(1 - tanh(10*x)); 78 | end 79 | 80 | dydt(1) = pi/12* (xc + mu*(1/3*x+4/3*x^3-256/105*x^7) + B + Ns); 81 | dydt(2) = pi/12* (q*B*xc - x*((24/(0.99729*tx))^2 + k*B)); 82 | dydt(3) = 60*(alph*(1-n) - b*n); 83 | 84 | dydt = dydt'; 85 | end 86 | 87 | function dydt = kronauerJewett(t,y,u) 88 | % Kronauer-Jewett Model 89 | x = y(1); 90 | xc = y(2); 91 | n = y(3); 92 | 93 | alph = interp1(u.time,u.light,t); 94 | tx = tau; % 24.2; 95 | G = 19.875; 96 | k = .55; 97 | b = 0.013; 98 | 99 | mu = .1300; 100 | q = 1/3; 101 | Bh = G*alph*(1-n); 102 | B = Bh*(1-0.4*x)*(1-0.4*xc); 103 | dydt(1) = pi/12* (xc + mu*(1/3*x+4/3*x^3-256/105*x^7) + B); 104 | dydt(2) = pi/12* (q*B*xc - x*((24/(0.99729*tx))^2 + k*B)); 105 | dydt(3) = 60*(alph*(1-n) - b*n); 106 | 107 | dydt = dydt'; 108 | end 109 | 110 | end 111 | 112 | 113 | 114 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/analysis/classification/test_classifier_input_builder.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | import numpy as np 3 | 4 | from source.analysis.classification.classifier_input_builder import ClassifierInputBuilder 5 | from source.analysis.setup.feature_type import FeatureType 6 | from source.analysis.setup.subject import Subject 7 | 8 | 9 | class TestClassifierInputBuilder(TestCase): 10 | 11 | @mock.patch("source.analysis.classification.classifier_input_builder.ClassifierInputBuilder.get_array") 12 | def test_sleep_wake_inputs(self, mock_get_array): 13 | labels = np.array([1, 2, 3, 4, 5, 0, 0, 0, 0]) 14 | binarized_labels = np.array([1, 1, 1, 1, 1, 0, 0, 0, 0]) 15 | values = np.array([3, 4]) 16 | mock_get_array.return_value = values, labels 17 | subject_ids = [] 18 | subject_dictionary = {} 19 | feature_set = [] 20 | returned_values, returned_labels = ClassifierInputBuilder.get_sleep_wake_inputs(subject_ids, 21 | subject_dictionary, 22 | feature_set) 23 | self.assertListEqual(binarized_labels.tolist(), returned_labels.tolist()) 24 | self.assertListEqual(values.tolist(), returned_values.tolist()) 25 | 26 | def test_get_array(self): 27 | subject_ids = ["subjectA", "subjectB"] 28 | subject_dictionary = { 29 | "subjectA": Subject(subject_id="subjectA", 30 | labeled_sleep=np.array([[0], [1]]), 31 | feature_dictionary={FeatureType.count: np.array([[0], [1]]), 32 | FeatureType.motion: np.array([[2], [3]]), 33 | FeatureType.heart_rate: np.array([[4], [5]]), 34 | FeatureType.cosine: np.array([[6], [7]]), 35 | FeatureType.circadian_model: np.array([[8], [9]]), 36 | FeatureType.time: np.array([[10], [11]]), 37 | }), 38 | "subjectB": Subject(subject_id="subjectB", 39 | labeled_sleep=np.array([[1], [1]]), 40 | feature_dictionary={FeatureType.count: np.array([[100], [101]]), 41 | FeatureType.motion: np.array([[102], [103]]), 42 | FeatureType.heart_rate: np.array([[104], [105]]), 43 | FeatureType.cosine: np.array([[106], [107]]), 44 | FeatureType.circadian_model: np.array( 45 | [[108], [109]]), 46 | FeatureType.time: np.array([[110], [111]]), 47 | }) 48 | } 49 | 50 | feature_set = [FeatureType.count, FeatureType.cosine] 51 | 52 | features, labels = ClassifierInputBuilder.get_array(subject_ids, subject_dictionary, feature_set) 53 | 54 | self.assertEqual(np.array([[0, 6], [1, 7], [100, 106], [101, 107]]).tolist(), features.tolist()) 55 | self.assertEqual(np.array([[0], [1], [1], [1]]).tolist(), labels.tolist()) 56 | -------------------------------------------------------------------------------- /data_loader/sliding_window.py: -------------------------------------------------------------------------------- 1 | # from http://www.johnvinyard.com/blog/?p=268 2 | 3 | import numpy as np 4 | from numpy.lib.stride_tricks import as_strided as ast 5 | 6 | def norm_shape(shape): 7 | ''' 8 | Normalize numpy array shapes so they're always expressed as a tuple, 9 | even for one-dimensional shapes. 10 | 11 | Parameters 12 | shape - an int, or a tuple of ints 13 | 14 | Returns 15 | a shape tuple 16 | ''' 17 | try: 18 | i = int(shape) 19 | return (i,) 20 | except TypeError: 21 | # shape was not a number 22 | pass 23 | 24 | try: 25 | t = tuple(shape) 26 | return t 27 | except TypeError: 28 | # shape was not iterable 29 | pass 30 | 31 | raise TypeError('shape must be an int, or a tuple of ints') 32 | 33 | def sliding_window(a,ws,ss = None,flatten = True): 34 | ''' 35 | Return a sliding window over a in any number of dimensions 36 | 37 | Parameters: 38 | a - an n-dimensional numpy array 39 | ws - an int (a is 1D) or tuple (a is 2D or greater) representing the size 40 | of each dimension of the window 41 | ss - an int (a is 1D) or tuple (a is 2D or greater) representing the 42 | amount to slide the window in each dimension. If not specified, it 43 | defaults to ws. 44 | flatten - if True, all slices are flattened, otherwise, there is an 45 | extra dimension for each dimension of the input. 46 | 47 | Returns 48 | an array containing each n-dimensional window from a 49 | ''' 50 | 51 | if None is ss: 52 | # ss was not provided. the windows will not overlap in any direction. 53 | ss = ws 54 | ws = norm_shape(ws) 55 | ss = norm_shape(ss) 56 | 57 | # convert ws, ss, and a.shape to numpy arrays so that we can do math in every 58 | # dimension at once. 59 | ws = np.array(ws) 60 | ss = np.array(ss) 61 | shape = np.array(a.shape) 62 | 63 | 64 | # ensure that ws, ss, and a.shape all have the same number of dimensions 65 | ls = [len(shape),len(ws),len(ss)] 66 | if 1 != len(set(ls)): 67 | raise ValueError(\ 68 | 'a.shape, ws and ss must all have the same length. They were %s' % str(ls)) 69 | 70 | # ensure that ws is smaller than a in every dimension 71 | if np.any(ws > shape): 72 | raise ValueError(\ 73 | 'ws cannot be larger than a in any dimension.\ 74 | a.shape was %s and ws was %s' % (str(a.shape),str(ws))) 75 | 76 | # how many slices will there be in each dimension? 77 | newshape = norm_shape(((shape - ws) // ss) + 1) 78 | # the shape of the strided array will be the number of slices in each dimension 79 | # plus the shape of the window (tuple addition) 80 | newshape += norm_shape(ws) 81 | # the strides tuple will be the array's strides multiplied by step size, plus 82 | # the array's strides (tuple addition) 83 | newstrides = norm_shape(np.array(a.strides) * ss) + a.strides 84 | strided = ast(a,shape = newshape,strides = newstrides) 85 | if not flatten: 86 | return strided 87 | 88 | # Collapse strided so that it has one more dimension than the window. I.e., 89 | # the new array is a flat list of slices. 90 | meat = len(ws) if ws.shape else 0 91 | firstdim = (np.product(newshape[:-meat]),) if ws.shape else () 92 | dim = firstdim + (newshape[-meat:]) 93 | # remove any dimensions with size 1 94 | # Ben: migrated from python 2 to python 3 95 | # dim = filter(lambda i: i != 1, dim) 96 | dim = [x for x in dim if x!=1] 97 | return strided.reshape(dim) -------------------------------------------------------------------------------- /applewatch_dataprocessing/archive/scripts/get_parameters.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import time 3 | 4 | from sklearn.model_selection import GridSearchCV 5 | from sklearn.neural_network import MLPClassifier 6 | from sklearn.ensemble import RandomForestClassifier 7 | from sklearn.linear_model import LogisticRegression 8 | from sklearn.neighbors import KNeighborsClassifier 9 | 10 | from sklearn.utils import class_weight 11 | 12 | import utilities 13 | import classify_sleep 14 | 15 | verbose = False 16 | 17 | 18 | def find_best(method_key, feature_set, training_subjects): 19 | # Load up all the data 20 | data_dict = utilities.build_data_dictionary(feature_set) 21 | 22 | # Initialize holders 23 | training_set_features = np.array([]) 24 | training_set_labels = np.array([]) 25 | 26 | # Build vectors for training subjects 27 | for subject in training_subjects: 28 | score_features, full_features = utilities.get_features(subject, data_dict) 29 | if np.shape(training_set_features)[0] == 0: 30 | training_set_features = full_features 31 | training_set_labels = score_features 32 | else: 33 | training_set_features = np.vstack((training_set_features, full_features)) 34 | training_set_labels = np.vstack((training_set_labels, score_features)) 35 | 36 | # Convert raw scores from 0-5 to binary,or 0-2 37 | training_set_labels = utilities.process_raw_scores(training_set_labels, classify_sleep.run_flag) 38 | 39 | if method_key == 'Logistic Regression': 40 | parameters = {'C': [0.001, 0.01, 0.1, 1, 10, 100], 'penalty': ['l1', 'l2']} 41 | classifier = LogisticRegression() 42 | 43 | if method_key == 'KNeighbors': 44 | parameters = {'n_neighbors': [500, 1000, 2000]} 45 | classifier = KNeighborsClassifier() 46 | 47 | if method_key == 'MLP': 48 | parameters = {'solver': ['lbfgs'], 'max_iter': [1000], 'alpha': 10.0 ** -np.arange(1, 4), 49 | 'hidden_layer_sizes': [(30, 30, 30)]} 50 | classifier = MLPClassifier() 51 | 52 | if method_key == 'Random Forest': 53 | max_depth = [int(x) for x in np.linspace(10, 110, num=2)] 54 | max_depth.append(None) 55 | max_depth = [10, 50, 100] 56 | min_samples_split = [10] 57 | min_samples_leaf = [32] 58 | parameters = {'n_estimators': [50], 'max_features': [None], 'max_depth': max_depth, 59 | 'min_samples_split': min_samples_split, 'min_samples_leaf': min_samples_leaf, 'bootstrap': [True]} 60 | classifier = RandomForestClassifier() 61 | 62 | class_weights = class_weight.compute_class_weight('balanced', 63 | np.unique(training_set_labels), 64 | training_set_labels) 65 | class_weight_dict = {0: class_weights[0], 1: class_weights[1]} 66 | 67 | if len(class_weights) > 2: 68 | class_weight_dict = {0: class_weights[0], 1: class_weights[1], 2: class_weights[2]} 69 | 70 | classifier.class_weight = class_weight_dict 71 | 72 | if classify_sleep.run_flag == utilities.RUN_REM: 73 | scoring = 'neg_log_loss' 74 | else: 75 | scoring = 'roc_auc' 76 | 77 | clf = GridSearchCV(classifier, parameters, scoring=scoring) 78 | 79 | clf.fit(training_set_features, training_set_labels) 80 | 81 | if verbose: 82 | print('Best parameters for set:') 83 | print(clf.best_params_) 84 | print('Score on training data: ' + str(clf.score(training_set_features, training_set_labels))) 85 | 86 | save_name = 'parameters/' + method_key + utilities.string_from_features(feature_set) + '.npy' 87 | np.save(save_name, clf.best_params_) 88 | 89 | return clf.best_params_ 90 | 91 | 92 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/psg/test_psg_service.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | 3 | from source.constants import Constants 4 | from source.preprocessing.epoch import Epoch 5 | from source.preprocessing.interval import Interval 6 | from source.preprocessing.psg.psg_raw_data_collection import PSGRawDataCollection 7 | from source.preprocessing.psg.psg_service import PSGService 8 | import numpy as np 9 | 10 | from source.preprocessing.psg.stage_item import StageItem 11 | from source.sleep_stage import SleepStage 12 | from test.test_helper import TestHelper 13 | 14 | 15 | class TestPSGService(TestCase): 16 | 17 | def test_crop(self): 18 | subject_id = 'subjectA' 19 | crop_interval = Interval(start_time=50, end_time=101) 20 | data = [StageItem(epoch=Epoch(timestamp=10, index=2), stage=SleepStage.n1), 21 | StageItem(epoch=Epoch(timestamp=50, index=3), stage=SleepStage.n2), 22 | StageItem(epoch=Epoch(timestamp=100, index=4), stage=SleepStage.n3), 23 | StageItem(epoch=Epoch(timestamp=170, index=5), stage=SleepStage.rem)] 24 | expected_data = [StageItem(epoch=Epoch(timestamp=50, index=3), stage=SleepStage.n2), 25 | StageItem(epoch=Epoch(timestamp=100, index=4), stage=SleepStage.n3), ] 26 | 27 | input_raw_data_collection = PSGRawDataCollection(subject_id=subject_id, data=data) 28 | 29 | returned_psg_raw_collection = PSGService.crop(input_raw_data_collection, crop_interval) 30 | 31 | self.assertEqual(subject_id, returned_psg_raw_collection.subject_id) 32 | TestHelper.assert_models_equal(self, expected_data[0], returned_psg_raw_collection.data[0]) 33 | TestHelper.assert_models_equal(self, expected_data[1], returned_psg_raw_collection.data[1]) 34 | 35 | @mock.patch('source.preprocessing.psg.psg_service.np') 36 | def test_write(self, mock_np): 37 | subject_id = 'subjectA' 38 | mock_np.array.return_value = returned_array = np.array([[1, 2], [3, 4]]) 39 | list_data = [[10, 1], [50, 2], [100, 3], [170, 5]] 40 | data = [StageItem(epoch=Epoch(timestamp=10, index=2), stage=SleepStage.n1), 41 | StageItem(epoch=Epoch(timestamp=50, index=3), stage=SleepStage.n2), 42 | StageItem(epoch=Epoch(timestamp=100, index=4), stage=SleepStage.n3), 43 | StageItem(epoch=Epoch(timestamp=170, index=5), stage=SleepStage.rem)] 44 | psg_raw_data_collection = PSGRawDataCollection(subject_id=subject_id, data=data) 45 | psg_output_path = Constants.CROPPED_FILE_PATH.joinpath("subjectA_cleaned_psg.out") 46 | 47 | PSGService.write(psg_raw_data_collection) 48 | 49 | mock_np.array.assert_called_once_with(list_data) 50 | mock_np.savetxt.assert_called_once_with(psg_output_path, returned_array, fmt='%f') 51 | 52 | @mock.patch('source.preprocessing.psg.psg_service.pd') 53 | def test_load_cropped_array(self, mock_pd): 54 | subject_id = 'subject100' 55 | cropped_psg_path = Constants.CROPPED_FILE_PATH.joinpath(subject_id + "_cleaned_psg.out") 56 | mock_pd.read_csv.return_value.values = np.array([[1, 2], [4, 5], [10, 1]]) 57 | 58 | psg_collection = PSGService.load_cropped(subject_id) 59 | 60 | mock_pd.read_csv.assert_called_once_with(str(cropped_psg_path), delimiter=' ') 61 | TestHelper.assert_models_equal(self, StageItem(epoch=Epoch(timestamp=1, index=0), stage=SleepStage.n2), 62 | psg_collection.data[0]) 63 | TestHelper.assert_models_equal(self, StageItem(epoch=Epoch(timestamp=4, index=1), stage=SleepStage.rem), 64 | psg_collection.data[1]) 65 | TestHelper.assert_models_equal(self, StageItem(epoch=Epoch(timestamp=10, index=2), stage=SleepStage.n1), 66 | psg_collection.data[2]) 67 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ubi-SleepNet 2 | This is the code repository for the paper: Ubi-SleepNet: Advanced Multimodal Fusion Techniques for 3 | Three-stage Sleep Classification Using Ubiquitous Sensing. 4 | ## Updates 5 | * 2022-04-20 Uploaded a preprocessed and windowed Apple Watch dataset based on raw accelerometer data and heart rate. 6 | * 2022-05-05 Uploaded preprocessed CSV files of the Apple Watch dataset 7 | * 2024-04-02 Uploaded the missing feature extraction code for the Apple Watch dataset. And the 8 | ## Dataset Download 9 | * [MESA Dataset](https://sleepdata.org/datasets/mesa) 10 | * [Apple Watch](https://physionet.org/content/sleep-accel/1.0.0/heart_rate/) 11 | * [Preprocessed Apple Watch H5 file used in the paper](https://drive.google.com/drive/folders/1GDPVpUZMes8FZz1fieGQt0eYBEiakUke?usp=sharing) Please download the H5 files and put it into the Dataset folder 12 | * [Preprocessed Apple Watch CSV file used in the paper](https://drive.google.com/drive/folders/1nZ3Bu0P6z_7jM-eFiNTvDmNtshMOzhZC?usp=sharing) Please download the H5 files and put it into the Dataset folder 13 | ## Models Trained in the paper: 14 | 15 | 16 | ## Dataset Building 17 | * Data Pre-processing code is available at:[MakeSenseOfSleep](https://github.com/bzhai/multimodal_sleep_stage_benchmark) which should produce a h5 file for MESA dataset. 18 | * This repository includes the builders for MESA statistic features and Apple Watch dataset with statistic features 19 | * For the MESA with HRV feature set, please go to [MakeSenseOfSleep](https://github.com/bzhai/multimodal_sleep_stage_benchmark). It includes a dataset builder that can build the HRV feature dataset. 20 | * This repository also includes a data builder to build the dataset that uses the raw accelerometer 21 | and HR data collected from the Apple Watch dataset. 22 | * The data processing code was adapted from [link](https://github.com/ojwalch/sleep_classifiers.git) and located under the `applewatch_dataprocessing` folder. I also updated the processed data in `outputs\features`. 23 | * #### Apple Watch Dataset Building Process 24 | * If you use Pycharm, please make sure you open the project from the "applewatch_dataprocessing" folder. This will eliminate the "import package" issue. 25 | * Once you have done the previous step, the details regarding the dataset-building pipeline can be found in its _readme.md_ file 26 | 27 | ## Set up environment 28 | To ensure the experiments run smoothly, please create a **python 3.8** environment, and please be aware that the `pytables` and `h5py` require to be installed via `conda` . 29 | 30 | ## Running Experiments 31 | you could run a non-attention-based model by: 32 | 33 | python -m train_val_test --nn_type Vggacc79f174_7 --epochs 20 --dataset mesa 34 | 35 | To run the attention model, the modality should be specified. The code below is an example: 36 | 37 | python -m train_val_test --nn_type VggAcc79F174_SplitModal_SANTimeDimMatrixAttOnMod1NLayer1 --epochs 20 --dataset mesa --att_on_modality act 38 | 39 | ## Citation 40 | If you found this paper is helpful and like it. Please don't mind citing it and thank you. 41 | ``` 42 | @article{10.1145/3494961, 43 | author = {Zhai, Bing and Guan, Yu and Catt, Michael and Pl\"{o}tz, Thomas}, 44 | title = {Ubi-SleepNet: Advanced Multimodal Fusion Techniques for Three-Stage Sleep Classification Using Ubiquitous Sensing}, 45 | year = {2022}, 46 | issue_date = {Dec 2021}, 47 | publisher = {Association for Computing Machinery}, 48 | address = {New York, NY, USA}, 49 | volume = {5}, 50 | number = {4}, 51 | url = {https://doi.org/10.1145/3494961}, 52 | doi = {10.1145/3494961}, 53 | journal = {Proc. ACM Interact. Mob. Wearable Ubiquitous Technol.}, 54 | month = {dec}, 55 | articleno = {191}, 56 | numpages = {33}, 57 | keywords = {Multimodal Fusion, Wearable, Neural Networks, Sleep Monitoring, Apple Watch, Deep Learning, Three Sleep Stages, MESA, Heart Rate Variability, Heart Rate, Ubiquitous Sensing} 58 | } 59 | 60 | ``` 61 | -------------------------------------------------------------------------------- /applewatch_dataprocessing/test/preprocessing/activity_count/test_activity_count_feature_service.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | from unittest.mock import MagicMock, call 3 | 4 | import numpy as np 5 | from source.constants import Constants 6 | from source.preprocessing.activity_count.activity_count_feature_service import ActivityCountFeatureService 7 | from source.preprocessing.activity_count.activity_count_service import ActivityCountService 8 | from source.preprocessing.epoch import Epoch 9 | from source.preprocessing.activity_count.activity_count_collection import ActivityCountCollection 10 | 11 | 12 | class TestActivityCountFeatureService(TestCase): 13 | 14 | @mock.patch('source.preprocessing.activity_count.activity_count_feature_service.pd') 15 | def test_load(self, mock_pd): 16 | mock_pd.read_csv.return_value = mock_return = MagicMock() 17 | mock_return.values = expected_return = np.array([1, 2, 3, 4, 5]) 18 | actual_returned_value = ActivityCountFeatureService.load("subjectA") 19 | 20 | self.assertListEqual(expected_return.tolist(), actual_returned_value.tolist()) 21 | mock_pd.read_csv.assert_called_once_with(str(ActivityCountFeatureService.get_path("subjectA"))) 22 | 23 | def test_get_path(self): 24 | expected_path = Constants.FEATURE_FILE_PATH.joinpath("subjectA" + '_count_feature.out') 25 | 26 | self.assertEqual(expected_path, ActivityCountFeatureService.get_path("subjectA")) 27 | 28 | @mock.patch('source.preprocessing.activity_count.activity_count_feature_service.np') 29 | def test_write(self, mock_np): 30 | feature_to_write = np.array([1, 2, 3, 4]) 31 | subject_id = "subjectA" 32 | ActivityCountFeatureService.write(subject_id, feature_to_write) 33 | 34 | mock_np.savetxt.assert_called_once_with(ActivityCountFeatureService.get_path(subject_id), feature_to_write, 35 | fmt='%f') 36 | 37 | def test_get_window(self): 38 | timestamps = np.array([-2000, 22, 32, 50, 60, 800, 1000]) 39 | epoch = Epoch(timestamp=55, index=120) 40 | expected_indices_in_range = np.array([1, 2, 3, 4]) 41 | 42 | actual_indices_in_range = ActivityCountFeatureService.get_window(timestamps, epoch) 43 | 44 | self.assertEqual(expected_indices_in_range.tolist(), actual_indices_in_range.tolist()) 45 | 46 | @mock.patch.object(ActivityCountFeatureService, 'get_feature') 47 | @mock.patch.object(ActivityCountService, 'load_cropped') 48 | def test_build_feature_array(self, mock_load_cropped, mock_get_feature): 49 | subject_id = "subjectA" 50 | data = np.array( 51 | [[1, 10], [10, 220], [20, 0], [40, 500], [70, 200], [90, 0], [100, 0], [120, 4]]) 52 | activity_count_collection = ActivityCountCollection(subject_id=subject_id, data=data) 53 | mock_load_cropped.return_value = activity_count_collection 54 | expected_features = [np.array([0.1]), np.array([0.2])] 55 | mock_get_feature.side_effect = expected_features 56 | expected_feature_array = np.array(expected_features) 57 | 58 | valid_epochs = [Epoch(timestamp=4, index=1), Epoch(timestamp=50, index=2)] 59 | 60 | returned_feature_array = ActivityCountFeatureService.build(subject_id, valid_epochs) 61 | 62 | self.assertEqual(expected_feature_array.tolist(), returned_feature_array.tolist()) 63 | 64 | def test_interpolate(self): 65 | subject_id = "subjectA" 66 | data = np.array([[1, 0], [10, 9]]) 67 | activity_count_collection = ActivityCountCollection(subject_id=subject_id, data=data) 68 | 69 | interpolated_timestamps, interpolated_counts = ActivityCountFeatureService.interpolate( 70 | activity_count_collection) 71 | 72 | self.assertListEqual([0, 1, 2, 3, 4, 5, 6, 7, 8], interpolated_counts.tolist()) 73 | self.assertListEqual([1, 2, 3, 4, 5, 6, 7, 8, 9], interpolated_timestamps.tolist()) -------------------------------------------------------------------------------- /evaluation_metrics.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | from imblearn.metrics import specificity_score 4 | from sklearn import metrics 5 | from joblib import Parallel, delayed 6 | import multiprocessing 7 | 8 | def eval_precision(gt, pred, average='macro'): 9 | if type(gt) is pd.core.frame.DataFrame or type(gt) is pd.core.frame.Series: 10 | return metrics.precision_score(gt.fillna(0.0), pred.fillna(0.0), average=average)*100.0 11 | else: 12 | return metrics.precision_score(gt, pred, average=average)*100.0 13 | 14 | 15 | def eval_acc(gt, pred, average='macro'): 16 | if type(gt) is pd.core.frame.DataFrame or type(gt) is pd.core.frame.Series: 17 | return metrics.accuracy_score(gt.fillna(0.0), pred.fillna(0.0))*100.0 18 | else: 19 | return metrics.accuracy_score(gt, pred)*100.0 20 | 21 | def eval_cohen(gt, pred, average='quadratic'): 22 | 23 | if average != 'binary': 24 | weight_method = "quadratic" 25 | else: 26 | weight_method = None 27 | if type(gt) is pd.core.frame.DataFrame or type(gt) is pd.core.frame.Series: 28 | return metrics.cohen_kappa_score(gt.fillna(0.0), pred.fillna(0.0), weights=weight_method)*100.0 29 | else: 30 | return metrics.cohen_kappa_score(gt, pred, weights=weight_method)*100.0 31 | 32 | def eval_acc_multiple_classes(gt, pred, label=0): 33 | tmp_pd = pd.DataFrame() 34 | tmp_pd['gt'] = gt 35 | tmp_pd['pred'] = pred 36 | tmp_pd = tmp_pd[(tmp_pd['gt'] == label) | (tmp_pd['pred'] == label)] 37 | tmp_pd['pred'] = tmp_pd['pred'].apply(lambda x: 1 if x == label else 0) 38 | tmp_pd['gt'] = tmp_pd['gt'].apply(lambda x: 1 if x == label else 0) 39 | return metrics.accuracy_score(tmp_pd['gt'], tmp_pd['pred']) 40 | 41 | def eval_recall(gt, pred, average='macro'): 42 | if type(gt) is pd.core.frame.DataFrame or type(gt) is pd.core.frame.Series: 43 | return metrics.recall_score(gt.fillna(0.0), pred.fillna(0.0), average=average)* 100.0 44 | else: 45 | return metrics.recall_score(gt, pred, average=average) * 100.0 46 | 47 | 48 | def eval_specificity(gt, pred, average='macro'): 49 | if type(gt) is pd.core.frame.DataFrame or type(gt) is pd.core.frame.Series: 50 | return specificity_score(gt.fillna(0.0), pred.fillna(0.0), average=average) * 100.0 51 | else: 52 | return specificity_score(gt, pred, average=average) * 100.0 53 | 54 | def eval_f1(gt, pred, average='macro'): 55 | if type(gt) is pd.core.frame.DataFrame or type(gt) is pd.core.frame.Series: 56 | return metrics.f1_score(gt.fillna(0.0), pred.fillna(0.0), average=average) * 100.0 57 | else: 58 | return metrics.f1_score(gt, pred, average='macro') * 100.0 59 | 60 | 61 | def applyParallel(dfGrouped, func): 62 | retLst = Parallel(n_jobs=multiprocessing.cpu_count())(delayed(func)(group) for name, group in dfGrouped) 63 | return pd.concat(retLst) 64 | 65 | def applyParallel_1(dfGrouped, index, func, alg1, alg2): 66 | retLst = Parallel(n_jobs=multiprocessing.cpu_count()/2)(delayed(func)(group, alg1, alg2) for name, group in dfGrouped) 67 | t1 = pd.Series(retLst, index) 68 | return t1 69 | 70 | def eval_acc_df(df,alg1,alg2, average='binary'): 71 | return metrics.accuracy_score(df[alg1].fillna(0.0), df[alg2].fillna(0.0)) 72 | 73 | def calc_metrics(y_gt, y_pred_all, avg_method = "macro"): 74 | """ 75 | average method should be "binary" or "macro" 76 | """ 77 | accuracy = eval_acc(y_gt, y_pred_all) 78 | macro_f1 = eval_f1(y_gt, y_pred_all, average=avg_method) 79 | specificity = eval_specificity(y_gt, y_pred_all, average=avg_method) 80 | precision = eval_precision(y_gt, y_pred_all, average=avg_method) 81 | cohen = eval_cohen(y_gt, y_pred_all, average=avg_method) 82 | recall = eval_recall(y_gt, y_pred_all, average=avg_method) 83 | return {avg_method + '_accuracy': accuracy, avg_method+'_specificity': specificity, 84 | avg_method+'_precision': precision, avg_method+'_f1': macro_f1, 85 | avg_method+'_cohen': cohen, avg_method+'_recall': recall 86 | } 87 | -------------------------------------------------------------------------------- /data_loader/MESA_Acc_HRStatistic_build_h5_file.py: -------------------------------------------------------------------------------- 1 | from sleep_stage_config import Config 2 | from utilities.utils import * 3 | 4 | 5 | class H5DatasetBuilder(object): 6 | def __init__(self, csv_path, output_path_name, feat_list_path="", pre_split=""): 7 | self.csv_path = csv_path 8 | self.output_path_name = output_path_name 9 | self.pre_split = pre_split 10 | self.feat_list_path = feat_list_path 11 | 12 | def build_mesa_h5file(self): 13 | """ 14 | This function is designed to build a H5 file to speed up IO for experiment 15 | :param data_path: 16 | :param feat_list_path: 17 | :param saveCache: 18 | :param cacheName: 19 | :return: 20 | """ 21 | tmp = [] 22 | all_files = os.listdir(self.csv_path) 23 | csv_raw_files = [] 24 | for idx, f in enumerate(all_files): 25 | if ".csv" in f: 26 | csv_raw_files.append(os.path.join(self.csv_path, f)) 27 | csv_raw_files.sort() 28 | feature_list = pd.read_csv(self.feat_list_path)['feature_list'].values.tolist() 29 | # for file_name in glob(os.path.join(path, "*"))[:]: # this is the Joao's code 30 | for file_name in csv_raw_files: 31 | print(file_name) 32 | df_tmp = pd.read_csv(file_name) 33 | # the sleep block is actually the start of sleep onset and end of sleep onset 34 | # creates a gt_block 35 | gt_true = df_tmp[df_tmp["stages"] > 0] 36 | if gt_true.empty: 37 | print("Ignoring file %s" % file_name) 38 | continue 39 | start_block = df_tmp.index.get_loc(gt_true.index[0]) 40 | end_block = df_tmp.index.get_loc(gt_true.index[-1]) # the 41 | df_tmp["gt_sleep_block"] = make_one_block(df_tmp["stages"], start_block, end_block) 42 | tmp.append(df_tmp) 43 | whole_df = pd.concat(tmp) 44 | del tmp 45 | whole_df = whole_df.reset_index(drop=True) 46 | whole_df["binterval"] = whole_df["interval"].replace("ACTIVE", 0).replace("REST", 1).replace("REST-S", 1) 47 | test_proportion = 0.2 48 | uids = whole_df.mesaid.unique().astype(int) 49 | np.random.seed(9) 50 | np.random.shuffle(uids) 51 | test_idx = int(uids.shape[0] * test_proportion) 52 | uids_test, uids_train = uids[:test_idx], uids[test_idx:] 53 | if len(self.pre_split) > 0: # load pre-calculated train test PID list 54 | uids_train, uids_test = load_pre_splited_train_test_ids(self.pre_split) 55 | train_idx = whole_df[whole_df["mesaid"].apply(lambda x: x in uids_train)].index 56 | dftrain = whole_df.iloc[train_idx].copy() 57 | test_idx = whole_df[whole_df["mesaid"].apply(lambda x: x in uids_test)].index 58 | dftest = whole_df.iloc[test_idx].copy() 59 | print("start standardisation on df_train....") 60 | # standardises the whole training df is too time consuming, so 2019-09-27 using simple method 61 | scaler = standardize_df_given_feature(dftrain, feature_list, df_name="dftrain", simple_method=True) 62 | print("start standardisation on df_test....") 63 | standardize_df_given_feature(dftest, feature_list, scaler, df_name="dftest", simple_method=True) 64 | 65 | store = pd.HDFStore(os.path.join(self.output_path_name), 'w') 66 | store["train"] = dftrain 67 | store["test"] = dftest 68 | store["featnames"] = pd.Series(feature_list) 69 | store.close() 70 | print('h5 dataset is saved to %s' % os.path.join(self.output_path_name)) 71 | with open(os.path.join(self.output_path_name + '_std_transformer'), "wb") as f: 72 | pickle.dump(scaler, f) 73 | return dftrain, dftest, feature_list 74 | 75 | 76 | if __name__ == '__main__': 77 | config = Config() 78 | builder = H5DatasetBuilder(csv_path=config.MESA_ACC_HR_STATISTIC_CSV_ALIGNED, 79 | output_path_name=config.MESA_ACC_HR_STATISTICS_STD_DATA_PATH, 80 | feat_list_path=config.MESA_ACC_HR_STATISTIC_FEATURE, pre_split=config.TRAIN_TEST_SPLIT) 81 | builder.build_mesa_h5file() 82 | 83 | --------------------------------------------------------------------------------