├── LICENSE ├── MelFilterBank.py ├── README.md ├── extract_features.py ├── load_data.m ├── reconstructWave.py ├── reconstruction_minimal.py └── viz_results.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Neural Interfacing Lab 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MelFilterBank.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import math 3 | 4 | class MelFilterBank(): 5 | def __init__(self, specSize, numCoefficients, sampleRate): 6 | numBands = int(numCoefficients) 7 | 8 | # Set up center frequencies 9 | minMel = 0 10 | maxMel = self.freqToMel(sampleRate / 2.0) 11 | melStep = (maxMel - minMel) / (numBands + 1) 12 | 13 | melFilterEdges = np.arange(0, numBands + 2) * melStep 14 | 15 | # Convert center frequencies to indices in spectrum 16 | centerIndices = list(map(lambda x: self.freqToBin(math.floor(self.melToFreq(x)), sampleRate, specSize), melFilterEdges)) 17 | 18 | # Prepare matrix 19 | filterMatrix = np.zeros((numBands, specSize)) 20 | 21 | # Construct matrix with triangular filters 22 | for i in range(numBands): 23 | start, center, end = centerIndices[i:i + 3] 24 | k1 = np.float(center - start) 25 | k2 = np.float(end - center) 26 | up = (np.array(range(start, center)) - start) / k1 27 | down = (end - np.array(range(center, end))) / k2 28 | 29 | filterMatrix[i][start:center] = up 30 | filterMatrix[i][center:end] = down 31 | 32 | # Save matrix and its best-effort inverse 33 | self.melMatrix = filterMatrix.transpose() 34 | self.melMatrix = self.makeNormal(self.melMatrix / self.normSum(self.melMatrix)) 35 | 36 | self.melInvMatrix = self.melMatrix.transpose() 37 | self.melInvMatrix = self.makeNormal(self.melInvMatrix / self.normSum(self.melInvMatrix)) 38 | 39 | def normSum(self, x): 40 | retSum = np.sum(x, axis = 0) 41 | retSum[np.where(retSum == 0)] = 1.0 42 | return retSum 43 | 44 | def fuzz(self, x): 45 | return x + 0.0000001 46 | 47 | def freqToBin(self, freq, sampleRate, specSize): 48 | return int(math.floor((freq / (sampleRate / 2.0)) * specSize)) 49 | 50 | def freqToMel(self, freq): 51 | return 2595.0 * math.log10(1.0 + freq / 700.0) 52 | 53 | def melToFreq(self, mel): 54 | return 700.0 * (math.pow(10.0, mel / 2595.0) - 1.0) 55 | 56 | def toMelScale(self, spectrogram): 57 | return(np.dot(spectrogram, self.melMatrix)) 58 | 59 | def fromMelScale(self, melSpectrogram): 60 | return(np.dot(melSpectrogram, self.melInvMatrix)) 61 | 62 | 63 | def makeNormal(self, x): 64 | nanIdx = np.isnan(x) 65 | x[nanIdx] = 0 66 | 67 | infIdx = np.isinf(x) 68 | x[infIdx] = 0 69 | 70 | return(x) 71 | 72 | def toMels(self, spectrogram): 73 | return(self.toMelScale(spectrogram)) 74 | 75 | def fromMels(self, melSpectrogram): 76 | return(self.fromMelScale(melSpectrogram)) 77 | 78 | def toLogMels(self, spectrogram): 79 | return(self.makeNormal(np.log(self.fuzz(self.toMelScale(spectrogram))))) 80 | 81 | def fromLogMels(self, melSpectrogram): 82 | return(self.makeNormal(self.fromMelScale(np.exp(melSpectrogram)))) 83 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SingleWordProductionDutch 2 | 3 | Scripts to work with the intracranial EEG data from [here](https://osf.io/nrgx6/) described in this [article](https://www.nature.com/articles/s41597-022-01542-9). 4 | 5 | ## Dependencies 6 | The scripts require Python >= 3.6 and the following packages 7 | * [numpy](http://www.numpy.org/) 8 | * [scipy](https://www.scipy.org/scipylib/index.html) 9 | * [scikit-learn](https://scikit-learn.org/stable/) 10 | * [pandas](https://pandas.pydata.org/) 11 | * [pynwb](https://github.com/NeurodataWithoutBorders/pynwb) 12 | 13 | ## Repository content 14 | To recreate the experiments, run the following scripts. 15 | * __extract_features.py__: Reads in the iBIDS dataset and extracts features which are then saved to './features' 16 | 17 | * __reconstruction_minimal.py__: Reconstructs the spectrogram from the neural features in a 10-fold cross-validation and synthesizes the audio using the Method described by Griffin and Lim. 18 | 19 | * __viz_results.py__: Can then be used to plot the results figure from the paper. 20 | 21 | * __reconstuctWave.py__: Synthesizes an audio waveform using the method described by Griffin-Lim 22 | 23 | * __MelFilterBank.py__: Applies mel filter banks to spectrograms. 24 | 25 | * __load_data.mat__: Example of how to load data using Matlab instead of Python -------------------------------------------------------------------------------- /extract_features.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pandas as pd 4 | import numpy as np 5 | import numpy.matlib as matlib 6 | import scipy 7 | import scipy.signal 8 | import scipy.stats 9 | import scipy.io.wavfile 10 | import scipy.fftpack 11 | 12 | from pynwb import NWBHDF5IO 13 | import MelFilterBank as mel 14 | 15 | #Small helper function to speed up the hilbert transform by extending the length of data to the next power of 2 16 | hilbert3 = lambda x: scipy.signal.hilbert(x, scipy.fftpack.next_fast_len(len(x)),axis=0)[:len(x)] 17 | 18 | def extractHG(data, sr, windowLength=0.05, frameshift=0.01): 19 | """ 20 | Window data and extract frequency-band envelope using the hilbert transform 21 | 22 | Parameters 23 | ---------- 24 | data: array (samples, channels) 25 | EEG time series 26 | sr: int 27 | Sampling rate of the data 28 | windowLength: float 29 | Length of window (in seconds) in which spectrogram will be calculated 30 | frameshift: float 31 | Shift (in seconds) after which next window will be extracted 32 | Returns 33 | ---------- 34 | feat: array (windows, channels) 35 | Frequency-band feature matrix 36 | """ 37 | #Linear detrend 38 | data = scipy.signal.detrend(data,axis=0) 39 | #Number of windows 40 | numWindows = int(np.floor((data.shape[0]-windowLength*sr)/(frameshift*sr))) 41 | #Filter High-Gamma Band 42 | sos = scipy.signal.iirfilter(4, [70/(sr/2),170/(sr/2)],btype='bandpass',output='sos') 43 | data = scipy.signal.sosfiltfilt(sos,data,axis=0) 44 | #Attenuate first harmonic of line noise 45 | sos = scipy.signal.iirfilter(4, [98/(sr/2),102/(sr/2)],btype='bandstop',output='sos') 46 | data = scipy.signal.sosfiltfilt(sos,data,axis=0) 47 | #Attenuate second harmonic of line noise 48 | sos = scipy.signal.iirfilter(4, [148/(sr/2),152/(sr/2)],btype='bandstop',output='sos') 49 | data = scipy.signal.sosfiltfilt(sos,data,axis=0) 50 | #Create feature space 51 | data = np.abs(hilbert3(data)) 52 | feat = np.zeros((numWindows,data.shape[1])) 53 | for win in range(numWindows): 54 | start= int(np.floor((win*frameshift)*sr)) 55 | stop = int(np.floor(start+windowLength*sr)) 56 | feat[win,:] = np.mean(data[start:stop,:],axis=0) 57 | return feat 58 | 59 | def stackFeatures(features, modelOrder=4, stepSize=5): 60 | """ 61 | Add temporal context to each window by stacking neighboring feature vectors 62 | 63 | Parameters 64 | ---------- 65 | features: array (windows, channels) 66 | Feature time series 67 | modelOrder: int 68 | Number of temporal context to include prior to and after current window 69 | stepSize: float 70 | Number of temporal context to skip for each next context (to compensate for frameshift) 71 | Returns 72 | ---------- 73 | featStacked: array (windows, feat*(2*modelOrder+1)) 74 | Stacked feature matrix 75 | """ 76 | featStacked=np.zeros((features.shape[0]-(2*modelOrder*stepSize),(2*modelOrder+1)*features.shape[1])) 77 | for fNum,i in enumerate(range(modelOrder*stepSize,features.shape[0]-modelOrder*stepSize)): 78 | ef=features[i-modelOrder*stepSize:i+modelOrder*stepSize+1:stepSize,:] 79 | featStacked[fNum,:]=ef.flatten() #Add 'F' if stacked the same as matlab 80 | return featStacked 81 | 82 | def downsampleLabels(labels, sr, windowLength=0.05, frameshift=0.01): 83 | """ 84 | Downsamples non-numerical data by using the mode 85 | 86 | Parameters 87 | ---------- 88 | labels: array of str 89 | Label time series 90 | sr: int 91 | Sampling rate of the data 92 | windowLength: float 93 | Length of window (in seconds) in which mode will be used 94 | frameshift: float 95 | Shift (in seconds) after which next window will be extracted 96 | Returns 97 | ---------- 98 | newLabels: array of str 99 | Downsampled labels 100 | """ 101 | numWindows=int(np.floor((labels.shape[0]-windowLength*sr)/(frameshift*sr))) 102 | newLabels = np.empty(numWindows, dtype="S15") 103 | for w in range(numWindows): 104 | start = int(np.floor((w*frameshift)*sr)) 105 | stop = int(np.floor(start+windowLength*sr)) 106 | newLabels[w]=scipy.stats.mode(labels[start:stop])[0][0].encode("ascii", errors="ignore").decode() 107 | return newLabels 108 | 109 | def extractMelSpecs(audio, sr, windowLength=0.05, frameshift=0.01): 110 | """ 111 | Extract logarithmic mel-scaled spectrogram, traditionally used to compress audio spectrograms 112 | 113 | Parameters 114 | ---------- 115 | audio: array 116 | Audio time series 117 | sr: int 118 | Sampling rate of the audio 119 | windowLength: float 120 | Length of window (in seconds) in which spectrogram will be calculated 121 | frameshift: float 122 | Shift (in seconds) after which next window will be extracted 123 | numFilter: int 124 | Number of triangular filters in the mel filterbank 125 | Returns 126 | ---------- 127 | spectrogram: array (numWindows, numFilter) 128 | Logarithmic mel scaled spectrogram 129 | """ 130 | numWindows=int(np.floor((audio.shape[0]-windowLength*sr)/(frameshift*sr))) 131 | win = scipy.hanning(np.floor(windowLength*sr + 1))[:-1] 132 | spectrogram = np.zeros((numWindows, int(np.floor(windowLength*sr / 2 + 1))),dtype='complex') 133 | for w in range(numWindows): 134 | start_audio = int(np.floor((w*frameshift)*sr)) 135 | stop_audio = int(np.floor(start_audio+windowLength*sr)) 136 | a = audio[start_audio:stop_audio] 137 | spec = np.fft.rfft(win*a) 138 | spectrogram[w,:] = spec 139 | mfb = mel.MelFilterBank(spectrogram.shape[1], 23, sr) 140 | spectrogram = np.abs(spectrogram) 141 | spectrogram = (mfb.toLogMels(spectrogram)).astype('float') 142 | return spectrogram 143 | 144 | def nameVector(elecs, modelOrder=4): 145 | """ 146 | Creates list of electrode names 147 | 148 | Parameters 149 | ---------- 150 | elecs: array of str 151 | Original electrode names 152 | modelOrder: int 153 | Temporal context stacked prior and after current window 154 | Will be added as T-modelOrder, T-(modelOrder+1), ..., T0, ..., T+modelOrder 155 | to the elctrode names 156 | Returns 157 | ---------- 158 | names: array of str 159 | List of electrodes including contexts, will have size elecs.shape[0]*(2*modelOrder+1) 160 | """ 161 | names = matlib.repmat(elecs.astype(np.dtype(('U', 10))),1,2 * modelOrder +1).T 162 | for i, off in enumerate(range(-modelOrder,modelOrder+1)): 163 | names[i,:] = [e[0] + 'T' + str(off) for e in elecs] 164 | return names.flatten() #Add 'F' if stacked the same as matlab 165 | 166 | 167 | if __name__=="__main__": 168 | winL = 0.05 169 | frameshift = 0.01 170 | modelOrder = 4 171 | stepSize = 5 172 | path_bids = r'./SingleWordProductionDutch-iBIDS' 173 | path_output = r'./features' 174 | participants = pd.read_csv(os.path.join(path_bids,'participants.tsv'), delimiter='\t') 175 | for p_id, participant in enumerate(participants['participant_id']): 176 | 177 | #Load data 178 | io = NWBHDF5IO(os.path.join(path_bids,participant,'ieeg',f'{participant}_task-wordProduction_ieeg.nwb'), 'r') 179 | nwbfile = io.read() 180 | #sEEG 181 | eeg = nwbfile.acquisition['iEEG'].data[:] 182 | eeg_sr = 1024 183 | #audio 184 | audio = nwbfile.acquisition['Audio'].data[:] 185 | audio_sr = 48000 186 | #words (markers) 187 | words = nwbfile.acquisition['Stimulus'].data[:] 188 | words = np.array(words, dtype=str) 189 | io.close() 190 | #channels 191 | channels = pd.read_csv(os.path.join(path_bids,participant,'ieeg',f'{participant}_task-wordProduction_channels.tsv'), delimiter='\t') 192 | channels = np.array(channels['name']) 193 | 194 | #Extract HG features 195 | feat = extractHG(eeg,eeg_sr, windowLength=winL,frameshift=frameshift) 196 | 197 | #Stack features 198 | feat = stackFeatures(feat,modelOrder=modelOrder,stepSize=stepSize) 199 | 200 | #Process Audio 201 | target_SR = 16000 202 | audio = scipy.signal.decimate(audio,int(audio_sr / target_SR)) 203 | audio_sr = target_SR 204 | scaled = np.int16(audio/np.max(np.abs(audio)) * 32767) 205 | os.makedirs(os.path.join(path_output), exist_ok=True) 206 | scipy.io.wavfile.write(os.path.join(path_output,f'{participant}_orig_audio.wav'),audio_sr,scaled) 207 | 208 | #Extract spectrogram 209 | melSpec = extractMelSpecs(scaled,audio_sr,windowLength=winL,frameshift=frameshift) 210 | 211 | #Align to EEG features 212 | words = downsampleLabels(words,eeg_sr,windowLength=winL,frameshift=frameshift) 213 | words = words[modelOrder*stepSize:words.shape[0]-modelOrder*stepSize] 214 | melSpec = melSpec[modelOrder*stepSize:melSpec.shape[0]-modelOrder*stepSize,:] 215 | #adjust length (differences might occur due to rounding in the number of windows) 216 | if melSpec.shape[0]!=feat.shape[0]: 217 | tLen = np.min([melSpec.shape[0],feat.shape[0]]) 218 | melSpec = melSpec[:tLen,:] 219 | feat = feat[:tLen,:] 220 | 221 | #Create feature names by appending the temporal shift 222 | feature_names = nameVector(channels[:,None], modelOrder=modelOrder) 223 | 224 | #Save everything 225 | np.save(os.path.join(path_output,f'{participant}_feat.npy'), feat) 226 | np.save(os.path.join(path_output,f'{participant}_procWords.npy'), words) 227 | np.save(os.path.join(path_output,f'{participant}_spec.npy'), melSpec) 228 | np.save(os.path.join(path_output,f'{participant}_feat_names.npy'), feature_names) -------------------------------------------------------------------------------- /load_data.m: -------------------------------------------------------------------------------- 1 | % Download MatNWB (https://neurodatawithoutborders.github.io/matnwb/) 2 | 3 | nwbfile = nwbRead('...\SingleWordProductionDutch-iBIDS\sub-01\ieeg\sub-01_task-wordProduction_ieeg.nwb') 4 | eeg = nwbfile.acquisition.get('iEEG').data.load; 5 | audio = nwbfile.acquisition.get('Audio').data.load; 6 | words = nwbfile.acquisition.get('Stimulus').data.load; 7 | -------------------------------------------------------------------------------- /reconstructWave.py: -------------------------------------------------------------------------------- 1 | import scipy, numpy as np 2 | import scipy.io.wavfile as wavefile 3 | 4 | def stft(x, fftsize=1024, overlap=4): 5 | """Returns short time fourier transform of a signal x 6 | """ 7 | hop = int(fftsize / overlap) 8 | w = scipy.hanning(fftsize+1)[:-1] # better reconstruction with this trick +1)[:-1] 9 | return np.array([np.fft.rfft(w*x[i:i+int(fftsize)]) for i in range(0, len(x)-int(fftsize), hop)]) 10 | 11 | def istft(X, overlap=4): 12 | """Returns inverse short time fourier transform of a complex spectrum X 13 | """ 14 | fftsize=(X.shape[1]-1)*2 15 | hop = int(fftsize / overlap) 16 | w = scipy.hanning(fftsize+1)[:-1] 17 | x = scipy.zeros(X.shape[0]*hop) 18 | wsum = scipy.zeros(X.shape[0]*hop) 19 | for n,i in enumerate(range(0, len(x)-fftsize, hop)): 20 | x[i:i+fftsize] += scipy.real(np.fft.irfft(X[n])) * w # overlap-add 21 | wsum[i:i+fftsize] += w ** 2. 22 | #Could be improved 23 | #pos = wsum != 0 24 | #x[pos] /= wsum[pos] 25 | return x 26 | 27 | def reconstructWavFromSpectrogram(spec,lenWaveFile,fftsize=1024,overlap=4,numIterations=8): 28 | """Returns a reconstructed waveform from the spectrogram 29 | using the method in 30 | Griffin, Lim: Signal estimation from modified short-time Fourier transform, 31 | IEEE Transactions on Acoustics Speech and Signal Processing, 1984 32 | algo described here: 33 | Bayram, Ilker. "An analytic wavelet transform with a flexible time-frequency covering." 34 | Signal Processing, IEEE Transactions on 61.5 (2013): 1131-1142. 35 | """ 36 | reconstructedWav = np.random.rand(lenWaveFile*2) 37 | #while(stft(reconstructedWav,fftsize=fftsize,overlap=overlap).shape[0]