├── .gitignore ├── .gitmodules ├── Arduino └── Arduino.ino ├── Common ├── CircularShortTimeFourierTransform.swift ├── Common-Bridging-Header.h ├── Common.swift ├── NeuralNet.swift ├── Resampler.swift ├── StreamReader.swift ├── SyllableDetector.swift ├── SyllableDetectorConfig.swift └── TPCircularBuffer │ ├── TPCircularBuffer.c │ └── TPCircularBuffer.h ├── LICENSE ├── README.md ├── SyllableDetector.xcodeproj ├── project.pbxproj └── project.xcworkspace │ ├── contents.xcworkspacedata │ └── xcshareddata │ └── SyllableDetector.xcscmblueprint ├── SyllableDetector ├── AppDelegate.swift ├── ArduinoIO.swift ├── Assets.xcassets │ └── AppIcon.appiconset │ │ └── Contents.json ├── AudioInterface.swift ├── Base.lproj │ └── Main.storyboard ├── Info.plist ├── Processor.swift ├── SummaryStat.swift ├── Time.swift ├── ViewControllerMenu.swift ├── ViewControllerProcessor.swift ├── ViewControllerSimulator.swift └── WindowControllerProcessor.swift ├── SyllableDetectorCLI ├── OutputStream.swift ├── TrackDetector.swift └── main.swift ├── convert_to_text.m └── sample.txt /.gitignore: -------------------------------------------------------------------------------- 1 | SyllableDetector.xcodeproj/xcuserdata 2 | SyllableDetector.xcodeproj/project.xcworkspace/xcuserdata 3 | Examples 4 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "Frameworks/ORSSerialPort"] 2 | path = Frameworks/ORSSerialPort 3 | url = https://github.com/armadsen/ORSSerialPort.git 4 | [submodule "Frameworks/Moderator"] 5 | path = Frameworks/Moderator 6 | url = https://github.com/kareman/Moderator.git 7 | -------------------------------------------------------------------------------- /Arduino/Arduino.ino: -------------------------------------------------------------------------------- 1 | 2 | //--------------------------------------------------------------------// 3 | // FINCHSCOPE ARDUINO MK 1.0 4 | //--------------------------------------------------------------------// 5 | 6 | /* MODIFIED by WALIII, based on cobe by Analog and Digital Input and 7 | Output Server for MATLA Giampiero Campa, Copyright 2013 The MathWorks, Inc */ 8 | 9 | 10 | 11 | 12 | /* This file is meant to be used with the MATLAB arduino IO 13 | package, however, it can be used from the IDE environment 14 | (or any other serial terminal) by typing commands like: 15 | 16 | 0e0 : assigns digital pin #4 (e) as input 17 | 0f1 : assigns digital pin #5 (f) as output 18 | 0n1 : assigns digital pin #13 (n) as output 19 | 20 | 1c : reads digital pin #2 (c) 21 | 1e : reads digital pin #4 (e) 22 | 2n0 : sets digital pin #13 (n) low 23 | 2n1 : sets digital pin #13 (n) high 24 | 2f1 : sets digital pin #5 (f) high 25 | 2f0 : sets digital pin #5 (f) low 26 | 4j2 : sets digital pin #9 (j) to 50=ascii(2) over 255 27 | 4jz : sets digital pin #9 (j) to 122=ascii(z) over 255 28 | 3a : reads analog pin #0 (a) 29 | 3f : reads analog pin #5 (f) 30 | 31 | R0 : sets analog reference to DEFAULT 32 | R1 : sets analog reference to INTERNAL 33 | R2 : sets analog reference to EXTERNAL 34 | 35 | X3 : roundtrip example case returning the input (ascii(3)) 36 | 99 : returns script type (0 adio.pde ... 3 motor.pde ) */ 37 | 38 | /* define internal for the MEGA as 1.1V (as as for the 328) */ 39 | #if defined(__AVR_ATmega1280__) || defined(__AVR_ATmega2560__) 40 | #define INTERNAL INTERNAL1V1 41 | #endif 42 | 43 | void setup() { 44 | 45 | // set PIN13 to PWM to 62khz 46 | int myEraser = 7; // this is 111 in binary and is used as an eraser 47 | TCCR0B &= ~myEraser; // this operation (AND plus NOT), set the three bits in TCCR2B to 0 48 | int myPrescaler = 1; // this could be a number in [1 , 6]. In this case, 3 corresponds in binary to 011. 49 | TCCR0B |= myPrescaler; //this operation (OR), replaces the last three bits in TCCR2B with our new value 011 50 | /* initialize serial */ 51 | Serial.begin(115200); 52 | } 53 | 54 | 55 | void loop() { 56 | 57 | /* variables declaration and initialization */ 58 | 59 | static int s = -1; /* state */ 60 | static int pin = 13; /* generic pin number */ 61 | static int cnt = 0; 62 | 63 | int val = 0; /* generic value read from serial */ 64 | int agv = 0; /* generic analog value */ 65 | int dgv = 0; /* generic digital value */ 66 | 67 | 68 | /* The following instruction constantly checks if anything 69 | is available on the serial port. Nothing gets executed in 70 | the loop if nothing is available to be read, but as soon 71 | as anything becomes available, then the part coded after 72 | the if statement (that is the real stuff) gets executed */ 73 | 74 | if (Serial.available() >0) { 75 | 76 | /* whatever is available from the serial is read here */ 77 | val = Serial.read(); 78 | 79 | /* This part basically implements a state machine that 80 | reads the serial port and makes just one transition 81 | to a new state, depending on both the previous state 82 | and the command that is read from the serial port. 83 | Some commands need additional inputs from the serial 84 | port, so they need 2 or 3 state transitions (each one 85 | happening as soon as anything new is available from 86 | the serial port) to be fully executed. After a command 87 | is fully executed the state returns to its initial 88 | value s=-1 */ 89 | 90 | switch (s) { 91 | 92 | /* s=-1 means NOTHING RECEIVED YET ******************* */ 93 | case -1: 94 | 95 | /* calculate next state */ 96 | if (val>47 && val<90) { 97 | /* the first received value indicates the mode 98 | 49 is ascii for 1, ... 90 is ascii for Z 99 | s=0 is change-pin mode; 100 | s=10 is DI; s=20 is DO; s=30 is AI; s=40 is AO; 101 | s=90 is query script type (1 basic, 2 motor); 102 | s=340 is change analog reference; 103 | s=400 example echo returning the input argument; 104 | */ 105 | s=10*(val-48); 106 | } 107 | 108 | /* the following statements are needed to handle 109 | unexpected first values coming from the serial (if 110 | the value is unrecognized then it defaults to s=-1) */ 111 | if ((s>50 && s<90) || (s>90 && s!=340 && s!=400)) { 112 | s=-1; 113 | } 114 | 115 | /* the break statements gets out of the switch-case, so 116 | /* we go back and wait for new serial data */ 117 | break; /* s=-1 (initial state) taken care of */ 118 | 119 | 120 | 121 | /* s=0 or 1 means CHANGE PIN MODE */ 122 | 123 | case 0: 124 | /* the second received value indicates the pin 125 | from abs('c')=99, pin 2, to abs('¦')=166, pin 69 */ 126 | if (val>98 && val<167) { 127 | pin=val-97; /* calculate pin */ 128 | s=1; /* next we will need to get 0 or 1 from serial */ 129 | } 130 | else { 131 | s=-1; /* if value is not a pin then return to -1 */ 132 | } 133 | break; /* s=0 taken care of */ 134 | 135 | 136 | case 1: 137 | /* the third received value indicates the value 0 or 1 */ 138 | if (val>47 && val<50) { 139 | /* set pin mode */ 140 | if (val==48) { 141 | pinMode(pin,INPUT); 142 | } 143 | else { 144 | pinMode(pin,OUTPUT); 145 | } 146 | } 147 | s=-1; /* we are done with CHANGE PIN so go to -1 */ 148 | break; /* s=1 taken care of */ 149 | 150 | 151 | 152 | /* s=10 means DIGITAL INPUT ************************** */ 153 | 154 | case 10: 155 | /* the second received value indicates the pin 156 | from abs('c')=99, pin 2, to abs('¦')=166, pin 69 */ 157 | if (val>98 && val<167) { 158 | pin=val-97; /* calculate pin */ 159 | dgv=digitalRead(pin); /* perform Digital Input */ 160 | Serial.println(dgv); /* send value via serial */ 161 | } 162 | s=-1; /* we are done with DI so next state is -1 */ 163 | break; /* s=10 taken care of */ 164 | 165 | 166 | 167 | /* s=20 or 21 means DIGITAL OUTPUT ******************* */ 168 | 169 | case 20: 170 | /* the second received value indicates the pin 171 | from abs('c')=99, pin 2, to abs('¦')=166, pin 69 */ 172 | if (val>98 && val<167) { 173 | pin=val-97; /* calculate pin */ 174 | s=21; /* next we will need to get 0 or 1 from serial */ 175 | } 176 | else { 177 | s=-1; /* if value is not a pin then return to -1 */ 178 | } 179 | break; /* s=20 taken care of */ 180 | 181 | case 21: 182 | /* the third received value indicates the value 0 or 1 */ 183 | if (val>47 && val<50) { 184 | dgv=val-48; /* calculate value */ 185 | digitalWrite(pin,dgv); /* perform Digital Output */ 186 | } 187 | s=-1; /* we are done with DO so next state is -1 */ 188 | break; /* s=21 taken care of */ 189 | 190 | 191 | 192 | /* s=50 means DIGITAL PULSE ******************* */ 193 | 194 | case 50: 195 | /* the second received value indicates the pin 196 | from abs('c')=99, pin 2, to abs('¦')=166, pin 69 */ 197 | if (val>98 && val<167) { 198 | pin=val-97; /* calculate pin */ 199 | digitalWrite(pin, HIGH); /* perform digital output */ 200 | delay(1); 201 | digitalWrite(pin, LOW); /* perform digital output */ 202 | } 203 | s = -1; 204 | break; /* s=50 taken care of */ 205 | 206 | /* s=30 means ANALOG INPUT *************************** */ 207 | 208 | case 30: 209 | /* the second received value indicates the pin 210 | from abs('a')=97, pin 0, to abs('p')=112, pin 15 */ 211 | if (val>96 && val<113) { 212 | pin=val-97; /* calculate pin */ 213 | agv=analogRead(pin); /* perform Analog Input */ 214 | Serial.println(agv); /* send value via serial */ 215 | } 216 | s=-1; /* we are done with AI so next state is -1 */ 217 | break; /* s=30 taken care of */ 218 | 219 | 220 | 221 | /* s=40 or 41 means ANALOG OUTPUT ******************** */ 222 | 223 | case 40: 224 | /* the second received value indicates the pin 225 | from abs('c')=99, pin 2, to abs('¦')=166, pin 69 */ 226 | if (val>98 && val<167) { 227 | pin=val-97; /* calculate pin */ 228 | s=41; /* next we will need to get value from serial */ 229 | } 230 | else { 231 | s=-1; /* if value is not a pin then return to -1 */ 232 | } 233 | break; /* s=40 taken care of */ 234 | 235 | 236 | case 41: 237 | /* the third received value indicates the analog value */ 238 | analogWrite(pin,val); /* perform Analog Output */ 239 | s=-1; /* we are done with AO so next state is -1 */ 240 | break; /* s=41 taken care of */ 241 | 242 | 243 | 244 | /* s=90 means Query Script Type: 245 | (0 adio, 1 adioenc, 2 adiosrv, 3 motor) */ 246 | 247 | case 90: 248 | if (val==57) { 249 | /* if string sent is 99 send script type via serial */ 250 | Serial.println(0); 251 | } 252 | s=-1; /* we are done with this so next state is -1 */ 253 | break; /* s=90 taken care of */ 254 | 255 | 256 | 257 | /* s=340 or 341 means ANALOG REFERENCE *************** */ 258 | 259 | case 340: 260 | /* the second received value indicates the reference, 261 | which is encoded as is 0,1,2 for DEFAULT, INTERNAL 262 | and EXTERNAL, respectively. Note that this function 263 | is ignored for boards not featuring AVR or PIC32 */ 264 | 265 | #if defined(__AVR__) || defined(__PIC32MX__) 266 | 267 | switch (val) { 268 | 269 | case 48: 270 | analogReference(DEFAULT); 271 | break; 272 | 273 | case 49: 274 | analogReference(INTERNAL); 275 | break; 276 | 277 | case 50: 278 | analogReference(EXTERNAL); 279 | break; 280 | 281 | default: /* unrecognized, no action */ 282 | break; 283 | } 284 | 285 | #endif 286 | 287 | s=-1; /* we are done with this so next state is -1 */ 288 | break; /* s=341 taken care of */ 289 | 290 | 291 | 292 | /* s=400 roundtrip example function (returns the input)*/ 293 | 294 | case 400: 295 | /* the second value (val) can really be anything here */ 296 | 297 | /* This is an auxiliary function that returns the ASCII 298 | value of its first argument. It is provided as an 299 | example for people that want to add their own code */ 300 | 301 | /* your own code goes here instead of the serial print */ 302 | Serial.println(val); 303 | 304 | s=-1; /* we are done with the aux function so -1 */ 305 | break; /* s=400 taken care of */ 306 | 307 | 308 | 309 | /* ******* UNRECOGNIZED STATE, go back to s=-1 ******* */ 310 | 311 | default: 312 | /* we should never get here but if we do it means we 313 | are in an unexpected state so whatever is the second 314 | received value we get out of here and back to s=-1 */ 315 | 316 | s=-1; /* go back to the initial state, break unneeded */ 317 | 318 | 319 | 320 | } /* end switch on state s */ 321 | 322 | } /* end if serial available */ 323 | 324 | } /* end loop statement */ 325 | 326 | -------------------------------------------------------------------------------- /Common/CircularShortTimeFourierTransform.swift: -------------------------------------------------------------------------------- 1 | // 2 | // CircularShortTimeFourierTransform.swift 3 | // SongDetector 4 | // 5 | // Created by Nathan Perkins on 9/4/15. 6 | // Copyright © 2015 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import Accelerate 11 | 12 | enum WindowType 13 | { 14 | case none 15 | case hamming 16 | case hanning 17 | case blackman 18 | 19 | func createWindow(_ pointer: UnsafeMutablePointer, len: Int) { 20 | switch self { 21 | case .none: 22 | var one: Float = 1.0 23 | vDSP_vfill(&one, pointer, 1, vDSP_Length(len)) 24 | case .hamming: vDSP_hamm_window(pointer, vDSP_Length(len), 0) 25 | case .hanning: vDSP_hann_window(pointer, vDSP_Length(len), 0) 26 | case .blackman: vDSP_blkman_window(pointer, vDSP_Length(len), 0) 27 | } 28 | } 29 | } 30 | 31 | class CircularShortTimeFourierTransform 32 | { 33 | private var buffer: TPCircularBuffer 34 | 35 | let lengthFft: Int // power of 2 36 | let lengthWindow: Int 37 | 38 | // only one can be non-zero 39 | let gap: Int // gaps between samples 40 | let overlap: Int // overlap between samples 41 | 42 | private let fftSize: vDSP_Length 43 | private let fftSetup: FFTSetup 44 | 45 | var windowType = WindowType.hanning { 46 | didSet { 47 | resetWindow() 48 | } 49 | } 50 | 51 | // store actual window 52 | private let window: UnsafeMutablePointer 53 | 54 | // store windowed values 55 | private let samplesWindowed: UnsafeMutablePointer 56 | 57 | // reusable memory 58 | private var complexBufferA: DSPSplitComplex 59 | private var complexBufferT: DSPSplitComplex 60 | 61 | init(windowLength lengthWindow: Int = 1024, withOverlap overlap: Int = 0, fftSizeOf theLengthFft: Int? = nil, buffer: Int = 409600) { 62 | // length of the fourier transform (must be a power of 2) 63 | self.lengthWindow = lengthWindow 64 | 65 | // if negative overlap, interpret that as a gap 66 | if overlap < 0 { 67 | self.gap = 0 - overlap 68 | self.overlap = 0 69 | } 70 | else { 71 | self.overlap = overlap 72 | self.gap = 0 73 | } 74 | 75 | // sanity check 76 | if overlap >= lengthWindow { 77 | fatalError("Invalid overlap value.") 78 | } 79 | 80 | // calculate fft 81 | if let v = theLengthFft { 82 | guard v.isPowerOfTwo() else { 83 | fatalError("The FFT size must be a power of 2.") 84 | } 85 | 86 | guard lengthWindow <= v else { 87 | fatalError("The FFT size must be greater than or equal to the window length.") 88 | } 89 | 90 | lengthFft = v 91 | fftSize = vDSP_Length(ceil(log2(CDouble(v)))) 92 | } 93 | else { 94 | // automatically calculate 95 | fftSize = vDSP_Length(ceil(log2(CDouble(lengthWindow)))) 96 | lengthFft = 1 << Int(fftSize) 97 | } 98 | 99 | // maybe use lazy instantion? 100 | 101 | // setup fft 102 | fftSetup = vDSP_create_fftsetup(fftSize, FFTRadix(kFFTRadix2))! 103 | 104 | // setup window 105 | window = UnsafeMutablePointer.allocate(capacity: lengthWindow) 106 | windowType.createWindow(window, len: lengthWindow) 107 | 108 | // setup windowed samples 109 | samplesWindowed = UnsafeMutablePointer.allocate(capacity: lengthFft) 110 | vDSP_vclr(samplesWindowed, 1, vDSP_Length(lengthFft)) 111 | 112 | // half length (for buffer allocation) 113 | let halfLength = lengthFft / 2 114 | 115 | // setup complex buffers 116 | complexBufferA = DSPSplitComplex(realp: UnsafeMutablePointer.allocate(capacity: halfLength), imagp: UnsafeMutablePointer.allocate(capacity: halfLength)) 117 | // to get desired alignment.. 118 | let alignment: Int = 0x10 119 | let ptrReal = UnsafeMutableRawPointer.allocate(byteCount: halfLength * MemoryLayout.stride, alignment: alignment) 120 | let ptrImag = UnsafeMutableRawPointer.allocate(byteCount: halfLength * MemoryLayout.stride, alignment: alignment) 121 | 122 | complexBufferT = DSPSplitComplex(realp: ptrReal.bindMemory(to: Float.self, capacity: halfLength), imagp: ptrImag.bindMemory(to: Float.self, capacity: halfLength)) 123 | 124 | // create the circular buffer 125 | self.buffer = TPCircularBuffer() 126 | if !TPCircularBufferInit(&self.buffer, Int32(buffer)) { 127 | fatalError("Unable to allocate circular buffer.") 128 | } 129 | } 130 | 131 | deinit { 132 | // half length (for buffer allocation) 133 | let halfLength = lengthFft / 2 134 | 135 | // free the complex buffer 136 | complexBufferA.realp.deinitialize(count: halfLength) 137 | complexBufferA.realp.deallocate() 138 | complexBufferA.imagp.deinitialize(count: halfLength) 139 | complexBufferA.imagp.deallocate() 140 | complexBufferT.realp.deinitialize(count: halfLength) 141 | complexBufferT.realp.deallocate() 142 | complexBufferT.imagp.deinitialize(count: halfLength) 143 | complexBufferT.imagp.deallocate() 144 | 145 | // free the FFT setup 146 | vDSP_destroy_fftsetup(fftSetup) 147 | 148 | // free the memory used to store the samples 149 | samplesWindowed.deinitialize(count: lengthFft) 150 | samplesWindowed.deallocate() 151 | 152 | // free the window 153 | window.deinitialize(count: lengthWindow) 154 | window.deallocate() 155 | 156 | // release the circular buffer 157 | TPCircularBufferCleanup(&self.buffer) 158 | } 159 | 160 | func frequenciesForSampleRate(_ rate: Double) -> [Double] { 161 | let halfLength = lengthFft / 2 162 | let toSampleRate = rate / Double(lengthFft) 163 | return (0.. (Int, Int)? { 167 | // sensible inputs 168 | guard startFreq >= 0.0 && endFreq > startFreq else { 169 | return nil 170 | } 171 | 172 | // helpful numbers 173 | let halfLength = lengthFft / 2 174 | let fromFrequency = Double(lengthFft) / rate 175 | 176 | // calculate start index 177 | let startIndex = Int(ceil(fromFrequency * startFreq)) 178 | if startIndex >= halfLength { 179 | return nil 180 | } 181 | 182 | // calculate end index + 1 183 | let endIndex = Int(floor(fromFrequency * endFreq)) + 1 184 | if endIndex < startIndex { 185 | return nil 186 | } 187 | if endIndex > halfLength { 188 | return (startIndex, halfLength) 189 | } 190 | return (startIndex, endIndex) 191 | } 192 | 193 | func resetWindow() { 194 | windowType.createWindow(window, len: lengthWindow) 195 | } 196 | 197 | func appendData(_ data: UnsafeMutablePointer, withSamples numSamples: Int) { 198 | if !TPCircularBufferProduceBytes(&self.buffer, data, Int32(numSamples * MemoryLayout.stride)) { 199 | fatalError("Insufficient space on buffer.") 200 | } 201 | } 202 | 203 | func appendInterleavedData(_ data: UnsafeMutablePointer, withSamples numSamples: Int, fromChannel channel: Int, ofTotalChannels totalChannels: Int) { 204 | // get head of circular buffer 205 | var space: Int32 = 0 206 | let head = TPCircularBufferHead(&self.buffer, &space) 207 | if Int(space) < numSamples * MemoryLayout.stride { 208 | fatalError("Insufficient space on buffer.") 209 | } 210 | 211 | // use vDSP to perform copy with stride 212 | var zero: Float = 0.0 213 | vDSP_vsadd(data + channel, vDSP_Stride(totalChannels), &zero, head!.bindMemory(to: Float.self, capacity: numSamples), 1, vDSP_Length(numSamples)) 214 | 215 | // move head forward 216 | TPCircularBufferProduce(&self.buffer, Int32(numSamples * MemoryLayout.stride)) 217 | } 218 | 219 | // TODO: write better functions that can help avoid double copying 220 | 221 | func extractMagnitude() -> [Float]? { 222 | // get buffer read point and available bytes 223 | var availableBytes: Int32 = 0 224 | let tail = TPCircularBufferTail(&buffer, &availableBytes) 225 | 226 | // not enough available bytes 227 | if Int(availableBytes) < ((gap + lengthWindow) * MemoryLayout.stride) { 228 | return nil 229 | } 230 | 231 | // make samples 232 | var samples = tail!.bindMemory(to: Float.self, capacity: Int(availableBytes) / MemoryLayout.stride) 233 | 234 | // skip gap 235 | if 0 < gap { 236 | samples = samples + gap 237 | } 238 | 239 | // mark circular buffer as consumed at END of excution 240 | defer { 241 | // mark as consumed 242 | TPCircularBufferConsume(&buffer, Int32((gap + lengthWindow - overlap) * MemoryLayout.stride)) 243 | } 244 | 245 | // get half length 246 | let halfLength = lengthFft / 2 247 | 248 | // prepare output 249 | var output = [Float](repeating: 0.0, count: halfLength) 250 | 251 | // window the samples 252 | vDSP_vmul(samples, 1, window, 1, samplesWindowed, 1, UInt(lengthWindow)) 253 | 254 | // pack samples into complex values (use stride 2 to fill just reals 255 | samplesWindowed.withMemoryRebound(to: DSPComplex.self, capacity: halfLength) { 256 | vDSP_ctoz($0, 2, &complexBufferA, 1, UInt(halfLength)) 257 | } 258 | 259 | // perform FFT 260 | // TODO: potentially use vDSP_fftm_zrip 261 | vDSP_fft_zript(fftSetup, &complexBufferA, 1, &complexBufferT, fftSize, FFTDirection(FFT_FORWARD)) 262 | 263 | // clear imagp, represents frequency at midpoint of symmetry, due to packing of array 264 | complexBufferA.imagp[0] = 0 265 | 266 | output.withUnsafeMutableBufferPointer() { 267 | guard let ba = $0.baseAddress else { return } 268 | 269 | // convert to magnitudes 270 | vDSP_zvmags(&complexBufferA, 1, ba, 1, UInt(halfLength)) 271 | 272 | // scaling unit 273 | var scale: Float = 4.0 274 | vDSP_vsdiv(ba, 1, &scale, ba, 1, UInt(halfLength)) 275 | } 276 | 277 | return output 278 | } 279 | 280 | func extractPower() -> [Float]? { 281 | // get buffer read point and available bytes 282 | var availableBytes: Int32 = 0 283 | let tail = TPCircularBufferTail(&buffer, &availableBytes) 284 | 285 | // not enough available bytes 286 | if Int(availableBytes) < ((gap + lengthWindow) * MemoryLayout.stride) { 287 | return nil 288 | } 289 | 290 | // make samples 291 | var samples = tail!.bindMemory(to: Float.self, capacity: Int(availableBytes) / MemoryLayout.stride) 292 | 293 | // skip gap 294 | if 0 < gap { 295 | samples = samples + gap 296 | } 297 | 298 | // mark circular buffer as consumed at END of excution 299 | defer { 300 | // mark as consumed 301 | TPCircularBufferConsume(&buffer, Int32((gap + lengthWindow - overlap) * MemoryLayout.stride)) 302 | } 303 | 304 | // get half length 305 | let halfLength = lengthFft / 2 306 | 307 | // prepare output 308 | var output = [Float](repeating: 0.0, count: halfLength) 309 | 310 | // window the samples 311 | vDSP_vmul(samples, 1, window, 1, samplesWindowed, 1, UInt(lengthWindow)) 312 | 313 | // pack samples into complex values (use stride 2 to fill just reals 314 | samplesWindowed.withMemoryRebound(to: DSPComplex.self, capacity: halfLength) { 315 | vDSP_ctoz($0, 2, &complexBufferA, 1, UInt(halfLength)) 316 | } 317 | 318 | // perform FFT 319 | // TODO: potentially use vDSP_fftm_zrip 320 | vDSP_fft_zript(fftSetup, &complexBufferA, 1, &complexBufferT, fftSize, FFTDirection(FFT_FORWARD)) 321 | 322 | // clear imagp, represents frequency at midpoint of symmetry, due to packing of array 323 | complexBufferA.imagp[0] = 0 324 | 325 | output.withUnsafeMutableBufferPointer() { 326 | guard let ba = $0.baseAddress else { return } 327 | 328 | // convert to magnitudes 329 | vDSP_zvabs(&complexBufferA, 1, ba, 1, UInt(halfLength)) 330 | 331 | // scaling unit 332 | var scale: Float = 2.0 333 | vDSP_vsdiv(ba, 1, &scale, ba, 1, UInt(halfLength)) 334 | } 335 | 336 | return output 337 | } 338 | } 339 | -------------------------------------------------------------------------------- /Common/Common-Bridging-Header.h: -------------------------------------------------------------------------------- 1 | // 2 | // Use this file to import your target's public headers that you would like to expose to Swift. 3 | // 4 | 5 | #include "TPCircularBuffer.h" 6 | -------------------------------------------------------------------------------- /Common/Common.swift: -------------------------------------------------------------------------------- 1 | // Common.swift 2 | // VideoCapture 3 | // 4 | // Created by L. Nathan Perkins on 7/2/15. 5 | // Copyright © 2015 6 | 7 | import Foundation 8 | 9 | /// A logging function that only executes in debugging mode. 10 | func DLog(_ message: String, function: String = #function ) { 11 | #if DEBUG 12 | print("\(function): \(message)") 13 | #endif 14 | } 15 | 16 | extension String { 17 | func trim() -> String { 18 | return self.trimmingCharacters(in: CharacterSet.whitespacesAndNewlines) 19 | } 20 | 21 | func splitAtCharacter(_ char: Character) -> [String] { 22 | return self.split { $0 == char } .map(String.init) 23 | } 24 | } 25 | 26 | extension Int { 27 | func isPowerOfTwo() -> Bool { 28 | return (self != 0) && (self & (self - 1)) == 0 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /Common/NeuralNet.swift: -------------------------------------------------------------------------------- 1 | // 2 | // NeuralNet.swift 3 | // SongDetector 4 | // 5 | // Created by Nathan Perkins on 9/22/15. 6 | // Copyright © 2015 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import Accelerate 11 | 12 | // mapping function protocol 13 | protocol InputProcessingFunction { 14 | func applyInPlace(_ values: UnsafeMutablePointer, count: Int) 15 | func applyAndCopy(_ values: UnsafePointer, count: Int, to destination: UnsafeMutablePointer) 16 | } 17 | 18 | protocol OutputProcessingFunction { 19 | func reverseInPlace(_ values: UnsafeMutablePointer, count: Int) 20 | func reverseAndCopy(_ values: UnsafePointer, count: Int, to destination: UnsafeMutablePointer) 21 | } 22 | 23 | class PassThrough: InputProcessingFunction, OutputProcessingFunction { 24 | func applyInPlace(_ values: UnsafeMutablePointer, count: Int) { 25 | // do nothing 26 | } 27 | 28 | func applyAndCopy(_ values: UnsafePointer, count: Int, to destination: UnsafeMutablePointer) { 29 | memcpy(destination, values, count * MemoryLayout.stride) 30 | } 31 | 32 | func reverseInPlace(_ values: UnsafeMutablePointer, count: Int) { 33 | // do nothing 34 | } 35 | 36 | func reverseAndCopy(_ values: UnsafePointer, count: Int, to destination: UnsafeMutablePointer) { 37 | memcpy(destination, values, count * MemoryLayout.stride) 38 | } 39 | } 40 | 41 | class L2Normalize: InputProcessingFunction { 42 | func applyInPlace(_ values: UnsafeMutablePointer, count: Int) { 43 | // vDSP functions in the copy version support in place operations 44 | applyAndCopy(values, count: count, to: values) 45 | } 46 | 47 | func applyAndCopy(_ values: UnsafePointer, count: Int, to destination: UnsafeMutablePointer) { 48 | let len = vDSP_Length(count) 49 | 50 | // sum of squares 51 | var sumsq: Float = 0.0, sqrtsumsq: Float = 0.0 52 | vDSP_svesq(values, 1, &sumsq, len) 53 | 54 | // get square root 55 | sqrtsumsq = sqrt(sumsq) 56 | 57 | // divide by sum of squares 58 | vDSP_vsdiv(values, 1, &sqrtsumsq, destination, 1, len) 59 | } 60 | 61 | } 62 | 63 | class Normalize: InputProcessingFunction { 64 | func applyInPlace(_ values: UnsafeMutablePointer, count: Int) { 65 | // vDSP functions in the copy version support in place operations 66 | applyAndCopy(values, count: count, to: values) 67 | } 68 | 69 | func applyAndCopy(_ values: UnsafePointer, count: Int, to destination: UnsafeMutablePointer) { 70 | let len = vDSP_Length(count) 71 | 72 | // max and min values 73 | var mx: Float = 0.0, mn: Float = 0.0 74 | var slope: Float, intercept: Float 75 | 76 | // calculate min and max 77 | vDSP_minv(values, 1, &mn, len) 78 | vDSP_maxv(values, 1, &mx, len) 79 | 80 | // calculate range 81 | let range = mx - mn 82 | 83 | // no range? fill with -1 84 | if 0 == range { 85 | var val: Float = -1.0 86 | vDSP_vfill(&val, destination, 1, len) 87 | return 88 | } 89 | 90 | // calculate slope 91 | slope = 2.0 / range 92 | intercept = (0 - mn - mx) / range; 93 | 94 | // scalar multiply and add 95 | vDSP_vsmsa(values, 1, &slope, &intercept, destination, 1, len) 96 | } 97 | } 98 | 99 | class NormalizeStd: InputProcessingFunction { 100 | func applyInPlace(_ values: UnsafeMutablePointer, count: Int) { 101 | // vDSP functions in the copy version support in place operations 102 | applyAndCopy(values, count: count, to: values) 103 | } 104 | 105 | func applyAndCopy(_ values: UnsafePointer, count: Int, to destination: UnsafeMutablePointer) { 106 | var mean: Float = 0.0, stddev: Float = 0.0 107 | vDSP_normalize(values, 1, destination, 1, &mean, &stddev, vDSP_Length(count)) 108 | } 109 | } 110 | 111 | class MapMinMax: InputProcessingFunction, OutputProcessingFunction { 112 | var gains: [Float] 113 | var xOffsets: [Float] 114 | var yMin: Float 115 | 116 | init(xOffsets: [Float], gains: [Float], yMin: Float) { 117 | self.xOffsets = xOffsets 118 | self.gains = gains 119 | self.yMin = yMin 120 | } 121 | 122 | func applyInPlace(_ values: UnsafeMutablePointer, count: Int) { 123 | // vDSP functions in the copy version support in place operations 124 | applyAndCopy(values, count: count, to: values) 125 | } 126 | 127 | func applyAndCopy(_ values: UnsafePointer, count: Int, to destination: UnsafeMutablePointer) { 128 | // (values - xOffsets) * gain + yMin 129 | vDSP_vsbm(values, 1, &xOffsets, 1, &gains, 1, destination, 1, vDSP_Length(count)) 130 | vDSP_vsadd(destination, 1, &yMin, destination, 1, vDSP_Length(count)) 131 | } 132 | 133 | func reverseInPlace(_ values: UnsafeMutablePointer, count: Int) { 134 | // vDSP functions in the copy version support in place operations 135 | reverseAndCopy(values, count: count, to: values) 136 | } 137 | 138 | func reverseAndCopy(_ values: UnsafePointer, count: Int, to destination: UnsafeMutablePointer) { 139 | var negYMin = 0 - yMin 140 | vDSP_vsadd(values, 1, &negYMin, destination, 1, vDSP_Length(count)) 141 | vDSP_vdiv(&gains, 1, destination, 1, destination, 1, vDSP_Length(count)) 142 | vDSP_vadd(destination, 1, &xOffsets, 1, destination, 1, vDSP_Length(count)) 143 | } 144 | } 145 | 146 | class MapStd: InputProcessingFunction, OutputProcessingFunction { 147 | var gains: [Float] 148 | var xOffsets: [Float] 149 | var yMean: Float 150 | 151 | init(xOffsets: [Float], gains: [Float], yMean: Float) { 152 | self.xOffsets = xOffsets 153 | self.gains = gains 154 | self.yMean = yMean 155 | } 156 | 157 | func applyInPlace(_ values: UnsafeMutablePointer, count: Int) { 158 | // vDSP functions in the copy version support in place operations 159 | applyAndCopy(values, count: count, to: values) 160 | } 161 | 162 | func applyAndCopy(_ values: UnsafePointer, count: Int, to destination: UnsafeMutablePointer) { 163 | // (values - xOffsets) * gain + yMean 164 | vDSP_vsbm(values, 1, &xOffsets, 1, &gains, 1, destination, 1, vDSP_Length(count)) 165 | 166 | if 0 != yMean { 167 | vDSP_vsadd(destination, 1, &yMean, destination, 1, vDSP_Length(count)) 168 | } 169 | } 170 | 171 | func reverseInPlace(_ values: UnsafeMutablePointer, count: Int) { 172 | // vDSP functions in the copy version support in place operations 173 | reverseAndCopy(values, count: count, to: values) 174 | } 175 | 176 | func reverseAndCopy(_ values: UnsafePointer, count: Int, to destination: UnsafeMutablePointer) { 177 | var negYMean = 0 - yMean 178 | vDSP_vsadd(values, 1, &negYMean, destination, 1, vDSP_Length(count)) 179 | vDSP_vdiv(&gains, 1, destination, 1, destination, 1, vDSP_Length(count)) 180 | vDSP_vadd(destination, 1, &xOffsets, 1, destination, 1, vDSP_Length(count)) 181 | } 182 | } 183 | 184 | // transfer function protocol 185 | protocol TransferFunction { 186 | func applyInPlace(_ values: UnsafeMutablePointer, count: Int) 187 | } 188 | 189 | struct TanSig: TransferFunction { 190 | func applyInPlace(_ values: UnsafeMutablePointer, count: Int) { 191 | var c = Int32(count) 192 | vvtanhf(values, values, &c) 193 | } 194 | } 195 | 196 | struct LogSig: TransferFunction { 197 | func applyInPlace(_ values: UnsafeMutablePointer, count: Int) { 198 | let len = vDSP_Length(count) 199 | 200 | // invert sign 201 | var negOne: Float = -1.0 202 | vDSP_vsmul(values, 1, &negOne, values, 1, len) 203 | 204 | // exponent 205 | var c = Int32(count) 206 | vvexpf(values, values, &c) 207 | 208 | // add one 209 | var one: Float = 1.0 210 | vDSP_vsadd(values, 1, &one, values, 1, len) 211 | 212 | // invert 213 | vDSP_svdiv(&one, values, 1, values, 1, len) 214 | } 215 | } 216 | 217 | struct PureLin: TransferFunction { 218 | func applyInPlace(_ values: UnsafeMutablePointer, count: Int) { 219 | // dp nothing 220 | } 221 | } 222 | 223 | struct SatLin: TransferFunction { 224 | func applyInPlace(_ values: UnsafeMutablePointer, count: Int) { 225 | var zero: Float = 0.0, one: Float = 1.0 226 | vDSP_vclip(values, 1, &zero, &one, values, 1, vDSP_Length(count)) 227 | } 228 | } 229 | 230 | /// Handles an extremely limited subset of neural networks from matlab of the form I -> L -> ... -> L -> O. That is, all inputs 231 | /// connect to the first layer only. Each layer only receives values from the prior layer. And the output only receives values 232 | /// from the last layer. This makes memory management and sharing easy(ier). 233 | class NeuralNet 234 | { 235 | let inputs: Int 236 | let outputs: Int 237 | let inputProcessing: [InputProcessingFunction] 238 | let outputProcessing: [OutputProcessingFunction] 239 | let layers: [NeuralNetLayer] 240 | 241 | private var bufferInput: UnsafeMutablePointer 242 | 243 | init(layers: [NeuralNetLayer], inputProcessing: [InputProcessingFunction] = [], outputProcessing: [OutputProcessingFunction] = []) { 244 | guard 0 < layers.count else { 245 | fatalError("Neural network must have 1 or more layers.") 246 | } 247 | 248 | for (i, l) in layers.enumerated() { 249 | if 0 < i { 250 | if layers[i - 1].outputs != l.inputs { 251 | fatalError("Number of inputs for layer \(i) does not match previous outputs.") 252 | } 253 | } 254 | } 255 | 256 | self.inputs = layers[0].inputs 257 | self.outputs = layers[layers.count - 1].outputs 258 | self.layers = layers 259 | 260 | // default input and output processing functions 261 | if 0 < inputProcessing.count { 262 | self.inputProcessing = inputProcessing 263 | } 264 | else { 265 | self.inputProcessing = [PassThrough()] 266 | } 267 | if 0 < outputProcessing.count { 268 | self.outputProcessing = outputProcessing 269 | } 270 | else { 271 | self.outputProcessing = [PassThrough()] 272 | } 273 | 274 | // memory for input layer 275 | bufferInput = UnsafeMutablePointer.allocate(capacity: inputs) 276 | } 277 | 278 | deinit { 279 | // free the window 280 | bufferInput.deinitialize(count: inputs) 281 | bufferInput.deallocate() 282 | } 283 | 284 | func test(_ val: Float) { 285 | let inp = [Float](repeating: val, count: inputs) 286 | inp.withUnsafeBufferPointer() { 287 | guard let ba = $0.baseAddress else { return } 288 | DLog("\(ba.pointee)") 289 | let out = self.apply(ba) 290 | print("\(out)") 291 | } 292 | } 293 | 294 | func apply(_ input: UnsafePointer) -> [Float] { 295 | // pointer to current buffer 296 | var currentBuffer: UnsafeMutablePointer = bufferInput 297 | var curOutput = [Float](repeating: 0.0, count: outputs) 298 | 299 | // create input 300 | for (k, ip) in inputProcessing.enumerated() { 301 | if k == 0 { 302 | ip.applyAndCopy(input, count: inputs, to: currentBuffer) 303 | } 304 | else { 305 | ip.applyInPlace(currentBuffer, count: inputs) 306 | } 307 | } 308 | 309 | 310 | for layer in layers { 311 | // apply the layer and move the content buffer 312 | currentBuffer = layer.apply(currentBuffer) 313 | } 314 | 315 | // create output 316 | for (k, op) in outputProcessing.enumerated() { 317 | if k == 0 { 318 | op.reverseAndCopy(currentBuffer, count: outputs, to: &curOutput) 319 | } 320 | else { 321 | op.reverseInPlace(&curOutput, count: outputs) 322 | } 323 | } 324 | 325 | return curOutput 326 | } 327 | } 328 | 329 | class NeuralNetLayer 330 | { 331 | let inputs: Int 332 | let outputs: Int 333 | var weights: [Float] // matrix of size inputs × outputs; should not change! just var for vDSP functions 334 | var biases: [Float] // should not change! just var for vDSP functions 335 | let transferFunction: TransferFunction 336 | 337 | private var bufferIntermediate: UnsafeMutablePointer 338 | 339 | init(inputs: Int, weights: [Float], biases: [Float], outputs: Int, transferFunction: TransferFunction) { 340 | guard 0 < inputs && 0 < outputs else { 341 | fatalError("Each layer must have at least one input and at least one output.") 342 | } 343 | guard weights.count == (inputs * outputs) else { 344 | fatalError("Weights must be a matrix with \(inputs * outputs) elements.") 345 | } 346 | guard biases.count == outputs else { 347 | fatalError("Biases must be a vector with \(outputs) element(s).") 348 | } 349 | 350 | self.inputs = inputs 351 | self.outputs = outputs 352 | self.weights = weights 353 | self.biases = biases 354 | self.transferFunction = transferFunction 355 | 356 | // memory for intermediate values and output 357 | bufferIntermediate = UnsafeMutablePointer.allocate(capacity: outputs) 358 | } 359 | 360 | deinit { 361 | // free the window 362 | bufferIntermediate.deinitialize(count: outputs) 363 | bufferIntermediate.deallocate() 364 | } 365 | 366 | func apply(_ input: UnsafeMutablePointer) -> UnsafeMutablePointer { 367 | // transform inputs 368 | vDSP_mmul(&weights, 1, input, 1, bufferIntermediate, 1, vDSP_Length(outputs), 1, vDSP_Length(inputs)) 369 | 370 | // add biases 371 | vDSP_vadd(bufferIntermediate, 1, &biases, 1, bufferIntermediate, 1, vDSP_Length(outputs)) 372 | 373 | // apply transfer function 374 | transferFunction.applyInPlace(bufferIntermediate, count: outputs) 375 | 376 | return bufferIntermediate 377 | } 378 | } 379 | -------------------------------------------------------------------------------- /Common/Resampler.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Resampler.swift 3 | // SyllableDetector 4 | // 5 | // Created by Nathan Perkins on 1/5/16. 6 | // Copyright © 2016 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import Accelerate 11 | 12 | protocol Resampler { 13 | func resampleVector(_ data: UnsafePointer, ofLength numSamples: Int) -> [Float] 14 | // func resampleVector(data: UnsafePointer, ofLength numSamples: Int, toOutput: UnsafeMutablePointer) -> Int 15 | } 16 | 17 | // potentially use: http://www.mega-nerd.com/SRC/api_misc.html#Converters 18 | 19 | /// Terrible quality, very fast. 20 | class ResamplerLinear: Resampler { 21 | let samplingRateIn: Double 22 | let samplingRateOut: Double 23 | 24 | private let step: Float 25 | private var last: Float = 0.0 // used for interpolating across samples 26 | private var offset: Float = 0.0 27 | 28 | init(fromRate samplingRateIn: Double, toRate samplingRateOut: Double) { 29 | self.samplingRateIn = samplingRateIn 30 | self.samplingRateOut = samplingRateOut 31 | 32 | self.step = Float(samplingRateIn / samplingRateOut) 33 | } 34 | 35 | func resampleVector(_ data: UnsafePointer, ofLength numSamplesIn: Int) -> [Float] { 36 | // need to interpolate across last set of samples 37 | let interpolateAcross = (offset < 0) 38 | 39 | // expected number of samples from current 40 | let numSamplesOut = Int((Float(numSamplesIn) - offset) / step) 41 | 42 | // return list 43 | var ret = [Float](repeating: 0.0, count: numSamplesOut) 44 | 45 | // indices 46 | let indices = UnsafeMutablePointer.allocate(capacity: numSamplesOut) 47 | var t_offset = offset, t_step = step 48 | defer { 49 | indices.deinitialize(count: numSamplesOut) 50 | indices.deallocate() 51 | } 52 | vDSP_vramp(&t_offset, &t_step, indices, 1, vDSP_Length(numSamplesOut)) 53 | 54 | if interpolateAcross { 55 | indices[0] = 0.0 56 | } 57 | 58 | // interpolate 59 | vDSP_vlint(data, indices, 1, &ret, 1, vDSP_Length(numSamplesOut), vDSP_Length(numSamplesIn)) 60 | 61 | if interpolateAcross { 62 | ret[0] = (last * (0 - offset)) + (data[0] * (1 + offset)) 63 | } 64 | 65 | offset = indices[numSamplesOut - 1] + step - Float(numSamplesIn - 1) 66 | last = data[numSamplesIn - 1] 67 | //print("\(indices[numSamplesOut - 1]) \(numSamplesIn) \(offset)") 68 | 69 | return ret 70 | } 71 | 72 | func resampleArray(_ arr: [Float]) -> [Float] { 73 | // used for testing 74 | var arr = arr 75 | return self.resampleVector(&arr, ofLength: arr.count) 76 | } 77 | 78 | // func resampleVector(data: UnsafePointer, ofLength numSamples: Int, toOutput: UnsafeMutablePointer) -> Int { 79 | // 80 | // } 81 | } 82 | 83 | -------------------------------------------------------------------------------- /Common/StreamReader.swift: -------------------------------------------------------------------------------- 1 | // 2 | // StreamReader.swift 3 | // SyllableDetector 4 | // 5 | // From Stack Overflow answer: 6 | // http://stackoverflow.com/questions/24581517/read-a-file-url-line-by-line-in-swift 7 | 8 | import Foundation 9 | 10 | class StreamReader { 11 | let encoding : String.Encoding 12 | let chunkSize : Int 13 | 14 | var fileHandle : FileHandle! 15 | let buffer : NSMutableData! 16 | let delimData : Data! 17 | var atEof : Bool = false 18 | 19 | init?(path: String, delimiter: String = "\n", encoding: String.Encoding = .utf8, chunkSize : Int = 4096) { 20 | self.chunkSize = chunkSize 21 | self.encoding = encoding 22 | 23 | if let fileHandle = FileHandle(forReadingAtPath: path), 24 | let delimData = delimiter.data(using: encoding), 25 | let buffer = NSMutableData(capacity: chunkSize) 26 | { 27 | self.fileHandle = fileHandle 28 | self.delimData = delimData 29 | self.buffer = buffer 30 | } else { 31 | self.fileHandle = nil 32 | self.delimData = nil 33 | self.buffer = nil 34 | return nil 35 | } 36 | } 37 | 38 | deinit { 39 | self.close() 40 | } 41 | 42 | /// Return next line, or nil on EOF. 43 | func nextLine() -> String? { 44 | precondition(fileHandle != nil, "Attempt to read from closed file") 45 | 46 | if atEof { 47 | return nil 48 | } 49 | 50 | // Read data chunks from file until a line delimiter is found: 51 | var range = buffer.range(of: delimData, options: [], in: NSMakeRange(0, buffer.length)) 52 | while range.location == NSNotFound { 53 | let tmpData = fileHandle.readData(ofLength: chunkSize) 54 | if tmpData.count == 0 { 55 | // EOF or read error. 56 | atEof = true 57 | if buffer.length > 0 { 58 | // Buffer contains last line in file (not terminated by delimiter). 59 | let line = NSString(data: buffer as Data, encoding: encoding.rawValue) 60 | 61 | buffer.length = 0 62 | return line as String? 63 | } 64 | // No more lines. 65 | return nil 66 | } 67 | buffer.append(tmpData) 68 | range = buffer.range(of: delimData, options: [], in: NSMakeRange(0, buffer.length)) 69 | } 70 | 71 | // Convert complete line (excluding the delimiter) to a string: 72 | let line = NSString(data: buffer.subdata(with: NSMakeRange(0, range.location)), 73 | encoding: encoding.rawValue) 74 | // Remove line (and the delimiter) from the buffer: 75 | buffer.replaceBytes(in: NSMakeRange(0, range.location + range.length), withBytes: nil, length: 0) 76 | 77 | return line as String? 78 | } 79 | 80 | /// Start reading from the beginning of file. 81 | func rewind() -> Void { 82 | fileHandle.seek(toFileOffset: 0) 83 | buffer.length = 0 84 | atEof = false 85 | } 86 | 87 | /// Close the underlying file. No reading must be done after calling this method. 88 | func close() -> Void { 89 | fileHandle?.closeFile() 90 | fileHandle = nil 91 | } 92 | } 93 | 94 | extension StreamReader : Sequence { 95 | func makeIterator() -> AnyIterator { 96 | return AnyIterator { 97 | return self.nextLine() 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /Common/SyllableDetector.swift: -------------------------------------------------------------------------------- 1 | // 2 | // SyllableDetector.swift 3 | // SongDetector 4 | // 5 | // Created by Nathan Perkins on 9/18/15. 6 | // Copyright © 2015 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import Accelerate 11 | import AVFoundation 12 | 13 | class SyllableDetector: NSObject, AVCaptureAudioDataOutputSampleBufferDelegate 14 | { 15 | // should be constant, but sampling rate can be changed when initializing 16 | let config: SyllableDetectorConfig 17 | 18 | // very specific audio settings required, since down sampling signal 19 | var audioSettings: [String: AnyObject] { 20 | get { 21 | return [AVFormatIDKey: NSNumber(value: kAudioFormatLinearPCM), AVLinearPCMBitDepthKey: NSNumber(value: 32), AVLinearPCMIsFloatKey: true as AnyObject, AVLinearPCMIsNonInterleaved: true as AnyObject, AVSampleRateKey: NSNumber(value: config.samplingRate)] 22 | } 23 | } 24 | 25 | // last values 26 | var lastOutputs: [Float] 27 | var lastDetected: Bool { 28 | get { 29 | return (Double(lastOutputs[0]) >= config.thresholds[0]) 30 | } 31 | } 32 | 33 | private let shortTimeFourierTransform: CircularShortTimeFourierTransform 34 | private let freqIndices: (Int, Int) // default: (26, 90) 35 | private var buffer: TPCircularBuffer 36 | 37 | init(config: SyllableDetectorConfig) { 38 | // set configuration 39 | self.config = config 40 | 41 | // initialize the FFT 42 | shortTimeFourierTransform = CircularShortTimeFourierTransform(windowLength: config.windowLength, withOverlap: config.windowOverlap, fftSizeOf: config.fourierLength) 43 | shortTimeFourierTransform.windowType = WindowType.hamming 44 | 45 | // store frequency indices 46 | guard let idx = shortTimeFourierTransform.frequencyIndexRange(from: config.freqRange.0, through: config.freqRange.1, forSampleRate: config.samplingRate) else { 47 | fatalError("The frequency range is invalid.") 48 | } 49 | freqIndices = idx 50 | 51 | // check that matches input size 52 | let expectedInputs = (freqIndices.1 - freqIndices.0) * config.timeRange 53 | guard expectedInputs == config.net.inputs else { 54 | fatalError("The neural network has \(config.net.inputs) inputs, but the configuration settings suggest there should be \(expectedInputs).") 55 | } 56 | 57 | // check that the threshold count matches the output size 58 | guard config.thresholds.count == config.net.outputs else { 59 | fatalError("The neural network has \(config.net.outputs) outputs, but the configuration settings suggest there should be \(config.thresholds.count).") 60 | } 61 | 62 | // create the circular buffer 63 | let bufferCapacity = 512 // hold several full sets of data (could be 2 easily, maybe even 1, for live processing) 64 | buffer = TPCircularBuffer() 65 | if !TPCircularBufferInit(&buffer, Int32((freqIndices.1 - freqIndices.0) * config.timeRange * bufferCapacity)) { 66 | fatalError("Unable to allocate circular buffer.") 67 | } 68 | 69 | // no last output 70 | lastOutputs = [Float](repeating: 0.0, count: config.net.outputs) 71 | 72 | // call super 73 | super.init() 74 | } 75 | 76 | deinit { 77 | // release the circular buffer 78 | TPCircularBufferCleanup(&buffer) 79 | } 80 | 81 | func processSampleBuffer(_ sampleBuffer: CMSampleBuffer) { 82 | // has samples 83 | let numSamples = CMSampleBufferGetNumSamples(sampleBuffer) 84 | guard 0 < numSamples else { 85 | return 86 | } 87 | 88 | // get format 89 | guard let format = CMSampleBufferGetFormatDescription(sampleBuffer) else { 90 | DLog("Unable to get format information.") 91 | return 92 | } 93 | let audioDescription = CMAudioFormatDescriptionGetStreamBasicDescription(format) 94 | 95 | // is interleaved 96 | let isInterleaved = 1 < (audioDescription?[0].mChannelsPerFrame)! && 0 == ((audioDescription?[0].mFormatFlags)! & kAudioFormatFlagIsNonInterleaved) 97 | let isFloat = 0 < ((audioDescription?[0].mFormatFlags)! & kAudioFormatFlagIsFloat) 98 | 99 | // checks 100 | guard audioDescription?[0].mFormatID == kAudioFormatLinearPCM && isFloat && !isInterleaved && audioDescription?[0].mBitsPerChannel == 32 else { 101 | fatalError("Invalid audio format.") 102 | } 103 | 104 | // get audio buffer 105 | guard let audioBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) else { 106 | DLog("Unable to get audio buffer.") 107 | return 108 | } 109 | 110 | // get data pointer 111 | var lengthAtOffset: Int = 0, totalLength: Int = 0 112 | var inSamples: UnsafeMutablePointer? = nil 113 | CMBlockBufferGetDataPointer(audioBuffer, 0, &lengthAtOffset, &totalLength, &inSamples) 114 | 115 | // append it to fourier transform 116 | inSamples!.withMemoryRebound(to: Float.self, capacity: numSamples) { 117 | shortTimeFourierTransform.appendData($0, withSamples: numSamples) 118 | } 119 | } 120 | 121 | func captureOutput(_ captureOutput: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { 122 | // append sample data 123 | processSampleBuffer(sampleBuffer) 124 | 125 | // process immediately 126 | while processNewValue() {} 127 | } 128 | 129 | func appendAudioData(_ data: UnsafeMutablePointer, withSamples numSamples: Int) { 130 | // add to short-time fourier transform 131 | shortTimeFourierTransform.appendData(data, withSamples: numSamples) 132 | } 133 | 134 | private func processFourierData() -> Bool { 135 | // get the power information 136 | guard var powr = shortTimeFourierTransform.extractPower() else { 137 | return false 138 | } 139 | 140 | let lengthPerTime = freqIndices.1 - freqIndices.0 141 | 142 | // append data to local circular buffer 143 | withUnsafePointer(to: &powr[freqIndices.0]) { 144 | up in 145 | if !TPCircularBufferProduceBytes(&self.buffer, up, Int32(lengthPerTime * MemoryLayout.stride)) { 146 | fatalError("Insufficient space on buffer.") 147 | } 148 | } 149 | 150 | return true 151 | } 152 | 153 | func processNewValue() -> Bool { 154 | // append all new fourier data 155 | while processFourierData() {} 156 | 157 | // get data counts 158 | let lengthPerTime = freqIndices.1 - freqIndices.0 159 | let lengthTotal = lengthPerTime * config.timeRange 160 | 161 | // let UnsafeMutablePointer: samples 162 | var availableBytes: Int32 = 0 163 | let samples: UnsafeMutablePointer 164 | guard let p = TPCircularBufferTail(&buffer, &availableBytes) else { 165 | return false 166 | } 167 | samples = p.bindMemory(to: Float.self, capacity: Int(availableBytes) / MemoryLayout.stride) 168 | 169 | // not enough available bytes 170 | if Int(availableBytes) < (lengthTotal * MemoryLayout.stride) { 171 | return false 172 | } 173 | 174 | // mark circular buffer as consumed at END of excution 175 | defer { 176 | // mark as consumed, one time per-time length 177 | TPCircularBufferConsume(&buffer, Int32(lengthPerTime * MemoryLayout.stride)) 178 | } 179 | 180 | /// samples now points to a vector of `lengthTotal` bytes of power data for the last `timeRange` outputs of the short-timer fourier transform 181 | /// view as a column vector 182 | 183 | let scaledSamples: UnsafeMutablePointer 184 | switch config.spectrogramScaling { 185 | case .db: 186 | // temporary memory 187 | scaledSamples = UnsafeMutablePointer.allocate(capacity: lengthTotal) 188 | defer { 189 | scaledSamples.deinitialize(count: lengthTotal) 190 | scaledSamples.deallocate() 191 | } 192 | 193 | // convert to db with amplitude flag 194 | var one: Float = 1.0 195 | vDSP_vdbcon(samples, 1, &one, scaledSamples, 1, vDSP_Length(lengthTotal), 1) 196 | 197 | case .log: 198 | // temporary memory 199 | scaledSamples = UnsafeMutablePointer.allocate(capacity: lengthTotal) 200 | defer { 201 | scaledSamples.deinitialize(count: lengthTotal) 202 | scaledSamples.deallocate() 203 | } 204 | 205 | // natural log 206 | var c = Int32(lengthTotal) 207 | vvlogf(samples, scaledSamples, &c) 208 | 209 | case .linear: 210 | // no copy needed 211 | scaledSamples = samples 212 | } 213 | 214 | lastOutputs = config.net.apply(scaledSamples) 215 | 216 | return true 217 | } 218 | 219 | // Returns true if a syllable seen since last call to this function. 220 | func seenSyllable() -> Bool { 221 | var ret = false 222 | 223 | while processNewValue() { 224 | if lastDetected { 225 | ret = true 226 | } 227 | } 228 | 229 | return ret 230 | } 231 | } 232 | -------------------------------------------------------------------------------- /Common/SyllableDetectorConfig.swift: -------------------------------------------------------------------------------- 1 | // 2 | // SyllableDetectorConfig.swift 3 | // SongDetector 4 | // 5 | // Created by Nathan Perkins on 9/22/15. 6 | // Copyright © 2015 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | 11 | struct SyllableDetectorConfig 12 | { 13 | enum Scaling { 14 | case linear 15 | case log 16 | case db 17 | 18 | init?(fromName name: String) { 19 | switch name { 20 | case "linear": 21 | self = .linear 22 | case "log": 23 | self = .log 24 | case "db": 25 | self = .db 26 | default: 27 | return nil 28 | } 29 | } 30 | } 31 | 32 | let samplingRate: Double // eqv: samplerate 33 | let fourierLength: Int // eqv: FFT_SIZE 34 | let windowLength: Int 35 | let windowOverlap: Int // eqv: NOVERLAP = FFT_SIZE - (floor(samplerate * FFT_TIME_SHIFT)) 36 | 37 | let freqRange: (Double, Double) // eqv: freq_range 38 | let timeRange: Int // eqv: time_window_steps = double(floor(time_window / timestep)) 39 | 40 | let spectrogramScaling: Scaling 41 | 42 | let thresholds: [Double] // eqv: trigger threshold 43 | 44 | let net: NeuralNet 45 | } 46 | 47 | // make parsable 48 | extension SyllableDetectorConfig 49 | { 50 | enum ParseError: Error { 51 | case unableToOpenPath(String) 52 | case missingValue(String) 53 | case invalidValue(String) 54 | case mismatchedLength(String) 55 | } 56 | 57 | private static func parseString(_ nm: String, from data: [String: String]) throws -> String { 58 | guard let v = data[nm] else { throw ParseError.missingValue(nm) } 59 | return v 60 | } 61 | 62 | private static func parseDouble(_ nm: String, from data: [String: String]) throws -> Double { 63 | guard let v = data[nm] else { throw ParseError.missingValue(nm) } 64 | guard let d = Double(v) else { throw ParseError.invalidValue(nm) } 65 | return d 66 | } 67 | 68 | private static func parseFloat(_ nm: String, from data: [String: String]) throws -> Float { 69 | guard let v = data[nm] else { throw ParseError.missingValue(nm) } 70 | guard let f = Float(v) else { throw ParseError.invalidValue(nm) } 71 | return f 72 | } 73 | 74 | private static func parseInt(_ nm: String, from data: [String: String]) throws -> Int { 75 | guard let v = data[nm] else { throw ParseError.missingValue(nm) } 76 | guard let i = Int(v) else { throw ParseError.invalidValue(nm) } 77 | return i 78 | } 79 | 80 | private static func parseDoubleArray(_ nm: String, withCount cnt: Int? = nil, from data: [String: String]) throws -> [Double] { 81 | guard let v = data[nm] else { throw ParseError.missingValue(nm) } 82 | 83 | // split into doubles 84 | let stringParts = v.splitAtCharacter(",").map { $0.trim() } 85 | let doubleParts = stringParts.compactMap(Double.init) 86 | 87 | // compare lengths to make sure all doubles were parsed 88 | if stringParts.count != doubleParts.count { throw ParseError.invalidValue(nm) } 89 | 90 | // check count 91 | if let desiredCnt = cnt { 92 | if doubleParts.count != desiredCnt { throw ParseError.mismatchedLength(nm) } 93 | } 94 | 95 | return doubleParts 96 | } 97 | 98 | private static func parseFloatArray(_ nm: String, withCount cnt: Int, from data: [String: String]) throws -> [Float] { 99 | guard let v = data[nm] else { throw ParseError.missingValue(nm) } 100 | 101 | // split into doubles 102 | let stringParts = v.splitAtCharacter(",").map { $0.trim() } 103 | let floatParts = stringParts.compactMap(Float.init) 104 | 105 | // compare lengths to make sure all doubles were parsed 106 | if stringParts.count != floatParts.count { throw ParseError.invalidValue(nm) } 107 | 108 | // check count 109 | if floatParts.count != cnt { throw ParseError.mismatchedLength(nm) } 110 | 111 | return floatParts 112 | } 113 | 114 | private static func parseMapMinMax(_ nm: String, withCount cnt: Int, from data: [String: String]) throws -> MapMinMax { 115 | let xOffsets = try SyllableDetectorConfig.parseFloatArray("\(nm).xOffsets", withCount: cnt, from: data) 116 | let gains = try SyllableDetectorConfig.parseFloatArray("\(nm).gains", withCount: cnt, from: data) 117 | let yMin = try SyllableDetectorConfig.parseFloat("\(nm).yMin", from: data) 118 | return MapMinMax(xOffsets: xOffsets, gains: gains, yMin: yMin) 119 | } 120 | 121 | private static func parseMapStd(_ nm: String, withCount cnt: Int, from data: [String: String]) throws -> MapStd { 122 | let xOffsets = try SyllableDetectorConfig.parseFloatArray("\(nm).xOffsets", withCount: cnt, from: data) 123 | let gains = try SyllableDetectorConfig.parseFloatArray("\(nm).gains", withCount: cnt, from: data) 124 | let yMean = try SyllableDetectorConfig.parseFloat("\(nm).yMean", from: data) 125 | return MapStd(xOffsets: xOffsets, gains: gains, yMean: yMean) 126 | } 127 | 128 | private static func parseInputProcessingFunction(_ nm: String, withCount cnt: Int, from data: [String: String]) throws -> InputProcessingFunction { 129 | // get processing function 130 | // TODO: add a default processing function that passes through values 131 | let functionName = try SyllableDetectorConfig.parseString("\(nm).function", from: data) 132 | 133 | switch functionName { 134 | case "mapminmax": 135 | return try SyllableDetectorConfig.parseMapMinMax(nm, withCount: cnt, from: data) 136 | 137 | case "mapstd": 138 | return try SyllableDetectorConfig.parseMapStd(nm, withCount: cnt, from: data) 139 | 140 | case "l2normalize": 141 | return L2Normalize() 142 | 143 | case "normalize": 144 | return Normalize() 145 | 146 | case "normalizestd": 147 | return NormalizeStd() 148 | 149 | default: 150 | throw ParseError.invalidValue("\(nm).function") 151 | } 152 | } 153 | 154 | private static func parseOutputProcessingFunction(_ nm: String, withCount cnt: Int, from data: [String: String]) throws -> OutputProcessingFunction { 155 | // get processing function 156 | let functionName = try SyllableDetectorConfig.parseString("\(nm).function", from: data) 157 | 158 | switch functionName { 159 | case "mapminmax": 160 | return try SyllableDetectorConfig.parseMapMinMax(nm, withCount: cnt, from: data) 161 | 162 | case "mapstd": 163 | return try SyllableDetectorConfig.parseMapStd(nm, withCount: cnt, from: data) 164 | 165 | default: 166 | throw ParseError.invalidValue("\(nm).function") 167 | } 168 | } 169 | 170 | init(fromTextFile path: String) throws { 171 | // get stream 172 | guard let stream = StreamReader(path: path) else { 173 | throw ParseError.unableToOpenPath(path) 174 | } 175 | 176 | // automatically close 177 | defer { 178 | stream.close() 179 | } 180 | 181 | // read line 182 | var data = [String: String]() 183 | for line in stream { 184 | // split string into parts 185 | let parts = line.splitAtCharacter("=") 186 | if 2 == parts.count { 187 | data[parts[0].trim()] = parts[1].trim() 188 | } 189 | } 190 | 191 | // read data 192 | // THIS SHOULD ALL BE REWRITTEN IN SOME SORT OF SYSTEMETIZED 193 | 194 | // sampling rate: double 195 | samplingRate = try SyllableDetectorConfig.parseDouble("samplingRate", from: data) 196 | 197 | // fourier length: int 198 | fourierLength = try SyllableDetectorConfig.parseInt("fourierLength", from: data) 199 | if !fourierLength.isPowerOfTwo() { 200 | throw SyllableDetectorConfig.ParseError.invalidValue("fourierLength") 201 | } 202 | 203 | // window length: int, defaults to fourierLength 204 | if nil == data["windowLength"] { 205 | windowLength = fourierLength 206 | } 207 | else { 208 | windowLength = try SyllableDetectorConfig.parseInt("windowLength", from: data) 209 | } 210 | 211 | // fourier length: int 212 | windowOverlap = try SyllableDetectorConfig.parseInt("windowOverlap", from: data) 213 | 214 | // frequency range: double, double 215 | let potentialFreqRange = try SyllableDetectorConfig.parseDoubleArray("freqRange", withCount: 2, from: data) 216 | if 2 != potentialFreqRange.count { throw ParseError.mismatchedLength("freqRange") } 217 | freqRange = (potentialFreqRange[0], potentialFreqRange[1]) 218 | 219 | // time range: int 220 | timeRange = try SyllableDetectorConfig.parseInt("timeRange", from: data) 221 | 222 | // threshold: double 223 | do { 224 | thresholds = try SyllableDetectorConfig.parseDoubleArray("thresholds", from: data) 225 | } 226 | catch { 227 | // backwards compatibility 228 | thresholds = try SyllableDetectorConfig.parseDoubleArray("threshold", from: data) 229 | } 230 | 231 | // read scaling 232 | if let scaling = Scaling(fromName: try SyllableDetectorConfig.parseString("scaling", from: data)) { 233 | spectrogramScaling = scaling 234 | } 235 | else { 236 | throw ParseError.invalidValue("scaling") 237 | } 238 | 239 | // get layers 240 | let layerCount = try SyllableDetectorConfig.parseInt("layers", from: data) 241 | let layers = try (0.. NeuralNetLayer in 243 | let inputs = try SyllableDetectorConfig.parseInt("layer\(i).inputs", from: data) 244 | let outputs = try SyllableDetectorConfig.parseInt("layer\(i).outputs", from: data) 245 | let weights = try SyllableDetectorConfig.parseFloatArray("layer\(i).weights", withCount: (inputs * outputs), from: data) 246 | let biases = try SyllableDetectorConfig.parseFloatArray("layer\(i).biases", withCount: outputs, from: data) 247 | 248 | // get transfer function 249 | let transferFunction: TransferFunction 250 | switch try SyllableDetectorConfig.parseString("layer\(i).transferFunction", from: data) { 251 | case "TanSig": transferFunction = TanSig() 252 | case "LogSig": transferFunction = LogSig() 253 | case "PureLin": transferFunction = PureLin() 254 | case "SatLin": transferFunction = SatLin() 255 | default: throw ParseError.invalidValue("layer\(i).transferFunction") 256 | } 257 | 258 | return NeuralNetLayer(inputs: inputs, weights: weights, biases: biases, outputs: outputs, transferFunction: transferFunction) 259 | } 260 | 261 | // get input mapping 262 | let processInputCount = try SyllableDetectorConfig.parseInt("processInputsCount", from: data) 263 | let processInputs = try (0.. InputProcessingFunction in 265 | return try SyllableDetectorConfig.parseInputProcessingFunction("processInputs\(i)", withCount: layers[0].inputs, from: data) 266 | } 267 | 268 | // get output mapping 269 | let processOutputCount = try SyllableDetectorConfig.parseInt("processOutputsCount", from: data) 270 | let processOutputs = try (0.. OutputProcessingFunction in 272 | return try SyllableDetectorConfig.parseOutputProcessingFunction("processOutputs\(i)", withCount: layers[layerCount - 1].outputs, from: data) 273 | } 274 | 275 | // create network 276 | net = NeuralNet(layers: layers, inputProcessing: processInputs, outputProcessing: processOutputs) 277 | } 278 | } 279 | -------------------------------------------------------------------------------- /Common/TPCircularBuffer/TPCircularBuffer.c: -------------------------------------------------------------------------------- 1 | // 2 | // TPCircularBuffer.c 3 | // Circular/Ring buffer implementation 4 | // 5 | // https://github.com/michaeltyson/TPCircularBuffer 6 | // 7 | // Created by Michael Tyson on 10/12/2011. 8 | // 9 | // Copyright (C) 2012-2013 A Tasty Pixel 10 | // 11 | // This software is provided 'as-is', without any express or implied 12 | // warranty. In no event will the authors be held liable for any damages 13 | // arising from the use of this software. 14 | // 15 | // Permission is granted to anyone to use this software for any purpose, 16 | // including commercial applications, and to alter it and redistribute it 17 | // freely, subject to the following restrictions: 18 | // 19 | // 1. The origin of this software must not be misrepresented; you must not 20 | // claim that you wrote the original software. If you use this software 21 | // in a product, an acknowledgment in the product documentation would be 22 | // appreciated but is not required. 23 | // 24 | // 2. Altered source versions must be plainly marked as such, and must not be 25 | // misrepresented as being the original software. 26 | // 27 | // 3. This notice may not be removed or altered from any source distribution. 28 | // 29 | 30 | #include "TPCircularBuffer.h" 31 | #include 32 | #include 33 | 34 | #define reportResult(result,operation) (_reportResult((result),(operation),strrchr(__FILE__, '/')+1,__LINE__)) 35 | static inline bool _reportResult(kern_return_t result, const char *operation, const char* file, int line) { 36 | if ( result != ERR_SUCCESS ) { 37 | printf("%s:%d: %s: %s\n", file, line, operation, mach_error_string(result)); 38 | return false; 39 | } 40 | return true; 41 | } 42 | 43 | bool TPCircularBufferInit(TPCircularBuffer *buffer, int32_t length) { 44 | 45 | // Keep trying until we get our buffer, needed to handle race conditions 46 | int retries = 3; 47 | while ( true ) { 48 | 49 | buffer->length = (int32_t)round_page(length); // We need whole page sizes 50 | 51 | // Temporarily allocate twice the length, so we have the contiguous address space to 52 | // support a second instance of the buffer directly after 53 | vm_address_t bufferAddress; 54 | kern_return_t result = vm_allocate(mach_task_self(), 55 | &bufferAddress, 56 | buffer->length * 2, 57 | VM_FLAGS_ANYWHERE); // allocate anywhere it'll fit 58 | if ( result != ERR_SUCCESS ) { 59 | if ( retries-- == 0 ) { 60 | reportResult(result, "Buffer allocation"); 61 | return false; 62 | } 63 | // Try again if we fail 64 | continue; 65 | } 66 | 67 | // Now replace the second half of the allocation with a virtual copy of the first half. Deallocate the second half... 68 | result = vm_deallocate(mach_task_self(), 69 | bufferAddress + buffer->length, 70 | buffer->length); 71 | if ( result != ERR_SUCCESS ) { 72 | if ( retries-- == 0 ) { 73 | reportResult(result, "Buffer deallocation"); 74 | return false; 75 | } 76 | // If this fails somehow, deallocate the whole region and try again 77 | vm_deallocate(mach_task_self(), bufferAddress, buffer->length); 78 | continue; 79 | } 80 | 81 | // Re-map the buffer to the address space immediately after the buffer 82 | vm_address_t virtualAddress = bufferAddress + buffer->length; 83 | vm_prot_t cur_prot, max_prot; 84 | result = vm_remap(mach_task_self(), 85 | &virtualAddress, // mirror target 86 | buffer->length, // size of mirror 87 | 0, // auto alignment 88 | 0, // force remapping to virtualAddress 89 | mach_task_self(), // same task 90 | bufferAddress, // mirror source 91 | 0, // MAP READ-WRITE, NOT COPY 92 | &cur_prot, // unused protection struct 93 | &max_prot, // unused protection struct 94 | VM_INHERIT_DEFAULT); 95 | if ( result != ERR_SUCCESS ) { 96 | if ( retries-- == 0 ) { 97 | reportResult(result, "Remap buffer memory"); 98 | return false; 99 | } 100 | // If this remap failed, we hit a race condition, so deallocate and try again 101 | vm_deallocate(mach_task_self(), bufferAddress, buffer->length); 102 | continue; 103 | } 104 | 105 | if ( virtualAddress != bufferAddress+buffer->length ) { 106 | // If the memory is not contiguous, clean up both allocated buffers and try again 107 | if ( retries-- == 0 ) { 108 | printf("Couldn't map buffer memory to end of buffer\n"); 109 | return false; 110 | } 111 | 112 | vm_deallocate(mach_task_self(), virtualAddress, buffer->length); 113 | vm_deallocate(mach_task_self(), bufferAddress, buffer->length); 114 | continue; 115 | } 116 | 117 | buffer->buffer = (void*)bufferAddress; 118 | buffer->fillCount = 0; 119 | buffer->head = buffer->tail = 0; 120 | 121 | return true; 122 | } 123 | return false; 124 | } 125 | 126 | void TPCircularBufferCleanup(TPCircularBuffer *buffer) { 127 | vm_deallocate(mach_task_self(), (vm_address_t)buffer->buffer, buffer->length * 2); 128 | memset(buffer, 0, sizeof(TPCircularBuffer)); 129 | } 130 | 131 | void TPCircularBufferClear(TPCircularBuffer *buffer) { 132 | int32_t fillCount; 133 | if ( TPCircularBufferTail(buffer, &fillCount) ) { 134 | TPCircularBufferConsume(buffer, fillCount); 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /Common/TPCircularBuffer/TPCircularBuffer.h: -------------------------------------------------------------------------------- 1 | // 2 | // TPCircularBuffer.h 3 | // Circular/Ring buffer implementation 4 | // 5 | // https://github.com/michaeltyson/TPCircularBuffer 6 | // 7 | // Created by Michael Tyson on 10/12/2011. 8 | // 9 | // 10 | // This implementation makes use of a virtual memory mapping technique that inserts a virtual copy 11 | // of the buffer memory directly after the buffer's end, negating the need for any buffer wrap-around 12 | // logic. Clients can simply use the returned memory address as if it were contiguous space. 13 | // 14 | // The implementation is thread-safe in the case of a single producer and single consumer. 15 | // 16 | // Virtual memory technique originally proposed by Philip Howard (http://vrb.slashusr.org/), and 17 | // adapted to Darwin by Kurt Revis (http://www.snoize.com, 18 | // http://www.snoize.com/Code/PlayBufferedSoundFile.tar.gz) 19 | // 20 | // 21 | // Copyright (C) 2012-2013 A Tasty Pixel 22 | // 23 | // This software is provided 'as-is', without any express or implied 24 | // warranty. In no event will the authors be held liable for any damages 25 | // arising from the use of this software. 26 | // 27 | // Permission is granted to anyone to use this software for any purpose, 28 | // including commercial applications, and to alter it and redistribute it 29 | // freely, subject to the following restrictions: 30 | // 31 | // 1. The origin of this software must not be misrepresented; you must not 32 | // claim that you wrote the original software. If you use this software 33 | // in a product, an acknowledgment in the product documentation would be 34 | // appreciated but is not required. 35 | // 36 | // 2. Altered source versions must be plainly marked as such, and must not be 37 | // misrepresented as being the original software. 38 | // 39 | // 3. This notice may not be removed or altered from any source distribution. 40 | // 41 | 42 | #ifndef TPCircularBuffer_h 43 | #define TPCircularBuffer_h 44 | 45 | #include 46 | #include 47 | #include 48 | 49 | #ifdef __cplusplus 50 | extern "C" { 51 | #endif 52 | 53 | typedef struct { 54 | void *buffer; 55 | int32_t length; 56 | int32_t tail; 57 | int32_t head; 58 | volatile int32_t fillCount; 59 | } TPCircularBuffer; 60 | 61 | /*! 62 | * Initialise buffer 63 | * 64 | * Note that the length is advisory only: Because of the way the 65 | * memory mirroring technique works, the true buffer length will 66 | * be multiples of the device page size (e.g. 4096 bytes) 67 | * 68 | * @param buffer Circular buffer 69 | * @param length Length of buffer 70 | */ 71 | bool TPCircularBufferInit(TPCircularBuffer *buffer, int32_t length); 72 | 73 | /*! 74 | * Cleanup buffer 75 | * 76 | * Releases buffer resources. 77 | */ 78 | void TPCircularBufferCleanup(TPCircularBuffer *buffer); 79 | 80 | /*! 81 | * Clear buffer 82 | * 83 | * Resets buffer to original, empty state. 84 | * 85 | * This is safe for use by consumer while producer is accessing 86 | * buffer. 87 | */ 88 | void TPCircularBufferClear(TPCircularBuffer *buffer); 89 | 90 | // Reading (consuming) 91 | 92 | /*! 93 | * Access end of buffer 94 | * 95 | * This gives you a pointer to the end of the buffer, ready 96 | * for reading, and the number of available bytes to read. 97 | * 98 | * @param buffer Circular buffer 99 | * @param availableBytes On output, the number of bytes ready for reading 100 | * @return Pointer to the first bytes ready for reading, or NULL if buffer is empty 101 | */ 102 | static __inline__ __attribute__((always_inline)) void* TPCircularBufferTail(TPCircularBuffer *buffer, int32_t* availableBytes) { 103 | *availableBytes = buffer->fillCount; 104 | if ( *availableBytes == 0 ) return NULL; 105 | return (void*)((char*)buffer->buffer + buffer->tail); 106 | } 107 | 108 | /*! 109 | * Consume bytes in buffer 110 | * 111 | * This frees up the just-read bytes, ready for writing again. 112 | * 113 | * @param buffer Circular buffer 114 | * @param amount Number of bytes to consume 115 | */ 116 | static __inline__ __attribute__((always_inline)) void TPCircularBufferConsume(TPCircularBuffer *buffer, int32_t amount) { 117 | buffer->tail = (buffer->tail + amount) % buffer->length; 118 | OSAtomicAdd32Barrier(-amount, &buffer->fillCount); 119 | assert(buffer->fillCount >= 0); 120 | } 121 | 122 | /*! 123 | * Version of TPCircularBufferConsume without the memory barrier, for more optimal use in single-threaded contexts 124 | */ 125 | static __inline__ __attribute__((always_inline)) void TPCircularBufferConsumeNoBarrier(TPCircularBuffer *buffer, int32_t amount) { 126 | buffer->tail = (buffer->tail + amount) % buffer->length; 127 | buffer->fillCount -= amount; 128 | assert(buffer->fillCount >= 0); 129 | } 130 | 131 | /*! 132 | * Access front of buffer 133 | * 134 | * This gives you a pointer to the front of the buffer, ready 135 | * for writing, and the number of available bytes to write. 136 | * 137 | * @param buffer Circular buffer 138 | * @param availableBytes On output, the number of bytes ready for writing 139 | * @return Pointer to the first bytes ready for writing, or NULL if buffer is full 140 | */ 141 | static __inline__ __attribute__((always_inline)) void* TPCircularBufferHead(TPCircularBuffer *buffer, int32_t* availableBytes) { 142 | *availableBytes = (buffer->length - buffer->fillCount); 143 | if ( *availableBytes == 0 ) return NULL; 144 | return (void*)((char*)buffer->buffer + buffer->head); 145 | } 146 | 147 | // Writing (producing) 148 | 149 | /*! 150 | * Produce bytes in buffer 151 | * 152 | * This marks the given section of the buffer ready for reading. 153 | * 154 | * @param buffer Circular buffer 155 | * @param amount Number of bytes to produce 156 | */ 157 | static __inline__ __attribute__((always_inline)) void TPCircularBufferProduce(TPCircularBuffer *buffer, int32_t amount) { 158 | buffer->head = (buffer->head + amount) % buffer->length; 159 | OSAtomicAdd32Barrier(amount, &buffer->fillCount); 160 | assert(buffer->fillCount <= buffer->length); 161 | } 162 | 163 | /*! 164 | * Version of TPCircularBufferProduce without the memory barrier, for more optimal use in single-threaded contexts 165 | */ 166 | static __inline__ __attribute__((always_inline)) void TPCircularBufferProduceNoBarrier(TPCircularBuffer *buffer, int32_t amount) { 167 | buffer->head = (buffer->head + amount) % buffer->length; 168 | buffer->fillCount += amount; 169 | assert(buffer->fillCount <= buffer->length); 170 | } 171 | 172 | /*! 173 | * Helper routine to copy bytes to buffer 174 | * 175 | * This copies the given bytes to the buffer, and marks them ready for writing. 176 | * 177 | * @param buffer Circular buffer 178 | * @param src Source buffer 179 | * @param len Number of bytes in source buffer 180 | * @return true if bytes copied, false if there was insufficient space 181 | */ 182 | static __inline__ __attribute__((always_inline)) bool TPCircularBufferProduceBytes(TPCircularBuffer *buffer, const void* src, int32_t len) { 183 | int32_t space; 184 | void *ptr = TPCircularBufferHead(buffer, &space); 185 | if ( space < len ) return false; 186 | memcpy(ptr, src, len); 187 | TPCircularBufferProduce(buffer, len); 188 | return true; 189 | } 190 | 191 | #ifdef __cplusplus 192 | } 193 | #endif 194 | 195 | #endif 196 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 L. Nathan Perkins 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Syllable Detector 2 | ================= 3 | 4 | The syllable detector is a Mac app that uses CoreAudio to perform low-latency 5 | syllable detection based on a simple Matlab neural network trained using the 6 | [training code](https://github.com/gardner-lab/syllable-detector-learn) 7 | created by @bwpearre. Audio sampling is highly tunable to tradeoff between detection 8 | latency/jitter and processing power. The app allows running multiple detectors through 9 | multiple audio devices and/or channels. 10 | 11 | Installation 12 | ------------ 13 | 14 | A binary version of the software can be downloaded from the 15 | [**releases** tab](https://github.com/gardner-lab/syllable-detector-swift/releases). 16 | 17 | Usage 18 | ----- 19 | 20 | **Connection:** The syllable detector software can process any standard audio input 21 | source. As a result, you can either connect a microphone or a line in to the 22 | standard ports on the computer, or you can use an external audio interface. If the 23 | input source supports multiple channels, instances of the detector can be 24 | specified independently for each channel. 25 | 26 | To generate an output TTL signal, the application supports two options. An output signal 27 | can be sent via an audio channel through any valid audio interface (headphone jack or an 28 | external audio interface). The audio output must have at least the same number of 29 | channels as the input. Because of mixing and signal processing, the audio output signal 30 | can introduce an added delay of up to 5ms. 31 | 32 | Alternatively, the output TTL signal can be sent via an Arduino pin. Load the MATLAB 33 | Arduino IO sketch onto the Arduino (the sketch is available in this repository, and 34 | enables controlling pins through a basic serial interface). Connect the Arduino to the 35 | computer via USB. Output TTL pulses for the first channel will be sent via pin 7, for the second channel via pin 8, etc. 36 | 37 | **Preparing the network:** After 38 | [using the training code](https://github.com/gardner-lab/syllable-detector-learn) to 39 | train a detector, use the `convert_to_text.m` file included in the repository to convert 40 | the detect to a text format that can be easily read by the Swift software. 41 | 42 | **Running:** 43 | 44 | 1. Launch the software. 45 | 2. The first window will provide a network to select an input source (listing all audio 46 | sources available), as well as an output source (listing both audio outputs and any 47 | detected Arduino serial ports). 48 | 3. Once you select both an input and output, a new window launches. From here, you can 49 | see all available input and output channels. Double click a channel to load a text 50 | version of a trained detector. 51 | 4. Once configured, press run to begin monitoring the inputs. 52 | 53 | -------------------------------------------------------------------------------- /SyllableDetector.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /SyllableDetector.xcodeproj/project.xcworkspace/xcshareddata/SyllableDetector.xcscmblueprint: -------------------------------------------------------------------------------- 1 | { 2 | "DVTSourceControlWorkspaceBlueprintPrimaryRemoteRepositoryKey" : "0DC6A8484D9B4C6C2590765C4EC90EC5CF35B881", 3 | "DVTSourceControlWorkspaceBlueprintWorkingCopyRepositoryLocationsKey" : { 4 | 5 | }, 6 | "DVTSourceControlWorkspaceBlueprintWorkingCopyStatesKey" : { 7 | "CBF5F70D4EDC8047BC75C25236002B0EA7F6B15B" : 0, 8 | "5A811E91F5DFF59170683E14D2C5755F19DE18CB" : 0, 9 | "0DC6A8484D9B4C6C2590765C4EC90EC5CF35B881" : 0 10 | }, 11 | "DVTSourceControlWorkspaceBlueprintIdentifierKey" : "D06BF7B2-80D5-418B-90F3-F6E9E23620BD", 12 | "DVTSourceControlWorkspaceBlueprintWorkingCopyPathsKey" : { 13 | "CBF5F70D4EDC8047BC75C25236002B0EA7F6B15B" : "SyllableDetector\/Frameworks\/CommandLine\/", 14 | "5A811E91F5DFF59170683E14D2C5755F19DE18CB" : "SyllableDetector\/Frameworks\/ORSSerialPort\/", 15 | "0DC6A8484D9B4C6C2590765C4EC90EC5CF35B881" : "SyllableDetector\/" 16 | }, 17 | "DVTSourceControlWorkspaceBlueprintNameKey" : "SyllableDetector", 18 | "DVTSourceControlWorkspaceBlueprintVersion" : 204, 19 | "DVTSourceControlWorkspaceBlueprintRelativePathToProjectKey" : "SyllableDetector.xcodeproj", 20 | "DVTSourceControlWorkspaceBlueprintRemoteRepositoriesKey" : [ 21 | { 22 | "DVTSourceControlWorkspaceBlueprintRemoteRepositoryURLKey" : "https:\/\/github.com\/nathanntg\/syllable-detector.git", 23 | "DVTSourceControlWorkspaceBlueprintRemoteRepositorySystemKey" : "com.apple.dt.Xcode.sourcecontrol.Git", 24 | "DVTSourceControlWorkspaceBlueprintRemoteRepositoryIdentifierKey" : "0DC6A8484D9B4C6C2590765C4EC90EC5CF35B881" 25 | }, 26 | { 27 | "DVTSourceControlWorkspaceBlueprintRemoteRepositoryURLKey" : "https:\/\/github.com\/armadsen\/ORSSerialPort.git", 28 | "DVTSourceControlWorkspaceBlueprintRemoteRepositorySystemKey" : "com.apple.dt.Xcode.sourcecontrol.Git", 29 | "DVTSourceControlWorkspaceBlueprintRemoteRepositoryIdentifierKey" : "5A811E91F5DFF59170683E14D2C5755F19DE18CB" 30 | }, 31 | { 32 | "DVTSourceControlWorkspaceBlueprintRemoteRepositoryURLKey" : "https:\/\/github.com\/jatoben\/CommandLine.git", 33 | "DVTSourceControlWorkspaceBlueprintRemoteRepositorySystemKey" : "com.apple.dt.Xcode.sourcecontrol.Git", 34 | "DVTSourceControlWorkspaceBlueprintRemoteRepositoryIdentifierKey" : "CBF5F70D4EDC8047BC75C25236002B0EA7F6B15B" 35 | } 36 | ] 37 | } -------------------------------------------------------------------------------- /SyllableDetector/AppDelegate.swift: -------------------------------------------------------------------------------- 1 | // 2 | // AppDelegate.swift 3 | // SyllableDetector 4 | // 5 | // Created by Nathan Perkins on 10/28/15. 6 | // Copyright © 2015 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Cocoa 10 | 11 | @NSApplicationMain 12 | class AppDelegate: NSObject, NSApplicationDelegate { 13 | 14 | 15 | 16 | func applicationDidFinishLaunching(_ aNotification: Notification) { 17 | // Insert code here to initialize your application 18 | } 19 | 20 | func applicationWillTerminate(_ aNotification: Notification) { 21 | // Insert code here to tear down your application 22 | } 23 | 24 | 25 | } 26 | 27 | -------------------------------------------------------------------------------- /SyllableDetector/ArduinoIO.swift: -------------------------------------------------------------------------------- 1 | // 2 | // ArduinoIO.swift 3 | // SyllableDetector 4 | // 5 | // Created by Nathan Perkins on 1/8/16. 6 | // Copyright © 2016 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import ORSSerial 11 | 12 | let kStartupTime = 2.0 13 | let kTimeoutDuration: TimeInterval = 0.5 14 | 15 | /// The state of the arduino and serial port. 16 | enum ArduinoIOState: Equatable { 17 | case closed 18 | case opened 19 | case waitingToOpen // Because of potential startup time, there is an inbetween period of 2 seconds during which requests are queued. 20 | //case WaitingToClose 21 | case error 22 | case uninitialized 23 | } 24 | 25 | /// Request information used to handle response. 26 | private enum ArduinoIORequest { 27 | case sketchInitialize 28 | case readDigital(Int, (Bool?) -> Void) 29 | case readAnalog(Int, (UInt16?) -> Void) 30 | } 31 | 32 | /// Errors associated with input and output. 33 | enum ArduinoIOError: Error, CustomStringConvertible { 34 | case unknownError 35 | case unableToOpenPath 36 | case portNotOpen 37 | case invalidPin(Int) 38 | case invalidMode // invalid pin mode 39 | case invalidValue 40 | 41 | var description: String { 42 | switch self { 43 | case .unknownError: return "Unknown error" 44 | case .unableToOpenPath: return "Unable to open path" 45 | case .portNotOpen: return "Port not open" 46 | case .invalidPin(let p): return "Invalid pin (\(p))" 47 | case .invalidMode: return "Invalid mode" 48 | case .invalidValue: return "Invalid value" 49 | } 50 | } 51 | } 52 | 53 | 54 | /// Used to track the Sketch type 55 | enum ArduinoIOSketch: CustomStringConvertible { 56 | case unknown 57 | case io 58 | case encoderIO 59 | case servoEncoderIO 60 | case motorShield1 61 | case motorShield2 62 | 63 | var description: String { 64 | switch self { 65 | case .unknown: return "Unknown" 66 | case .io: return "Analog & Digital I/O (adio.pde)" 67 | case .encoderIO: return "Analog & Digital I/O + Encoder (arioe.pde)" 68 | case .servoEncoderIO: return "Analog & Digital I/O + Encoder + Servos (arioes.pde)" 69 | case .motorShield1: return "Motor Shield V1" 70 | case .motorShield2: return "Motor Shield V2" 71 | } 72 | } 73 | } 74 | 75 | enum ArduinoIOQueue { 76 | case request(ORSSerialRequest) 77 | case send(Data) 78 | } 79 | 80 | enum ArduinoIOPin: Int, CustomStringConvertible { 81 | case unassigned = -1 82 | case input = 0 83 | case output = 1 84 | 85 | var description: String { 86 | switch self { 87 | case .unassigned: return "Unassigned" 88 | case .input: return "Input" 89 | case .output: return "Output" 90 | } 91 | } 92 | } 93 | 94 | enum ArduinoIODevice { 95 | case detached 96 | case attached 97 | } 98 | 99 | 100 | protocol ArduinoIODelegate: class { 101 | //func arduinoStateChangedFrom(oldState: ArduinoIOState, newState: ArduinoIOState) 102 | 103 | func arduinoError(_ message: String, isPermanent: Bool) 104 | } 105 | 106 | /// An example of an extension of the ORSSerialPacketDescriptor that enables identifying delimited packets. 107 | class DelimitedSerialPacketDescriptor: ORSSerialPacketDescriptor { 108 | var delimiter: Data? 109 | 110 | convenience init(delimiter: Data, maximumPacketLength maxPacketLength: UInt, userInfo: AnyObject?, responseEvaluator: @escaping ORSSerialPacketEvaluator) { 111 | self.init(maximumPacketLength: maxPacketLength, userInfo: userInfo, responseEvaluator: responseEvaluator) 112 | 113 | // set delimiter 114 | self.delimiter = delimiter 115 | } 116 | 117 | convenience init(delimiterString: String, maximumPacketLength maxPacketLength: UInt, userInfo: AnyObject?, responseEvaluator: @escaping ORSSerialPacketEvaluator) { 118 | self.init(maximumPacketLength: maxPacketLength, userInfo: userInfo, responseEvaluator: responseEvaluator) 119 | 120 | // set delimiter 121 | self.delimiter = delimiterString.data(using: String.Encoding.utf8) 122 | } 123 | 124 | private func packetMatchingExcludingFinalDelimiter(_ buffer: Data) -> Data? { 125 | // only use log if delimiter is provided (should only be called if delimiter exists) 126 | guard let delimiter = delimiter else { 127 | return nil 128 | } 129 | 130 | // empty buffer? potentially valid 131 | if buffer.count == 0 { 132 | if dataIsValidPacket(buffer) { 133 | return buffer 134 | } 135 | return nil 136 | } 137 | 138 | // work back from the end of the buffer 139 | for i in 0...buffer.count { 140 | // check for delimiter if not reading from the beginning of the buffer 141 | if i < buffer.count { 142 | // not enough space for the delimiter 143 | if i + delimiter.count > buffer.count { 144 | continue 145 | } 146 | 147 | // check for proceeding delimiter 148 | // (could be more lenient and just check for the end of the delimiter) 149 | let windowDel = buffer.subdata(in: (buffer.count - i - delimiter.count)..<(buffer.count - i)) 150 | 151 | // does not match? continue 152 | if windowDel != delimiter { 153 | continue 154 | } 155 | } 156 | 157 | // make window 158 | let window = buffer.subdata(in: (buffer.count - i).. Data? { 168 | // only use log if delimiter is provided 169 | guard let delimiter = delimiter else { 170 | // otherwise inherit normal behavior 171 | return super.packetMatching(atEndOfBuffer: buffer) 172 | } 173 | 174 | // unwrap buffer 175 | guard let buffer = buffer else { return nil } 176 | 177 | // space for delimiter 178 | if buffer.count < delimiter.count { 179 | return nil 180 | } 181 | 182 | // ensure buffer ends with delimiter 183 | let windowFinalDel = buffer.subdata(in: (buffer.count - delimiter.count).. Bool in 227 | guard let data = d else { 228 | return false 229 | } 230 | return data.count > 0 231 | }) 232 | 233 | // used to hold requests while waiting to open 234 | private var pendingConnection: [ArduinoIOQueue] = [] 235 | private var requestInfo = [Int: ArduinoIORequest]() 236 | private var requestInfoId = 1 237 | 238 | class func getSerialPorts() -> [ORSSerialPort] { 239 | return ORSSerialPortManager.shared().availablePorts 240 | } 241 | 242 | init(serial: ORSSerialPort) { 243 | super.init() 244 | 245 | // set delegate 246 | serial.delegate = self 247 | 248 | // store and open 249 | self.serial = serial 250 | self.open() 251 | } 252 | 253 | deinit { 254 | // close 255 | close() 256 | } 257 | 258 | convenience init(path: String) throws { 259 | if let port = ORSSerialPort(path: path) { 260 | self.init(serial: port) 261 | return 262 | } 263 | throw ArduinoIOError.unableToOpenPath 264 | } 265 | 266 | private func send(_ data: Data, withRequest req: ArduinoIORequest) { 267 | requestInfoId += 1 268 | let num: Int = requestInfoId 269 | requestInfo[num] = req 270 | 271 | // send request 272 | let serialReq = ORSSerialRequest(dataToSend: data, userInfo: num as AnyObject, timeoutInterval: kTimeoutDuration, responseDescriptor: responseDescription) 273 | send(serialReq) 274 | } 275 | 276 | private func send(_ req: ORSSerialRequest) { 277 | if state == .opened { 278 | if let serialPort = serial { 279 | serialPort.send(req) 280 | } 281 | } 282 | else if state == .waitingToOpen { 283 | pendingConnection.append(ArduinoIOQueue.request(req)) 284 | } 285 | } 286 | 287 | private func send(_ data: Data) { 288 | if state == .opened { 289 | if let serialPort = serial { 290 | serialPort.send(data) 291 | } 292 | } 293 | else if state == .waitingToOpen { 294 | pendingConnection.append(ArduinoIOQueue.send(data)) 295 | } 296 | } 297 | 298 | private func open() { 299 | guard state == .uninitialized else { 300 | return 301 | } 302 | guard let serialPort = serial else { 303 | return 304 | } 305 | 306 | // open serial port 307 | serialPort.baudRate = 115200 308 | serialPort.open() 309 | 310 | // set waiting to open state 311 | state = .waitingToOpen 312 | 313 | // setup timer 314 | Timer.scheduledTimer(timeInterval: kStartupTime, target: self, selector: #selector(ArduinoIO.completeOpen(_:)), userInfo: nil, repeats: false) 315 | } 316 | 317 | /// Opening process takes 2~6 seconds. Inital requests are held until Arduino is online. 318 | @objc func completeOpen(_ timer: Timer!) { 319 | guard self.state == .waitingToOpen else { 320 | return 321 | } 322 | 323 | DLog("ARDUINO OPEN") 324 | 325 | // set state to opened 326 | state = .opened 327 | 328 | // send request to complete opening process 329 | let data = "99".data(using: String.Encoding.ascii)! 330 | send(data, withRequest: ArduinoIORequest.sketchInitialize) 331 | } 332 | 333 | private func runPendingConnectionQueue() { 334 | guard let serialPort = serial else { 335 | return 336 | } 337 | guard self.state == .opened else { 338 | pendingConnection.removeAll() 339 | return 340 | } 341 | 342 | 343 | // clear pending requests 344 | for entry in pendingConnection { 345 | switch entry { 346 | case ArduinoIOQueue.send(let data): 347 | serialPort.send(data) 348 | case ArduinoIOQueue.request(let req): 349 | serialPort.send(req) 350 | } 351 | } 352 | pendingConnection.removeAll() 353 | } 354 | 355 | func canInteract() -> Bool { 356 | return state == .opened || state == .waitingToOpen 357 | } 358 | 359 | func isOpen() -> Bool { 360 | return state == .opened 361 | } 362 | 363 | func close() { 364 | switch state { 365 | case .closed, .error: 366 | return 367 | case .uninitialized: 368 | state = .closed 369 | return 370 | case .opened: 371 | // leave in a good state 372 | for i in 2...69 { 373 | do { 374 | switch pins[i] { 375 | case .unassigned: continue 376 | case .output: 377 | try writeTo(i, digitalValue: false) 378 | case .input: 379 | try setPinMode(i, to: ArduinoIOPin.output) 380 | try writeTo(i, digitalValue: false) 381 | } 382 | } 383 | catch { 384 | break 385 | } 386 | } 387 | 388 | serial?.close() 389 | serial = nil 390 | state = .closed 391 | 392 | return 393 | case .waitingToOpen: 394 | serial?.close() 395 | serial = nil 396 | state = .closed 397 | return 398 | } 399 | } 400 | 401 | // MARK: - Interface 402 | 403 | private func isValidPin(_ pin: Int) -> Bool { 404 | return pin >= 2 && pin <= 69 405 | } 406 | 407 | func setPinMode(_ pin: Int, to: ArduinoIOPin) throws { 408 | guard canInteract() else { 409 | throw ArduinoIOError.portNotOpen 410 | } 411 | guard isValidPin(pin) else { 412 | throw ArduinoIOError.invalidPin(pin) 413 | } 414 | guard to != .unassigned else { 415 | throw ArduinoIOError.invalidMode 416 | } 417 | guard nil != serial else { 418 | throw ArduinoIOError.portNotOpen 419 | } 420 | 421 | DLog("ARDUINO CONFIG \(pin): \(to)") 422 | 423 | // build data to change pin mode 424 | let dataBytes: [UInt8] = [48, 97 + UInt8(pin), 48 + UInt8(to.rawValue)] 425 | let data = Data(bytes: UnsafePointer(dataBytes), count: dataBytes.count) 426 | send(data) 427 | 428 | // set the internal representation 429 | pins[pin] = to 430 | 431 | // TODO: potentially dettach servo 432 | } 433 | 434 | func getPinMode(_ pin: Int) -> ArduinoIOPin { 435 | if pin >= 2 && pin <= 69 { 436 | return pins[pin] 437 | } 438 | return .unassigned 439 | } 440 | 441 | func writeTo(_ pin: Int, digitalValue: Bool) throws { 442 | guard canInteract() else { 443 | throw ArduinoIOError.portNotOpen 444 | } 445 | guard isValidPin(pin) else { 446 | throw ArduinoIOError.invalidPin(pin) 447 | } 448 | guard pins[pin] == .output else { 449 | throw ArduinoIOError.invalidMode 450 | } 451 | guard nil != serial else { 452 | throw ArduinoIOError.portNotOpen 453 | } 454 | 455 | // build data to change pin mode 456 | let dataBytes: [UInt8] = [50, 97 + UInt8(pin), 48 + UInt8(digitalValue ? 1 : 0)] 457 | let data = Data(bytes: UnsafePointer(dataBytes), count: dataBytes.count) 458 | send(data) 459 | 460 | DLog("ARDUINO WRITE \(pin): \(digitalValue)") 461 | } 462 | 463 | func readDigitalValueFrom(_ pin: Int, andExecute cb: @escaping (Bool?) -> Void) throws { 464 | guard canInteract() else { 465 | throw ArduinoIOError.portNotOpen 466 | } 467 | guard isValidPin(pin) else { 468 | throw ArduinoIOError.invalidPin(pin) 469 | } 470 | guard pins[pin] == .input else { 471 | throw ArduinoIOError.invalidMode 472 | } 473 | guard nil != serial else { 474 | throw ArduinoIOError.portNotOpen 475 | } 476 | 477 | // build data to change pin mode 478 | let dataBytes: [UInt8] = [49, 97 + UInt8(pin)] 479 | let data = Data(bytes: UnsafePointer(dataBytes), count: dataBytes.count) 480 | send(data, withRequest: ArduinoIORequest.readDigital(pin, cb)) 481 | } 482 | 483 | func writeTo(_ pin: Int, analogValue: UInt8) throws { 484 | guard canInteract() else { 485 | throw ArduinoIOError.portNotOpen 486 | } 487 | guard (pin >= 2 && pin <= 13) || (pin >= 44 && pin <= 46) else { 488 | throw ArduinoIOError.invalidPin(pin) 489 | } 490 | guard pins[pin] == .output else { 491 | throw ArduinoIOError.invalidMode 492 | } 493 | guard nil != serial else { 494 | throw ArduinoIOError.portNotOpen 495 | } 496 | 497 | // build data to change pin mode 498 | let dataBytes: [UInt8] = [52, 97 + UInt8(pin), analogValue] 499 | let data = Data(bytes: UnsafePointer(dataBytes), count: dataBytes.count) 500 | send(data) 501 | 502 | DLog("ARDUINO WRITE \(pin): \(analogValue)") 503 | } 504 | 505 | func readAnalogValueFrom(_ pin: Int, andExecute cb: @escaping (UInt16?) -> Void) throws { 506 | guard canInteract() else { 507 | throw ArduinoIOError.portNotOpen 508 | } 509 | guard pin >= 0 && pin <= 15 else { 510 | throw ArduinoIOError.invalidPin(pin) 511 | } 512 | guard pin < 2 || pins[pin] == .input else { 513 | throw ArduinoIOError.invalidMode 514 | } 515 | guard nil != serial else { 516 | throw ArduinoIOError.portNotOpen 517 | } 518 | 519 | // build data to change pin mode 520 | let dataBytes: [UInt8] = [51, 97 + UInt8(pin)] 521 | let data = Data(bytes: UnsafePointer(dataBytes), count: dataBytes.count) 522 | send(data, withRequest: ArduinoIORequest.readAnalog(pin, cb)) 523 | } 524 | 525 | // MARK: - ORSSerialPortDelegate 526 | 527 | func serialPortWasOpened(_ serialPort: ORSSerialPort) { 528 | DLog("SERIAL OPENED: \(serialPort)") 529 | } 530 | 531 | func serialPortWasClosed(_ serialPort: ORSSerialPort) { 532 | DLog("SERIAL CLOSED: \(serialPort)") 533 | } 534 | 535 | func serialPort(_ serialPort: ORSSerialPort, didReceive data: Data) { 536 | // debugging 537 | // if let string = NSString(data: data, encoding: NSUTF8StringEncoding) { 538 | // DLog("SERIAL \(serialPort) RECEIVED: \(string)") 539 | // } 540 | } 541 | 542 | func serialPort(_ serialPort: ORSSerialPort, didReceiveResponse responseData: Data, to request: ORSSerialRequest) { 543 | guard let info = request.userInfo, let reqId = info as? Int, let reqType = requestInfo[reqId] else { 544 | return 545 | } 546 | 547 | // remove value 548 | requestInfo.removeValue(forKey: reqId) 549 | 550 | // convert to NSString 551 | guard let s = NSString(data: responseData, encoding: String.Encoding.ascii.rawValue) else { 552 | return 553 | } 554 | 555 | let dataAsString: String = (s as String).trimmingCharacters(in: CharacterSet.newlines) 556 | 557 | switch reqType { 558 | case .sketchInitialize: 559 | // get sketch identifier 560 | switch dataAsString { 561 | case "0": sketch = .io 562 | case "1": sketch = .encoderIO 563 | case "2": sketch = .servoEncoderIO 564 | case "3": sketch = .motorShield1 565 | case "4": sketch = .motorShield2 566 | default: sketch = .unknown 567 | } 568 | 569 | // log sketch 570 | DLog("ARDUINO SKETCH: \(sketch)") 571 | 572 | if sketch == .unknown { 573 | // send to delegate 574 | delegate?.arduinoError("Unknown Sketch", isPermanent: true) 575 | 576 | // close connection 577 | close() 578 | } 579 | 580 | // run queue 581 | runPendingConnectionQueue() 582 | 583 | case .readDigital(let pin, let cb): 584 | DLog("ARDUINO READ \(pin): \(dataAsString)") 585 | switch dataAsString { 586 | case "0": cb(false) 587 | case "1": cb(true) 588 | default: cb(nil) 589 | } 590 | 591 | case .readAnalog(let pin, let cb): 592 | DLog("ARDUINO READ \(pin): \(dataAsString)") 593 | if let val = Int(dataAsString), val >= 0 && val <= 1023 { 594 | cb(UInt16(val)) 595 | } 596 | else { 597 | cb(nil) 598 | } 599 | } 600 | } 601 | 602 | func serialPort(_ serialPort: ORSSerialPort, requestDidTimeout request: ORSSerialRequest) { 603 | guard let info = request.userInfo, let reqId = info as? Int, let reqType = requestInfo[reqId] else { 604 | return 605 | } 606 | 607 | // remove value 608 | requestInfo.removeValue(forKey: reqId) 609 | 610 | // log it 611 | DLog("ARDUINO TIMEOUT: \(reqType)") 612 | 613 | switch reqType { 614 | case .sketchInitialize: 615 | // send to delegate 616 | delegate?.arduinoError("Initialization Timeout", isPermanent: true) 617 | 618 | // close connection 619 | close() 620 | 621 | case .readAnalog(_, let cb): 622 | // send to delegate 623 | delegate?.arduinoError("Timeout \(reqType)", isPermanent: false) 624 | 625 | // callback with no value 626 | cb(nil) 627 | 628 | case .readDigital(_, let cb): 629 | // send to delegate 630 | delegate?.arduinoError("Timeout \(reqType)", isPermanent: false) 631 | 632 | // callback with no value 633 | cb(nil) 634 | } 635 | } 636 | 637 | func serialPortWasRemovedFromSystem(_ serialPort: ORSSerialPort) { 638 | DLog("SERIAL \(serialPort) REMOVED") 639 | 640 | if state == .waitingToOpen || state == .opened { 641 | // send to delegate 642 | delegate?.arduinoError("Disconnected", isPermanent: true) 643 | } 644 | 645 | // close everything 646 | serial = nil 647 | close() 648 | } 649 | 650 | func serialPort(_ serialPort: ORSSerialPort, didEncounterError error: Error) { 651 | DLog("SERIAL \(serialPort) ERROR: \(error)") 652 | 653 | // send to delegate 654 | delegate?.arduinoError("Error: \(error.localizedDescription)", isPermanent: false) 655 | } 656 | } 657 | -------------------------------------------------------------------------------- /SyllableDetector/Assets.xcassets/AppIcon.appiconset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "idiom" : "mac", 5 | "size" : "16x16", 6 | "scale" : "1x" 7 | }, 8 | { 9 | "idiom" : "mac", 10 | "size" : "16x16", 11 | "scale" : "2x" 12 | }, 13 | { 14 | "idiom" : "mac", 15 | "size" : "32x32", 16 | "scale" : "1x" 17 | }, 18 | { 19 | "idiom" : "mac", 20 | "size" : "32x32", 21 | "scale" : "2x" 22 | }, 23 | { 24 | "idiom" : "mac", 25 | "size" : "128x128", 26 | "scale" : "1x" 27 | }, 28 | { 29 | "idiom" : "mac", 30 | "size" : "128x128", 31 | "scale" : "2x" 32 | }, 33 | { 34 | "idiom" : "mac", 35 | "size" : "256x256", 36 | "scale" : "1x" 37 | }, 38 | { 39 | "idiom" : "mac", 40 | "size" : "256x256", 41 | "scale" : "2x" 42 | }, 43 | { 44 | "idiom" : "mac", 45 | "size" : "512x512", 46 | "scale" : "1x" 47 | }, 48 | { 49 | "idiom" : "mac", 50 | "size" : "512x512", 51 | "scale" : "2x" 52 | } 53 | ], 54 | "info" : { 55 | "version" : 1, 56 | "author" : "xcode" 57 | } 58 | } -------------------------------------------------------------------------------- /SyllableDetector/AudioInterface.swift: -------------------------------------------------------------------------------- 1 | // 2 | // AudioInterface.swift 3 | // SongDetector 4 | // 5 | // Created by Nathan Perkins on 10/22/15. 6 | // Copyright © 2015 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import AudioToolbox 11 | import Accelerate 12 | 13 | func renderOutput(_ inRefCon:UnsafeMutableRawPointer, actionFlags: UnsafeMutablePointer, timeStamp: UnsafePointer, busNumber: UInt32, frameCount: UInt32, data: UnsafeMutablePointer?) -> OSStatus { 14 | 15 | // get audio out interface 16 | let aoi = unsafeBitCast(inRefCon, to: AudioOutputInterface.self) 17 | let usableBufferList = UnsafeMutableAudioBufferListPointer(data!) 18 | 19 | // number of frames 20 | let frameCountAsInt = Int(frameCount) 21 | 22 | // fill output 23 | for (channel, buffer) in usableBufferList.enumerated() { 24 | let data = buffer.mData!.bindMemory(to: Float.self, capacity: frameCountAsInt) 25 | let high = aoi.outputHighFor[channel] 26 | 27 | // decrement high for 28 | if 0 < high { 29 | aoi.outputHighFor[channel] = high - min(high, frameCountAsInt) 30 | } 31 | 32 | // write data out 33 | for i in 0.., timeStamp: UnsafePointer, busNumber: UInt32, frameCount: UInt32, data: UnsafeMutablePointer?) -> OSStatus { 43 | 44 | // get audio in interface 45 | let aii = unsafeBitCast(inRefCon, to: AudioInputInterface.self) 46 | 47 | // number of channels 48 | let numberOfChannels = Int(aii.inputFormat.mChannelsPerFrame) 49 | 50 | // set buffer data size 51 | for channel in 0.. Void))>]() 96 | 97 | struct AudioDevice { 98 | let deviceID: AudioDeviceID 99 | let deviceUID: String 100 | let deviceName: String 101 | let deviceManufacturer: String 102 | let streamsInput: Int 103 | let streamsOutput: Int 104 | let sampleRateInput: Float64 105 | let sampleRateOutput: Float64 106 | let buffersInput: [AudioBuffer] 107 | let buffersOutput: [AudioBuffer] 108 | 109 | init?(deviceID: AudioDeviceID) { 110 | self.deviceID = deviceID 111 | 112 | // property address 113 | var propertyAddress = AudioObjectPropertyAddress(mSelector: kAudioDevicePropertyDeviceUID, mScope: kAudioObjectPropertyScopeGlobal, mElement: kAudioObjectPropertyElementMaster) 114 | 115 | // size and status variables 116 | var status: OSStatus 117 | var size: UInt32 = UInt32(MemoryLayout.size) 118 | 119 | // get device UID 120 | var deviceUID = "" as CFString 121 | propertyAddress.mSelector = kAudioDevicePropertyDeviceUID 122 | status = AudioObjectGetPropertyData(deviceID, &propertyAddress, 0, nil, &size, &deviceUID) 123 | guard noErr == status else { return nil } 124 | self.deviceUID = String(deviceUID) 125 | 126 | // get deivce name 127 | var deviceName = "" as CFString 128 | propertyAddress.mSelector = kAudioDevicePropertyDeviceNameCFString 129 | status = AudioObjectGetPropertyData(deviceID, &propertyAddress, 0, nil, &size, &deviceName) 130 | guard noErr == status else { return nil } 131 | self.deviceName = String(deviceName) 132 | 133 | // get deivce manufacturer 134 | var deviceManufacturer = "" as CFString 135 | propertyAddress.mSelector = kAudioDevicePropertyDeviceManufacturerCFString 136 | status = AudioObjectGetPropertyData(deviceID, &propertyAddress, 0, nil, &size, &deviceManufacturer) 137 | guard noErr == status else { return nil } 138 | self.deviceManufacturer = String(deviceManufacturer) 139 | 140 | // get number of streams 141 | // LAST AS IT CHANGES THE SCOPE OF THE PROPERTY ADDRESS 142 | propertyAddress.mSelector = kAudioDevicePropertyStreams 143 | propertyAddress.mScope = kAudioDevicePropertyScopeInput 144 | status = AudioObjectGetPropertyDataSize(deviceID, &propertyAddress, 0, nil, &size) 145 | guard noErr == status else { return nil } 146 | self.streamsInput = Int(size) / MemoryLayout.size 147 | 148 | if 0 < self.streamsInput { 149 | // get sample rate 150 | size = UInt32(MemoryLayout.size) 151 | var sampleRateInput: Float64 = 0.0 152 | propertyAddress.mSelector = kAudioDevicePropertyNominalSampleRate 153 | status = AudioObjectGetPropertyData(deviceID, &propertyAddress, 0, nil, &size, &sampleRateInput) 154 | guard noErr == status else { return nil } 155 | self.sampleRateInput = sampleRateInput 156 | 157 | // get stream configuration 158 | size = 0 159 | propertyAddress.mSelector = kAudioDevicePropertyStreamConfiguration 160 | status = AudioObjectGetPropertyDataSize(deviceID, &propertyAddress, 0, nil, &size) 161 | guard noErr == status else { DLog("d \(status)"); return nil } 162 | 163 | // allocate 164 | // is it okay to assume binding? or should bind (but if so, what capacity)? 165 | var bufferList = UnsafeMutableRawPointer.allocate(byteCount: Int(size), alignment: MemoryLayout.alignment).assumingMemoryBound(to: AudioBufferList.self) 166 | status = AudioObjectGetPropertyData(deviceID, &propertyAddress, 0, nil, &size, bufferList) 167 | defer { 168 | free(bufferList) 169 | } 170 | guard noErr == status else { DLog("e"); return nil } 171 | 172 | // turn into something swift usable 173 | let usableBufferList = UnsafeMutableAudioBufferListPointer(bufferList) 174 | 175 | // add device buffers 176 | var buffersInput = [AudioBuffer]() 177 | for ab in usableBufferList { 178 | buffersInput.append(ab) 179 | } 180 | self.buffersInput = buffersInput 181 | } 182 | else { 183 | self.buffersInput = [] 184 | self.sampleRateInput = 0.0 185 | } 186 | 187 | propertyAddress.mSelector = kAudioDevicePropertyStreams 188 | propertyAddress.mScope = kAudioDevicePropertyScopeOutput 189 | status = AudioObjectGetPropertyDataSize(deviceID, &propertyAddress, 0, nil, &size) 190 | guard noErr == status else { return nil } 191 | self.streamsOutput = Int(size) / MemoryLayout.size 192 | 193 | if 0 < self.streamsOutput { 194 | // get sample rate 195 | size = UInt32(MemoryLayout.size) 196 | var sampleRateOutput: Float64 = 0.0 197 | propertyAddress.mSelector = kAudioDevicePropertyNominalSampleRate 198 | status = AudioObjectGetPropertyData(deviceID, &propertyAddress, 0, nil, &size, &sampleRateOutput) 199 | guard noErr == status else { return nil } 200 | self.sampleRateOutput = sampleRateOutput 201 | 202 | // get stream configuration 203 | size = 0 204 | propertyAddress.mSelector = kAudioDevicePropertyStreamConfiguration 205 | status = AudioObjectGetPropertyDataSize(deviceID, &propertyAddress, 0, nil, &size) 206 | guard noErr == status else { DLog("d \(status)"); return nil } 207 | 208 | // allocate 209 | // is it okay to assume binding? or should bind (but if so, what capacity)? 210 | var bufferList = UnsafeMutableRawPointer.allocate(byteCount: Int(size), alignment: MemoryLayout.alignment).assumingMemoryBound(to: AudioBufferList.self) 211 | status = AudioObjectGetPropertyData(deviceID, &propertyAddress, 0, nil, &size, bufferList) 212 | defer { 213 | free(bufferList) 214 | } 215 | guard noErr == status else { DLog("e"); return nil } 216 | 217 | // turn into something swift usable 218 | let usableBufferList = UnsafeMutableAudioBufferListPointer(bufferList) 219 | 220 | // add device buffers 221 | var buffersOutput = [AudioBuffer]() 222 | for ab in usableBufferList { 223 | buffersOutput.append(ab) 224 | } 225 | self.buffersOutput = buffersOutput 226 | } 227 | else { 228 | self.buffersOutput = [] 229 | self.sampleRateOutput = 0.0 230 | } 231 | } 232 | } 233 | 234 | var audioUnit: AudioComponentInstance? = nil 235 | 236 | static func devices() throws -> [AudioDevice] { 237 | // property address 238 | var propertyAddress = AudioObjectPropertyAddress(mSelector: kAudioHardwarePropertyDevices, mScope: kAudioObjectPropertyScopeGlobal, mElement: kAudioObjectPropertyElementMaster) 239 | var size: UInt32 = 0 240 | 241 | // get input size 242 | try checkError(AudioObjectGetPropertyDataSize(AudioObjectID(kAudioObjectSystemObject), &propertyAddress, 0, nil, &size)) 243 | 244 | // number of devices 245 | let deviceCount = Int(size) / MemoryLayout.size 246 | var audioDevices = [AudioDeviceID](repeating: AudioDeviceID(0), count: deviceCount) 247 | 248 | // get device ids 249 | try checkError(AudioObjectGetPropertyData(AudioObjectID(kAudioObjectSystemObject), &propertyAddress, 0, nil, &size, &audioDevices[0])) 250 | 251 | return audioDevices.compactMap { 252 | return AudioDevice(deviceID: $0) 253 | } 254 | } 255 | 256 | static func createListenerForDeviceChange(_ cb: @escaping (Void) -> Void, withIdentifier unique: T) throws { 257 | let selector = kAudioHardwarePropertyDevices 258 | 259 | // alread has listener 260 | if var cur = listeners[selector] { 261 | // add callback 262 | cur.append((unique.hashValue, cb)) 263 | 264 | // update listeners 265 | listeners[selector] = cur 266 | 267 | return 268 | } 269 | 270 | // property address 271 | var propertyAddress = AudioObjectPropertyAddress(mSelector: selector, mScope: kAudioObjectPropertyScopeGlobal, mElement: kAudioObjectPropertyElementMaster) 272 | let onAudioObject = AudioObjectID(bitPattern: kAudioObjectSystemObject) 273 | 274 | // create listener 275 | try checkError(AudioObjectAddPropertyListenerBlock(onAudioObject, &propertyAddress, nil, dispatchEvent)) 276 | 277 | // create callbacks array 278 | let cbs: Array<(Int, ((Void) -> Void))> = [(unique.hashValue, cb)] 279 | listeners[selector] = cbs 280 | } 281 | 282 | static func destroyListenerForDeviceChange(withIdentifier unique: T) { 283 | let selector = kAudioHardwarePropertyDevices 284 | 285 | guard var cur = listeners[selector] else { 286 | // nothing to remove 287 | return 288 | } 289 | 290 | // hash to remove 291 | let hashToRemove = unique.hashValue 292 | 293 | // remove from listeners 294 | cur = cur.filter() { 295 | return $0.0 != hashToRemove 296 | } 297 | 298 | // if empty 299 | if cur.isEmpty { 300 | // remove listener 301 | listeners.removeValue(forKey: selector) 302 | 303 | // property address 304 | var propertyAddress = AudioObjectPropertyAddress(mSelector: selector, mScope: kAudioObjectPropertyScopeGlobal, mElement: kAudioObjectPropertyElementMaster) 305 | let onAudioObject = AudioObjectID(bitPattern: kAudioObjectSystemObject) 306 | 307 | do { 308 | try checkError(AudioObjectRemovePropertyListenerBlock(onAudioObject, &propertyAddress, nil, dispatchEvent)) 309 | } 310 | catch { 311 | DLog("unable to remove listener") 312 | } 313 | } 314 | else { 315 | // update listeners 316 | listeners[selector] = cur 317 | } 318 | } 319 | 320 | static func dispatchEvent(_ numAddresses: UInt32, addresses: UnsafePointer) { 321 | for i: UInt32 in 0...size))) 372 | 373 | // get the audio format 374 | var size: UInt32 = UInt32(MemoryLayout.size) 375 | try checkError(AudioUnitGetProperty(audioUnit!, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, outputBus, &outputFormat, &size)) 376 | 377 | // print format information for debugging 378 | //DLog("OUT:OUT \(outputFormat)") 379 | 380 | // check for expected format 381 | guard outputFormat.mFormatID == kAudioFormatLinearPCM && outputFormat.mFramesPerPacket == 1 && outputFormat.mFormatFlags == kAudioFormatFlagsNativeFloatPacked else { 382 | throw AudioInterfaceError.unsupportedFormat 383 | } 384 | 385 | // get the audio format 386 | //size = UInt32(sizeof(AudioStreamBasicDescription)) 387 | //try checkError(AudioUnitGetProperty(audioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, outputBus, &inputFormat, &size)) 388 | //DLog("OUT:INi \(inputFormat)") 389 | 390 | // configure input format 391 | inputFormat.mFormatID = kAudioFormatLinearPCM 392 | inputFormat.mFormatFlags = kAudioFormatFlagsNativeFloatPacked | kLinearPCMFormatFlagIsNonInterleaved 393 | inputFormat.mSampleRate = outputFormat.mSampleRate 394 | inputFormat.mFramesPerPacket = 1 395 | inputFormat.mBytesPerPacket = UInt32(MemoryLayout.size) 396 | inputFormat.mBytesPerFrame = UInt32(MemoryLayout.size) 397 | inputFormat.mChannelsPerFrame = outputFormat.mChannelsPerFrame 398 | inputFormat.mBitsPerChannel = UInt32(8 * MemoryLayout.size) 399 | 400 | // print format information for debugging 401 | //DLog("OUT:IN \(inputFormat)") 402 | 403 | // set the audio format 404 | size = UInt32(MemoryLayout.size) 405 | try checkError(AudioUnitSetProperty(audioUnit!, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, outputBus, &inputFormat, size)) 406 | 407 | // initiate output array 408 | outputHighFor = [Int](repeating: 0, count: Int(outputFormat.mChannelsPerFrame)) 409 | 410 | // set frame size 411 | var frameSize = UInt32(self.frameSize) 412 | try checkError(AudioUnitSetProperty(audioUnit!, kAudioDevicePropertyBufferFrameSize, kAudioUnitScope_Global, outputBus, &frameSize, UInt32(MemoryLayout.size))) 413 | 414 | // setup playback callback 415 | var callbackStruct = AURenderCallbackStruct(inputProc: renderOutput, inputProcRefCon: unsafeBitCast(Unmanaged.passUnretained(self), to: UnsafeMutableRawPointer.self)) 416 | try checkError(AudioUnitSetProperty(audioUnit!, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Global, outputBus, &callbackStruct, UInt32(MemoryLayout.size))) 417 | 418 | // initialize audio unit 419 | AudioUnitInitialize(audioUnit!) 420 | 421 | // start playback 422 | AudioOutputUnitStart(audioUnit!) 423 | } 424 | 425 | func tearDownAudio() { 426 | if nil == audioUnit { 427 | return 428 | } 429 | 430 | // stop playback 431 | AudioOutputUnitStop(audioUnit!) 432 | 433 | // uninitialize audio unit 434 | AudioUnitUninitialize(audioUnit!) 435 | 436 | // dispose 437 | AudioComponentInstanceDispose(audioUnit!) 438 | 439 | audioUnit = nil 440 | } 441 | 442 | func createHighOutput(_ channel: Int, forDuration duration: Double) { 443 | guard channel < Int(outputFormat.mChannelsPerFrame) else { return } 444 | outputHighFor[channel] = Int(duration * outputFormat.mSampleRate) 445 | } 446 | 447 | static func defaultOutputDevice() throws -> AudioDeviceID { 448 | var size: UInt32 449 | size = UInt32(MemoryLayout.size) 450 | var outputDevice = AudioDeviceID() 451 | var address = AudioObjectPropertyAddress(mSelector: kAudioHardwarePropertyDefaultOutputDevice, mScope: kAudioObjectPropertyScopeGlobal, mElement: kAudioObjectPropertyElementMaster) 452 | try checkError(AudioObjectGetPropertyData(AudioObjectID(kAudioObjectSystemObject), &address, 0, nil, &size, &outputDevice)) 453 | return outputDevice 454 | } 455 | } 456 | 457 | protocol AudioInputInterfaceDelegate: class 458 | { 459 | func receiveAudioFrom(_ interface: AudioInputInterface, fromChannel: Int, withData data: UnsafeMutablePointer, ofLength: Int) 460 | } 461 | 462 | class AudioInputInterface: AudioInterface 463 | { 464 | let deviceID: AudioDeviceID 465 | let frameSize: Int 466 | 467 | weak var delegate: AudioInputInterfaceDelegate? = nil 468 | 469 | var inputFormat: AudioStreamBasicDescription = AudioStreamBasicDescription() 470 | var outputFormat: AudioStreamBasicDescription = AudioStreamBasicDescription() 471 | 472 | var bufferList: UnsafeMutableAudioBufferListPointer! 473 | 474 | init(deviceID: AudioDeviceID, frameSize: Int = 32) { 475 | self.deviceID = deviceID 476 | self.frameSize = frameSize 477 | } 478 | 479 | deinit { 480 | tearDownAudio() 481 | } 482 | 483 | func initializeAudio() throws { 484 | // set output bus 485 | let inputBus: AudioUnitElement = 1 486 | 487 | // describe component 488 | var componentDescription = AudioComponentDescription(componentType: kAudioUnitType_Output, componentSubType: kAudioUnitSubType_HALOutput, componentManufacturer: kAudioUnitManufacturer_Apple, componentFlags: 0, componentFlagsMask: 0) 489 | 490 | // get input component 491 | let inputComponent = AudioComponentFindNext(nil, &componentDescription) 492 | 493 | // check found 494 | if nil == inputComponent { 495 | throw AudioInterfaceError.noComponentFound 496 | } 497 | 498 | // make audio unit 499 | try checkError(AudioComponentInstanceNew(inputComponent!, &audioUnit)) 500 | 501 | var size: UInt32 502 | 503 | // enable input 504 | size = UInt32(MemoryLayout.size) 505 | var enableIO: UInt32 = 1 506 | try checkError(AudioUnitSetProperty(audioUnit!, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Input, inputBus, &enableIO, size)) 507 | 508 | // disable output 509 | enableIO = 0 510 | try checkError(AudioUnitSetProperty(audioUnit!, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Output, 0, &enableIO, size)) 511 | 512 | // set input device 513 | var inputDevice = deviceID 514 | try checkError(AudioUnitSetProperty(audioUnit!, kAudioOutputUnitProperty_CurrentDevice, kAudioUnitScope_Global, 0, &inputDevice, UInt32(MemoryLayout.size))) 515 | 516 | // get the audio format 517 | size = UInt32(MemoryLayout.size) 518 | try checkError(AudioUnitGetProperty(audioUnit!, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, inputBus, &inputFormat, &size)) 519 | 520 | // print format information for debugging 521 | //DLog("IN:IN \(inputFormat)") 522 | 523 | // check for expected format 524 | guard inputFormat.mFormatID == kAudioFormatLinearPCM && inputFormat.mFramesPerPacket == 1 && inputFormat.mFormatFlags == kAudioFormatFlagsNativeFloatPacked else { 525 | throw AudioInterfaceError.unsupportedFormat 526 | } 527 | 528 | // get the audio format 529 | //size = UInt32(sizeof(AudioStreamBasicDescription)) 530 | //try checkError(AudioUnitGetProperty(audioUnit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, inputBus, &outputFormat, &size)) 531 | //DLog("IN:OUTi \(outputFormat)") 532 | 533 | // configure output format 534 | outputFormat.mFormatID = kAudioFormatLinearPCM 535 | outputFormat.mFormatFlags = kAudioFormatFlagsNativeFloatPacked | kLinearPCMFormatFlagIsNonInterleaved 536 | outputFormat.mSampleRate = inputFormat.mSampleRate 537 | outputFormat.mFramesPerPacket = 1 538 | outputFormat.mBytesPerPacket = UInt32(MemoryLayout.size) 539 | outputFormat.mBytesPerFrame = UInt32(MemoryLayout.size) 540 | outputFormat.mChannelsPerFrame = inputFormat.mChannelsPerFrame 541 | outputFormat.mBitsPerChannel = UInt32(8 * MemoryLayout.size) 542 | 543 | // print format information for debugging 544 | //DLog("IN:OUT \(outputFormat)") 545 | 546 | // set the audio format 547 | size = UInt32(MemoryLayout.size) 548 | try checkError(AudioUnitSetProperty(audioUnit!, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, inputBus, &outputFormat, size)) 549 | 550 | // get maximum frame size 551 | var maxFrameSize: UInt32 = 0 552 | size = UInt32(MemoryLayout.size) 553 | try checkError(AudioUnitGetProperty(audioUnit!, kAudioUnitProperty_MaximumFramesPerSlice, kAudioUnitScope_Global, 0, &maxFrameSize, &size)) 554 | 555 | // create buffers 556 | bufferList = AudioBufferList.allocate(maximumBuffers: Int(outputFormat.mChannelsPerFrame)) 557 | bufferList.count = Int(outputFormat.mChannelsPerFrame) 558 | for channel in 0 ..< Int(outputFormat.mChannelsPerFrame) { 559 | // build buffer 560 | var buffer = AudioBuffer() 561 | buffer.mDataByteSize = outputFormat.mBytesPerFrame * maxFrameSize 562 | buffer.mNumberChannels = 1 // since non-interleaved 563 | buffer.mData = malloc(Int(outputFormat.mBytesPerFrame * maxFrameSize)) 564 | bufferList[channel] = buffer 565 | } 566 | 567 | // set frame size 568 | var frameSize: UInt32 = UInt32(self.frameSize) 569 | try checkError(AudioUnitSetProperty(audioUnit!, kAudioDevicePropertyBufferFrameSize, kAudioUnitScope_Global, 0, &frameSize, UInt32(MemoryLayout.size))) 570 | 571 | // setup playback callback 572 | var callbackStruct = AURenderCallbackStruct(inputProc: processInput, inputProcRefCon: unsafeBitCast(Unmanaged.passUnretained(self), to: UnsafeMutableRawPointer.self)) 573 | try checkError(AudioUnitSetProperty(audioUnit!, kAudioOutputUnitProperty_SetInputCallback, kAudioUnitScope_Global, 0, &callbackStruct, UInt32(MemoryLayout.size))) 574 | 575 | // initialize audio unit 576 | AudioUnitInitialize(audioUnit!) 577 | 578 | // start playback 579 | AudioOutputUnitStart(audioUnit!) 580 | } 581 | 582 | func tearDownAudio() { 583 | if nil == audioUnit { 584 | return 585 | } 586 | 587 | // free buffer 588 | for b in bufferList { 589 | free(b.mData) 590 | } 591 | free(bufferList.unsafeMutablePointer) 592 | 593 | // stop playback 594 | AudioOutputUnitStop(audioUnit!) 595 | 596 | // uninitialize audio unit 597 | AudioUnitUninitialize(audioUnit!) 598 | 599 | // dispose 600 | AudioComponentInstanceDispose(audioUnit!) 601 | 602 | audioUnit = nil 603 | } 604 | 605 | static func defaultInputDevice() throws -> AudioDeviceID { 606 | var size: UInt32 607 | size = UInt32(MemoryLayout.size) 608 | var inputDevice = AudioDeviceID() 609 | var address = AudioObjectPropertyAddress(mSelector: kAudioHardwarePropertyDefaultInputDevice, mScope: kAudioObjectPropertyScopeGlobal, mElement: kAudioObjectPropertyElementMaster) 610 | try checkError(AudioObjectGetPropertyData(AudioObjectID(kAudioObjectSystemObject), &address, 0, nil, &size, &inputDevice)) 611 | return inputDevice 612 | } 613 | } 614 | -------------------------------------------------------------------------------- /SyllableDetector/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | en 7 | CFBundleExecutable 8 | $(EXECUTABLE_NAME) 9 | CFBundleIconFile 10 | 11 | CFBundleIdentifier 12 | $(PRODUCT_BUNDLE_IDENTIFIER) 13 | CFBundleInfoDictionaryVersion 14 | 6.0 15 | CFBundleName 16 | $(PRODUCT_NAME) 17 | CFBundlePackageType 18 | APPL 19 | CFBundleShortVersionString 20 | 0.3.2 21 | CFBundleSignature 22 | ???? 23 | CFBundleVersion 24 | 14 25 | LSApplicationCategoryType 26 | public.app-category.education 27 | LSMinimumSystemVersion 28 | $(MACOSX_DEPLOYMENT_TARGET) 29 | NSHumanReadableCopyright 30 | Copyright © 2015 Gardner Lab. All rights reserved. 31 | NSMainStoryboardFile 32 | Main 33 | NSPrincipalClass 34 | NSApplication 35 | 36 | 37 | -------------------------------------------------------------------------------- /SyllableDetector/Processor.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Processor.swift 3 | // SyllableDetector 4 | // 5 | // Created by Nathan Perkins on 7/28/16. 6 | // Copyright © 2016 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import Accelerate 11 | import ORSSerial 12 | 13 | struct ProcessorEntry { 14 | let inputChannel: Int 15 | var network: String = "" 16 | var config: SyllableDetectorConfig? 17 | var resampler: Resampler? 18 | let outputChannel: Int 19 | 20 | init(inputChannel: Int, outputChannel: Int) { 21 | self.inputChannel = inputChannel 22 | self.outputChannel = outputChannel 23 | } 24 | } 25 | 26 | protocol Processor: AudioInputInterfaceDelegate { 27 | func setUp() throws 28 | func tearDown() 29 | 30 | func getInputForChannel(_ channel: Int) -> Double? 31 | func getOutputForChannel(_ channel: Int) -> Double? 32 | } 33 | 34 | class ProcessorBase: Processor, AudioInputInterfaceDelegate { 35 | // input interface 36 | let interfaceInput: AudioInputInterface 37 | 38 | // processor entries 39 | let entries: [ProcessorEntry] 40 | let detectors: [SyllableDetector] 41 | let channels: [Int] 42 | 43 | // stats 44 | let statInput: [SummaryStat] 45 | let statOutput: [SummaryStat] 46 | 47 | // dispatch queue 48 | let queueProcessing: DispatchQueue 49 | 50 | init(deviceInput: AudioInterface.AudioDevice, entries: [ProcessorEntry]) throws { 51 | // setup processor entries 52 | self.entries = entries.filter { 53 | return $0.config != nil 54 | } 55 | 56 | // setup processor detectors 57 | self.detectors = self.entries.map { 58 | return SyllableDetector(config: $0.config!) 59 | } 60 | 61 | // setup channels 62 | var channels = [Int](repeating: -1, count: 1 + (self.entries.map { return $0.inputChannel }.max() ?? -1)) 63 | for (i, p) in self.entries.enumerated() { 64 | channels[p.inputChannel] = i 65 | } 66 | self.channels = channels 67 | 68 | // setup stats 69 | var statInput = [SummaryStat]() 70 | var statOutput = [SummaryStat]() 71 | for _ in 0.., ofLength length: Int) { 103 | // valid channel 104 | guard channel < channels.count else { return } 105 | 106 | // get index 107 | let index = channels[channel] 108 | guard index >= 0 else { return } 109 | 110 | // get audio data 111 | var sum: Float = 0.0 112 | vDSP_svesq(data, 1, &sum, vDSP_Length(length)) 113 | statInput[index].writeValue(Double(sum) / Double(length)) 114 | 115 | // resample 116 | if let r = entries[index].resampler { 117 | var resampledData = r.resampleVector(data, ofLength: length) 118 | 119 | // append audio samples 120 | detectors[index].appendAudioData(&resampledData, withSamples: resampledData.count) 121 | } 122 | else { 123 | // append audio samples 124 | detectors[index].appendAudioData(data, withSamples: length) 125 | } 126 | 127 | // process 128 | queueProcessing.async { 129 | // detector 130 | let d = self.detectors[index] 131 | 132 | // seen syllable 133 | var seen = false 134 | 135 | // while there are new values 136 | while self.detectors[index].processNewValue() { 137 | // send to output 138 | self.statOutput[index].writeValue(Double(d.lastOutputs[0])) 139 | 140 | // update detected 141 | if !seen && d.lastDetected { 142 | seen = true 143 | } 144 | } 145 | 146 | // if seen, send output 147 | self.prepareOutputFor(index: index, seenSyllable: seen) 148 | } 149 | } 150 | 151 | func prepareOutputFor(index: Int, seenSyllable seen: Bool) { 152 | // log 153 | if seen { 154 | DLog("\(index) play") 155 | } 156 | } 157 | 158 | func getInputForChannel(_ channel: Int) -> Double? { 159 | // valid channel 160 | guard channel < channels.count else { return nil } 161 | 162 | // get index 163 | let index = channels[channel] 164 | guard index >= 0 else { return nil } 165 | 166 | // output stat 167 | if let meanSquareLevel = statInput[index].readStatAndReset() { 168 | return sqrt(meanSquareLevel) // RMS 169 | } 170 | 171 | return nil 172 | } 173 | 174 | func getOutputForChannel(_ channel: Int) -> Double? { 175 | // valid channel 176 | guard channel < channels.count else { return nil } 177 | 178 | // get index 179 | let index = channels[channel] 180 | guard index >= 0 else { return nil } 181 | 182 | // output stat 183 | return statOutput[index].readStatAndReset() 184 | } 185 | } 186 | 187 | final class ProcessorAudio: ProcessorBase { 188 | // output interface 189 | let interfaceOutput: AudioOutputInterface 190 | 191 | // high duration 192 | let highDuration = 0.001 // 1ms 193 | 194 | init(deviceInput: AudioInterface.AudioDevice, deviceOutput: AudioInterface.AudioDevice, entries: [ProcessorEntry]) throws { 195 | // set up output interface 196 | interfaceOutput = AudioOutputInterface(deviceID: deviceOutput.deviceID) 197 | 198 | // call parent 199 | try super.init(deviceInput: deviceInput, entries: entries) 200 | } 201 | 202 | deinit { 203 | DLog("deinit processorAudio") 204 | interfaceOutput.tearDownAudio() 205 | } 206 | 207 | override func setUp() throws { 208 | try interfaceOutput.initializeAudio() 209 | try super.setUp() 210 | } 211 | 212 | override func tearDown() { 213 | super.tearDown() 214 | interfaceOutput.tearDownAudio() 215 | } 216 | 217 | override func prepareOutputFor(index: Int, seenSyllable seen: Bool) { 218 | if seen { 219 | // create output high 220 | interfaceOutput.createHighOutput(entries[index].outputChannel, forDuration: highDuration) 221 | } 222 | 223 | // call super (just for logging) 224 | super.prepareOutputFor(index: index, seenSyllable: seen) 225 | } 226 | } 227 | 228 | final class ProcessorArduino: ProcessorBase { 229 | // output interface 230 | let interfaceOutput: ArduinoIO 231 | 232 | // high duration 233 | let highSteps = 20 234 | var highCount = [Int]() 235 | 236 | // triggering queue 237 | let queueTriggering: DispatchQueue 238 | 239 | init(deviceInput: AudioInterface.AudioDevice, deviceOutput: ORSSerialPort, entries: [ProcessorEntry]) throws { 240 | // create high count 241 | highCount = [Int](repeating: 0, count: entries.count) 242 | 243 | // create output interface 244 | interfaceOutput = ArduinoIO(serial: deviceOutput) 245 | 246 | // create triggering queue 247 | queueTriggering = DispatchQueue(label: "TriggerQueue") 248 | 249 | // call parent 250 | try super.init(deviceInput: deviceInput, entries: entries) 251 | } 252 | 253 | deinit { 254 | DLog("deinit processorArduino") 255 | } 256 | 257 | override func setUp() throws { 258 | // configure pins 259 | try entries.forEach { 260 | try interfaceOutput.setPinMode(7 + $0.outputChannel, to: .output) 261 | } 262 | 263 | try super.setUp() 264 | } 265 | 266 | override func prepareOutputFor(index: Int, seenSyllable seen: Bool) { 267 | if seen { 268 | // start high pulse 269 | if 0 == highCount[index] { 270 | do { 271 | try self.interfaceOutput.writeTo(7 + entries[index].outputChannel, digitalValue: true) 272 | } 273 | catch { 274 | DLog("ERROR: \(error)") 275 | } 276 | } 277 | 278 | // set counter 279 | highCount[index] = 20 280 | } 281 | else if 0 < highCount[index] { 282 | highCount[index] -= 1 283 | if 0 == highCount[index] { 284 | do { 285 | try self.interfaceOutput.writeTo(7 + entries[index].outputChannel, digitalValue: false) 286 | } 287 | catch { 288 | DLog("ERROR: \(error)") 289 | } 290 | } 291 | } 292 | 293 | super.prepareOutputFor(index: index, seenSyllable: seen) 294 | } 295 | } 296 | -------------------------------------------------------------------------------- /SyllableDetector/SummaryStat.swift: -------------------------------------------------------------------------------- 1 | // 2 | // SummaryStat.swift 3 | // SyllableDetector 4 | // 5 | // Created by Nathan Perkins on 11/11/15. 6 | // Copyright © 2015 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | 11 | protocol Stat 12 | { 13 | mutating func appendValue(_ value: Double) 14 | func readStat() -> Double? 15 | mutating func resetStat() 16 | } 17 | 18 | struct StatMean: Stat 19 | { 20 | var sum: Double = 0.0 21 | var count: Int = 0 22 | 23 | mutating func appendValue(_ value: Double) { 24 | sum += value 25 | count += 1 26 | } 27 | 28 | func readStat() -> Double? { 29 | guard 0 < count else { return nil } 30 | return sum / Double(count) 31 | } 32 | 33 | mutating func resetStat() { 34 | sum = 0.0 35 | count = 0 36 | } 37 | } 38 | 39 | struct StatMax: Stat 40 | { 41 | var largest: Double? 42 | 43 | mutating func appendValue(_ value: Double) { 44 | if let cur = largest { 45 | if value > cur { 46 | largest = value 47 | } 48 | } 49 | else { 50 | largest = value 51 | } 52 | } 53 | 54 | func readStat() -> Double? { 55 | return largest 56 | } 57 | 58 | mutating func resetStat() { 59 | largest = nil 60 | } 61 | } 62 | 63 | class SummaryStat 64 | { 65 | private var stat: Stat 66 | private var queue: DispatchQueue 67 | 68 | init(withStat stat: Stat) { 69 | self.stat = stat 70 | self.queue = DispatchQueue(label: "SummaryStat\(stat)") 71 | } 72 | 73 | func writeValue(_ value: Double) { 74 | queue.async { 75 | self.stat.appendValue(value) 76 | } 77 | } 78 | 79 | func readStatAndReset() -> Double? { 80 | var ret: Double? 81 | queue.sync { 82 | ret = self.stat.readStat() 83 | self.stat.resetStat() 84 | } 85 | return ret 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /SyllableDetector/Time.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Time.swift 3 | // SyllableDetector 4 | // 5 | // Created by Nathan Perkins on 3/30/16. 6 | // Copyright © 2016 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import Darwin 11 | 12 | class Time 13 | { 14 | private var timeStart: UInt64 = 0 15 | private var timeStop: UInt64 = 0 16 | 17 | private static var timeBase: Double? 18 | 19 | private static var globalTimers = [String: Time]() 20 | private static var globalStats = [String: [Double]]() 21 | 22 | private static func getTimeBase() -> Double { 23 | if let base = self.timeBase { 24 | return base 25 | } 26 | 27 | var info = mach_timebase_info(numer: 0, denom: 0) 28 | mach_timebase_info(&info) 29 | 30 | // calculate base 31 | let base = Double(info.numer) / Double(info.denom) 32 | self.timeBase = base 33 | return base 34 | } 35 | 36 | func start() { 37 | timeStart = mach_absolute_time() 38 | } 39 | 40 | func stop() { 41 | timeStop = mach_absolute_time() 42 | } 43 | 44 | var nanoseconds: Double { 45 | return Double(timeStop - timeStart) * Time.getTimeBase() 46 | } 47 | 48 | static func startWithName(_ key: String) { 49 | if let t = Time.globalTimers[key] { 50 | t.start() 51 | } 52 | else { 53 | let t = Time() 54 | Time.globalTimers[key] = t 55 | t.start() 56 | } 57 | } 58 | 59 | static func stopWithName(_ key: String) -> Double { 60 | if let t = Time.globalTimers[key] { 61 | t.stop() 62 | return t.nanoseconds 63 | } 64 | 65 | return -1.0 66 | } 67 | 68 | static func stopAndSaveWithName(_ key: String) { 69 | if let t = Time.globalTimers[key] { 70 | t.stop() 71 | 72 | if nil == Time.globalStats[key] { 73 | Time.globalStats[key] = [Double]() 74 | } 75 | 76 | Time.globalStats[key]!.append(t.nanoseconds) 77 | } 78 | } 79 | 80 | static func saveWithName(_ key: String, andValue value: Double) { 81 | if nil == Time.globalStats[key] { 82 | Time.globalStats[key] = [Double]() 83 | } 84 | 85 | Time.globalStats[key]!.append(value) 86 | } 87 | 88 | static func stopAndPrintWithName(_ key: String) { 89 | if let t = Time.globalTimers[key] { 90 | t.stop() 91 | print("\(key): \(t.nanoseconds)ns") 92 | } 93 | } 94 | 95 | static func printAll() { 96 | for (k, a) in Time.globalStats { 97 | print("\(k):") 98 | a.forEach { print("\($0)") } 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /SyllableDetector/ViewControllerMenu.swift: -------------------------------------------------------------------------------- 1 | // 2 | // ViewController.swift 3 | // SyllableDetector 4 | // 5 | // Created by Nathan Perkins on 10/28/15. 6 | // Copyright © 2015 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Cocoa 10 | import AudioToolbox 11 | import ORSSerial 12 | 13 | class ViewControllerMenu: NSViewController, WindowControllerProcessorDelegate { 14 | @IBOutlet weak var selectInput: NSPopUpButton! 15 | @IBOutlet weak var selectOutput: NSPopUpButton! 16 | @IBOutlet weak var buttonLaunch: NSButton! 17 | 18 | var openProcessors = [NSWindowController]() 19 | var openSimulators = [NSWindowController]() 20 | 21 | @objc class DeviceRepresentation: NSObject { 22 | var audioDeviceID: AudioDeviceID? 23 | var arduinoDevicePath: String? 24 | } 25 | 26 | override func viewDidLoad() { 27 | super.viewDidLoad() 28 | 29 | // reload devices 30 | buttonLaunch.isEnabled = false 31 | } 32 | 33 | override func viewDidAppear() { 34 | super.viewDidAppear() 35 | 36 | // listen for serial changes 37 | let nc = NotificationCenter.default 38 | nc.addObserver(self, selector: #selector(ViewControllerMenu.serialPortsWereConnected(_:)), name: NSNotification.Name.ORSSerialPortsWereConnected, object: nil) 39 | nc.addObserver(self, selector: #selector(ViewControllerMenu.serialPortsWereDisconnected(_:)), name: NSNotification.Name.ORSSerialPortsWereDisconnected, object: nil) 40 | 41 | // listen 42 | do { 43 | try AudioInterface.createListenerForDeviceChange({ 44 | DLog("refreshing device") 45 | self.reloadDevices() 46 | }, withIdentifier: self) 47 | } 48 | catch { 49 | DLog("Unable to add device change listener: \(error)") 50 | } 51 | 52 | // reload devices 53 | reloadDevices() 54 | } 55 | 56 | override func viewDidDisappear() { 57 | // remove notification center 58 | NotificationCenter.default.removeObserver(self) 59 | 60 | // remove listener 61 | AudioInterface.destroyListenerForDeviceChange(withIdentifier: self) 62 | 63 | // terminate 64 | if 0 == openProcessors.count && 0 == openSimulators.count { 65 | NSApp.terminate(nil) 66 | } 67 | } 68 | 69 | // serial port 70 | @objc func serialPortsWereConnected(_ notification: Notification) { 71 | if let userInfo = (notification as NSNotification).userInfo { 72 | let connectedPorts = userInfo[ORSConnectedSerialPortsKey] as! [ORSSerialPort] 73 | DLog("Ports were connected: \(connectedPorts)") 74 | reloadDevices() 75 | } 76 | } 77 | 78 | @objc func serialPortsWereDisconnected(_ notification: Notification) { 79 | if let userInfo = (notification as NSNotification).userInfo { 80 | let disconnectedPorts: [ORSSerialPort] = userInfo[ORSDisconnectedSerialPortsKey] as! [ORSSerialPort] 81 | DLog("Ports were disconnected: \(disconnectedPorts)") 82 | reloadDevices() 83 | } 84 | } 85 | 86 | func reloadDevices() { 87 | // fetch list of devices 88 | let devices: [AudioInterface.AudioDevice] 89 | do { 90 | devices = try AudioInterface.devices() 91 | } 92 | catch { 93 | DLog("Unable to reload devices: \(error)") 94 | return 95 | } 96 | 97 | // get input 98 | // let selectedInput = selectInput.selectedItem?.representedObject 99 | // let selectedOutput = selectOutput.selectedItem?.representedObject 100 | 101 | // rebuild inputs 102 | selectInput.removeAllItems() 103 | selectInput.addItem(withTitle: "Input") 104 | for d in devices { 105 | if 0 < d.streamsInput { 106 | // representation 107 | let obj = DeviceRepresentation() 108 | obj.audioDeviceID = d.deviceID 109 | 110 | // menu item 111 | let item = NSMenuItem() 112 | item.title = d.deviceName 113 | item.representedObject = obj 114 | selectInput.menu?.addItem(item) 115 | } 116 | } 117 | // selectInput.selectItem(withTag: selectedInput) 118 | selectInput.synchronizeTitleAndSelectedItem() 119 | 120 | // rebuild outputs 121 | selectOutput.removeAllItems() 122 | selectOutput.addItem(withTitle: "Output") 123 | for d in devices { 124 | if 0 < d.streamsOutput { 125 | // representation 126 | let obj = DeviceRepresentation() 127 | obj.audioDeviceID = d.deviceID 128 | 129 | // menu item 130 | let item = NSMenuItem() 131 | item.title = d.deviceName 132 | item.representedObject = obj 133 | selectOutput.menu?.addItem(item) 134 | } 135 | } 136 | for port in ORSSerialPortManager.shared().availablePorts { 137 | // representation 138 | let obj = DeviceRepresentation() 139 | obj.arduinoDevicePath = port.path 140 | 141 | // menu item 142 | let item = NSMenuItem() 143 | item.title = "Arduino (\(port.name))" 144 | item.representedObject = obj 145 | selectOutput.menu?.addItem(item) 146 | } 147 | // selectOutput.selectItem(withTag: selectedOutput) 148 | selectOutput.synchronizeTitleAndSelectedItem() 149 | } 150 | 151 | @IBAction func selectDevice(_ sender: NSPopUpButton) { 152 | buttonLaunch.isEnabled = (nil != selectInput.selectedItem?.representedObject && nil != selectOutput.selectedItem?.representedObject) 153 | } 154 | 155 | @IBAction func buttonSimulate(_ sender: NSButton) { 156 | guard let sb = storyboard, let controller = sb.instantiateController(withIdentifier: NSStoryboard.SceneIdentifier(rawValue: "Simulator")) as? NSWindowController else { return } 157 | 158 | // launch 159 | controller.showWindow(sender) 160 | openSimulators.append(controller) 161 | } 162 | 163 | @IBAction func buttonLaunch(_ sender: NSButton) { 164 | guard let sb = storyboard, let controller = sb.instantiateController(withIdentifier: NSStoryboard.SceneIdentifier(rawValue: "Processor")) as? WindowControllerProcessor else { return } 165 | 166 | // get input device representation 167 | guard let menuItemInput = selectInput.selectedItem, let objInput = menuItemInput.representedObject, let deviceRepresentationInput = objInput as? DeviceRepresentation else { 168 | return 169 | } 170 | 171 | // get output device representation 172 | guard let menuItemOutput = selectOutput.selectedItem, let objOutput = menuItemOutput.representedObject, let deviceRepresentationOutput = objOutput as? DeviceRepresentation else { 173 | return 174 | } 175 | 176 | // get input device 177 | guard let inputAudioID = deviceRepresentationInput.audioDeviceID, let deviceInput = AudioInterface.AudioDevice(deviceID: inputAudioID) else { 178 | DLog("input device no longer valid") 179 | reloadDevices() 180 | return 181 | } 182 | 183 | // get output device 184 | let deviceOutput: ViewControllerProcessor.OutputDevice 185 | if let outputAudioID = deviceRepresentationOutput.audioDeviceID { 186 | guard let o = AudioInterface.AudioDevice(deviceID: outputAudioID) else { 187 | DLog("output device no longer valid") 188 | reloadDevices() 189 | return 190 | } 191 | deviceOutput = .audio(interface: o) 192 | } 193 | else if let outputArduinoPath = deviceRepresentationOutput.arduinoDevicePath { 194 | guard let o = ORSSerialPort(path: outputArduinoPath) else { 195 | DLog("output device no longer valid") 196 | reloadDevices() 197 | return 198 | } 199 | deviceOutput = .arduino(port: o) 200 | } 201 | else { 202 | DLog("no output device found") 203 | return 204 | } 205 | 206 | DLog("\(deviceOutput)") 207 | 208 | // setup controller 209 | if let vc = controller.contentViewController, let vcp = vc as? ViewControllerProcessor { 210 | vcp.setupEntries(input: deviceInput, output: deviceOutput) 211 | } 212 | else { 213 | DLog("unknown error") 214 | return 215 | } 216 | 217 | controller.delegate = self // custom delegate used to clean up open processor list when windows are closed 218 | controller.showWindow(sender) 219 | openProcessors.append(controller) 220 | 221 | // reset selector 222 | selectInput.selectItem(at: 0) 223 | selectOutput.selectItem(at: 0) 224 | buttonLaunch.isEnabled = false 225 | } 226 | 227 | func windowControllerDone(_ controller: WindowControllerProcessor) { 228 | // window controller closed, clean from open processor list 229 | openProcessors = openProcessors.filter { 230 | return $0 !== controller 231 | } 232 | } 233 | } 234 | 235 | -------------------------------------------------------------------------------- /SyllableDetector/ViewControllerProcessor.swift: -------------------------------------------------------------------------------- 1 | // 2 | // ViewController.swift 3 | // SyllableDetector 4 | // 5 | // Created by Nathan Perkins on 10/28/15. 6 | // Copyright © 2015 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Cocoa 10 | import ORSSerial 11 | 12 | class ViewControllerProcessor: NSViewController, NSTableViewDelegate, NSTableViewDataSource { 13 | @IBOutlet weak var buttonLoad: NSButton! 14 | @IBOutlet weak var buttonToggle: NSButton! 15 | @IBOutlet weak var tableChannels: NSTableView! 16 | 17 | enum OutputDevice { 18 | case audio(interface: AudioInterface.AudioDevice) 19 | case arduino(port: ORSSerialPort) 20 | 21 | var outputChannels: Int { 22 | get { 23 | switch self { 24 | case .audio(let interface): 25 | if 0 < interface.buffersOutput.count { 26 | return Int(interface.buffersOutput[0].mNumberChannels) 27 | } 28 | return 0 29 | case .arduino(port: _): 30 | return 7 31 | } 32 | } 33 | } 34 | } 35 | 36 | // devices 37 | var deviceInput: AudioInterface.AudioDevice! 38 | var deviceOutput: OutputDevice! 39 | 40 | var processorEntries = [ProcessorEntry]() 41 | var processor: Processor? 42 | 43 | // timer to redraw interface (saves time) 44 | var timerRedraw: Timer? 45 | 46 | var isRunning = false { 47 | didSet { 48 | if oldValue == isRunning { return } 49 | 50 | // update interface 51 | tableChannels.isEnabled = !isRunning 52 | buttonLoad.isEnabled = !isRunning 53 | buttonToggle.title = (isRunning ? "Stop" : "Start") 54 | 55 | // start or stop timer 56 | if isRunning { 57 | timerRedraw = Timer.scheduledTimer(timeInterval: 0.1, target: self, selector: #selector(ViewControllerProcessor.timerUpdateValues(_:)), userInfo: nil, repeats: true) 58 | } 59 | else { 60 | timerRedraw?.invalidate() 61 | timerRedraw = nil 62 | } 63 | } 64 | } 65 | 66 | override func viewDidLoad() { 67 | super.viewDidLoad() 68 | 69 | tableChannels.target = self 70 | tableChannels.doubleAction = #selector(ViewControllerProcessor.tableRowDoubleClicked) 71 | } 72 | 73 | override func viewDidAppear() { 74 | super.viewDidAppear() 75 | 76 | if nil == deviceInput || nil == deviceOutput { 77 | fatalError("Input and output devices must be already defined.") 78 | } 79 | 80 | // reload table 81 | tableChannels.reloadData() 82 | } 83 | 84 | override func viewWillDisappear() { 85 | // clear processor 86 | processor = nil 87 | isRunning = false 88 | 89 | super.viewWillDisappear() 90 | } 91 | 92 | func setupEntries(input deviceInput: AudioInterface.AudioDevice, output deviceOutput: OutputDevice) { 93 | // store input and output 94 | self.deviceInput = deviceInput 95 | self.deviceOutput = deviceOutput 96 | 97 | // get input channels 98 | let inputChannels: Int 99 | if 0 < deviceInput.buffersInput.count { 100 | inputChannels = Int(deviceInput.buffersInput[0].mNumberChannels) 101 | } 102 | else { 103 | inputChannels = 0 104 | } 105 | 106 | // get output channels 107 | let outputChannels = deviceOutput.outputChannels 108 | 109 | // for each pair, create an entry 110 | let numEntries = min(inputChannels, outputChannels) 111 | processorEntries = (0.. Int { 157 | let inputChannels: Int, outputChannels: Int 158 | 159 | if nil != deviceInput && 0 < deviceInput.buffersInput.count { 160 | inputChannels = Int(deviceInput.buffersInput[0].mNumberChannels) 161 | } 162 | else { 163 | inputChannels = 0 164 | } 165 | 166 | if nil != deviceOutput { 167 | outputChannels = deviceOutput.outputChannels 168 | } 169 | else { 170 | outputChannels = 0 171 | } 172 | 173 | return min(inputChannels, outputChannels) 174 | } 175 | 176 | func tableView(_ tableView: NSTableView, objectValueFor tableColumn: NSTableColumn?, row: Int) -> Any? { 177 | guard let identifier = tableColumn?.identifier else { return nil } 178 | guard row < processorEntries.count else { return nil } 179 | 180 | switch identifier.rawValue { 181 | case "ColumnInput", "ColumnOutput": return "Channel \(row + 1)" 182 | case "ColumnInLevel": 183 | if let p = processor { 184 | return NSNumber(value: 100.0 * (p.getInputForChannel(row) ?? 0.0)) 185 | } 186 | return NSNumber(value: 0.00) 187 | case "ColumnOutLevel": 188 | if let p = processor { 189 | return NSNumber(value: 100.0 * (p.getOutputForChannel(row) ?? 0.0)) 190 | } 191 | return NSNumber(value: 0.00) 192 | case "ColumnNetwork": return nil == processorEntries[row].config ? "Not Selected" : processorEntries[row].network 193 | default: return nil 194 | } 195 | } 196 | 197 | @objc func tableRowDoubleClicked() { 198 | guard !isRunning else { return } // can not select when running 199 | guard 0 <= tableChannels.clickedColumn else { return } // valid column 200 | guard "ColumnNetwork" == tableChannels.tableColumns[tableChannels.clickedColumn].identifier.rawValue else { return } // double clicked network column 201 | 202 | // show network selector 203 | loadNetworkForRow(tableChannels.clickedRow) 204 | } 205 | 206 | @IBAction func loadNetwork(_ sender: NSButton) { 207 | if 0 > tableChannels.selectedRow { // no row selected... 208 | // find next row needing a network 209 | for (i, p) in processorEntries.enumerated() { 210 | if nil == p.config { 211 | loadNetworkForRow(i) 212 | break 213 | } 214 | } 215 | } 216 | else { 217 | // load network for row 218 | loadNetworkForRow(tableChannels.selectedRow) 219 | } 220 | } 221 | 222 | func loadNetworkForRow(_ row: Int) { 223 | guard !isRunning else { return } // can not select when running 224 | guard row < processorEntries.count else { return } 225 | 226 | let panel = NSOpenPanel() 227 | panel.title = "Select Network Definition" 228 | panel.allowedFileTypes = ["txt"] 229 | panel.allowsOtherFileTypes = false 230 | 231 | // callback for handling response 232 | let cb = { 233 | (result: NSApplication.ModalResponse) -> Void in 234 | // check again, just in case 235 | guard !self.isRunning else { return } // can not select when running 236 | guard row < self.processorEntries.count else { return } 237 | 238 | // make sure ok was pressed 239 | if NSApplication.ModalResponse.OK == result { 240 | if let url = panel.url { 241 | let path = url.path 242 | do { 243 | // load file 244 | let config = try SyllableDetectorConfig(fromTextFile: path) 245 | 246 | // check sampling rate 247 | if (1 < abs(config.samplingRate - self.deviceInput.sampleRateInput)) { 248 | print("Mismatched sampling rates. Expecting: \(config.samplingRate). Device: \(self.deviceInput.sampleRateInput).") 249 | self.processorEntries[row].resampler = ResamplerLinear(fromRate: self.deviceInput.sampleRateInput, toRate: config.samplingRate) 250 | } 251 | 252 | self.processorEntries[row].config = config 253 | self.processorEntries[row].network = url.lastPathComponent 254 | } 255 | catch { 256 | // unable to load 257 | let alert = NSAlert() 258 | alert.messageText = "Unable to load" 259 | alert.informativeText = "The text file could not be successfully loaded: \(error)." 260 | alert.addButton(withTitle: "Ok") 261 | alert.beginSheetModal(for: self.view.window!, completionHandler:nil) 262 | 263 | // clear selected 264 | self.processorEntries[row].network = "" 265 | self.processorEntries[row].config = nil 266 | } 267 | 268 | // reload table 269 | self.tableChannels.reloadData() 270 | } 271 | } 272 | } 273 | 274 | // show 275 | panel.beginSheetModal(for: self.view.window!, completionHandler: cb) 276 | } 277 | 278 | @objc func timerUpdateValues(_ timer: Timer!) { 279 | // create column indices 280 | let indexes = IndexSet([1, 4]) 281 | 282 | // reload data 283 | tableChannels.reloadData(forRowIndexes: IndexSet(integersIn: 0.. Void in 44 | // make sure ok was pressed 45 | if NSApplication.ModalResponse.OK == result { 46 | if let url = panel.url { 47 | let path = url.path 48 | do { 49 | // load file 50 | let _ = try SyllableDetectorConfig(fromTextFile: path) 51 | 52 | // confirm loaded 53 | self.pathNetwork.url = url 54 | 55 | // update buttons 56 | self.buttonRun.isEnabled = (self.pathNetwork.url != nil && self.pathAudio.url != nil) 57 | } 58 | catch { 59 | // unable to load 60 | let alert = NSAlert() 61 | alert.messageText = "Unable to load" 62 | alert.informativeText = "The text file could not be successfully loaded: \(error)." 63 | alert.addButton(withTitle: "Ok") 64 | alert.beginSheetModal(for: self.view.window!, completionHandler:nil) 65 | } 66 | } 67 | } 68 | } 69 | 70 | // show 71 | panel.beginSheetModal(for: self.view.window!, completionHandler: cb) 72 | } 73 | 74 | @IBAction func loadAudio(_ sender: NSButton) { 75 | let panel = NSOpenPanel() 76 | panel.allowedFileTypes = [AVFileType.wav.rawValue, AVFileType.m4a.rawValue] 77 | panel.title = "Select Audio File" 78 | 79 | // callback for handling response 80 | let cb = { 81 | (result: NSApplication.ModalResponse) -> Void in 82 | // make sure ok was pressed 83 | if NSApplication.ModalResponse.OK == result { 84 | if let url = panel.url { 85 | // store audio path 86 | self.pathAudio.url = url 87 | 88 | // update buttons 89 | self.buttonRun.isEnabled = (self.pathNetwork.url != nil && self.pathAudio.url != nil) 90 | } 91 | } 92 | } 93 | 94 | // show 95 | panel.beginSheetModal(for: self.view.window!, completionHandler: cb) 96 | } 97 | 98 | @IBAction func run(_ sender: NSButton) { 99 | // confirm there are URLs 100 | guard let urlNetwork = pathNetwork.url, let urlAudio = pathAudio.url else { 101 | return 102 | } 103 | 104 | // diable all 105 | buttonRun.isEnabled = false 106 | buttonLoadAudio.isEnabled = false 107 | buttonLoadNetwork.isEnabled = false 108 | 109 | let panel = NSSavePanel() 110 | panel.allowedFileTypes = [AVFileType.wav.rawValue] 111 | panel.allowsOtherFileTypes = false 112 | panel.title = "Save Output File" 113 | 114 | // callback for handling response 115 | let cb = { 116 | (result: NSApplication.ModalResponse) -> Void in 117 | // make sure ok was pressed 118 | if NSApplication.ModalResponse.OK == result { 119 | if let url = panel.url { 120 | // simulate 121 | self.simulateNetwork(urlNetwork, withAudio: urlAudio, writeTo: url) 122 | } 123 | 124 | // enable all 125 | self.buttonRun.isEnabled = true 126 | self.buttonLoadAudio.isEnabled = true 127 | self.buttonLoadNetwork.isEnabled = true 128 | } 129 | } 130 | 131 | // show 132 | panel.beginSheetModal(for: self.view.window!, completionHandler: cb) 133 | } 134 | 135 | func simulateNetwork(_ urlNetwork: URL, withAudio urlAudio: URL, writeTo urlOutput: URL) { 136 | // convert to path 137 | let pathNetwork = urlNetwork.path 138 | 139 | // 1. LOAD AUDIO INPUT 140 | let assetRead = AVAsset(url: urlAudio) 141 | let avReader: AVAssetReader 142 | do { 143 | avReader = try AVAssetReader(asset: assetRead) 144 | } 145 | catch { 146 | DLog("\(error)") 147 | return 148 | } 149 | 150 | // get number of audio tracks 151 | let tracksAudio = assetRead.tracks(withMediaType: AVMediaType.audio) 152 | guard 0 < tracksAudio.count else { 153 | DLog("no audio tracks") 154 | return 155 | } 156 | 157 | if 1 < tracksAudio.count { 158 | DLog("Only processing the first track") 159 | } 160 | 161 | // 2. LOAD SYLLABLE DETECTOR 162 | let config: SyllableDetectorConfig 163 | do { 164 | // load file 165 | config = try SyllableDetectorConfig(fromTextFile: pathNetwork) 166 | } 167 | catch { 168 | DLog("unable to load the syllabe detector") 169 | return 170 | } 171 | 172 | let sd = SyllableDetector(config: config) 173 | 174 | // 3. CONFIGURE READER 175 | // track reader 176 | let avReaderOutput = AVAssetReaderTrackOutput(track: tracksAudio[0], outputSettings: sd.audioSettings) 177 | if avReader.canAdd(avReaderOutput) { 178 | avReader.add(avReaderOutput) 179 | } 180 | else { 181 | DLog("Unable to add reader output.") 182 | return 183 | } 184 | 185 | // 4. START WRITER 186 | // create asset and asset writer 187 | let avWriter: AVAssetWriter 188 | do { 189 | avWriter = try AVAssetWriter(url: urlOutput, fileType: AVFileType.wav) 190 | } 191 | catch { 192 | DLog("\(error)") 193 | return 194 | } 195 | 196 | // create channel layout 197 | var monoChannelLayout = AudioChannelLayout() 198 | monoChannelLayout.mChannelLayoutTag = kAudioChannelLayoutTag_Mono 199 | monoChannelLayout.mChannelBitmap = AudioChannelBitmap(rawValue: 0) 200 | monoChannelLayout.mNumberChannelDescriptions = 0 201 | 202 | // audio settings 203 | var compressionAudioSettings: [String: AnyObject] = [:] 204 | withUnsafePointer(to: &monoChannelLayout) { 205 | compressionAudioSettings = [ 206 | AVFormatIDKey: NSNumber(value: kAudioFormatLinearPCM), 207 | AVLinearPCMBitDepthKey: NSNumber(value: 16), 208 | AVLinearPCMIsFloatKey: false as AnyObject, 209 | AVLinearPCMIsBigEndianKey: false as AnyObject, 210 | AVLinearPCMIsNonInterleaved: false as AnyObject, 211 | AVSampleRateKey: NSNumber(value: sd.config.samplingRate), 212 | AVChannelLayoutKey: Data(bytes: UnsafeRawPointer($0), count: MemoryLayout.size) as AnyObject, 213 | AVNumberOfChannelsKey: NSNumber(value: 1) 214 | ] 215 | } 216 | 217 | // make writer input 218 | let avWriterInput = AVAssetWriterInput(mediaType: AVMediaType.audio, outputSettings: compressionAudioSettings) 219 | avWriterInput.expectsMediaDataInRealTime = true 220 | if avWriter.canAdd(avWriterInput) { 221 | avWriter.add(avWriterInput) 222 | } 223 | else { 224 | DLog("Can not add input to writer.") 225 | return 226 | } 227 | 228 | // 5. START PROCESSING 229 | if !avReader.startReading() { 230 | DLog("Unable to read: \(String(describing: avReader.error))") 231 | return 232 | } 233 | 234 | // start writing 235 | if !avWriter.startWriting() { 236 | DLog("Unable to write: \(String(describing: avWriter.error))") 237 | return 238 | } 239 | 240 | avWriter.startSession(atSourceTime: kCMTimeZero) 241 | 242 | // DESCRIBE OUTPUT FORMAT 243 | 244 | // output format description 245 | var outputAudioFormatDescription = AudioStreamBasicDescription(mSampleRate: sd.config.samplingRate, mFormatID: kAudioFormatLinearPCM, mFormatFlags: kLinearPCMFormatFlagIsFloat | kLinearPCMFormatFlagIsAlignedHigh, mBytesPerPacket: 4, mFramesPerPacket: 1, mBytesPerFrame: 4, mChannelsPerFrame: 1, mBitsPerChannel: 32, mReserved: 0) 246 | var outputFormatDescription: CMAudioFormatDescription? = nil 247 | var status = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &outputAudioFormatDescription, 0, nil, 0, nil, nil, &outputFormatDescription) 248 | assert(status == noErr) 249 | 250 | // processing 251 | var nextCount: Int = sd.config.windowLength + ((sd.config.windowLength - sd.config.windowOverlap) * (sd.config.timeRange - 1)), nextValue: Float = 0.0 252 | if sd.config.windowOverlap < 0 { 253 | nextCount = nextCount - sd.config.windowOverlap // since gap is applied even to the first data set 254 | } 255 | var samplePosition: Int64 = 0 256 | let gcdGroup = DispatchGroup() 257 | let gcdQueue = DispatchQueue(label: "Encode") 258 | avWriterInput.requestMediaDataWhenReady(on: gcdQueue) { 259 | // probably not needed 260 | gcdGroup.enter() 261 | defer { 262 | gcdGroup.leave() 263 | } 264 | 265 | var completedOrFailed = false 266 | var status: OSStatus 267 | 268 | while avWriterInput.isReadyForMoreMediaData && !completedOrFailed { 269 | // check status 270 | guard avReader.status == AVAssetReaderStatus.reading else { 271 | DLog("STATUS changed to \(avReader.status)") 272 | completedOrFailed = true 273 | break 274 | } 275 | 276 | // copy next sample buffer 277 | guard let sampleBuffer = avReaderOutput.copyNextSampleBuffer() else { 278 | completedOrFailed = true 279 | break 280 | } 281 | 282 | // get number of samples 283 | let numSamples = CMSampleBufferGetNumSamples(sampleBuffer) 284 | guard 0 < numSamples else { 285 | continue 286 | } 287 | 288 | DLog("READ: \(numSamples) samples") 289 | 290 | // run song detector 291 | Time.startWithName("ingest") 292 | sd.processSampleBuffer(sampleBuffer) 293 | Time.stopAndSaveWithName("ingest") 294 | 295 | // make floats 296 | // released by buffer block 297 | let newSamples = UnsafeMutablePointer.allocate(capacity: numSamples) 298 | 299 | // encode previous values 300 | var i = 0 301 | while 0 < nextCount && i < numSamples { 302 | newSamples[i] = nextValue 303 | i += 1 304 | nextCount -= 1 305 | } 306 | 307 | // still more to write? don't process any 308 | while 0 == nextCount { 309 | let t = Time() 310 | t.start() 311 | let ret = sd.processNewValue() 312 | t.stop() 313 | if ret { 314 | Time.saveWithName("process", andValue: t.nanoseconds) 315 | } 316 | else { 317 | Time.saveWithName("skip", andValue: t.nanoseconds) 318 | break 319 | } 320 | 321 | // value to write 322 | var v = sd.lastOutputs[0] / Float(sd.config.thresholds[0]) 323 | if v > 1.0 { 324 | v = 1.0 325 | } 326 | else if v < 0.0 { 327 | v = 0.0 328 | } 329 | 330 | // length 331 | var l = sd.config.windowLength - sd.config.windowOverlap 332 | 333 | while 0 < l && i < numSamples { 334 | newSamples[i] = v 335 | i += 1 336 | l -= 1 337 | } 338 | 339 | if 0 < l { 340 | nextCount = l 341 | nextValue = v 342 | break 343 | } 344 | } 345 | 346 | // make block buffer 347 | var newBlockBuffer: CMBlockBuffer? = nil 348 | status = CMBlockBufferCreateWithMemoryBlock(nil, UnsafeMutableRawPointer(newSamples), numSamples * MemoryLayout.stride, nil, nil, 0, numSamples * MemoryLayout.stride, 0, &newBlockBuffer) 349 | assert(status == noErr) 350 | 351 | // timestamp for output 352 | let timestamp = CMTimeMake(samplePosition, Int32(outputAudioFormatDescription.mSampleRate)) 353 | samplePosition += Int64(numSamples) 354 | 355 | // get sample buffer 356 | var newSampleBuffer: CMSampleBuffer? = nil 357 | status = CMAudioSampleBufferCreateWithPacketDescriptions(kCFAllocatorDefault, newBlockBuffer, true, nil, nil, outputFormatDescription!, numSamples, timestamp, nil, &newSampleBuffer) 358 | assert(status == noErr) 359 | 360 | // append sample buffer 361 | if !avWriterInput.append(newSampleBuffer!) { 362 | DLog("ERROR writing \(avWriter.status) \(String(describing: avWriter.error))") 363 | avReader.cancelReading() // cancel reading 364 | completedOrFailed = true 365 | } 366 | } 367 | 368 | if completedOrFailed { 369 | avWriterInput.markAsFinished() 370 | avWriter.finishWriting { 371 | DLog("done!") 372 | } 373 | } 374 | } 375 | 376 | } 377 | } 378 | -------------------------------------------------------------------------------- /SyllableDetector/WindowControllerProcessor.swift: -------------------------------------------------------------------------------- 1 | // 2 | // WindowControllerProcessor.swift 3 | // SyllableDetector 4 | // 5 | // Created by Nathan Perkins on 11/1/15. 6 | // Copyright © 2015 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Cocoa 10 | 11 | protocol WindowControllerProcessorDelegate: class { 12 | func windowControllerDone(_ controller: WindowControllerProcessor) 13 | } 14 | 15 | class WindowControllerProcessor: NSWindowController, NSWindowDelegate { 16 | weak var delegate: WindowControllerProcessorDelegate? 17 | 18 | override func windowDidLoad() { 19 | super.windowDidLoad() 20 | 21 | window?.delegate = self 22 | } 23 | 24 | func windowWillClose(_ notification: Notification) { 25 | // when window will close, pass it up the chain 26 | delegate?.windowControllerDone(self) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /SyllableDetectorCLI/OutputStream.swift: -------------------------------------------------------------------------------- 1 | // 2 | // OutputStream.swift 3 | // SyllableDetector 4 | // 5 | // Created by Nathan Perkins on 6/2/16. 6 | // Copyright © 2016 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | 11 | extension TextOutputStream { 12 | mutating func writeLine(string: String) { 13 | self.write("\(string)\n") 14 | } 15 | } 16 | 17 | extension FileHandle { 18 | func writeLine(_ string: String) { 19 | write("\(string)\n".data(using: String.Encoding.utf8)!) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /SyllableDetectorCLI/TrackDetector.swift: -------------------------------------------------------------------------------- 1 | // 2 | // TrackDetector.swift 3 | // SyllableDetector 4 | // 5 | // Created by Nathan Perkins on 6/2/16. 6 | // Copyright © 2016 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import AVFoundation 11 | 12 | class TrackDetector 13 | { 14 | let track: AVAssetTrack 15 | let reader: AVAssetReaderTrackOutput 16 | let detector: SyllableDetector 17 | let channel: Int 18 | var debounceFrames = 0 19 | var debounceTime: Double { 20 | get { 21 | return Double(debounceFrames) / detector.config.samplingRate 22 | } 23 | set { 24 | debounceFrames = Int(newValue * detector.config.samplingRate) 25 | } 26 | } 27 | 28 | private var nextOutput: Int // counter until next sd output 29 | private var totalSamples: Int = 0 30 | private var debounceUntil: Int = -1 31 | 32 | init(track: AVAssetTrack, config: SyllableDetectorConfig, channel: Int = 0) { 33 | detector = SyllableDetector(config: config) 34 | self.track = track 35 | self.reader = AVAssetReaderTrackOutput(track: track, outputSettings: detector.audioSettings) 36 | self.channel = channel 37 | 38 | // next output is equal to one full window, plus the non-overlapping value for each subsequent time range 39 | nextOutput = config.windowLength + ((config.windowLength - config.windowOverlap) * (config.timeRange - 1)) 40 | if config.windowOverlap < 0 { 41 | nextOutput = nextOutput - config.windowOverlap // since gap is applied even to the first data set 42 | } 43 | } 44 | 45 | func process() { 46 | // copy next sample buffer 47 | guard let sampleBuffer = reader.copyNextSampleBuffer() else { 48 | return 49 | } 50 | 51 | // get number of samples 52 | let numSamples = CMSampleBufferGetNumSamples(sampleBuffer) 53 | //print("\(numSamples)") 54 | guard 0 < numSamples else { 55 | return 56 | } 57 | 58 | // get timing information 59 | let presentationTimestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) 60 | 61 | // process sample buffer 62 | detector.processSampleBuffer(sampleBuffer) 63 | 64 | // get output 65 | while detector.processNewValue() { 66 | // get curoutput sample number and increment next ouput sample number 67 | let curOutput = nextOutput 68 | nextOutput += detector.config.windowLength - detector.config.windowOverlap 69 | 70 | // look for detection 71 | var hasDetection = false 72 | for (i, d) in detector.lastOutputs.enumerated() { 73 | if Double(d) >= detector.config.thresholds[i] { 74 | hasDetection = true 75 | break 76 | } 77 | } 78 | 79 | // detection 80 | if hasDetection && debounceUntil < curOutput { 81 | // get sample number within current buffer 82 | let curSample = curOutput - totalSamples 83 | if curSample >= numSamples { 84 | fatalError("Unexpected sample number.") 85 | } 86 | 87 | // get presentation time 88 | let curTime = CMTime(value: presentationTimestamp.value + Int64(curSample), timescale: presentationTimestamp.timescale) 89 | let curTimeSeconds = CMTimeGetSeconds(curTime) 90 | 91 | // print results 92 | print("\(channel),\(curOutput),\(curTimeSeconds)", terminator: "") 93 | for d in detector.lastOutputs { 94 | print(",\(d)", terminator: "") 95 | } 96 | print("") 97 | 98 | // start debounce counter 99 | debounceUntil = curOutput + debounceFrames 100 | } 101 | } 102 | 103 | // increment number of samples 104 | totalSamples += numSamples 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /SyllableDetectorCLI/main.swift: -------------------------------------------------------------------------------- 1 | // 2 | // main.swift 3 | // SyllableDetectorCLI 4 | // 5 | // Created by Nathan Perkins on 6/2/16. 6 | // Copyright © 2016 Gardner Lab. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | import AVFoundation 11 | import Moderator 12 | 13 | // UTILITY 14 | 15 | private var stderr = FileHandle.standardError 16 | 17 | // PARSE COMMAND LINE 18 | 19 | let cli = Moderator(description: "") 20 | 21 | let argNetworkPath = cli.add(Argument.optionWithValue("n", "net", description: "Path to trained network file.").required()) 22 | let argAudioPaths = cli.add(Argument.optionWithValue("a", "audio", description: "Path to the audio file to process.").repeat()) 23 | let argDebounceTime = cli.add(Argument.optionWithValue("d", "debounce", description: "Number of seconds to debounce triggers.")) 24 | 25 | do { 26 | try cli.parse() 27 | } 28 | catch { 29 | // if help 30 | print(cli.usagetext) 31 | print("The command line will write a comma-separated list of detection events (when the network has at least one output above threshold) to standard out. For example, it might output:") 32 | print("") 33 | print("\t0,1593298,36.1292063492063,0.918557") 34 | print("") 35 | print("The columns are:") 36 | print("1. The track or channel number from the audio file (starting with 0).") 37 | print("2. The sample number from the audio when detection occurred.") 38 | print("3. The timestamp from the audio when detection occurred.") 39 | print("4. The first neural network output. Note that there may be additional columns for additional outputs.") 40 | exit(EX_USAGE) 41 | } 42 | 43 | let audioPaths = argAudioPaths.value 44 | let networkPath = argNetworkPath.value 45 | let debounceTime: Double? = argDebounceTime.value != nil ? Double.init(argDebounceTime.value!) : nil 46 | 47 | // RUN 48 | 49 | // 1. load network 50 | 51 | let config: SyllableDetectorConfig 52 | do { 53 | // load file 54 | config = try SyllableDetectorConfig(fromTextFile: networkPath) 55 | } 56 | catch { 57 | stderr.writeLine("Unable to load the network configuration: \(error)") 58 | fatalError() 59 | } 60 | 61 | // 2. read in the audio 62 | 63 | audioPaths.forEach { 64 | audioPath in 65 | 66 | // 2b. open asset 67 | 68 | let assetRead = AVAsset(url: URL(fileURLWithPath: audioPath)) 69 | let avReader: AVAssetReader 70 | do { 71 | avReader = try AVAssetReader(asset: assetRead) 72 | } 73 | catch { 74 | stderr.writeLine("Unable to read \(audioPath): \(error)") 75 | return 76 | } 77 | 78 | // get number of audio tracks 79 | let tracksAudio = assetRead.tracks(withMediaType: AVMediaType.audio) 80 | guard 0 < tracksAudio.count else { 81 | stderr.writeLine("No audio tracks found in \(audioPath).") 82 | return 83 | } 84 | 85 | // make detectors 86 | let potentialTrackDetectors = tracksAudio.enumerated().map { 87 | (i, track) in 88 | return TrackDetector(track: track, config: config, channel: i) 89 | } 90 | 91 | // validate 92 | let trackDetectors = potentialTrackDetectors.filter { 93 | return avReader.canAdd($0.reader) 94 | } 95 | if trackDetectors.count == 0 { 96 | stderr.writeLine("Can not read audio tracks found in \(audioPath).") 97 | return 98 | } 99 | if trackDetectors.count < potentialTrackDetectors.count { 100 | stderr.writeLine("Can not read from \(potentialTrackDetectors.count - trackDetectors.count) audio track(s) in \(audioPath). Skipping those tracks.") 101 | } 102 | 103 | // add all 104 | trackDetectors.forEach { 105 | // configure 106 | if let seconds = debounceTime { 107 | $0.debounceTime = seconds 108 | } 109 | 110 | // add it 111 | avReader.add($0.reader) 112 | } 113 | 114 | // start reading 115 | if !avReader.startReading() { 116 | stderr.writeLine("Can not start reading \(audioPath): \(String(describing: avReader.error)).") 117 | return 118 | } 119 | 120 | // 2c. iterate over audio 121 | 122 | if 1 < audioPaths.count { 123 | print("\(audioPath)") 124 | } 125 | 126 | while avReader.status == AVAssetReaderStatus.reading { 127 | for trackDetector in trackDetectors { 128 | trackDetector.process() 129 | } 130 | } 131 | } 132 | 133 | -------------------------------------------------------------------------------- /convert_to_text.m: -------------------------------------------------------------------------------- 1 | function convert_to_text(fn, mat, varargin) 2 | 3 | % extra input processing functions 4 | prepend_input_processing = {}; 5 | 6 | nparams=length(varargin); 7 | 8 | if 0 < mod(nparams, 2) 9 | error('Parameters must be specified as parameter/value pairs'); 10 | end 11 | for i = 1:2:nparams 12 | nm = lower(varargin{i}); 13 | switch nm 14 | case 'prepend_input_processing' 15 | if ischar(varargin{i+1}) 16 | prepend_input_processing = varargin(i+1); 17 | else 18 | prepend_input_processing = varargin{i+1}; 19 | end 20 | otherwise 21 | if ~exist(nm, 'var') 22 | error('Invalid parameter: %s.', nm); 23 | end 24 | eval([nm ' = varargin{i+1};']); 25 | end 26 | end 27 | 28 | %% LOAD NETWORK 29 | 30 | % load network definition file 31 | f = load(mat); 32 | 33 | % default to same window size 34 | if ~isfield(f, 'win_size') 35 | f.win_size = f.fft_size; 36 | end 37 | 38 | %% CHECKS 39 | 40 | % FFT msut be a power of 2 41 | if f.fft_size ~= 2^nextpow2(f.fft_size) 42 | error('Only FFT sizes that are a power of two are supported.'); 43 | end 44 | 45 | % FFT must be longer than or equal to the window size 46 | if f.win_size > f.fft_size 47 | error('The window size must be less than or equal to the FFT size.'); 48 | end 49 | 50 | % handle weird spectrogram behavior 51 | if 256 > f.fft_size 52 | warning('The spectrogram defaults to using an FFT size of 256. As a result, the provided FFT size will be ignored.'); 53 | f.fft_size = 256; 54 | end 55 | 56 | %% WRITE TEXT FILE 57 | 58 | % open file for writing 59 | fh = fopen(fn, 'w'); 60 | 61 | fprintf(fh, '# AUTOMATICALLY GENERATED SYLLABLE DETECTOR CONFIGURATION\n'); 62 | fprintf(fh, 'samplingRate = %.1f\n', f.samplerate); 63 | fprintf(fh, 'fourierLength = %d\n', f.fft_size); 64 | fprintf(fh, 'windowLength = %d\n', f.win_size); 65 | fprintf(fh, 'windowOverlap = %d\n', f.fft_size - f.fft_time_shift); 66 | 67 | fprintf(fh, 'freqRange = %.1f, %.1f\n', f.freq_range(1), f.freq_range(end)); 68 | fprintf(fh, 'timeRange = %d\n', f.time_window_steps); 69 | 70 | thresholds = sprintf('%.15g, ', reshape(f.trigger_thresholds, [], 1)); 71 | thresholds = thresholds(1:end - 2); % remove final comma 72 | fprintf(fh, 'thresholds = %s\n', thresholds); 73 | 74 | fprintf(fh, 'scaling = %s\n', f.scaling); 75 | 76 | % build neural network 77 | 78 | % input mapping 79 | convert_processing_functions(fh, 'processInputs', f.net.input, prepend_input_processing); 80 | 81 | % output mapping 82 | convert_processing_functions(fh, 'processOutputs', f.net.output); 83 | 84 | fprintf(fh, 'layers = %d\n', length(f.net.layers)); 85 | 86 | % layers 87 | layers = {}; 88 | for i = 1:length(f.net.layers) 89 | % add layer 90 | name = sprintf('layer%d', i - 1); 91 | layers{i} = name; 92 | 93 | % check for non-consecutive weights 94 | if any(cellfun(@numel, f.net.LW(i, 1:length(f.net.layers) ~= i - 1))) 95 | error('Networks with only connections between consecutive layers supported.'); 96 | end 97 | 98 | % get weights 99 | if 1 == i 100 | w = f.net.IW{i}; 101 | else 102 | w = f.net.LW{i, i - 1}; 103 | if 0 < length(f.net.IW{i}) 104 | error('Found unexpected input weights for layer 1.'); 105 | end 106 | end 107 | b = f.net.b{i}; 108 | 109 | % add layer 110 | convert_layer(fh, name, f.net.layers{i}, w, b); 111 | end 112 | 113 | % close file handle 114 | fclose(fh); 115 | 116 | %% HELPER FUNCTIONS 117 | 118 | function convert_processing_functions(fh, nm, put, pre, post) 119 | l = length(put.processFcns); 120 | 121 | if exist('pre', 'var') 122 | l = l + length(pre); 123 | end 124 | if exist('post', 'var') 125 | l = l + length(post); 126 | end 127 | 128 | if l == 0 129 | warning('Zero processing functions no longer results in linear normalization of input vectors.'); 130 | end 131 | 132 | fprintf(fh, '%sCount = %d\n', nm, l); 133 | 134 | k = 0; 135 | 136 | if exist('pre', 'var') 137 | for j = 1:length(pre) 138 | % TODO: eventually support more than just strings here 139 | fprintf(fh, '%s%d.function = %s\n', nm, k, pre{j}); 140 | k = k + 1; 141 | end 142 | end 143 | 144 | for j = 1:length(put.processFcns) 145 | switch put.processFcns{j} 146 | case 'mapminmax' 147 | offsets = sprintf('%.15g, ', put.processSettings{j}.xoffset); 148 | offsets = offsets(1:end - 2); % remove final comma 149 | gains = sprintf('%.15g, ', put.processSettings{j}.gain); 150 | gains = gains(1:end - 2); % remove final comma 151 | 152 | fprintf(fh, '%s%d.function = mapminmax\n', nm, k); 153 | fprintf(fh, '%s%d.xOffsets = %s\n', nm, k, offsets); 154 | fprintf(fh, '%s%d.gains = %s\n', nm, k, gains); 155 | fprintf(fh, '%s%d.yMin = %.15g\n', nm, k, put.processSettings{j}.ymin); 156 | 157 | case 'mapstd' 158 | offsets = sprintf('%.15g, ', put.processSettings{j}.xoffset); 159 | offsets = offsets(1:end - 2); % remove final comma 160 | gains = sprintf('%.15g, ', put.processSettings{j}.gain); 161 | gains = gains(1:end - 2); % remove final comma 162 | 163 | fprintf(fh, '%s%d.function = mapstd\n', nm, k); 164 | fprintf(fh, '%s%d.xOffsets = %s\n', nm, k, offsets); 165 | fprintf(fh, '%s%d.gains = %s\n', nm, k, gains); 166 | fprintf(fh, '%s%d.yMean = %.15g\n', nm, k, put.processSettings{j}.ymean); 167 | 168 | otherwise 169 | error('Invalid processing function: %s.', put.processFcns{j}); 170 | end 171 | 172 | k = k + 1; 173 | end 174 | 175 | if exist('post', 'var') 176 | for j = 1:length(post) 177 | % TODO: eventually support more than just strings here 178 | fprintf(fh, '%s%d.function = %s\n', nm, k, post{j}); 179 | k = k + 1; 180 | end 181 | end 182 | end 183 | 184 | function convert_layer(fh, nm, layer, w, b) 185 | if ~strcmp(layer.netInputFcn, 'netsum') 186 | error('Invalid input function: %s. Expected netsum.', layer.netInputFcn); 187 | end 188 | 189 | if strcmp(layer.transferFcn, 'tansig') 190 | tf = 'TanSig'; 191 | elseif strcmp(layer.transferFcn, 'logsig') 192 | tf = 'LogSig'; 193 | elseif strcmp(layer.transferFcn, 'purelin') 194 | tf = 'PureLin'; 195 | elseif strcmp(layer.transferFcn, 'satlin') 196 | tf = 'SatLin'; 197 | else 198 | error('Invalid transfer function: %s.', layer.transferFcn); 199 | end 200 | 201 | % have to flip weights before resizing to print row by row 202 | weights = sprintf('%.15g, ', reshape(w', [], 1)); 203 | weights = weights(1:end - 2); % remove final comma 204 | biases = sprintf('%.15g, ', b); 205 | biases = biases(1:end - 2); % remove final comma 206 | 207 | fprintf(fh, '%s.inputs = %d\n', nm, size(w, 2)); 208 | fprintf(fh, '%s.outputs = %d\n', nm, size(w, 1)); 209 | fprintf(fh, '%s.weights = %s\n', nm, weights); 210 | fprintf(fh, '%s.biases = %s\n', nm, biases); 211 | fprintf(fh, '%s.transferFunction = %s\n', nm, tf); 212 | end 213 | 214 | end 215 | --------------------------------------------------------------------------------