├── .gitattributes ├── .gitignore ├── AudioProcessor.cs ├── Example.cs └── README.md /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # Build results 11 | [Dd]ebug/ 12 | [Dd]ebugPublic/ 13 | [Rr]elease/ 14 | [Rr]eleases/ 15 | x64/ 16 | x86/ 17 | build/ 18 | bld/ 19 | [Bb]in/ 20 | [Oo]bj/ 21 | 22 | # Roslyn cache directories 23 | *.ide/ 24 | 25 | # MSTest test Results 26 | [Tt]est[Rr]esult*/ 27 | [Bb]uild[Ll]og.* 28 | 29 | #NUNIT 30 | *.VisualState.xml 31 | TestResult.xml 32 | 33 | # Build Results of an ATL Project 34 | [Dd]ebugPS/ 35 | [Rr]eleasePS/ 36 | dlldata.c 37 | 38 | *_i.c 39 | *_p.c 40 | *_i.h 41 | *.ilk 42 | *.meta 43 | *.obj 44 | *.pch 45 | *.pdb 46 | *.pgc 47 | *.pgd 48 | *.rsp 49 | *.sbr 50 | *.tlb 51 | *.tli 52 | *.tlh 53 | *.tmp 54 | *.tmp_proj 55 | *.log 56 | *.vspscc 57 | *.vssscc 58 | .builds 59 | *.pidb 60 | *.svclog 61 | *.scc 62 | 63 | # Chutzpah Test files 64 | _Chutzpah* 65 | 66 | # Visual C++ cache files 67 | ipch/ 68 | *.aps 69 | *.ncb 70 | *.opensdf 71 | *.sdf 72 | *.cachefile 73 | 74 | # Visual Studio profiler 75 | *.psess 76 | *.vsp 77 | *.vspx 78 | 79 | # TFS 2012 Local Workspace 80 | $tf/ 81 | 82 | # Guidance Automation Toolkit 83 | *.gpState 84 | 85 | # ReSharper is a .NET coding add-in 86 | _ReSharper*/ 87 | *.[Rr]e[Ss]harper 88 | *.DotSettings.user 89 | 90 | # JustCode is a .NET coding addin-in 91 | .JustCode 92 | 93 | # TeamCity is a build add-in 94 | _TeamCity* 95 | 96 | # DotCover is a Code Coverage Tool 97 | *.dotCover 98 | 99 | # NCrunch 100 | _NCrunch_* 101 | .*crunch*.local.xml 102 | 103 | # MightyMoose 104 | *.mm.* 105 | AutoTest.Net/ 106 | 107 | # Web workbench (sass) 108 | .sass-cache/ 109 | 110 | # Installshield output folder 111 | [Ee]xpress/ 112 | 113 | # DocProject is a documentation generator add-in 114 | DocProject/buildhelp/ 115 | DocProject/Help/*.HxT 116 | DocProject/Help/*.HxC 117 | DocProject/Help/*.hhc 118 | DocProject/Help/*.hhk 119 | DocProject/Help/*.hhp 120 | DocProject/Help/Html2 121 | DocProject/Help/html 122 | 123 | # Click-Once directory 124 | publish/ 125 | 126 | # Publish Web Output 127 | *.[Pp]ublish.xml 128 | *.azurePubxml 129 | # TODO: Comment the next line if you want to checkin your web deploy settings 130 | # but database connection strings (with potential passwords) will be unencrypted 131 | *.pubxml 132 | *.publishproj 133 | 134 | # NuGet Packages 135 | *.nupkg 136 | # The packages folder can be ignored because of Package Restore 137 | **/packages/* 138 | # except build/, which is used as an MSBuild target. 139 | !**/packages/build/ 140 | # If using the old MSBuild-Integrated Package Restore, uncomment this: 141 | #!**/packages/repositories.config 142 | 143 | # Windows Azure Build Output 144 | csx/ 145 | *.build.csdef 146 | 147 | # Windows Store app package directory 148 | AppPackages/ 149 | 150 | # Others 151 | sql/ 152 | *.Cache 153 | ClientBin/ 154 | [Ss]tyle[Cc]op.* 155 | ~$* 156 | *~ 157 | *.dbmdl 158 | *.dbproj.schemaview 159 | *.pfx 160 | *.publishsettings 161 | node_modules/ 162 | 163 | # RIA/Silverlight projects 164 | Generated_Code/ 165 | 166 | # Backup & report files from converting an old project file 167 | # to a newer Visual Studio version. Backup files are not needed, 168 | # because we have git ;-) 169 | _UpgradeReport_Files/ 170 | Backup*/ 171 | UpgradeLog*.XML 172 | UpgradeLog*.htm 173 | 174 | # SQL Server files 175 | *.mdf 176 | *.ldf 177 | 178 | # Business Intelligence projects 179 | *.rdl.data 180 | *.bim.layout 181 | *.bim_*.settings 182 | 183 | # Microsoft Fakes 184 | FakesAssemblies/ 185 | 186 | # ========================= 187 | # Operating System Files 188 | # ========================= 189 | 190 | # OSX 191 | # ========================= 192 | 193 | .DS_Store 194 | .AppleDouble 195 | .LSOverride 196 | 197 | # Thumbnails 198 | ._* 199 | 200 | # Files that might appear on external disk 201 | .Spotlight-V100 202 | .Trashes 203 | 204 | # Directories potentially created on remote AFP share 205 | .AppleDB 206 | .AppleDesktop 207 | Network Trash Folder 208 | Temporary Items 209 | .apdisk 210 | 211 | # Windows 212 | # ========================= 213 | 214 | # Windows image file caches 215 | Thumbs.db 216 | ehthumbs.db 217 | 218 | # Folder config file 219 | Desktop.ini 220 | 221 | # Recycle Bin used on file shares 222 | $RECYCLE.BIN/ 223 | 224 | # Windows Installer files 225 | *.cab 226 | *.msi 227 | *.msm 228 | *.msp 229 | 230 | # Windows shortcuts 231 | *.lnk 232 | -------------------------------------------------------------------------------- /AudioProcessor.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015 Allan Pichardo 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | using UnityEngine; 18 | using System.Collections; 19 | using System.Collections.Generic; 20 | 21 | [RequireComponent (typeof(AudioSource))] 22 | public class AudioProcessor : MonoBehaviour 23 | { 24 | public AudioSource audioSource; 25 | 26 | private long lastT, nowT, diff, entries, sum; 27 | 28 | public int bufferSize = 1024; 29 | // fft size 30 | private int samplingRate = 44100; 31 | // fft sampling frequency 32 | 33 | /* log-frequency averaging controls */ 34 | private int nBand = 12; 35 | // number of bands 36 | 37 | public float gThresh = 0.1f; 38 | // sensitivity 39 | 40 | int blipDelayLen = 16; 41 | int[] blipDelay; 42 | 43 | private int sinceLast = 0; 44 | // counter to suppress double-beats 45 | 46 | private float framePeriod; 47 | 48 | /* storage space */ 49 | private int colmax = 120; 50 | float[] spectrum; 51 | float[] averages; 52 | float[] acVals; 53 | float[] onsets; 54 | float[] scorefun; 55 | float[] dobeat; 56 | int now = 0; 57 | // time index for circular buffer within above 58 | 59 | float[] spec; 60 | // the spectrum of the previous step 61 | 62 | /* Autocorrelation structure */ 63 | int maxlag = 100; 64 | // (in frames) largest lag to track 65 | float decay = 0.997f; 66 | // smoothing constant for running average 67 | Autoco auco; 68 | 69 | private float alph; 70 | // trade-off constant between tempo deviation penalty and onset strength 71 | 72 | [Header ("Events")] 73 | public OnBeatEventHandler onBeat; 74 | public OnSpectrumEventHandler onSpectrum; 75 | 76 | ////////////////////////////////// 77 | private long getCurrentTimeMillis () 78 | { 79 | long milliseconds = System.DateTime.Now.Ticks / System.TimeSpan.TicksPerMillisecond; 80 | return milliseconds; 81 | } 82 | 83 | private void initArrays () 84 | { 85 | blipDelay = new int[blipDelayLen]; 86 | onsets = new float[colmax]; 87 | scorefun = new float[colmax]; 88 | dobeat = new float[colmax]; 89 | spectrum = new float[bufferSize]; 90 | averages = new float[12]; 91 | acVals = new float[maxlag]; 92 | alph = 100 * gThresh; 93 | } 94 | 95 | // Use this for initialization 96 | void Start () 97 | { 98 | initArrays (); 99 | 100 | audioSource = GetComponent (); 101 | samplingRate = audioSource.clip.frequency; 102 | 103 | framePeriod = (float)bufferSize / (float)samplingRate; 104 | 105 | //initialize record of previous spectrum 106 | spec = new float[nBand]; 107 | for (int i = 0; i < nBand; ++i) 108 | spec [i] = 100.0f; 109 | 110 | auco = new Autoco (maxlag, decay, framePeriod, getBandWidth ()); 111 | 112 | lastT = getCurrentTimeMillis (); 113 | } 114 | 115 | public void tapTempo () 116 | { 117 | nowT = getCurrentTimeMillis (); 118 | diff = nowT - lastT; 119 | lastT = nowT; 120 | sum = sum + diff; 121 | entries++; 122 | 123 | int average = (int)(sum / entries); 124 | 125 | Debug.Log ("average = " + average); 126 | } 127 | 128 | double[] toDoubleArray (float[] arr) 129 | { 130 | if (arr == null) 131 | return null; 132 | int n = arr.Length; 133 | double[] ret = new double[n]; 134 | for (int i = 0; i < n; i++) { 135 | ret [i] = (float)arr [i]; 136 | } 137 | return ret; 138 | } 139 | 140 | // Update is called once per frame 141 | void Update () 142 | { 143 | if (audioSource.isPlaying) { 144 | audioSource.GetSpectrumData (spectrum, 0, FFTWindow.BlackmanHarris); 145 | computeAverages (spectrum); 146 | onSpectrum.Invoke (averages); 147 | 148 | /* calculate the value of the onset function in this frame */ 149 | float onset = 0; 150 | for (int i = 0; i < nBand; i++) { 151 | float specVal = (float)System.Math.Max (-100.0f, 20.0f * (float)System.Math.Log10 (averages [i]) + 160); // dB value of this band 152 | specVal *= 0.025f; 153 | float dbInc = specVal - spec [i]; // dB increment since last frame 154 | spec [i] = specVal; // record this frome to use next time around 155 | onset += dbInc; // onset function is the sum of dB increments 156 | } 157 | 158 | onsets [now] = onset; 159 | 160 | /* update autocorrelator and find peak lag = current tempo */ 161 | auco.newVal (onset); 162 | // record largest value in (weighted) autocorrelation as it will be the tempo 163 | float aMax = 0.0f; 164 | int tempopd = 0; 165 | //float[] acVals = new float[maxlag]; 166 | for (int i = 0; i < maxlag; ++i) { 167 | float acVal = (float)System.Math.Sqrt (auco.autoco (i)); 168 | if (acVal > aMax) { 169 | aMax = acVal; 170 | tempopd = i; 171 | } 172 | // store in array backwards, so it displays right-to-left, in line with traces 173 | acVals [maxlag - 1 - i] = acVal; 174 | } 175 | 176 | /* calculate DP-ish function to update the best-score function */ 177 | float smax = -999999; 178 | int smaxix = 0; 179 | // weight can be varied dynamically with the mouse 180 | alph = 100 * gThresh; 181 | // consider all possible preceding beat times from 0.5 to 2.0 x current tempo period 182 | for (int i = tempopd / 2; i < System.Math.Min (colmax, 2 * tempopd); ++i) { 183 | // objective function - this beat's cost + score to last beat + transition penalty 184 | float score = onset + scorefun [(now - i + colmax) % colmax] - alph * (float)System.Math.Pow (System.Math.Log ((float)i / (float)tempopd), 2); 185 | // keep track of the best-scoring predecesor 186 | if (score > smax) { 187 | smax = score; 188 | smaxix = i; 189 | } 190 | } 191 | 192 | scorefun [now] = smax; 193 | // keep the smallest value in the score fn window as zero, by subtracing the min val 194 | float smin = scorefun [0]; 195 | for (int i = 0; i < colmax; ++i) 196 | if (scorefun [i] < smin) 197 | smin = scorefun [i]; 198 | for (int i = 0; i < colmax; ++i) 199 | scorefun [i] -= smin; 200 | 201 | /* find the largest value in the score fn window, to decide if we emit a blip */ 202 | smax = scorefun [0]; 203 | smaxix = 0; 204 | for (int i = 0; i < colmax; ++i) { 205 | if (scorefun [i] > smax) { 206 | smax = scorefun [i]; 207 | smaxix = i; 208 | } 209 | } 210 | 211 | // dobeat array records where we actally place beats 212 | dobeat [now] = 0; // default is no beat this frame 213 | ++sinceLast; 214 | // if current value is largest in the array, probably means we're on a beat 215 | if (smaxix == now) { 216 | //tapTempo(); 217 | // make sure the most recent beat wasn't too recently 218 | if (sinceLast > tempopd / 4) { 219 | onBeat.Invoke (); 220 | blipDelay [0] = 1; 221 | // record that we did actually mark a beat this frame 222 | dobeat [now] = 1; 223 | // reset counter of frames since last beat 224 | sinceLast = 0; 225 | } 226 | } 227 | 228 | /* update column index (for ring buffer) */ 229 | if (++now == colmax) 230 | now = 0; 231 | 232 | //Debug.Log(System.Math.Round(60 / (tempopd * framePeriod)) + " bpm"); 233 | //Debug.Log(System.Math.Round(auco.avgBpm()) + " bpm"); 234 | } 235 | } 236 | 237 | public void changeCameraColor () 238 | { 239 | //Debug.Log("camera"); 240 | float r = Random.Range (0f, 1f); 241 | float g = Random.Range (0f, 1f); 242 | float b = Random.Range (0f, 1f); 243 | 244 | //Debug.Log(r + "," + g + "," + b); 245 | Color color = new Color (r, g, b); 246 | 247 | GetComponent ().clearFlags = CameraClearFlags.Color; 248 | Camera.main.backgroundColor = color; 249 | 250 | //camera.backgroundColor = color; 251 | } 252 | 253 | public float getBandWidth () 254 | { 255 | return (2f / (float)bufferSize) * (samplingRate / 2f); 256 | } 257 | 258 | public int freqToIndex (int freq) 259 | { 260 | // special case: freq is lower than the bandwidth of spectrum[0] 261 | if (freq < getBandWidth () / 2) 262 | return 0; 263 | // special case: freq is within the bandwidth of spectrum[512] 264 | if (freq > samplingRate / 2 - getBandWidth () / 2) 265 | return (bufferSize / 2); 266 | // all other cases 267 | float fraction = (float)freq / (float)samplingRate; 268 | int i = (int)System.Math.Round (bufferSize * fraction); 269 | //Debug.Log("frequency: " + freq + ", index: " + i); 270 | return i; 271 | } 272 | 273 | public void computeAverages (float[] data) 274 | { 275 | for (int i = 0; i < 12; i++) { 276 | float avg = 0; 277 | int lowFreq; 278 | if (i == 0) 279 | lowFreq = 0; 280 | else 281 | lowFreq = (int)((samplingRate / 2) / (float)System.Math.Pow (2, 12 - i)); 282 | int hiFreq = (int)((samplingRate / 2) / (float)System.Math.Pow (2, 11 - i)); 283 | int lowBound = freqToIndex (lowFreq); 284 | int hiBound = freqToIndex (hiFreq); 285 | for (int j = lowBound; j <= hiBound; j++) { 286 | //Debug.Log("lowbound: " + lowBound + ", highbound: " + hiBound); 287 | avg += data [j]; 288 | } 289 | // line has been changed since discussion in the comments 290 | // avg /= (hiBound - lowBound); 291 | avg /= (hiBound - lowBound + 1); 292 | averages [i] = avg; 293 | } 294 | } 295 | 296 | float map (float s, float a1, float a2, float b1, float b2) 297 | { 298 | return b1 + (s - a1) * (b2 - b1) / (a2 - a1); 299 | } 300 | 301 | public float constrain (float value, float inclusiveMinimum, float inlusiveMaximum) 302 | { 303 | if (value >= inclusiveMinimum) { 304 | if (value <= inlusiveMaximum) { 305 | return value; 306 | } 307 | 308 | return inlusiveMaximum; 309 | } 310 | 311 | return inclusiveMinimum; 312 | } 313 | 314 | [System.Serializable] 315 | public class OnBeatEventHandler : UnityEngine.Events.UnityEvent 316 | { 317 | 318 | } 319 | 320 | [System.Serializable] 321 | public class OnSpectrumEventHandler : UnityEngine.Events.UnityEvent 322 | { 323 | 324 | } 325 | 326 | // class to compute an array of online autocorrelators 327 | private class Autoco 328 | { 329 | private int del_length; 330 | private float decay; 331 | private float[] delays; 332 | private float[] outputs; 333 | private int indx; 334 | 335 | private float[] bpms; 336 | private float[] rweight; 337 | private float wmidbpm = 120f; 338 | private float woctavewidth; 339 | 340 | public Autoco (int len, float alpha, float framePeriod, float bandwidth) 341 | { 342 | woctavewidth = bandwidth; 343 | decay = alpha; 344 | del_length = len; 345 | delays = new float[del_length]; 346 | outputs = new float[del_length]; 347 | indx = 0; 348 | 349 | // calculate a log-lag gaussian weighting function, to prefer tempi around 120 bpm 350 | bpms = new float[del_length]; 351 | rweight = new float[del_length]; 352 | for (int i = 0; i < del_length; ++i) { 353 | bpms [i] = 60.0f / (framePeriod * (float)i); 354 | //Debug.Log(bpms[i]); 355 | // weighting is Gaussian on log-BPM axis, centered at wmidbpm, SD = woctavewidth octaves 356 | rweight [i] = (float)System.Math.Exp (-0.5f * System.Math.Pow (System.Math.Log (bpms [i] / wmidbpm) / System.Math.Log (2.0f) / woctavewidth, 2.0f)); 357 | } 358 | } 359 | 360 | public void newVal (float val) 361 | { 362 | 363 | delays [indx] = val; 364 | 365 | // update running autocorrelator values 366 | for (int i = 0; i < del_length; ++i) { 367 | int delix = (indx - i + del_length) % del_length; 368 | outputs [i] += (1 - decay) * (delays [indx] * delays [delix] - outputs [i]); 369 | } 370 | 371 | if (++indx == del_length) 372 | indx = 0; 373 | } 374 | 375 | // read back the current autocorrelator value at a particular lag 376 | public float autoco (int del) 377 | { 378 | float blah = rweight [del] * outputs [del]; 379 | return blah; 380 | } 381 | 382 | public float avgBpm () 383 | { 384 | float sum = 0; 385 | for (int i = 0; i < bpms.Length; ++i) { 386 | sum += bpms [i]; 387 | } 388 | return sum / del_length; 389 | } 390 | } 391 | } 392 | 393 | 394 | -------------------------------------------------------------------------------- /Example.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015 Allan Pichardo 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | using UnityEngine; 18 | using System; 19 | 20 | public class Example : MonoBehaviour 21 | { 22 | 23 | void Start () 24 | { 25 | //Select the instance of AudioProcessor and pass a reference 26 | //to this object 27 | AudioProcessor processor = FindObjectOfType (); 28 | processor.onBeat.AddListener (onOnbeatDetected); 29 | processor.onSpectrum.AddListener (onSpectrum); 30 | } 31 | 32 | //this event will be called every time a beat is detected. 33 | //Change the threshold parameter in the inspector 34 | //to adjust the sensitivity 35 | void onOnbeatDetected () 36 | { 37 | Debug.Log ("Beat!!!"); 38 | } 39 | 40 | //This event will be called every frame while music is playing 41 | void onSpectrum (float[] spectrum) 42 | { 43 | //The spectrum is logarithmically averaged 44 | //to 12 bands 45 | 46 | for (int i = 0; i < spectrum.Length; ++i) { 47 | Vector3 start = new Vector3 (i, 0, 0); 48 | Vector3 end = new Vector3 (i, spectrum [i], 0); 49 | Debug.DrawLine (start, end); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Unity-Beat-Detection 2 | Musical beat detection and audio spectrum analysis for use with the Unity game engine. 3 | 4 | The AudioProcessor class contains an interface that can be implemented on your GameObject. 5 | 6 |

Usage

7 | Add the AudioProcessor script to your Main Camera object and 8 | adjust the threshold parameter to change the sensitivity. 9 | Then set a callback delegate on the audio processor's onBeat or onSpectrum events. 10 | 11 | ```c# 12 | public class Example : MonoBehaviour 13 | { 14 | 15 | void Start () 16 | { 17 | //Select the instance of AudioProcessor and pass a reference 18 | //to this object 19 | AudioProcessor processor = FindObjectOfType (); 20 | processor.onBeat.AddListener (onOnbeatDetected); 21 | processor.onSpectrum.AddListener (onSpectrum); 22 | } 23 | 24 | //this event will be called every time a beat is detected. 25 | //Change the threshold parameter in the inspector 26 | //to adjust the sensitivity 27 | void onOnbeatDetected () 28 | { 29 | Debug.Log ("Beat!!!"); 30 | } 31 | 32 | //This event will be called every frame while music is playing 33 | void onSpectrum (float[] spectrum) 34 | { 35 | //The spectrum is logarithmically averaged 36 | //to 12 bands 37 | 38 | for (int i = 0; i < spectrum.Length; ++i) { 39 | Vector3 start = new Vector3 (i, 0, 0); 40 | Vector3 end = new Vector3 (i, spectrum [i], 0); 41 | Debug.DrawLine (start, end); 42 | } 43 | } 44 | } 45 | ``` 46 | --------------------------------------------------------------------------------