├── .gitignore ├── LICENSE ├── README.md ├── analyser.js ├── examples ├── beats.js ├── cepstrum.js └── pitch.js ├── index.js ├── microphone.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | *.seed 3 | *.log 4 | *.csv 5 | *.dat 6 | *.out 7 | *.pid 8 | *.gz 9 | 10 | pids 11 | logs 12 | results 13 | 14 | npm-debug.log 15 | node_modules/* 16 | *.DS_Store -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2016 Mikola Lysenko 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # regl-audio 2 | Tools for working with audio in [regl](https://github.com/mikolalysenko/regl). This module has the following components: 3 | 4 | * An analyser 5 | * A microphone connection 6 | 7 | Examples: 8 | 9 | * [Beat detection](https://mikolalysenko.github.io/regl-audio/beats.html) 10 | * [Cepstrum](https://mikolalysenko.github.io/regl-audio/cepstrum.html) 11 | * [Pitch detection](https://mikolalysenko.github.io/regl-audio/pitch.html) 12 | 13 | ## Examples 14 | Here is a simple beat detector: 15 | 16 | ```javascript 17 | const regl = require('regl')() 18 | 19 | const drawBeats = regl({ 20 | vert: ` 21 | precision highp float; 22 | 23 | attribute vec2 position; 24 | varying vec2 uv; 25 | 26 | void main () { 27 | uv = position; 28 | gl_Position = vec4(position, 0, 1); 29 | } 30 | `, 31 | 32 | frag: ` 33 | precision highp float; 34 | 35 | varying vec2 uv; 36 | 37 | uniform float beats[16]; 38 | 39 | void main () { 40 | float intensity = 0.0; 41 | 42 | float bin = floor(8.0 * (1.0 + uv.x)); 43 | 44 | for (int i = 0; i < 16; ++i) { 45 | if (abs(float(i) - bin) < 0.25) { 46 | intensity += step(0.25 * abs(uv.y), beats[i]); 47 | } 48 | } 49 | gl_FragColor = vec4(intensity, 0, 0, 1); 50 | } 51 | `, 52 | 53 | attributes: { 54 | position: [ 55 | -4, 0, 56 | 4, 4, 57 | 4, -4 58 | ] 59 | }, 60 | 61 | count: 3 62 | }) 63 | 64 | require('regl-audio/microphone')({ 65 | regl, 66 | beats: 16, 67 | name: '', 68 | done: (microphone) => { 69 | regl.frame(() => { 70 | microphone(({beats}) => { 71 | drawBeats() 72 | }) 73 | }) 74 | } 75 | }) 76 | ``` 77 | 78 | ## regl-audio/analyser 79 | This module takes a WebAudio analyser node and returns a scope command giving convenient access to stats from the analyser. 80 | 81 | * PCM time domain data 82 | * STFT frequency domain data 83 | * Beats 84 | * Pitch 85 | 86 | ### API 87 | 88 | #### `const audio = require('regl-audio/analyser')(options)` 89 | The constructor for the analyser takes the following arguments: 90 | 91 | | Parameter | Description | Default | 92 | |-----------|-------------|---------| 93 | | `regl` | A handle to a regl instance | *Required* | 94 | | `analyser` | A WebAudio [analyser node](https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode) | *Required* | 95 | | `name` | A prefix for the analyser output. | `''` | 96 | | `sampleRate` | The sample rate of the audio source in Hz | `44100` | 97 | | `beats` | The number of beats to detect grouped by pitch | `16` | 98 | | `beatTime` | Duration of moving average for beats in seconds | `1` | 99 | | `beatThreshold` | Cutoff for beat detection (must be between 0.5 and 1) | `0.8` | 100 | | `pitches` | Number of pitches to detect | `4` | 101 | | `maxPitch` | Maximum detectable pitch in Hz | `10000` | 102 | | `pitchTime` | Duration of moving average for pitch in seconds | `0.25` | 103 | 104 | #### `audio(block)` 105 | The result is a `regl` scope command with the following properties: 106 | 107 | | Context | Description | 108 | |---------|-------------| 109 | | `sampleCount` | Number of samples | 110 | | `freq` | Array of frequencies | 111 | | `time` | Array of PCM time samples | 112 | | `cepstrum` | The cepstrum of the signal | 113 | | `timeTexture` | Current time information in texture | 114 | | `freqTexture` | Current frequency information in texture | 115 | | `volume` | Volume of the current signal | 116 | | `beats` | Array of detected beats sorted from low to high pitch. Each beat is a scalar in `[0, 1]` | 117 | | `pitches` | Array of detected pitches sorted from loudest to softest in Hz | 118 | 119 | | Uniform | Type | Description | 120 | |---------|------|-------------| 121 | | `sampleCount` | `float` | Number of samples in texture | 122 | | `time` | `sampler2D` | A sampler storing the PCM time data | 123 | | `freq` | `sampler2D` | A sampler storing the frequency data | 124 | | `volume` | `float` | Volume of the signal | 125 | | `beats` | `float[NUM_PITCHES]` | An array of beats | 126 | | `pitches` | `float[NUM_PITCHES]` | The array of detected pitches | 127 | 128 | Note that these context variables are optionally prefixed depending on the `name` parameter. 129 | 130 | ## regl-audio/microphone 131 | A short cut which gives you an analyser node connected to the microphone input from the computer. Note that this must be run on a secure domain. 132 | 133 | ### API 134 | 135 | #### `const mic = require('regl-audio/microphone')(options)` 136 | The options are the same as above, except that it takes a webaudio context via the `options.audioContext` parameter instead of an analyser node. By default everything is prefixed with `mic_` though this can be changed by passing some alternative to `name`. 137 | 138 | # License 139 | (c) 2016 Mikola Lysenko. MIT License 140 | -------------------------------------------------------------------------------- /analyser.js: -------------------------------------------------------------------------------- 1 | const fourierTransform = require('fourier-transform') 2 | 3 | function compareInt (a, b) { 4 | return a - b 5 | } 6 | 7 | function applyPrefix (prefix, object) { 8 | const result = {} 9 | Object.keys(object).forEach((name) => { 10 | result[prefix + name] = object[name] 11 | }) 12 | return result 13 | } 14 | 15 | module.exports = function (options) { 16 | // namespace 17 | const name = options.name || '' 18 | 19 | // 20 | // basic inputs 21 | // 22 | // regl instance 23 | const regl = options.regl 24 | // web audio analyser node 25 | const analyser = options.analyser 26 | // sample rate in Hz 27 | const sampleRate = options.sampleRate || 44100 28 | 29 | // 30 | // beat detection 31 | // 32 | // number of beat detection bins (set to 0 to disable) 33 | const binCount = 'beats' in options ? options.beats : 16 34 | // length of moving window in seconds 35 | const beatTime = options.beatTime || 1.0 36 | // strictness of beat detection (should be between 0.5 and 1) 37 | const beatThreshold = options.beatThreshold || 0.8 38 | 39 | // 40 | // pitch detection 41 | // 42 | // number of pitches to detect (set to 0 to disable) 43 | const pitchCount = 'pitches' in options ? options.pitches : 4 44 | // minimum detectable pitch in Hz 45 | const maxPitch = (options.maxPitch || 10000) 46 | // length of moving pitch window in seconds 47 | const pitchTime = (options.pitchTime || 0.25) 48 | 49 | // ------------------------------------------- 50 | // implementation stuff 51 | // ------------------------------------------- 52 | const prefix = name ? name + '_' : '' 53 | 54 | const N = analyser.frequencyBinCount 55 | 56 | const freq = new Uint8Array(N) 57 | const time = new Uint8Array(N) 58 | const cepstrum = new Float64Array(N / 2) 59 | 60 | const freqTexture = regl.texture({ 61 | shape: [N, 1, 1], 62 | type: 'uint8' 63 | }) 64 | 65 | const timeTexture = regl.texture({ 66 | shape: [N, 1, 1], 67 | type: 'uint8' 68 | }) 69 | 70 | const uniforms = { 71 | sampleCount: N, 72 | freq: freqTexture, 73 | time: timeTexture, 74 | volume: regl.prop('volume') 75 | } 76 | 77 | for (let i = 0; i < binCount; ++i) { 78 | uniforms['beats[' + i + ']'] = regl.prop('beats[' + i + ']') 79 | } 80 | 81 | for (let i = 0; i < pitchCount; ++i) { 82 | uniforms['pitches[' + i + ']'] = regl.prop('pitches[' + i + ']') 83 | } 84 | 85 | const setupAnalysis = regl({ 86 | context: applyPrefix(prefix, { 87 | sampleCount: N, 88 | freq, 89 | time, 90 | cepstrum: cepstrum, 91 | freqTexture, 92 | timeTexture, 93 | volume: regl.prop('volume'), 94 | beats: regl.prop('beats'), 95 | pitches: regl.prop('pitches') 96 | }), 97 | uniforms: applyPrefix(prefix, uniforms) 98 | }) 99 | 100 | const binSize = Math.floor(N / binCount) | 0 101 | const beatBufferSize = Math.ceil(beatTime * sampleRate / N) | 0 102 | 103 | const volumeHistory = new Float64Array(binCount * beatBufferSize) 104 | const beats = Array(binCount).fill(0) 105 | 106 | const medianArray = Array(beatBufferSize).fill(0) 107 | const cutoffIndex = Math.floor((beatBufferSize - 1) * beatThreshold) | 0 108 | 109 | let beatPtr = 0 110 | let volume = 0 111 | function estimateBeats () { 112 | let vol = 0.0 113 | for (var i = 0; i < binCount; ++i) { 114 | for (let j = i * beatBufferSize, k = 0; k < beatBufferSize; ++j, ++k) { 115 | medianArray[k] = volumeHistory[j] 116 | } 117 | medianArray.sort(compareInt) 118 | 119 | let sum = 0.0 120 | for (let j = binSize * i; j < binSize * (i + 1); ++j) { 121 | const x = Math.pow(freq[j] / 255.0, 2.0) 122 | sum += x 123 | vol += x 124 | } 125 | sum = Math.sqrt(sum) / binSize 126 | beats[i] = sum > medianArray[cutoffIndex] ? sum : 0 127 | volumeHistory[beatPtr + beatBufferSize * i] = sum 128 | } 129 | volume = Math.sqrt(volume) / N 130 | beatPtr = (beatPtr + 1) % beatBufferSize 131 | } 132 | 133 | const startQ = Math.min(Math.ceil(sampleRate / maxPitch), N / 2) | 0 134 | const endQ = (N / 2) | 0 135 | 136 | const pitches = Array(pitchCount).fill(0) 137 | 138 | const pitchWindow = Math.ceil(pitchTime * sampleRate / N) | 0 139 | const pitchHistogram = new Float64Array(N) 140 | const pitchHistory = new Float64Array(pitchWindow * pitchCount) 141 | let pitchWindowPtr = 0 142 | 143 | const pitchIndex = Array(N).fill(0).map(function (_, i) { 144 | return i 145 | }) 146 | function comparePitch (a, b) { 147 | return pitchHistogram[b] - pitchHistogram[a] 148 | } 149 | 150 | const pitchQ = Array(pitchCount).fill(0) 151 | const pitchW = Array(pitchCount).fill(0) 152 | 153 | const logFreq = new Float64Array(N) 154 | 155 | function estimatePitch () { 156 | for (let i = 0; i < N; ++i) { 157 | logFreq[i] = Math.log(1 + freq[i]) 158 | } 159 | fourierTransform(logFreq, cepstrum) 160 | 161 | for (let i = 0; i < pitchCount; ++i) { 162 | pitchQ[i] = 0 163 | pitchW[i] = -Infinity 164 | } 165 | 166 | for (let i = startQ; i < endQ; ++i) { 167 | const a = cepstrum[i - 1] 168 | const b = cepstrum[i] 169 | const c = cepstrum[i + 1] 170 | 171 | if (b > a && b > c) { 172 | for (let j = 0; j < pitchCount; ++j) { 173 | if (pitchW[j] < b) { 174 | for (let k = pitchCount - 1; k > j; --k) { 175 | pitchQ[k] = pitchQ[k - 1] 176 | pitchW[k] = pitchW[k - 1] 177 | } 178 | pitchQ[j] = i 179 | pitchW[j] = b 180 | break 181 | } 182 | } 183 | } 184 | } 185 | 186 | // Update histogram 187 | for (let j = 0; j < pitchCount; ++j) { 188 | const w = 1.0 / (1.0 + j) 189 | if (pitchQ[j]) { 190 | pitchHistogram[pitchQ[j]] += w 191 | } 192 | var prev = pitchHistory[pitchWindowPtr] 193 | if (prev) { 194 | pitchHistogram[prev] -= w 195 | } 196 | pitchHistory[pitchWindowPtr] = pitchQ[j] 197 | pitchWindowPtr = (pitchWindowPtr + 1) % pitchHistory.length 198 | } 199 | 200 | // Take top k pitch values for current pitch 201 | // FIXME: should use heap or insertion sort here 202 | pitchIndex.sort(comparePitch) 203 | 204 | for (let j = 0; j < pitchCount; ++j) { 205 | if (pitchIndex[j]) { 206 | pitches[j] = sampleRate / pitchIndex[j] 207 | } else { 208 | for (; j < pitchCount; ++j) { 209 | pitches[j] = 0 210 | } 211 | } 212 | } 213 | } 214 | 215 | return function (block) { 216 | // poll analyser 217 | analyser.getByteFrequencyData(freq) 218 | analyser.getByteTimeDomainData(time) 219 | 220 | // upload texture data 221 | freqTexture.subimage(freq) 222 | timeTexture.subimage(time) 223 | 224 | // update beat detection 225 | if (binCount) { 226 | estimateBeats() 227 | } 228 | 229 | // update pitch detection 230 | if (pitchCount) { 231 | estimatePitch() 232 | } 233 | 234 | setupAnalysis({ 235 | volume, 236 | beats, 237 | pitches 238 | }, 239 | block) 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /examples/beats.js: -------------------------------------------------------------------------------- 1 | const regl = require('regl')() 2 | 3 | const drawBeats = regl({ 4 | vert: ` 5 | precision highp float; 6 | 7 | attribute vec2 position; 8 | varying vec2 uv; 9 | 10 | void main () { 11 | uv = position; 12 | gl_Position = vec4(position, 0, 1); 13 | } 14 | `, 15 | 16 | frag: ` 17 | precision highp float; 18 | 19 | varying vec2 uv; 20 | 21 | uniform float beats[16]; 22 | 23 | void main () { 24 | float intensity = 0.0; 25 | 26 | float bin = floor(8.0 * (1.0 + uv.x)); 27 | 28 | for (int i = 0; i < 16; ++i) { 29 | if (abs(float(i) - bin) < 0.25) { 30 | intensity += step(0.25 * abs(uv.y), beats[i]); 31 | } 32 | } 33 | gl_FragColor = vec4(intensity, 0, 0, 1); 34 | } 35 | `, 36 | 37 | attributes: { 38 | position: [ 39 | -4, 0, 40 | 4, 4, 41 | 4, -4 42 | ] 43 | }, 44 | 45 | count: 3 46 | }) 47 | 48 | require('../microphone')({ 49 | regl, 50 | beats: 16, 51 | name: '', 52 | done: (microphone) => { 53 | regl.frame(() => { 54 | microphone(({beats}) => { 55 | drawBeats() 56 | }) 57 | }) 58 | } 59 | }) 60 | -------------------------------------------------------------------------------- /examples/cepstrum.js: -------------------------------------------------------------------------------- 1 | const regl = require('regl')() 2 | 3 | const drawCepstrum = regl({ 4 | vert: ` 5 | precision highp float; 6 | 7 | attribute float cep, que; 8 | 9 | void main () { 10 | gl_Position = vec4(cep, que, 0, 1); 11 | } 12 | `, 13 | 14 | frag: ` 15 | void main () { 16 | gl_FragColor = vec4(1, 1, 1, 1); 17 | } 18 | `, 19 | 20 | attributes: { 21 | cep: Array(512).fill(0).map((_, i) => i / 256 - 1.0), 22 | que: ({cepstrum}) => new Float32Array(cepstrum) 23 | }, 24 | 25 | count: ({sampleCount}) => sampleCount / 2, 26 | primitive: 'line strip' 27 | }) 28 | 29 | require('../microphone')({ 30 | regl, 31 | beats: 16, 32 | name: '', 33 | done: (microphone) => { 34 | regl.frame(() => { 35 | microphone(({beats}) => { 36 | regl.clear({ 37 | color: [0, 0, 0, 1] 38 | }) 39 | drawCepstrum() 40 | }) 41 | }) 42 | } 43 | }) 44 | -------------------------------------------------------------------------------- /examples/pitch.js: -------------------------------------------------------------------------------- 1 | const regl = require('regl')({ 2 | attributes: { 3 | preserveDrawingBuffer: true 4 | } 5 | }) 6 | 7 | const drawPitch = regl({ 8 | vert: ` 9 | precision highp float; 10 | attribute float pitch; 11 | uniform float column; 12 | void main () { 13 | gl_PointSize = 8.0; 14 | gl_Position = vec4(column, pitch / 5000.0 - 0.9, 0, 1); 15 | } 16 | `, 17 | 18 | frag: ` 19 | void main () { 20 | gl_FragColor = vec4(1, 1, 1, 1); 21 | } 22 | `, 23 | 24 | attributes: { 25 | pitch: regl.context('pitches') 26 | }, 27 | 28 | uniforms: { 29 | column: ({tick, viewportWidth}) => 2.0 * (tick % viewportWidth) / viewportWidth - 1.0 30 | }, 31 | 32 | count: 1, 33 | depth: {enable: false}, 34 | primitive: 'points' 35 | }) 36 | 37 | require('../microphone')({ 38 | regl, 39 | beats: 0, 40 | pitches: 4, 41 | name: '', 42 | done: (microphone) => { 43 | regl.clear({ 44 | color: [0, 0, 0, 1] 45 | }) 46 | regl.frame(() => { 47 | microphone(({pitches}) => { 48 | drawPitch() 49 | }) 50 | }) 51 | } 52 | }) 53 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | analyser: require('./analyser'), 3 | microphone: require('./microphone') 4 | } 5 | -------------------------------------------------------------------------------- /microphone.js: -------------------------------------------------------------------------------- 1 | const getUserMedia = require('getusermedia') 2 | const reglAnalyser = require('./analyser') 3 | 4 | module.exports = function (options) { 5 | getUserMedia({audio: true}, function (err, stream) { 6 | if (err) { 7 | options.error && options.error(err) 8 | return 9 | } 10 | const context = options.audioContext || (new window.AudioContext()) 11 | const analyser = context.createAnalyser() 12 | context.createMediaStreamSource(stream).connect(analyser) 13 | options.done(reglAnalyser(Object.assign({ 14 | analyser, 15 | sampleRate: context.sampleRate, 16 | name: 'mic' 17 | }, options))) 18 | }) 19 | } 20 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "regl-audio", 3 | "version": "1.0.0", 4 | "description": "Audio helper functions for regl", 5 | "main": "index.js", 6 | "directories": { 7 | "example": "examples" 8 | }, 9 | "dependencies": { 10 | "getusermedia": "2.0.0", 11 | "fourier-transform": "1.0.2" 12 | }, 13 | "devDependencies": { 14 | "regl": "^1.0.0" 15 | }, 16 | "scripts": { 17 | "test": "echo \"Error: no test specified\" && exit 1" 18 | }, 19 | "repository": { 20 | "type": "git", 21 | "url": "git+https://github.com/mikolalysenko/regl-audio.git" 22 | }, 23 | "keywords": [ 24 | "regl", 25 | "audio", 26 | "graphics", 27 | "live", 28 | "coding", 29 | "interactive" 30 | ], 31 | "author": "Mikola Lysenko", 32 | "license": "MIT", 33 | "bugs": { 34 | "url": "https://github.com/mikolalysenko/regl-audio/issues" 35 | }, 36 | "homepage": "https://github.com/mikolalysenko/regl-audio#readme" 37 | } 38 | --------------------------------------------------------------------------------