├── .gitignore ├── 2d.js ├── index.js ├── lib └── core.js ├── package.json ├── qa.md ├── readme.md └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | *.log 3 | .DS_Store 4 | ./bundle.js 5 | demo 6 | *.mp3 -------------------------------------------------------------------------------- /2d.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Lightweight canvas version of a spectrogram. 3 | * @module gl-spectrogram/2d 4 | */ 5 | 6 | var Spectrogram = require('./lib/core'); 7 | var parseColor = require('color-parse'); 8 | var clamp = require('mumath/clamp'); 9 | 10 | module.exports = Spectrogram; 11 | 12 | Spectrogram.prototype.context = '2d'; 13 | Spectrogram.prototype.autostart = false; 14 | 15 | Spectrogram.prototype.init = function () { 16 | var ctx = this.context; 17 | 18 | this.count = 0; 19 | 20 | //render only on pushes 21 | this.on('push', (magnitudes) => { 22 | //map mags to 0..255 range limiting by db subrange 23 | magnitudes = magnitudes.map((value) => clamp(255 * (1 + value / 100), 0, 255)); 24 | 25 | this.render(magnitudes); 26 | this.count++; 27 | }); 28 | 29 | 30 | //on color update 31 | this.on('update', () => { 32 | this.colorValues = parseColor(this.color).values; 33 | this.bgValues = parseColor(this.canvas.style.background).values; 34 | }); 35 | }; 36 | 37 | 38 | //get mapped frequency 39 | function lg (x) { 40 | return Math.log(x) / Math.log(10); 41 | } 42 | 43 | Spectrogram.prototype.f = function (ratio) { 44 | var halfRate = this.sampleRate * .5; 45 | if (this.logarithmic) { 46 | var logF = Math.pow(10., Math.log10(this.minFrequency) + ratio * (Math.log10(this.maxFrequency) - Math.log10(this.minFrequency)) ); 47 | ratio = (logF - this.minFrequency) / (this.maxFrequency - this.minFrequency); 48 | } 49 | 50 | var leftF = this.minFrequency / halfRate; 51 | var rightF = this.maxFrequency / halfRate; 52 | 53 | ratio = leftF + ratio * (rightF - leftF); 54 | 55 | return ratio; 56 | } 57 | 58 | //return color based on current palette 59 | Spectrogram.prototype.getColor = function (ratio) { 60 | var cm = this.fillData; 61 | ratio = clamp(ratio, 0, 1); 62 | var idx = ratio*(cm.length*.25 - 1); 63 | var amt = idx % 1; 64 | var left = cm.slice(Math.floor(idx)*4, Math.floor(idx)*4 + 4); 65 | var right = cm.slice(Math.ceil(idx)*4, Math.ceil(idx)*4 + 4); 66 | var values = left.map((v,i) => (v * (1 - amt) + right[i] * amt)|0 ); 67 | return values; 68 | } 69 | 70 | 71 | Spectrogram.prototype.draw = function (data) { 72 | var ctx = this.context; 73 | var width = this.viewport[2], 74 | height = this.viewport[3]; 75 | 76 | if (!this.bgValues) { 77 | return; 78 | } 79 | 80 | var padding = 5; 81 | 82 | if (this.count < this.viewport[1] + width - padding) { 83 | var offset = this.count; 84 | } 85 | else { 86 | //displace canvas 87 | var imgData = ctx.getImageData(this.viewport[0], this.viewport[1], width, height); 88 | ctx.putImageData(imgData, this.viewport[0]-1, this.viewport[1]); 89 | var offset = this.viewport[0] + width - padding - 1; 90 | } 91 | 92 | //put new slice 93 | var imgData = ctx.getImageData(offset, this.viewport[1], 1, height); 94 | var pixels = imgData.data; 95 | 96 | for (var i = 0; i < height; i++) { 97 | var ratio = i / height; 98 | var amt = data[(this.f(ratio) * data.length)|0] / 255; 99 | amt = (amt * 100. - 100 - this.minDecibels) / (this.maxDecibels - this.minDecibels); 100 | var values = this.getColor(amt); 101 | values[3] *= 255; 102 | pixels.set(values, (height - i - 1)*4); 103 | } 104 | ctx.putImageData(imgData, offset, 0); 105 | 106 | } -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * WebGL version 3 | * @module gl-spectrogram 4 | */ 5 | 6 | var Spectrogram = require('./lib/core'); 7 | var Component = require('gl-component'); 8 | var clamp = require('mumath/clamp'); 9 | 10 | module.exports = Spectrogram; 11 | 12 | //hook up webgl rendering routines 13 | Spectrogram.prototype.init = function () { 14 | var gl = this.gl; 15 | 16 | //preset colormap texture 17 | this.setTexture('fill', { 18 | unit: 3, 19 | type: gl.UNSIGNED_BYTE, 20 | filter: gl.LINEAR, 21 | wrap: gl.CLAMP_TO_EDGE, 22 | }); 23 | 24 | //save texture location 25 | this.textureLocation = gl.getUniformLocation(this.program, 'texture'); 26 | this.maxFrequencyLocation = gl.getUniformLocation(this.program, 'maxFrequency'); 27 | this.minFrequencyLocation = gl.getUniformLocation(this.program, 'minFrequency'); 28 | this.maxDecibelsLocation = gl.getUniformLocation(this.program, 'maxDecibels'); 29 | this.minDecibelsLocation = gl.getUniformLocation(this.program, 'minDecibels'); 30 | this.logarithmicLocation = gl.getUniformLocation(this.program, 'logarithmic'); 31 | this.sampleRateLocation = gl.getUniformLocation(this.program, 'sampleRate'); 32 | 33 | var size = this.size; 34 | 35 | this.shiftComponent = Component({ 36 | context: gl, 37 | textures: { 38 | texture: { 39 | unit: 0, 40 | data: null, 41 | format: gl.RGBA, 42 | type: gl.UNSIGNED_BYTE, 43 | filter: gl.NEAREST, 44 | wrap: gl.CLAMP_TO_EDGE, 45 | width: size[0], 46 | height: size[1] 47 | }, 48 | altTexture: { 49 | unit: 1, 50 | data: null, 51 | format: gl.RGBA, 52 | type: gl.UNSIGNED_BYTE, 53 | filter: gl.NEAREST, 54 | wrap: gl.CLAMP_TO_EDGE, 55 | width: size[0], 56 | height: size[1] 57 | }, 58 | magnitudes: { 59 | unit: 2, 60 | data: null, 61 | format: gl.ALPHA, 62 | type: gl.UNSIGNED_BYTE, 63 | filter: gl.NEAREST, 64 | wrap: gl.CLAMP_TO_EDGE 65 | } 66 | }, 67 | frag: ` 68 | precision highp float; 69 | 70 | uniform sampler2D texture; 71 | uniform sampler2D magnitudes; 72 | uniform vec4 viewport; 73 | uniform float count; 74 | 75 | const float padding = 1.; 76 | 77 | void main () { 78 | vec2 one = vec2(1) / viewport.zw; 79 | vec2 coord = gl_FragCoord.xy / viewport.zw; 80 | 81 | //do not shift if there is a room for the data 82 | if (count < viewport.z - padding + 1.) { 83 | vec3 color = texture2D(texture, coord).xyz; 84 | float mixAmt = step(count, gl_FragCoord.x); 85 | color = mix(color, texture2D(magnitudes, vec2(coord.y,.5)).www, mixAmt); 86 | mixAmt *= (- count - padding + gl_FragCoord.x) / padding; 87 | color = mix(color, vec3(0), mixAmt); 88 | gl_FragColor = vec4(color, 1); 89 | } 90 | else { 91 | coord.x += one.x; 92 | vec3 color = texture2D(texture, coord).xyz; 93 | float mixAmt = step(viewport.z - padding, gl_FragCoord.x); 94 | color = mix(color, texture2D(magnitudes, vec2(coord.y,.5)).www, mixAmt); 95 | mixAmt *= (- viewport.z + gl_FragCoord.x) / padding; 96 | color = mix(color, vec3(0), mixAmt); 97 | gl_FragColor = vec4(color, 1); 98 | } 99 | } 100 | `, 101 | phase: 0, 102 | spectrogram: this, 103 | framebuffer: gl.createFramebuffer(), 104 | render: function () { 105 | var gl = this.gl; 106 | 107 | gl.useProgram(this.program); 108 | 109 | //TODO: throttle rendering here 110 | gl.bindFramebuffer(gl.FRAMEBUFFER, this.framebuffer); 111 | gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, this.textures[this.phase ? 'texture' : 'altTexture'].texture, 0); 112 | gl.uniform1i(this.textures.texture.location, this.phase); 113 | 114 | this.phase = (this.phase + 1) % 2; 115 | 116 | //vp is unbound from canvas, so we have to manually set it 117 | gl.uniform4fv(gl.getUniformLocation(this.program, 'viewport'), [0,0,size[0], size[1]]); 118 | gl.viewport(0,0,size[0],size[1]); 119 | this.draw(this); 120 | 121 | gl.bindFramebuffer(gl.FRAMEBUFFER, null); 122 | 123 | gl.useProgram(this.spectrogram.program); 124 | gl.uniform1i(this.spectrogram.textureLocation, this.phase); 125 | }, 126 | autostart: false, 127 | float: this.float, 128 | antialias: this.antialias 129 | }); 130 | 131 | //hook up counter 132 | this.shiftComponent.count = 0; 133 | this.shiftComponent.countLocation = gl.getUniformLocation(this.shiftComponent.program, 'count'); 134 | 135 | //shift data on push 136 | this.on('push', (magnitudes) => { 137 | //map mags to 0..255 range limiting by db subrange 138 | magnitudes = magnitudes.map((value) => clamp(255 * (1 + value / 100), 0, 255)); 139 | 140 | this.shiftComponent.setTexture('magnitudes', magnitudes); 141 | 142 | //update count 143 | this.shiftComponent.count++; 144 | gl.uniform1f(this.shiftComponent.countLocation, this.shiftComponent.count); 145 | 146 | //do shift 147 | this.shiftComponent.render(); 148 | }); 149 | 150 | //update uniforms 151 | this.on('update', () => { 152 | gl.uniform1f(this.minFrequencyLocation, this.minFrequency); 153 | gl.uniform1f(this.maxFrequencyLocation, this.maxFrequency); 154 | gl.uniform1f(this.minDecibelsLocation, this.minDecibels); 155 | gl.uniform1f(this.maxDecibelsLocation, this.maxDecibels); 156 | gl.uniform1f(this.logarithmicLocation, this.logarithmic ? 1 : 0); 157 | gl.uniform1f(this.sampleRateLocation, this.sampleRate); 158 | }); 159 | }; 160 | 161 | //background texture size 162 | Spectrogram.prototype.size = [2048, 2048]; 163 | 164 | //default renderer just outputs active texture 165 | Spectrogram.prototype.frag = ` 166 | precision highp float; 167 | 168 | uniform sampler2D texture; 169 | uniform sampler2D fill; 170 | uniform vec4 viewport; 171 | uniform float sampleRate; 172 | uniform float maxFrequency; 173 | uniform float minFrequency; 174 | uniform float maxDecibels; 175 | uniform float minDecibels; 176 | uniform float logarithmic; 177 | 178 | 179 | const float log10 = ${Math.log(10)}; 180 | 181 | float lg (float x) { 182 | return log(x) / log10; 183 | } 184 | 185 | //return a or b based on weight 186 | float decide (float a, float b, float w) { 187 | return step(0.5, w) * b + step(w, 0.5) * a; 188 | } 189 | 190 | //get mapped frequency 191 | float f (float ratio) { 192 | float halfRate = sampleRate * .5; 193 | 194 | float logF = pow(10., lg(minFrequency) + ratio * (lg(maxFrequency) - lg(minFrequency)) ); 195 | 196 | ratio = decide(ratio, (logF - minFrequency) / (maxFrequency - minFrequency), logarithmic); 197 | 198 | float leftF = minFrequency / halfRate; 199 | float rightF = maxFrequency / halfRate; 200 | 201 | ratio = leftF + ratio * (rightF - leftF); 202 | 203 | return ratio; 204 | } 205 | 206 | void main () { 207 | vec2 coord = (gl_FragCoord.xy - viewport.xy) / viewport.zw; 208 | float intensity = texture2D(texture, vec2(coord.x, f(coord.y))).x; 209 | intensity = (intensity * 100. - minDecibels - 100.) / (maxDecibels - minDecibels); 210 | gl_FragColor = vec4(vec3(texture2D(fill, vec2(intensity, coord.y) )), 1); 211 | } 212 | `; 213 | -------------------------------------------------------------------------------- /lib/core.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @module gl-spectrogram/lib/core 3 | */ 4 | 5 | 6 | var extend = require('xtend/mutable'); 7 | var Component = require('gl-component'); 8 | var inherits = require('inherits'); 9 | var isBrowser = require('is-browser'); 10 | var createGrid = require('plot-grid'); 11 | var flatten = require('flatten'); 12 | var lg = require('mumath/lg'); 13 | var clamp = require('mumath/clamp'); 14 | var weighting = require('a-weighting'); 15 | var colormap = require('colormap'); 16 | var parseColor = require('color-parse'); 17 | var hsl = require('color-space/hsl'); 18 | var colorScales = require('colormap/colorScales'); 19 | 20 | module.exports = Spectrogram; 21 | 22 | 23 | 24 | /** 25 | * @contructor 26 | */ 27 | function Spectrogram (options) { 28 | if (!(this instanceof Spectrogram)) return new Spectrogram(options); 29 | 30 | Component.call(this, options); 31 | 32 | if (isBrowser) this.container.classList.add(this.className); 33 | 34 | this.init(); 35 | 36 | //preset initial freqs 37 | 38 | this.push(this.magnitudes); 39 | 40 | //init style props 41 | this.update(); 42 | } 43 | 44 | inherits(Spectrogram, Component); 45 | 46 | Spectrogram.prototype.className = 'gl-spectrogram'; 47 | 48 | Spectrogram.prototype.init = () => {}; 49 | 50 | Spectrogram.prototype.antialias = false; 51 | Spectrogram.prototype.premultipliedAlpha = true; 52 | Spectrogram.prototype.alpha = true; 53 | Spectrogram.prototype.float = false; 54 | 55 | Spectrogram.prototype.maxDecibels = -30; 56 | Spectrogram.prototype.minDecibels = -90; 57 | 58 | Spectrogram.prototype.maxFrequency = 20000; 59 | Spectrogram.prototype.minFrequency = 40; 60 | 61 | Spectrogram.prototype.smoothing = 0.75; 62 | Spectrogram.prototype.details = 1; 63 | 64 | Spectrogram.prototype.grid = true; 65 | Spectrogram.prototype.axes = false; 66 | Spectrogram.prototype.logarithmic = true; 67 | Spectrogram.prototype.weighting = 'itu'; 68 | Spectrogram.prototype.sampleRate = 44100; 69 | 70 | Spectrogram.prototype.fill = 'greys'; 71 | Spectrogram.prototype.background = undefined; 72 | 73 | 74 | 75 | //array with initial values of the last moment 76 | Spectrogram.prototype.magnitudes = Array(1024).fill(-150); 77 | 78 | 79 | //set last actual frequencies values 80 | Spectrogram.prototype.push = function (magnitudes) { 81 | if (!magnitudes) magnitudes = [-150]; 82 | 83 | var gl = this.gl; 84 | var halfRate = this.sampleRate * 0.5; 85 | var l = halfRate / this.magnitudes.length; 86 | var w = weighting[this.weighting] || weighting.z; 87 | 88 | magnitudes = [].slice.call(magnitudes); 89 | 90 | //apply weighting and clamping 91 | for (var i = 0; i < magnitudes.length; i++) { 92 | var v = magnitudes[i]; 93 | magnitudes[i] = clamp(clamp(v, -100, 0) + 20 * Math.log(w(i * l)) / Math.log(10), -200, 0); 94 | } 95 | 96 | var smoothing = this.smoothing; 97 | 98 | for (var i = 0; i < magnitudes.length; i++) { 99 | magnitudes[i] = magnitudes[i] * (1 - smoothing) + this.magnitudes[Math.floor(this.magnitudes.length * (i / magnitudes.length))] * smoothing; 100 | } 101 | 102 | //save actual magnitudes in db 103 | this.magnitudes = magnitudes; 104 | 105 | //find peak 106 | this.peak = this.magnitudes.reduce((prev, curr) => Math.max(curr, prev), -200); 107 | //emit magnitudes in db range 108 | this.emit('push', magnitudes, this.peak); 109 | 110 | return this; 111 | }; 112 | 113 | 114 | /** 115 | * Reset colormap 116 | */ 117 | Spectrogram.prototype.setFill = function (cm, inverse) { 118 | this.fill = cm; 119 | this.inversed = inverse; 120 | 121 | //named colormap 122 | if (typeof cm === 'string') { 123 | //a color scale 124 | if (colorScales[cm]) { 125 | var cm = (flatten(colormap({ 126 | colormap: cm, 127 | nshades: 128, 128 | format: 'rgba', 129 | alpha: 1 130 | })));//.map((v,i) => !((i + 1) % 4) ? v : v/255)); 131 | } 132 | //url 133 | else if (/\\|\//.test(cm)) { 134 | this.setTexture('fill', cm); 135 | return this; 136 | } 137 | //plain color or CSS color string 138 | else { 139 | var parsed = parseColor(cm); 140 | 141 | if (parsed.space === 'hsl') { 142 | cm = hsl.rgb(parsed.values); 143 | } 144 | else { 145 | cm = parsed.values; 146 | } 147 | } 148 | } 149 | else if (!cm) { 150 | if (!this.background) this.setBackground([0,0,0,1]); 151 | return this; 152 | } 153 | //image, canvas etc 154 | else if (!Array.isArray(cm)) { 155 | this.setTexture('fill', cm); 156 | 157 | return this; 158 | } 159 | //custom array, like palette etc. 160 | else { 161 | cm = flatten(cm); 162 | } 163 | 164 | if (inverse) { 165 | var reverse = cm.slice(); 166 | for (var i = 0; i < cm.length; i+=4){ 167 | reverse[cm.length - i - 1] = cm[i + 3]; 168 | reverse[cm.length - i - 2] = cm[i + 2]; 169 | reverse[cm.length - i - 3] = cm[i + 1]; 170 | reverse[cm.length - i - 4] = cm[i + 0]; 171 | } 172 | cm = reverse; 173 | } 174 | 175 | this.setTexture('fill', { 176 | data: cm, 177 | height: 1, 178 | width: (cm.length / 4)|0 179 | }); 180 | 181 | //ensure bg 182 | if (!this.background) { 183 | this.setBackground(cm.slice(0, 4)); 184 | } 185 | 186 | var mainColor = cm.slice(-4); 187 | this.color = `rgba(${mainColor})`; 188 | 189 | this.fillData = cm; 190 | 191 | //set grid color to colormap’s color 192 | if (this.gridComponent) { 193 | this.gridComponent.linesContainer.style.color = this.color; 194 | } 195 | 196 | return this; 197 | }; 198 | 199 | 200 | /** Set background */ 201 | Spectrogram.prototype.setBackground = function (bg) { 202 | if (this.background !== null) { 203 | var bgStyle = null; 204 | if (typeof bg === 'string') { 205 | bgStyle = bg; 206 | } 207 | else if (Array.isArray(bg)) { 208 | //map 0..1 range to 0..255 209 | if (bg[0] && bg[0] <= 1 && bg[1] && bg[1] <= 1 && bg[2] && bg[2] <= 1) { 210 | bg = [ 211 | bg[0] * 255, bg[1] * 255, bg[2] * 255, bg[3] || 1 212 | ]; 213 | } 214 | 215 | bgStyle = `rgba(${bg.slice(0,3).map(v => Math.round(v)).join(', ')}, ${bg[3]})`; 216 | } 217 | this.canvas.style.background = bgStyle; 218 | } 219 | 220 | return this; 221 | }; 222 | 223 | 224 | 225 | 226 | //update view 227 | Spectrogram.prototype.update = function () { 228 | var gl = this.gl; 229 | 230 | if (typeof this.smoothing === 'string') { 231 | this.smoothing = parseFloat(this.smoothing); 232 | } 233 | 234 | if (this.grid) { 235 | if (!this.gridComponent) { 236 | this.gridComponent = createGrid({ 237 | container: this.container, 238 | viewport: () => this.viewport, 239 | lines: Array.isArray(this.grid.lines) ? this.grid.lines : (this.grid.lines === undefined || this.grid.lines === true) && [{ 240 | min: this.minFrequency, 241 | max: this.maxFrequency, 242 | orientation: 'y', 243 | logarithmic: this.logarithmic, 244 | titles: function (value) { 245 | return (value >= 1000 ? ((value / 1000).toLocaleString() + 'k') : value.toLocaleString()) + 'Hz'; 246 | } 247 | }, this.logarithmic ? { 248 | min: this.minFrequency, 249 | max: this.maxFrequency, 250 | orientation: 'y', 251 | logarithmic: this.logarithmic, 252 | values: function (value) { 253 | var str = value.toString(); 254 | if (str[0] !== '1') return null; 255 | return value; 256 | }, 257 | titles: null, 258 | style: { 259 | borderLeftStyle: 'solid', 260 | pointerEvents: 'none', 261 | opacity: '0.08', 262 | display: this.logarithmic ? null :'none' 263 | } 264 | } : null], 265 | axes: Array.isArray(this.grid.axes) ? this.grid.axes : (this.grid.axes || this.axes) && [{ 266 | name: 'Frequency', 267 | labels: function (value, i, opt) { 268 | var str = value.toString(); 269 | if (str[0] !== '2' && str[0] !== '1' && str[0] !== '5') return null; 270 | return opt.titles[i]; 271 | } 272 | }] 273 | }); 274 | 275 | this.on('resize', () => { 276 | if (this.isPlannedGridUpdate) return; 277 | this.isPlannedGridUpdate = true; 278 | this.once('render', () => { 279 | this.isPlannedGridUpdate = false; 280 | this.gridComponent.update(); 281 | }); 282 | }); 283 | } 284 | else { 285 | this.gridComponent.linesContainer.style.display = 'block'; 286 | } 287 | } 288 | else if (this.gridComponent) { 289 | this.gridComponent.linesContainer.style.display = 'none'; 290 | } 291 | 292 | this.setBackground(this.background); 293 | this.setFill(this.fill, this.inversed); 294 | 295 | this.emit('update'); 296 | }; 297 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gl-spectrogram", 3 | "version": "1.1.8", 4 | "description": "Draw spectrogram with webgl or 2d-canvas", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "budo test.js", 8 | "test:browser": "budo test.js -- -g bubleify", 9 | "build": "browserify test.js -g bubleify -o demo/index.js" 10 | }, 11 | "browserify": { 12 | "transform": [ 13 | "bubleify" 14 | ] 15 | }, 16 | "repository": { 17 | "type": "git", 18 | "url": "git+https://github.com/audio-lab/gl-spectrogram.git" 19 | }, 20 | "keywords": [ 21 | "audio", 22 | "vis", 23 | "canvas2d", 24 | "webgl" 25 | ], 26 | "author": "ΔY ", 27 | "license": "MIT", 28 | "bugs": { 29 | "url": "https://github.com/audio-lab/gl-spectrogram/issues" 30 | }, 31 | "homepage": "https://github.com/audio-lab/gl-spectrogram#readme", 32 | "devDependencies": { 33 | "audio-context": "^0.1.0", 34 | "buble": "^0.11.6", 35 | "bubleify": "^0.3.1", 36 | "decibels": "^1.0.1", 37 | "fourier-transform": "^1.0.2", 38 | "is-mobile": "^0.2.2", 39 | "nice-color-palettes": "^1.0.1", 40 | "start-app": "^1.1.13" 41 | }, 42 | "dependencies": { 43 | "a-weighting": "^1.0.0", 44 | "color-parse": "^1.1.1", 45 | "color-space": "^1.14.3", 46 | "colormap": "^2.2.0", 47 | "flatten": "^1.0.2", 48 | "gl-component": "^1.4.11", 49 | "inherits": "^2.0.1", 50 | "is-browser": "^2.0.1", 51 | "mumath": "^2.2.1", 52 | "plot-grid": "^1.1.11", 53 | "xtend": "^4.0.1" 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /qa.md: -------------------------------------------------------------------------------- 1 | ## Q: what is the principle of rendering? 2 | 3 | 1. Render freq slice shifted to output texture. 4 | 2. Render frequency slice to 1px viewport 5 | 3. Render texture to renderbuffer. 6 | 4. Swap input and output textures. 7 | 8 | ## Q: how do we render complete waveform? 9 | 10 | 1. setData(data) method, filling last texture with the data (slice). 11 | - but we need fftSize then, to recognize height. Or passing size in some fashion. 12 | 2. isolate `.render` method to `.push` or alike, where audio data rendering is unbound from the raf. 13 | + that way, rendering full waveform is just a cycle of pushing freq slices, we don’t have to care about fftsize or etc 14 | + that way, raf automatically binds spectrum to realtime. 15 | + that way we avoid setting speed - it can be regulated by repeatable push, like push 10 times etc. 16 | + smoothing starts making sense, providing that distance between pushed data is constant. 17 | + that way we avoid playback API. -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # gl-spectrogram [![experimental](http://badges.github.io/stability-badges/dist/experimental.svg)](http://github.com/badges/stability-badges) 2 | 3 | Render spectrogram in webgl or 2d. 4 | 5 | [![Spectrogram](https://raw.githubusercontent.com/audio-lab/gl-spectrogram/gh-pages/preview.png "Spectrogram")](http://audio-lab.github.io/gl-spectrogram/) 6 | 7 | ## Usage 8 | 9 | [![npm install gl-spectrogram](https://nodei.co/npm/gl-spectrogram.png?mini=true)](https://npmjs.org/package/gl-spectrogram/) 10 | 11 | ```js 12 | var createSpectrogram = require('gl-spectrogram'); 13 | 14 | //lightweight 2d-canvas version 15 | //var createSpectrogram = require('gl-spectrogram/2d'); 16 | 17 | var spectrogram = createSpectrogram({ 18 | //placement settings 19 | container: document.body, 20 | canvas: canvas, 21 | context: 'webgl', 22 | 23 | //audio settings 24 | maxDecibels: -30, 25 | minDecibels: -100, 26 | maxFrequency: 20000, 27 | minFrequency: 20, 28 | sampleRate: 44100, 29 | weighting: 'itu', 30 | 31 | //grid settings 32 | grid: true, 33 | axes: false, 34 | logarithmic: true, 35 | 36 | //rendering settings 37 | smoothing: 0.5, 38 | fill: 'inferno', 39 | background: null, 40 | 41 | //useful only for webgl renderer, defines the size of data texture 42 | size: [1024, 1024] 43 | }); 44 | 45 | //push frequencies data, view is shifted for 1 slice 46 | spectrogram.push(data); 47 | 48 | //for even timeflow push data in setTimeout, stream data event, scriptProcessorCallback etc. 49 | setTimeout(() => { 50 | spectrogram.push(getData(data)); 51 | }, 100); 52 | 53 | //set data colormap or background 54 | spectrogram.setFill(colormap|color|pixels); 55 | spectrogram.setBackground(color|array); 56 | 57 | //update colors, grid view 58 | spectrogram.update(opts); 59 | 60 | //called when freqs being pushed 61 | spectrogram.on('push', magnitudes => {}); 62 | spectrogram.on('update', magnitudes => {}); 63 | 64 | //latest frequencies data in db 65 | spectrogram.magnitudes; 66 | spectrogram.peak; 67 | ``` 68 | 69 | ## Related 70 | 71 | * [gl-spectrum](https://github.com/audio-lab/gl-spectrum) 72 | * [plot-grid](https://github.com/audio-lab/plot-grid) 73 | * [start-app](https://github.com/audio-lab/start-app) 74 | 75 | ## Inspiration 76 | 77 | * [colormap](https://github.com/bpostlethwaite/colormap) 78 | * [nice-color-palettes](https://github.com/Jam3/nice-color-palettes) 79 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var startApp = require('start-app'); 2 | var Spectrogram = require('./2d'); 3 | var db = require('decibels'); 4 | var ft = require('fourier-transform'); 5 | var ctx = require('audio-context'); 6 | var colorScales = require('colormap/colorScales'); 7 | var palettes = require('nice-color-palettes/200'); 8 | var colorParse = require('color-parse'); 9 | var flatten = require('flatten'); 10 | var isMobile = require('is-mobile')(); 11 | 12 | palettes = palettes 13 | .map((palette) => { 14 | return palette.map(v => { 15 | var parsed = colorParse(v); 16 | parsed.values.push(1); 17 | return parsed.values; 18 | }) 19 | }) 20 | .filter((palette) => { 21 | var start = palette[0], end = palette[palette.length - 1]; 22 | var leftLightness = (start[0] * 299 + start[1] * 587 + start[2] * 114) / (1000); 23 | var rightLightness = (end[0] * 299 + end[1] * 587 + end[2] * 114) / (1000); 24 | if (Math.abs(leftLightness - rightLightness) < 128) { 25 | return false; 26 | } 27 | return true; 28 | }); 29 | 30 | //playback speed 31 | var speed = 100; 32 | 33 | //pick random palette 34 | var palette = palettes[0]//(Math.random() * palettes.length)|0]; 35 | 36 | //analyser 37 | var source = null; 38 | var analyser = ctx.createAnalyser(); 39 | analyser.fftSize = 4096; 40 | analyser.smoothingTimeConstant = .1; 41 | analyser.connect(ctx.destination); 42 | 43 | 44 | //generate input sine 45 | var N = 4096; 46 | var sine = Array(N); 47 | var saw = Array(N); 48 | var noise = Array(N); 49 | var rate = 44100; 50 | 51 | for (var i = 0; i < N; i++) { 52 | sine[i] = Math.sin(10000 * Math.PI * 2 * (i / rate)); 53 | saw[i] = 2 * ((1000 * i / rate) % 1) - 1; 54 | noise[i] = Math.random() * 2 - 1; 55 | } 56 | 57 | // var frequencies = ft(sine); 58 | // var frequencies = Array(1024).fill(-150); 59 | // var frequencies = ft(noise); 60 | //NOTE: ios does not allow setting too big this value 61 | var frequencies = new Float32Array(analyser.frequencyBinCount); 62 | // for (var i = 0; i < frequencies.length; i++) frequencies[i] = -150; 63 | // frequencies = frequencies.map((v) => db.fromGain(v)); 64 | 65 | 66 | var app = startApp({ 67 | github: 'audio-lab/gl-spectrogram', 68 | color: palette[palette.length - 1], 69 | // source: 'https://soundcloud.com/xlr8r/sets/xlr8r-top-10-downloads-of-may', 70 | source: isMobile ? './sample.mp3' : 'https://soundcloud.com/sincopat/alberto-sola-sincopat-podcast-157', 71 | history: false, 72 | // autoplay: false 73 | }); 74 | 75 | 76 | var spectrogram = Spectrogram({ 77 | smoothing: .1, 78 | fill: palette, 79 | maxDecibels: 0, 80 | minDecibels: -100, 81 | // logarithmic: false, 82 | // autostart: false 83 | // weighting: 84 | }); 85 | 86 | app.addParams({ 87 | fill: { 88 | type: 'select', 89 | values: (() => { 90 | var values = {}; 91 | for (var name in colorScales) { 92 | if (name === 'alpha') continue; 93 | if (name === 'hsv') continue; 94 | if (name === 'rainbow') continue; 95 | if (name === 'rainbow-soft') continue; 96 | if (name === 'phase') continue; 97 | values[name] = name; 98 | } 99 | return values; 100 | })(), 101 | value: 'greys', 102 | change: function (value, state) { 103 | spectrogram.setFill(value, this.getParamValue('inversed')); 104 | this.setColor(spectrogram.color); 105 | } 106 | }, 107 | inversed: { 108 | value: false, 109 | change: function (value) { 110 | spectrogram.setFill(this.getParamValue('fill'), value); 111 | this.setColor(spectrogram.color); 112 | } 113 | }, 114 | weighting: { 115 | values: { 116 | itu: 'itu', 117 | a: 'a', 118 | b: 'b', 119 | c: 'c', 120 | d: 'd', 121 | z: 'z' 122 | }, 123 | value: spectrogram.weighting, 124 | change: v => { 125 | spectrogram.weighting = v; 126 | } 127 | }, 128 | logarithmic: { 129 | value: spectrogram.logarithmic, 130 | change: v => { 131 | spectrogram.logarithmic = v; 132 | spectrogram.update(); 133 | } 134 | }, 135 | grid: { 136 | value: spectrogram.grid, 137 | change: v => { 138 | spectrogram.grid = v; 139 | spectrogram.update(); 140 | } 141 | }, 142 | // axes: spectrogram.axes, 143 | smoothing: { 144 | min: 0, 145 | max: 1, 146 | step: .01, 147 | value: spectrogram.smoothing, 148 | change: v => { 149 | spectrogram.smoothing = v; 150 | } 151 | }, 152 | speed: { 153 | type: 'range', 154 | value: speed, 155 | min: 1, 156 | //4ms is minimal interval for HTML5 (250 times per second) 157 | max: 250, 158 | change: (v) => { 159 | speed = v; 160 | } 161 | }, 162 | minDecibels: { 163 | type: 'range', 164 | value: spectrogram.minDecibels, 165 | min: -100, 166 | max: 0, 167 | change: (v) => { 168 | spectrogram.minDecibels = v; 169 | spectrogram.update(); 170 | } 171 | }, 172 | maxDecibels: { 173 | type: 'range', 174 | value: spectrogram.maxDecibels, 175 | min: -100, 176 | max: 0, 177 | change: (v) => { 178 | spectrogram.maxDecibels = v; 179 | spectrogram.update(); 180 | } 181 | } 182 | }); 183 | 184 | 185 | 186 | var pushIntervalId; 187 | app.on('source', function (node) { 188 | source = node; 189 | source.connect(analyser); 190 | }) 191 | .on('play', function () { 192 | pushChunk(); 193 | }) 194 | .on('pause', function () { 195 | clearInterval(pushIntervalId); 196 | }); 197 | 198 | function pushChunk () { 199 | // for (var i = 0; i < N; i++) { 200 | // frequencies[i] = Math.sin(10000 * Math.PI * 2 * (i / rate)); 201 | // } 202 | // frequencies = ft(frequencies).map(db.fromGain); 203 | 204 | analyser.getFloatFrequencyData(frequencies); 205 | spectrogram.push(frequencies); 206 | 207 | pushIntervalId = setTimeout(pushChunk, 1000 / speed); 208 | } 209 | --------------------------------------------------------------------------------