RAVE.js - RAVE online timbre transfer

├── Roboto-Light.ttf ├── buffer_to_wav.js ├── default.mp3 ├── favicon.png ├── index.html ├── models ├── cats.onnx ├── darbouka.onnx ├── dogs.onnx ├── jazz.onnx └── vctk.onnx ├── rave.js ├── rave_card.jpg ├── rave_cropped.png └── style.css /Roboto-Light.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caillonantoine/ravejs/fb8bf37596f0da0d0c0e08b4aadf5e8dc9d397dc/Roboto-Light.ttf -------------------------------------------------------------------------------- /buffer_to_wav.js: -------------------------------------------------------------------------------- 1 | // ADAPTED FROM https://www.russellgood.com/how-to-convert-audiobuffer-to-audio-file/ 2 | 3 | // Convert an AudioBuffer to a Blob using WAVE representation 4 | function bufferToWave(abuffer, len) { 5 | var numOfChan = abuffer.numberOfChannels, 6 | length = len * numOfChan * 2 + 44, 7 | buffer = new ArrayBuffer(length), 8 | view = new DataView(buffer), 9 | channels = [], 10 | i, 11 | sample, 12 | offset = 0, 13 | pos = 0; 14 | 15 | // write WAVE header 16 | setUint32(0x46464952); // "RIFF" 17 | setUint32(length - 8); // file length - 8 18 | setUint32(0x45564157); // "WAVE" 19 | 20 | setUint32(0x20746d66); // "fmt " chunk 21 | setUint32(16); // length = 16 22 | setUint16(1); // PCM (uncompressed) 23 | setUint16(numOfChan); 24 | setUint32(abuffer.sampleRate); 25 | setUint32(abuffer.sampleRate * 2 * numOfChan); // avg. bytes/sec 26 | setUint16(numOfChan * 2); // block-align 27 | setUint16(16); // 16-bit (hardcoded in this demo) 28 | 29 | setUint32(0x61746164); // "data" - chunk 30 | setUint32(length - pos - 4); // chunk length 31 | 32 | // write interleaved data 33 | for (i = 0; i < abuffer.numberOfChannels; i++) 34 | channels.push(abuffer.getChannelData(i)); 35 | 36 | while (pos < length) { 37 | for (i = 0; i < numOfChan; i++) { 38 | // interleave channels 39 | sample = Math.max(-1, Math.min(1, channels[i][offset])); // clamp 40 | sample = (0.5 + sample < 0 ? sample * 32768 : sample * 32767) | 0; // scale to 16-bit signed int 41 | view.setInt16(pos, sample, true); // write 16-bit sample 42 | pos += 2; 43 | } 44 | offset++; // next source sample 45 | } 46 | 47 | // create Blob 48 | return new Blob([buffer], { type: "audio/wav" }); 49 | 50 | function setUint16(data) { 51 | view.setUint16(pos, data, true); 52 | pos += 2; 53 | } 54 | 55 | function setUint32(data) { 56 | view.setUint32(pos, data, true); 57 | pos += 4; 58 | } 59 | } 60 | 61 | function make_download(abuffer, total_samples) { 62 | var new_file = URL.createObjectURL(bufferToWave(abuffer, total_samples)); 63 | 64 | var download_link = document.getElementById("download_link"); 65 | download_link.href = new_file; 66 | download_link.download = "rave_render.wav"; 67 | download_link.innerText = "Download result"; 68 | } 69 | -------------------------------------------------------------------------------- /default.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caillonantoine/ravejs/fb8bf37596f0da0d0c0e08b4aadf5e8dc9d397dc/default.mp3 -------------------------------------------------------------------------------- /favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caillonantoine/ravejs/fb8bf37596f0da0d0c0e08b4aadf5e8dc9d397dc/favicon.png -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | RAVE.js - RAVE online timbre transfer 8 | 9 | 10 | 14 | 18 | 22 | 26 | 27 | 28 | 29 | 30 | 31 | 32 |

33 |

34 | 35 |

36 |

I - Audio source selection

37 | 42 |

43 | 48 | 49 |

50 | 56 |

57 | 58 |

59 |

II - Audio transfer

60 | Select model : 61 | 68 | 69 |

70 | 77 | 84 | 91 |

92 | 93 |

94 |

95 | * We do not store any of your data, everything is processed locally on 96 | your computer ! 97 |

98 |

99 | 100 | 101 | -------------------------------------------------------------------------------- /models/cats.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caillonantoine/ravejs/fb8bf37596f0da0d0c0e08b4aadf5e8dc9d397dc/models/cats.onnx -------------------------------------------------------------------------------- /models/darbouka.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caillonantoine/ravejs/fb8bf37596f0da0d0c0e08b4aadf5e8dc9d397dc/models/darbouka.onnx -------------------------------------------------------------------------------- /models/dogs.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caillonantoine/ravejs/fb8bf37596f0da0d0c0e08b4aadf5e8dc9d397dc/models/dogs.onnx -------------------------------------------------------------------------------- /models/jazz.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caillonantoine/ravejs/fb8bf37596f0da0d0c0e08b4aadf5e8dc9d397dc/models/jazz.onnx -------------------------------------------------------------------------------- /models/vctk.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caillonantoine/ravejs/fb8bf37596f0da0d0c0e08b4aadf5e8dc9d397dc/models/vctk.onnx -------------------------------------------------------------------------------- /rave.js: -------------------------------------------------------------------------------- 1 | let audioCtx = null; 2 | let inputBuffer = null; 3 | let outputBuffer = null; 4 | let isRecording = false; 5 | let stream = null; 6 | let recorder = null; 7 | let chunks = []; 8 | 9 | // ENABLING AUDIO CONTEXT 10 | const enableAudioCtx = () => { 11 | if (audioCtx != null) return; 12 | console.log("enabling audio"); 13 | audioCtx = new (AudioContext || webkitAudioContext)(); 14 | }; 15 | 16 | // UPLOAD VARIOUS SOURCES TO MEMORY 17 | const toogleRecording = async () => { 18 | console.log("toogle recording"); 19 | let recordButton = document.getElementById("record-button"); 20 | if (!isRecording) { 21 | recordButton.value = "Stop recording"; 22 | isRecording = true; 23 | chunks = []; 24 | try { 25 | // GET INPUT STREAM AND CREATE RECORDER 26 | stream = await navigator.mediaDevices.getUserMedia({ audio: true }); 27 | recorder = new MediaRecorder(stream); 28 | 29 | // ON STOP FUNCTION 30 | recorder.onstop = function (e) { 31 | let blob = new Blob(chunks, { type: "audio/ogg; codecs=opus" }); 32 | chunks = []; 33 | let audioURL = URL.createObjectURL(blob); 34 | 35 | urlToBuffer(audioURL).then((buffer) => { 36 | inputBuffer = buffer; 37 | let playButton = document.getElementById("play_input"); 38 | let ravifyButton = document.getElementById("ravify_button"); 39 | playButton.disabled = false; 40 | ravifyButton.disabled = false; 41 | }); 42 | }; 43 | 44 | recorder.ondataavailable = function (e) { 45 | chunks.push(e.data); 46 | }; 47 | 48 | recorder.start(); 49 | /* use the stream */ 50 | } catch (err) { 51 | console.log(err); 52 | } 53 | } else { 54 | recordButton.value = "Record from microphone"; 55 | isRecording = false; 56 | recorder.stop(); 57 | stream.getTracks().forEach((track) => track.stop()); 58 | } 59 | }; 60 | 61 | const loadUploadedFile = () => { 62 | enableAudioCtx(); 63 | 64 | let fileInput = document.getElementById("audio-file"); 65 | let ravifyButton = document.getElementById("ravify_button"); 66 | if (fileInput.files[0] == null) return; 67 | 68 | let playButton = document.getElementById("play_input"); 69 | playButton.disabled = true; 70 | var reader1 = new FileReader(); 71 | reader1.onload = function (ev) { 72 | audioCtx.decodeAudioData(ev.target.result).then(function (buffer) { 73 | buffer = tensorToBuffer(bufferToTensor(buffer)); 74 | inputBuffer = buffer; 75 | playButton.disabled = false; 76 | ravifyButton.disabled = false; 77 | }); 78 | }; 79 | reader1.readAsArrayBuffer(fileInput.files[0]); 80 | }; 81 | 82 | const loadCantina = async () => { 83 | let playButton = document.getElementById("play_input"); 84 | let ravifyButton = document.getElementById("ravify_button"); 85 | playButton.disabled = true; 86 | 87 | let buffer = await urlToBuffer("/ravejs/default.mp3"); 88 | buffer = tensorToBuffer(bufferToTensor(buffer)); 89 | inputBuffer = buffer; 90 | playButton.disabled = false; 91 | ravifyButton.disabled = false; 92 | }; 93 | 94 | const urlToBuffer = async (url) => { 95 | enableAudioCtx(); 96 | const audioBuffer = await fetch(url) 97 | .then((res) => res.arrayBuffer()) 98 | .then((ArrayBuffer) => audioCtx.decodeAudioData(ArrayBuffer)); 99 | return audioBuffer; 100 | }; 101 | 102 | // PLAY BUFFER 103 | const playBuffer = (buffer) => { 104 | enableAudioCtx(); 105 | const source = audioCtx.createBufferSource(); 106 | source.buffer = buffer; 107 | source.connect(audioCtx.destination); 108 | source.start(); 109 | }; 110 | 111 | const playInput = () => { 112 | if (inputBuffer == null) return; 113 | playBuffer(inputBuffer); 114 | }; 115 | 116 | const playOutput = () => { 117 | if (outputBuffer == null) return; 118 | playBuffer(outputBuffer); 119 | }; 120 | 121 | // PROCESSING 122 | const transfer = async () => { 123 | if (inputBuffer == null) return; 124 | console.log("transfer in progress..."); 125 | outputBuffer = await raveForward(inputBuffer); 126 | make_download(outputBuffer, outputBuffer.getChannelData(0).length); 127 | }; 128 | 129 | const bufferToTensor = (buffer) => { 130 | let b = buffer.getChannelData(0); 131 | let cropped_data = []; 132 | let cropped_length = Math.min(10 * audioCtx.sampleRate, b.length); 133 | for (let i = 0; i < cropped_length; i++) { 134 | cropped_data.push(b[i]); 135 | } 136 | const inputTensor = new ort.Tensor("float32", cropped_data, [ 137 | 1, 138 | 1, 139 | cropped_length, 140 | ]); 141 | return inputTensor; 142 | }; 143 | 144 | const tensorToBuffer = (tensor) => { 145 | let len = tensor.dims[2]; 146 | let buffer = audioCtx.createBuffer(1, len, audioCtx.sampleRate); 147 | channel = buffer.getChannelData(0); 148 | for (let i = 0; i < buffer.length; i++) { 149 | channel[i] = isNaN(tensor.data[i]) ? 0 : tensor.data[i]; 150 | } 151 | return buffer; 152 | }; 153 | 154 | const raveForward = async (buffer) => { 155 | let model_name = document.getElementById("model"); 156 | let playButton = document.getElementById("play_output"); 157 | let ravifyButton = document.getElementById("ravify_button"); 158 | ravifyButton.disabled = true; 159 | playButton.disabled = true; 160 | let inputTensor = bufferToTensor(buffer); 161 | let session = await ort.InferenceSession.create(model_name.value); 162 | let feeds = { audio_in: inputTensor }; 163 | let audio_out = (await session.run(feeds)).audio_out; 164 | audio_out = tensorToBuffer(audio_out); 165 | playButton.disabled = false; 166 | ravifyButton.disabled = false; 167 | return audio_out; 168 | }; 169 | -------------------------------------------------------------------------------- /rave_card.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caillonantoine/ravejs/fb8bf37596f0da0d0c0e08b4aadf5e8dc9d397dc/rave_card.jpg -------------------------------------------------------------------------------- /rave_cropped.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caillonantoine/ravejs/fb8bf37596f0da0d0c0e08b4aadf5e8dc9d397dc/rave_cropped.png -------------------------------------------------------------------------------- /style.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --default-gap: 10px; 3 | --double-gap: 20px; 4 | --bg-color: #ffffff; 5 | --border-color: #6272a4; 6 | --fg-color: #282a36; 7 | } 8 | 9 | @font-face { 10 | font-family: 'Roboto-Light'; 11 | src: url('Roboto-Light.ttf') format("truetype"); 12 | } 13 | 14 | body { 15 | font-family: 'Roboto-Light'; 16 | color: var(--fg-color); 17 | background-color: var(--bg-color); 18 | padding-top: var(--default-gap); 19 | } 20 | 21 | h1 { 22 | font-size: 42px; 23 | margin-bottom: 0px; 24 | margin-top: 0px; 25 | } 26 | 27 | h2 { 28 | font-size: 28px; 29 | margin: 0px; 30 | margin-bottom: 15px; 31 | } 32 | 33 | .info { 34 | font-size: 16px; 35 | } 36 | 37 | .content { 38 | display: flex; 39 | flex-direction: column; 40 | justify-content: space-between; 41 | align-items: center; 42 | align-content: center; 43 | gap: 15px; 44 | height: 90vh; 45 | } 46 | 47 | .section { 48 | border-top: 5px dashed black; 49 | padding: 10px; 50 | width: min(80%,500px); 51 | margin: auto; 52 | display: flex; 53 | flex-direction: column; 54 | justify-content: center; 55 | align-items: center; 56 | align-content: center; 57 | gap: var(--default-gap); 58 | } 59 | 60 | .flexRow { 61 | display: flex; 62 | gap: var(--default-gap); 63 | } 64 | 65 | .group { 66 | border: 2px dashed black; 67 | padding: var(--default-gap); 68 | } 69 | 70 | 71 | input[type=button] { 72 | transition-duration: 0.2s; 73 | } 74 | 75 | input[type=button],select { 76 | background-color: #890010; 77 | border: none; 78 | color: white; 79 | padding: 15px 32px; 80 | text-align: center; 81 | text-decoration: none; 82 | display: inline-block; 83 | font-size: 16px; 84 | } 85 | 86 | input[type=button]:hover{ 87 | background-color: #c60017; /* Green */ 88 | color: white; 89 | } 90 | 91 | input[type=button]:disabled{ 92 | background-color: #c0c0c0; 93 | color: rgb(123, 123, 123); 94 | } --------------------------------------------------------------------------------