├── .gitignore ├── Pitch_Detection_with_SPICE.ipynb ├── README.md ├── app ├── .gitignore ├── build.gradle ├── proguard-rules.pro └── src │ ├── androidTest │ └── java │ │ └── com │ │ └── george │ │ └── pitch_estimator │ │ └── ExampleInstrumentedTest.kt │ ├── main │ ├── AndroidManifest.xml │ ├── assets │ │ ├── final1.txt │ │ └── lite-model_spice_1.tflite │ ├── ic_launcher-playstore.png │ ├── java │ │ └── com │ │ │ └── george │ │ │ └── pitch_estimator │ │ │ ├── PitchEstimatorApplication.kt │ │ │ ├── PitchModelExecutor.kt │ │ │ ├── SingRecorder.kt │ │ │ ├── binding_adapters │ │ │ └── BindingAdapters.kt │ │ │ ├── di │ │ │ └── SingingFragmentModule.kt │ │ │ ├── singingFragment │ │ │ ├── SingingFragment.kt │ │ │ └── SingingFragmentViewModel.kt │ │ │ └── ui │ │ │ ├── MainActivity.kt │ │ │ └── SecondFragment.kt │ └── res │ │ ├── anim │ │ ├── cycle.xml │ │ ├── scale_anim.xml │ │ └── shake.xml │ │ ├── drawable-v24 │ │ └── ic_launcher_foreground.xml │ │ ├── drawable │ │ ├── ic_launcher_background.xml │ │ ├── ic_shark.xml │ │ ├── round_button.xml │ │ └── round_button_colorless.xml │ │ ├── layout │ │ ├── activity_main.xml │ │ ├── content_main.xml │ │ ├── fragment_first.xml │ │ └── fragment_second.xml │ │ ├── menu │ │ └── menu_main.xml │ │ ├── mipmap-anydpi-v26 │ │ ├── ic_launcher.xml │ │ └── ic_launcher_round.xml │ │ ├── mipmap-hdpi │ │ ├── ic_launcher.png │ │ ├── ic_launcher_foreground.png │ │ └── ic_launcher_round.png │ │ ├── mipmap-mdpi │ │ ├── ic_launcher.png │ │ ├── ic_launcher_foreground.png │ │ └── ic_launcher_round.png │ │ ├── mipmap-xhdpi │ │ ├── ic_launcher.png │ │ ├── ic_launcher_foreground.png │ │ └── ic_launcher_round.png │ │ ├── mipmap-xxhdpi │ │ ├── ic_launcher.png │ │ ├── ic_launcher_foreground.png │ │ └── ic_launcher_round.png │ │ ├── mipmap-xxxhdpi │ │ ├── ic_launcher.png │ │ ├── ic_launcher_foreground.png │ │ └── ic_launcher_round.png │ │ ├── navigation │ │ └── nav_graph.xml │ │ └── values │ │ ├── colors.xml │ │ ├── dimens.xml │ │ ├── ic_launcher_background.xml │ │ ├── strings.xml │ │ └── styles.xml │ └── test │ └── java │ └── com │ └── george │ └── pitch_estimator │ └── ExampleUnitTest.kt ├── build.gradle ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle └── shot.jpg /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | .gradle 3 | /local.properties 4 | /.idea/caches 5 | /.idea/libraries 6 | /.idea/modules.xml 7 | /.idea/workspace.xml 8 | /.idea/navEditor.xml 9 | /.idea/assetWizardSettings.xml 10 | .DS_Store 11 | /build 12 | /captures 13 | .externalNativeBuild 14 | .cxx 15 | /.idea/ 16 | -------------------------------------------------------------------------------- /Pitch_Detection_with_SPICE.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "Pitch Detection with SPICE", 7 | "provenance": [], 8 | "private_outputs": true, 9 | "collapsed_sections": [], 10 | "toc_visible": true 11 | }, 12 | "kernelspec": { 13 | "display_name": "Python 3", 14 | "name": "python3" 15 | } 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "markdown", 20 | "metadata": { 21 | "colab_type": "text", 22 | "id": "aXehiGc3Kr2I" 23 | }, 24 | "source": [ 25 | "##### Copyright 2020 The TensorFlow Hub Authors.\n", 26 | "\n", 27 | "Licensed under the Apache License, Version 2.0 (the \"License\");" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "metadata": { 33 | "cellView": "form", 34 | "colab_type": "code", 35 | "id": "-6LKjmi8Ktoh", 36 | "colab": {} 37 | }, 38 | "source": [ 39 | "#@title Copyright 2020 The TensorFlow Hub Authors. All Rights Reserved.\n", 40 | "#\n", 41 | "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", 42 | "# you may not use this file except in compliance with the License.\n", 43 | "# You may obtain a copy of the License at\n", 44 | "#\n", 45 | "# http://www.apache.org/licenses/LICENSE-2.0\n", 46 | "#\n", 47 | "# Unless required by applicable law or agreed to in writing, software\n", 48 | "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", 49 | "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", 50 | "# See the License for the specific language governing permissions and\n", 51 | "# limitations under the License.\n", 52 | "# ==============================================================================" 53 | ], 54 | "execution_count": null, 55 | "outputs": [] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": { 60 | "colab_type": "text", 61 | "id": "MfBg1C5NB3X0" 62 | }, 63 | "source": [ 64 | "\n", 65 | " \n", 68 | " \n", 71 | " \n", 74 | " \n", 77 | "
\n", 66 | " View on TensorFlow.org\n", 67 | " \n", 69 | " Run in Google Colab\n", 70 | " \n", 72 | " View source on GitHub\n", 73 | " \n", 75 | " Download notebook\n", 76 | "
" 78 | ] 79 | }, 80 | { 81 | "cell_type": "markdown", 82 | "metadata": { 83 | "colab_type": "text", 84 | "id": "sPQKw4x4bL8w" 85 | }, 86 | "source": [ 87 | "# Pitch Detection with SPICE\n", 88 | "\n", 89 | "This colab will show you how to use the SPICE model downloaded from TensorFlow Hub." 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "metadata": { 95 | "colab_type": "code", 96 | "id": "rfKwZlPnPwD1", 97 | "colab": {} 98 | }, 99 | "source": [ 100 | "!sudo apt-get install -q -y timidity libsndfile1" 101 | ], 102 | "execution_count": null, 103 | "outputs": [] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "metadata": { 108 | "colab_type": "code", 109 | "id": "dYrIdOS8SW3b", 110 | "colab": {} 111 | }, 112 | "source": [ 113 | "# All the imports to deal with sound data\n", 114 | "!pip install pydub numba==0.48 librosa music21" 115 | ], 116 | "execution_count": null, 117 | "outputs": [] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "metadata": { 122 | "colab_type": "code", 123 | "id": "p09o78LGYdnz", 124 | "colab": {} 125 | }, 126 | "source": [ 127 | "import tensorflow as tf\n", 128 | "import tensorflow_hub as hub\n", 129 | "\n", 130 | "import numpy as np\n", 131 | "import matplotlib.pyplot as plt\n", 132 | "import librosa\n", 133 | "from librosa import display as librosadisplay\n", 134 | "\n", 135 | "import logging\n", 136 | "import math\n", 137 | "import statistics\n", 138 | "import sys\n", 139 | "\n", 140 | "from IPython.display import Audio, Javascript\n", 141 | "from scipy.io import wavfile\n", 142 | "\n", 143 | "from base64 import b64decode\n", 144 | "\n", 145 | "import music21\n", 146 | "from pydub import AudioSegment\n", 147 | "\n", 148 | "logger = logging.getLogger()\n", 149 | "logger.setLevel(logging.ERROR)\n", 150 | "\n", 151 | "print(\"tensorflow: %s\" % tf.__version__)\n", 152 | "#print(\"librosa: %s\" % librosa.__version__)" 153 | ], 154 | "execution_count": null, 155 | "outputs": [] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "metadata": { 160 | "colab_type": "text", 161 | "id": "wHxox8hXc3w1" 162 | }, 163 | "source": [ 164 | "# The audio input file\n", 165 | "Now the hardest part: Record your singing! :)\n", 166 | "\n", 167 | "We provide four methods to obtain an audio file:\n", 168 | "\n", 169 | "1. Record audio directly in colab\n", 170 | "2. Upload from your computer\n", 171 | "3. Use a file saved on Google Drive\n", 172 | "4. Download the file from the web\n", 173 | "\n", 174 | "Choose one of the four methods below." 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "metadata": { 180 | "cellView": "form", 181 | "colab_type": "code", 182 | "id": "HaCAHOqiVu5B", 183 | "colab": {} 184 | }, 185 | "source": [ 186 | "#@title [Run this] Definition of the JS code to record audio straight from the browser\n", 187 | "\n", 188 | "RECORD = \"\"\"\n", 189 | "const sleep = time => new Promise(resolve => setTimeout(resolve, time))\n", 190 | "const b2text = blob => new Promise(resolve => {\n", 191 | " const reader = new FileReader()\n", 192 | " reader.onloadend = e => resolve(e.srcElement.result)\n", 193 | " reader.readAsDataURL(blob)\n", 194 | "})\n", 195 | "var record = time => new Promise(async resolve => {\n", 196 | " stream = await navigator.mediaDevices.getUserMedia({ audio: true })\n", 197 | " recorder = new MediaRecorder(stream)\n", 198 | " chunks = []\n", 199 | " recorder.ondataavailable = e => chunks.push(e.data)\n", 200 | " recorder.start()\n", 201 | " await sleep(time)\n", 202 | " recorder.onstop = async ()=>{\n", 203 | " blob = new Blob(chunks)\n", 204 | " text = await b2text(blob)\n", 205 | " resolve(text)\n", 206 | " }\n", 207 | " recorder.stop()\n", 208 | "})\n", 209 | "\"\"\"\n", 210 | "\n", 211 | "def record(sec=5):\n", 212 | " try:\n", 213 | " from google.colab import output\n", 214 | " except ImportError:\n", 215 | " print('No possible to import output from google.colab')\n", 216 | " return ''\n", 217 | " else:\n", 218 | " print('Recording')\n", 219 | " display(Javascript(RECORD))\n", 220 | " s = output.eval_js('record(%d)' % (sec*1000))\n", 221 | " fname = 'recorded_audio.wav'\n", 222 | " print('Saving to', fname)\n", 223 | " b = b64decode(s.split(',')[1])\n", 224 | " with open(fname, 'wb') as f:\n", 225 | " f.write(b)\n", 226 | " return fname" 227 | ], 228 | "execution_count": null, 229 | "outputs": [] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "metadata": { 234 | "cellView": "both", 235 | "colab_type": "code", 236 | "id": "sBpWWkTzfUYR", 237 | "colab": {} 238 | }, 239 | "source": [ 240 | "#@title Select how to input your audio { run: \"auto\" }\n", 241 | "INPUT_SOURCE = 'https://storage.googleapis.com/download.tensorflow.org/data/c-scale-metronome.wav' #@param [\"https://storage.googleapis.com/download.tensorflow.org/data/c-scale-metronome.wav\", \"RECORD\", \"UPLOAD\", \"./drive/My Drive/YOUR_MUSIC_FILE.wav\"] {allow-input: true}\n", 242 | "\n", 243 | "print('You selected', INPUT_SOURCE)\n", 244 | "\n", 245 | "if INPUT_SOURCE == 'RECORD':\n", 246 | " uploaded_file_name = record(5)\n", 247 | "elif INPUT_SOURCE == 'UPLOAD':\n", 248 | " try:\n", 249 | " from google.colab import files\n", 250 | " except ImportError:\n", 251 | " print(\"ImportError: files from google.colab seems to not be available\")\n", 252 | " else:\n", 253 | " uploaded = files.upload()\n", 254 | " for fn in uploaded.keys():\n", 255 | " print('User uploaded file \"{name}\" with length {length} bytes'.format(\n", 256 | " name=fn, length=len(uploaded[fn])))\n", 257 | " uploaded_file_name = next(iter(uploaded))\n", 258 | " print('Uploaded file: ' + uploaded_file_name)\n", 259 | "elif INPUT_SOURCE.startswith('./drive/'):\n", 260 | " try:\n", 261 | " from google.colab import drive\n", 262 | " except ImportError:\n", 263 | " print(\"ImportError: files from google.colab seems to not be available\")\n", 264 | " else:\n", 265 | " drive.mount('/content/drive')\n", 266 | " # don't forget to change the name of the file you\n", 267 | " # will you here!\n", 268 | " gdrive_audio_file = 'YOUR_MUSIC_FILE.wav'\n", 269 | " uploaded_file_name = INPUT_SOURCE\n", 270 | "elif INPUT_SOURCE.startswith('http'):\n", 271 | " !wget --no-check-certificate 'https://storage.googleapis.com/download.tensorflow.org/data/c-scale-metronome.wav' -O c-scale.wav\n", 272 | " uploaded_file_name = 'c-scale.wav'\n", 273 | "else:\n", 274 | " print('Unrecognized input format!')\n", 275 | " print('Please select \"RECORD\", \"UPLOAD\", or specify a file hosted on Google Drive or a file from the web to download file to download')" 276 | ], 277 | "execution_count": null, 278 | "outputs": [] 279 | }, 280 | { 281 | "cell_type": "markdown", 282 | "metadata": { 283 | "colab_type": "text", 284 | "id": "4S2BvIoDf9nf" 285 | }, 286 | "source": [ 287 | "# Preparing the audio data\n", 288 | "\n", 289 | "Now we have the audio, let's convert it to the expected format and then listen to it!\n", 290 | "\n", 291 | "The SPICE model needs as input an audio file at a sampling rate of 16kHz and with only one channel (mono). \n", 292 | "\n", 293 | "To help you with this part, we created a function (`convert_audio_for_model`) to convert any wav file you have to the model's expected format:" 294 | ] 295 | }, 296 | { 297 | "cell_type": "code", 298 | "metadata": { 299 | "colab_type": "code", 300 | "id": "bQ1362i-JoFI", 301 | "colab": {} 302 | }, 303 | "source": [ 304 | "# Function that converts the user-created audio to the format that the model \n", 305 | "# expects: bitrate 16kHz and only one channel (mono).\n", 306 | "\n", 307 | "EXPECTED_SAMPLE_RATE = 16000\n", 308 | "\n", 309 | "def convert_audio_for_model(user_file, output_file='converted_audio_file.wav'):\n", 310 | " audio = AudioSegment.from_file(user_file)\n", 311 | " audio = audio.set_frame_rate(EXPECTED_SAMPLE_RATE).set_channels(1)\n", 312 | " audio.export(output_file, format=\"wav\")\n", 313 | " return output_file" 314 | ], 315 | "execution_count": null, 316 | "outputs": [] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "metadata": { 321 | "colab_type": "code", 322 | "id": "oL9pftZ2nPm9", 323 | "colab": {} 324 | }, 325 | "source": [ 326 | "# Converting to the expected format for the model\n", 327 | "# in all the input 4 input method before, the uploaded file name is at\n", 328 | "# the variable uploaded_file_name\n", 329 | "converted_audio_file = convert_audio_for_model(uploaded_file_name)" 330 | ], 331 | "execution_count": null, 332 | "outputs": [] 333 | }, 334 | { 335 | "cell_type": "code", 336 | "metadata": { 337 | "colab_type": "code", 338 | "id": "TslkX2AOZN0p", 339 | "colab": {} 340 | }, 341 | "source": [ 342 | "# Loading audio samples from the wav file:\n", 343 | "sample_rate, audio_samples = wavfile.read(converted_audio_file, 'rb')\n", 344 | "\n", 345 | "# Show some basic information about the audio.\n", 346 | "duration = len(audio_samples)/sample_rate\n", 347 | "print(f'Sample rate: {sample_rate} Hz')\n", 348 | "print(f'Total duration: {duration:.2f}s')\n", 349 | "print(f'Size of the input: {len(audio_samples)}')\n", 350 | "\n", 351 | "# Let's listen to the wav file.\n", 352 | "Audio(audio_samples, rate=sample_rate)" 353 | ], 354 | "execution_count": null, 355 | "outputs": [] 356 | }, 357 | { 358 | "cell_type": "markdown", 359 | "metadata": { 360 | "colab_type": "text", 361 | "id": "iBicZu5AgcpR" 362 | }, 363 | "source": [ 364 | "First thing, let's take a look at the waveform of our singing." 365 | ] 366 | }, 367 | { 368 | "cell_type": "code", 369 | "metadata": { 370 | "colab_type": "code", 371 | "id": "aAa2M3CLZcWW", 372 | "colab": {} 373 | }, 374 | "source": [ 375 | "# We can visualize the audio as a waveform.\n", 376 | "_ = plt.plot(audio_samples)" 377 | ], 378 | "execution_count": null, 379 | "outputs": [] 380 | }, 381 | { 382 | "cell_type": "markdown", 383 | "metadata": { 384 | "colab_type": "text", 385 | "id": "J1eI0b8qgn08" 386 | }, 387 | "source": [ 388 | "A more informative visualization is the [spectrogram](https://en.wikipedia.org/wiki/Spectrogram), which shows frequencies present over time.\n", 389 | "\n", 390 | "Here, we use a logarithmic frequency scale, to make the singing more clearly visible.\n" 391 | ] 392 | }, 393 | { 394 | "cell_type": "code", 395 | "metadata": { 396 | "colab_type": "code", 397 | "id": "fGR4UZtpZvWI", 398 | "colab": {} 399 | }, 400 | "source": [ 401 | "MAX_ABS_INT16 = 32768.0\n", 402 | "\n", 403 | "def plot_stft(x, sample_rate, show_black_and_white=False):\n", 404 | " x_stft = np.abs(librosa.stft(x, n_fft=2048))\n", 405 | " fig, ax = plt.subplots()\n", 406 | " fig.set_size_inches(20, 10)\n", 407 | " x_stft_db = librosa.amplitude_to_db(x_stft, ref=np.max)\n", 408 | " if(show_black_and_white):\n", 409 | " librosadisplay.specshow(data=x_stft_db, y_axis='log', \n", 410 | " sr=sample_rate, cmap='gray_r')\n", 411 | " else:\n", 412 | " librosadisplay.specshow(data=x_stft_db, y_axis='log', sr=sample_rate)\n", 413 | "\n", 414 | " plt.colorbar(format='%+2.0f dB')\n", 415 | "\n", 416 | "plot_stft(audio_samples / MAX_ABS_INT16 , sample_rate=EXPECTED_SAMPLE_RATE)\n", 417 | "plt.show()\n" 418 | ], 419 | "execution_count": null, 420 | "outputs": [] 421 | }, 422 | { 423 | "cell_type": "markdown", 424 | "metadata": { 425 | "colab_type": "text", 426 | "id": "MGCzo_cjjH-7" 427 | }, 428 | "source": [ 429 | "We need one last conversion here. The audio samples are in int16 format. They need to be normalized to floats between -1 and 1." 430 | ] 431 | }, 432 | { 433 | "cell_type": "code", 434 | "metadata": { 435 | "colab_type": "code", 436 | "id": "dv4H4O1Xb8T8", 437 | "colab": {} 438 | }, 439 | "source": [ 440 | "audio_samples = audio_samples / float(MAX_ABS_INT16)" 441 | ], 442 | "execution_count": null, 443 | "outputs": [] 444 | }, 445 | { 446 | "cell_type": "markdown", 447 | "metadata": { 448 | "colab_type": "text", 449 | "id": "yTdo_TwljVUV" 450 | }, 451 | "source": [ 452 | "# Executing the Model\n", 453 | "Now is the easy part, let's load the model with **TensorFlow Hub**, and feed the audio to it.\n", 454 | "SPICE will give us two outputs: pitch and uncertainty\n", 455 | "\n", 456 | "\n" 457 | ] 458 | }, 459 | { 460 | "cell_type": "markdown", 461 | "metadata": { 462 | "colab_type": "text", 463 | "id": "xUptYSTAbc3I" 464 | }, 465 | "source": [ 466 | "**TensorFlow Hub** is a library for the publication, discovery, and consumption of reusable parts of machine learning models. It makes easy to use machine learning to solve your challenges.\n", 467 | "\n", 468 | "To load the model you just need the Hub module and the URL pointing to the model:" 469 | ] 470 | }, 471 | { 472 | "cell_type": "code", 473 | "metadata": { 474 | "colab_type": "code", 475 | "id": "ri0A0DSXY_Yd", 476 | "colab": {} 477 | }, 478 | "source": [ 479 | "# Loading the SPICE model is easy:\n", 480 | "model = hub.load(\"https://tfhub.dev/google/spice/2\")" 481 | ], 482 | "execution_count": null, 483 | "outputs": [] 484 | }, 485 | { 486 | "cell_type": "markdown", 487 | "metadata": { 488 | "colab_type": "text", 489 | "id": "kQV5H6J4suMT" 490 | }, 491 | "source": [ 492 | "**Note:** An interesting detail here is that all the model urls from Hub can be used for download and also to read the documentation, so if you point your browser to that link you can read documentation on how to use the model and learn more about how it was trained." 493 | ] 494 | }, 495 | { 496 | "cell_type": "markdown", 497 | "metadata": { 498 | "colab_type": "text", 499 | "id": "GUVICjIps9hI" 500 | }, 501 | "source": [ 502 | "With the model loaded, data prepared, we need 3 lines to get the result: " 503 | ] 504 | }, 505 | { 506 | "cell_type": "code", 507 | "metadata": { 508 | "colab_type": "code", 509 | "id": "tP55fXBYcBhb", 510 | "colab": {} 511 | }, 512 | "source": [ 513 | "# We now feed the audio to the SPICE tf.hub model to obtain pitch and uncertainty outputs as tensors.\n", 514 | "model_output = model.signatures[\"serving_default\"](tf.constant(audio_samples, tf.float32))\n", 515 | "\n", 516 | "pitch_outputs = model_output[\"pitch\"]\n", 517 | "uncertainty_outputs = model_output[\"uncertainty\"]\n", 518 | "\n", 519 | "# 'Uncertainty' basically means the inverse of confidence.\n", 520 | "confidence_outputs = 1.0 - uncertainty_outputs\n", 521 | "\n", 522 | "fig, ax = plt.subplots()\n", 523 | "fig.set_size_inches(20, 10)\n", 524 | "plt.plot(pitch_outputs, label='pitch')\n", 525 | "plt.plot(confidence_outputs, label='confidence')\n", 526 | "plt.legend(loc=\"lower right\")\n", 527 | "plt.show()" 528 | ], 529 | "execution_count": null, 530 | "outputs": [] 531 | }, 532 | { 533 | "cell_type": "markdown", 534 | "metadata": { 535 | "colab_type": "text", 536 | "id": "blJwFWR4kMul" 537 | }, 538 | "source": [ 539 | "Let's make the results easier to understand by removing all pitch estimates with low confidence (confidence < 0.9) and plot the remaining ones.\n", 540 | "\n" 541 | ] 542 | }, 543 | { 544 | "cell_type": "code", 545 | "metadata": { 546 | "colab_type": "code", 547 | "id": "d1MRmcm2cEkM", 548 | "colab": {} 549 | }, 550 | "source": [ 551 | "confidence_outputs = list(confidence_outputs)\n", 552 | "pitch_outputs = [ float(x) for x in pitch_outputs]\n", 553 | "\n", 554 | "indices = range(len (pitch_outputs))\n", 555 | "confident_pitch_outputs = [ (i,p) \n", 556 | " for i, p, c in zip(indices, pitch_outputs, confidence_outputs) if c >= 0.9 ]\n", 557 | "confident_pitch_outputs_x, confident_pitch_outputs_y = zip(*confident_pitch_outputs)\n", 558 | " \n", 559 | "fig, ax = plt.subplots()\n", 560 | "fig.set_size_inches(20, 10)\n", 561 | "ax.set_ylim([0, 1])\n", 562 | "plt.scatter(confident_pitch_outputs_x, confident_pitch_outputs_y, )\n", 563 | "plt.scatter(confident_pitch_outputs_x, confident_pitch_outputs_y, c=\"r\")\n", 564 | "\n", 565 | "plt.show()" 566 | ], 567 | "execution_count": null, 568 | "outputs": [] 569 | }, 570 | { 571 | "cell_type": "markdown", 572 | "metadata": { 573 | "colab_type": "text", 574 | "id": "vNBZ7ZblkxOm" 575 | }, 576 | "source": [ 577 | "The pitch values returned by SPICE are in the range from 0 to 1. Let's convert them to absolute pitch values in Hz." 578 | ] 579 | }, 580 | { 581 | "cell_type": "code", 582 | "metadata": { 583 | "colab_type": "code", 584 | "id": "n-CnpKzmcQi9", 585 | "colab": {} 586 | }, 587 | "source": [ 588 | "def output2hz(pitch_output):\n", 589 | " # Constants taken from https://tfhub.dev/google/spice/2\n", 590 | " PT_OFFSET = 25.58\n", 591 | " PT_SLOPE = 63.07\n", 592 | " FMIN = 10.0;\n", 593 | " BINS_PER_OCTAVE = 12.0;\n", 594 | " cqt_bin = pitch_output * PT_SLOPE + PT_OFFSET;\n", 595 | " return FMIN * 2.0 ** (1.0 * cqt_bin / BINS_PER_OCTAVE)\n", 596 | " \n", 597 | "confident_pitch_values_hz = [ output2hz(p) for p in confident_pitch_outputs_y ]" 598 | ], 599 | "execution_count": null, 600 | "outputs": [] 601 | }, 602 | { 603 | "cell_type": "markdown", 604 | "metadata": { 605 | "colab_type": "text", 606 | "id": "24yK0a6HjCSZ" 607 | }, 608 | "source": [ 609 | "Now, let's see how good the prediction is: We will overlay the predicted pitches over the original spectrogram. To make the pitch predictions more visible, we changed the spectrogram to black and white." 610 | ] 611 | }, 612 | { 613 | "cell_type": "code", 614 | "metadata": { 615 | "colab_type": "code", 616 | "id": "L1kaAcX9rrDo", 617 | "colab": {} 618 | }, 619 | "source": [ 620 | "plot_stft(audio_samples / MAX_ABS_INT16 , \n", 621 | " sample_rate=EXPECTED_SAMPLE_RATE, show_black_and_white=True)\n", 622 | "# Note: conveniently, since the plot is in log scale, the pitch outputs \n", 623 | "# also get converted to the log scale automatically by matplotlib.\n", 624 | "plt.scatter(confident_pitch_outputs_x, confident_pitch_values_hz, c=\"r\")\n", 625 | "\n", 626 | "plt.show()" 627 | ], 628 | "execution_count": null, 629 | "outputs": [] 630 | }, 631 | { 632 | "cell_type": "markdown", 633 | "metadata": { 634 | "colab_type": "text", 635 | "id": "NskqpiHLxq6V" 636 | }, 637 | "source": [ 638 | "# Converting to musical notes\n", 639 | "\n", 640 | "Now that we have the pitch values, let's convert them to notes!\n", 641 | "This is part is challenging by itself. We have to take into account two things:\n", 642 | "1. the rests (when there's no singing) \n", 643 | "2. the size of each note (offsets) " 644 | ] 645 | }, 646 | { 647 | "cell_type": "markdown", 648 | "metadata": { 649 | "colab_type": "text", 650 | "id": "KDOlm9PLTTjt" 651 | }, 652 | "source": [ 653 | "### 1: Adding zeros to the output to indicate when there's no singing" 654 | ] 655 | }, 656 | { 657 | "cell_type": "code", 658 | "metadata": { 659 | "colab_type": "code", 660 | "id": "9uSQ3bJmTZmo", 661 | "colab": {} 662 | }, 663 | "source": [ 664 | "pitch_outputs_and_rests = [\n", 665 | " output2hz(p) if c >= 0.9 else 0\n", 666 | " for i, p, c in zip(indices, pitch_outputs, confidence_outputs)\n", 667 | "]" 668 | ], 669 | "execution_count": null, 670 | "outputs": [] 671 | }, 672 | { 673 | "cell_type": "markdown", 674 | "metadata": { 675 | "colab_type": "text", 676 | "id": "9fM0UwlsTt4w" 677 | }, 678 | "source": [ 679 | "### 2: Adding note offsets\n", 680 | "\n", 681 | "When a person sings freely, the melody may have an offset to the absolute pitch values that notes can represent.\n", 682 | "Hence, to convert predictions to notes, one needs to correct for this possible offset.\n", 683 | "This is what the following code computes." 684 | ] 685 | }, 686 | { 687 | "cell_type": "code", 688 | "metadata": { 689 | "colab_type": "code", 690 | "id": "fsJu-P5ksdFW", 691 | "colab": {} 692 | }, 693 | "source": [ 694 | "A4 = 440\n", 695 | "C0 = A4 * pow(2, -4.75)\n", 696 | "note_names = [\"C\", \"C#\", \"D\", \"D#\", \"E\", \"F\", \"F#\", \"G\", \"G#\", \"A\", \"A#\", \"B\"]\n", 697 | "\n", 698 | "def hz2offset(freq):\n", 699 | " # This measures the quantization error for a single note.\n", 700 | " if freq == 0: # Rests always have zero error.\n", 701 | " return None\n", 702 | " # Quantized note.\n", 703 | " h = round(12 * math.log2(freq / C0))\n", 704 | " return 12 * math.log2(freq / C0) - h\n", 705 | "\n", 706 | "\n", 707 | "# The ideal offset is the mean quantization error for all the notes\n", 708 | "# (excluding rests):\n", 709 | "offsets = [hz2offset(p) for p in pitch_outputs_and_rests if p != 0]\n", 710 | "print(\"offsets: \", offsets)\n", 711 | "\n", 712 | "ideal_offset = statistics.mean(offsets)\n", 713 | "print(\"ideal offset: \", ideal_offset)\n" 714 | ], 715 | "execution_count": null, 716 | "outputs": [] 717 | }, 718 | { 719 | "cell_type": "markdown", 720 | "metadata": { 721 | "colab_type": "text", 722 | "id": "K17It_qT2DtE" 723 | }, 724 | "source": [ 725 | "We can now use some heuristics to try and estimate the most likely sequence of notes that were sung.\n", 726 | "The ideal offset computed above is one ingredient - but we also need to know the speed (how many predictions make, say, an eighth?), and the time offset to start quantizing. To keep it simple, we'll just try different speeds and time offsets and measure the quantization error, using in the end the values that minimize this error." 727 | ] 728 | }, 729 | { 730 | "cell_type": "code", 731 | "metadata": { 732 | "colab_type": "code", 733 | "id": "eMUTI4L52ZHA", 734 | "colab": {} 735 | }, 736 | "source": [ 737 | "def quantize_predictions(group, ideal_offset):\n", 738 | " # Group values are either 0, or a pitch in Hz.\n", 739 | " non_zero_values = [v for v in group if v != 0]\n", 740 | " zero_values_count = len(group) - len(non_zero_values)\n", 741 | "\n", 742 | " # Create a rest if 80% is silent, otherwise create a note.\n", 743 | " if zero_values_count > 0.8 * len(group):\n", 744 | " # Interpret as a rest. Count each dropped note as an error, weighted a bit\n", 745 | " # worse than a badly sung note (which would 'cost' 0.5).\n", 746 | " return 0.51 * len(non_zero_values), \"Rest\"\n", 747 | " else:\n", 748 | " # Interpret as note, estimating as mean of non-rest predictions.\n", 749 | " h = round(\n", 750 | " statistics.mean([\n", 751 | " 12 * math.log2(freq / C0) - ideal_offset for freq in non_zero_values\n", 752 | " ]))\n", 753 | " octave = h // 12\n", 754 | " n = h % 12\n", 755 | " note = note_names[n] + str(octave)\n", 756 | " # Quantization error is the total difference from the quantized note.\n", 757 | " error = sum([\n", 758 | " abs(12 * math.log2(freq / C0) - ideal_offset - h)\n", 759 | " for freq in non_zero_values\n", 760 | " ])\n", 761 | " return error, note\n", 762 | "\n", 763 | "\n", 764 | "def get_quantization_and_error(pitch_outputs_and_rests, predictions_per_eighth,\n", 765 | " prediction_start_offset, ideal_offset):\n", 766 | " # Apply the start offset - we can just add the offset as rests.\n", 767 | " pitch_outputs_and_rests = [0] * prediction_start_offset + \\\n", 768 | " pitch_outputs_and_rests\n", 769 | " # Collect the predictions for each note (or rest).\n", 770 | " groups = [\n", 771 | " pitch_outputs_and_rests[i:i + predictions_per_eighth]\n", 772 | " for i in range(0, len(pitch_outputs_and_rests), predictions_per_eighth)\n", 773 | " ]\n", 774 | "\n", 775 | " quantization_error = 0\n", 776 | "\n", 777 | " notes_and_rests = []\n", 778 | " for group in groups:\n", 779 | " error, note_or_rest = quantize_predictions(group, ideal_offset)\n", 780 | " quantization_error += error\n", 781 | " notes_and_rests.append(note_or_rest)\n", 782 | "\n", 783 | " return quantization_error, notes_and_rests\n", 784 | "\n", 785 | "\n", 786 | "best_error = float(\"inf\")\n", 787 | "best_notes_and_rests = None\n", 788 | "best_predictions_per_note = None\n", 789 | "\n", 790 | "for predictions_per_note in range(20, 65, 1):\n", 791 | " for prediction_start_offset in range(predictions_per_note):\n", 792 | "\n", 793 | " error, notes_and_rests = get_quantization_and_error(\n", 794 | " pitch_outputs_and_rests, predictions_per_note,\n", 795 | " prediction_start_offset, ideal_offset)\n", 796 | "\n", 797 | " if error < best_error: \n", 798 | " best_error = error\n", 799 | " best_notes_and_rests = notes_and_rests\n", 800 | " best_predictions_per_note = predictions_per_note\n", 801 | "\n", 802 | "# At this point, best_notes_and_rests contains the best quantization.\n", 803 | "# Since we don't need to have rests at the beginning, let's remove these:\n", 804 | "while best_notes_and_rests[0] == 'Rest':\n", 805 | " best_notes_and_rests = best_notes_and_rests[1:]\n", 806 | "# Also remove silence at the end.\n", 807 | "while best_notes_and_rests[-1] == 'Rest':\n", 808 | " best_notes_and_rests = best_notes_and_rests[:-1]" 809 | ], 810 | "execution_count": null, 811 | "outputs": [] 812 | }, 813 | { 814 | "cell_type": "markdown", 815 | "metadata": { 816 | "colab_type": "text", 817 | "id": "vMZbWA3aVqee" 818 | }, 819 | "source": [ 820 | "Now let's write the quantized notes as sheet music score!\n", 821 | "\n", 822 | "To do it we will use two libraries: [music21](http://web.mit.edu/music21/) and [Open Sheet Music Display](https://github.com/opensheetmusicdisplay/opensheetmusicdisplay)\n", 823 | "\n", 824 | "**Note:** for simplicity, we assume here that all notes have the same duration (a half note)." 825 | ] 826 | }, 827 | { 828 | "cell_type": "code", 829 | "metadata": { 830 | "colab_type": "code", 831 | "id": "yVrk_IOIzpQR", 832 | "colab": {} 833 | }, 834 | "source": [ 835 | "# Creating the sheet music score.\n", 836 | "sc = music21.stream.Score()\n", 837 | "# Adjust the speed to match the actual singing.\n", 838 | "bpm = 60 * 60 / best_predictions_per_note\n", 839 | "print ('bpm: ', bpm)\n", 840 | "a = music21.tempo.MetronomeMark(number=bpm)\n", 841 | "sc.insert(0,a)\n", 842 | "\n", 843 | "for snote in best_notes_and_rests: \n", 844 | " d = 'half'\n", 845 | " if snote == 'Rest': \n", 846 | " sc.append(music21.note.Rest(type=d))\n", 847 | " else:\n", 848 | " sc.append(music21.note.Note(snote, type=d))" 849 | ], 850 | "execution_count": null, 851 | "outputs": [] 852 | }, 853 | { 854 | "cell_type": "code", 855 | "metadata": { 856 | "cellView": "both", 857 | "colab_type": "code", 858 | "id": "CEleCWHtG2s4", 859 | "colab": {} 860 | }, 861 | "source": [ 862 | "#@title [Run this] Helper function to use Open Sheet Music Display (JS code) to show a music score\n", 863 | "\n", 864 | "from IPython.core.display import display, HTML, Javascript\n", 865 | "import json, random\n", 866 | "\n", 867 | "def showScore(score):\n", 868 | " xml = open(score.write('musicxml')).read()\n", 869 | " showMusicXML(xml)\n", 870 | " \n", 871 | "def showMusicXML(xml):\n", 872 | " DIV_ID = \"OSMD_div\"\n", 873 | " display(HTML('
loading OpenSheetMusicDisplay
'))\n", 874 | " script = \"\"\"\n", 875 | " var div_id = {{DIV_ID}};\n", 876 | " function loadOSMD() { \n", 877 | " return new Promise(function(resolve, reject){\n", 878 | " if (window.opensheetmusicdisplay) {\n", 879 | " return resolve(window.opensheetmusicdisplay)\n", 880 | " }\n", 881 | " // OSMD script has a 'define' call which conflicts with requirejs\n", 882 | " var _define = window.define // save the define object \n", 883 | " window.define = undefined // now the loaded script will ignore requirejs\n", 884 | " var s = document.createElement( 'script' );\n", 885 | " s.setAttribute( 'src', \"https://cdn.jsdelivr.net/npm/opensheetmusicdisplay@0.7.6/build/opensheetmusicdisplay.min.js\" );\n", 886 | " //s.setAttribute( 'src', \"/custom/opensheetmusicdisplay.js\" );\n", 887 | " s.onload=function(){\n", 888 | " window.define = _define\n", 889 | " resolve(opensheetmusicdisplay);\n", 890 | " };\n", 891 | " document.body.appendChild( s ); // browser will try to load the new script tag\n", 892 | " }) \n", 893 | " }\n", 894 | " loadOSMD().then((OSMD)=>{\n", 895 | " window.openSheetMusicDisplay = new OSMD.OpenSheetMusicDisplay(div_id, {\n", 896 | " drawingParameters: \"compacttight\"\n", 897 | " });\n", 898 | " openSheetMusicDisplay\n", 899 | " .load({{data}})\n", 900 | " .then(\n", 901 | " function() {\n", 902 | " openSheetMusicDisplay.render();\n", 903 | " }\n", 904 | " );\n", 905 | " })\n", 906 | " \"\"\".replace('{{DIV_ID}}',DIV_ID).replace('{{data}}',json.dumps(xml))\n", 907 | " display(Javascript(script))\n", 908 | " return" 909 | ], 910 | "execution_count": null, 911 | "outputs": [] 912 | }, 913 | { 914 | "cell_type": "code", 915 | "metadata": { 916 | "colab_type": "code", 917 | "id": "WTu4phq4WeAI", 918 | "colab": {} 919 | }, 920 | "source": [ 921 | "# rendering the music score\n", 922 | "showScore(sc)\n", 923 | "print(best_notes_and_rests)" 924 | ], 925 | "execution_count": null, 926 | "outputs": [] 927 | }, 928 | { 929 | "cell_type": "markdown", 930 | "metadata": { 931 | "colab_type": "text", 932 | "id": "fGPXm6Z83U2g" 933 | }, 934 | "source": [ 935 | "Let's convert the music notes to a MIDI file and listen to it.\n", 936 | "\n", 937 | "To create this file, we can use the stream we created before." 938 | ] 939 | }, 940 | { 941 | "cell_type": "code", 942 | "metadata": { 943 | "colab_type": "code", 944 | "id": "klYoWjgmPaod", 945 | "colab": {} 946 | }, 947 | "source": [ 948 | "# Saving the recognized musical notes as a MIDI file\n", 949 | "converted_audio_file_as_midi = converted_audio_file[:-4] + '.mid'\n", 950 | "fp = sc.write('midi', fp=converted_audio_file_as_midi)" 951 | ], 952 | "execution_count": null, 953 | "outputs": [] 954 | }, 955 | { 956 | "cell_type": "code", 957 | "metadata": { 958 | "colab_type": "code", 959 | "id": "tz7Mj3Qx1lpR", 960 | "colab": {} 961 | }, 962 | "source": [ 963 | "wav_from_created_midi = converted_audio_file_as_midi.replace(' ', '_') + \"_midioutput.wav\"\n", 964 | "print(wav_from_created_midi)" 965 | ], 966 | "execution_count": null, 967 | "outputs": [] 968 | }, 969 | { 970 | "cell_type": "markdown", 971 | "metadata": { 972 | "colab_type": "text", 973 | "id": "ahss5EOiWDDp" 974 | }, 975 | "source": [ 976 | "To listen to it on colab, we need to convert it back to wav. An easy way of doing that is using Timidity." 977 | ] 978 | }, 979 | { 980 | "cell_type": "code", 981 | "metadata": { 982 | "colab_type": "code", 983 | "id": "XmeJ-UITV2nq", 984 | "colab": {} 985 | }, 986 | "source": [ 987 | "!timidity $converted_audio_file_as_midi -Ow -o $wav_from_created_midi" 988 | ], 989 | "execution_count": null, 990 | "outputs": [] 991 | }, 992 | { 993 | "cell_type": "markdown", 994 | "metadata": { 995 | "colab_type": "text", 996 | "id": "bnvwmyNj7kCC" 997 | }, 998 | "source": [ 999 | "And finally, listen the audio, created from notes, created via MIDI from the predicted pitches, inferred by the model!\n" 1000 | ] 1001 | }, 1002 | { 1003 | "cell_type": "code", 1004 | "metadata": { 1005 | "colab_type": "code", 1006 | "id": "qNLBB0zJV6vN", 1007 | "colab": {} 1008 | }, 1009 | "source": [ 1010 | "Audio(wav_from_created_midi)" 1011 | ], 1012 | "execution_count": null, 1013 | "outputs": [] 1014 | } 1015 | ] 1016 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pitch_Estimator 2 | Music Pitch detection using Tensorflow SPICE model. 3 | 4 | Pitch is a perceptual property of sounds that allows their ordering on a frequency-related scale, or more commonly, pitch is the quality that makes it possible to judge sounds as “higher” and “lower” in the sense associated with musical melodies. Pitch is a major auditory attribute of musical tones, along with [duration](https://en.wikipedia.org/wiki/Duration_(music)), [loudness](https://en.wikipedia.org/wiki/Loudness), and [timbre](https://en.wikipedia.org/wiki/Timbre), is quantified by frequency and measured in Hertz (Hz), where one Hz corresponds to one cycle per second. 5 | 6 | Pitch detection is an interesting challenge. Historically, the study of pitch and pitch perception has been a central problem in psychoacoustics, and has been instrumental in forming and testing theories of sound representation, [signal-processing algorithms](https://en.wikipedia.org/wiki/Pitch_detection_algorithm), and perception in the auditory system. A lot of [techniques](https://www.cs.uregina.ca/Research/Techreports/2003-06.pdf) have been used for this purpose. Efforts have also been made to separate the relevant frequency from background noise and backing instruments. 7 | 8 | Today, we can do that with machine learning, more specifically with the SPICE model ([SPICE: Self-Supervised Pitch Estimation](https://ai.googleblog.com/2019/11/spice-self-supervised-pitch-estimation.html)). This is a pretrained model that can recognize the fundamental pitch from mixed audio recordings (including noise and backing instruments).The model is available to use through [TensorFlow Hub](https://tfhub.dev/), on the web with [TensorFlow.js](https://tfhub.dev/google/tfjs-model/spice/1/default/1) and on mobile devices with [TensorFlow Lite](https://tfhub.dev/google/lite-model/spice/1). 9 | 10 | You can follow along with this [Colab notebook](https://colab.sandbox.google.com/github/tensorflow/hub/blob/master/examples/colab/spice.ipynb). 11 | 12 | **Note:** Application saves audio file in .wav format inside phone's internal storage memory in Pitch Detector folder so anyone can compare results with colab notebook output. For this purpose and for this demo version of the application, writing internal storage permission is mandatory for app to work! 13 | 14 | [Demo](https://youtu.be/v1d3o4r40PQ) of the project 15 | 16 | **Screenshot** 17 | 18 | 19 | -------------------------------------------------------------------------------- /app/.gitignore: -------------------------------------------------------------------------------- 1 | /build -------------------------------------------------------------------------------- /app/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'com.android.application' 2 | apply plugin: 'kotlin-android' 3 | apply plugin: 'kotlin-android-extensions' 4 | apply plugin: 'kotlin-kapt' 5 | 6 | android { 7 | compileSdkVersion 29 8 | buildToolsVersion "29.0.3" 9 | 10 | defaultConfig { 11 | applicationId "com.george.pitch_estimator" 12 | minSdkVersion 23 13 | targetSdkVersion 29 14 | versionCode 1 15 | versionName "1.0" 16 | 17 | testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" 18 | } 19 | 20 | buildTypes { 21 | release { 22 | minifyEnabled false 23 | proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' 24 | } 25 | } 26 | 27 | buildFeatures{ 28 | dataBinding = true 29 | // for view binding : 30 | // viewBinding = true 31 | } 32 | 33 | aaptOptions { 34 | noCompress "tflite" 35 | } 36 | 37 | compileOptions { 38 | sourceCompatibility JavaVersion.VERSION_1_8 39 | targetCompatibility JavaVersion.VERSION_1_8 40 | } 41 | 42 | kotlinOptions { 43 | jvmTarget = '1.8' 44 | } 45 | } 46 | 47 | dependencies { 48 | implementation fileTree(dir: "libs", include: ["*.jar"]) 49 | implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" 50 | implementation 'androidx.core:core-ktx:1.3.1' 51 | implementation 'androidx.appcompat:appcompat:1.2.0' 52 | implementation 'com.google.android.material:material:1.2.0' 53 | implementation 'androidx.constraintlayout:constraintlayout:1.1.3' 54 | implementation 'androidx.navigation:navigation-fragment-ktx:2.3.0' 55 | implementation 'androidx.navigation:navigation-ui-ktx:2.3.0' 56 | testImplementation 'junit:junit:4.12' 57 | androidTestImplementation 'androidx.test.ext:junit:1.1.1' 58 | androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' 59 | 60 | // Koin for Kotlin 61 | def koin_version = "2.1.5" 62 | implementation "org.koin:koin-core:$koin_version" 63 | // Koin for Android 64 | implementation "org.koin:koin-android:$koin_version" 65 | // Koin Android Scope features 66 | implementation "org.koin:koin-android-scope:$koin_version" 67 | // Koin Android ViewModel features 68 | implementation "org.koin:koin-android-viewmodel:$koin_version" 69 | 70 | // TF-Lite 71 | implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly' 72 | implementation 'org.tensorflow:tensorflow-lite-gpu:0.0.0-nightly' 73 | // For extra operations...it is mandatory for this project but adds significant size to final .apk 74 | implementation 'org.tensorflow:tensorflow-lite-select-tf-ops:0.0.0-nightly' 75 | 76 | } -------------------------------------------------------------------------------- /app/proguard-rules.pro: -------------------------------------------------------------------------------- 1 | # Add project specific ProGuard rules here. 2 | # You can control the set of applied configuration files using the 3 | # proguardFiles setting in build.gradle. 4 | # 5 | # For more details, see 6 | # http://developer.android.com/guide/developing/tools/proguard.html 7 | 8 | # If your project uses WebView with JS, uncomment the following 9 | # and specify the fully qualified class name to the JavaScript interface 10 | # class: 11 | #-keepclassmembers class fqcn.of.javascript.interface.for.webview { 12 | # public *; 13 | #} 14 | 15 | # Uncomment this to preserve the line number information for 16 | # debugging stack traces. 17 | #-keepattributes SourceFile,LineNumberTable 18 | 19 | # If you keep the line number information, uncomment this to 20 | # hide the original source file name. 21 | #-renamesourcefileattribute SourceFile -------------------------------------------------------------------------------- /app/src/androidTest/java/com/george/pitch_estimator/ExampleInstrumentedTest.kt: -------------------------------------------------------------------------------- 1 | package com.george.pitch_estimator 2 | 3 | import androidx.test.platform.app.InstrumentationRegistry 4 | import androidx.test.ext.junit.runners.AndroidJUnit4 5 | 6 | import org.junit.Test 7 | import org.junit.runner.RunWith 8 | 9 | import org.junit.Assert.* 10 | 11 | /** 12 | * Instrumented test, which will execute on an Android device. 13 | * 14 | * See [testing documentation](http://d.android.com/tools/testing). 15 | */ 16 | @RunWith(AndroidJUnit4::class) 17 | class ExampleInstrumentedTest { 18 | @Test 19 | fun useAppContext() { 20 | // Context of the app under test. 21 | val appContext = InstrumentationRegistry.getInstrumentation().targetContext 22 | assertEquals("com.george.pitch_estimator", appContext.packageName) 23 | } 24 | } -------------------------------------------------------------------------------- /app/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 17 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /app/src/main/assets/final1.txt: -------------------------------------------------------------------------------- 1 | 2 | 74 | 75 |
76 |
77 |
78 |
79 |
80 | 83 |

Musical Sheet

84 |
85 |
86 | 89 |

90 |
91 | 92 |
93 |
94 | 95 | 96 | 139 | 140 | 185 | 186 | -------------------------------------------------------------------------------- /app/src/main/assets/lite-model_spice_1.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farmaker47/Pitch_Estimator/63afeca2b19b1937c82bce2af71aa6af444020a5/app/src/main/assets/lite-model_spice_1.tflite -------------------------------------------------------------------------------- /app/src/main/ic_launcher-playstore.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/farmaker47/Pitch_Estimator/63afeca2b19b1937c82bce2af71aa6af444020a5/app/src/main/ic_launcher-playstore.png -------------------------------------------------------------------------------- /app/src/main/java/com/george/pitch_estimator/PitchEstimatorApplication.kt: -------------------------------------------------------------------------------- 1 | package com.george.pitch_estimator 2 | 3 | import android.app.Application 4 | import com.george.pitch_estimator.di.singingFragmentModule 5 | import kotlinx.coroutines.CoroutineScope 6 | import kotlinx.coroutines.Dispatchers 7 | import kotlinx.coroutines.launch 8 | import org.koin.android.ext.koin.androidContext 9 | import org.koin.core.context.startKoin 10 | 11 | class PitchEstimatorApplication : Application() { 12 | 13 | private val applicationScope = CoroutineScope(Dispatchers.Default) 14 | 15 | override fun onCreate() { 16 | super.onCreate() 17 | startKoin { 18 | //androidContext(applicationContext) 19 | androidContext(this@PitchEstimatorApplication) 20 | modules( 21 | singingFragmentModule 22 | ) 23 | } 24 | 25 | delayedInit() 26 | } 27 | 28 | private fun delayedInit() { 29 | applicationScope.launch { 30 | Thread.sleep(1_000) 31 | } 32 | } 33 | } -------------------------------------------------------------------------------- /app/src/main/java/com/george/pitch_estimator/PitchModelExecutor.kt: -------------------------------------------------------------------------------- 1 | package com.george.pitch_estimator 2 | 3 | import android.content.Context 4 | import android.util.Log 5 | import org.tensorflow.lite.Interpreter 6 | import org.tensorflow.lite.gpu.GpuDelegate 7 | import java.io.FileInputStream 8 | import java.io.IOException 9 | import java.nio.MappedByteBuffer 10 | import java.nio.channels.FileChannel 11 | import kotlin.math.* 12 | 13 | class PitchModelExecutor( 14 | context: Context, 15 | useGPU: Boolean 16 | ) { 17 | private var gpuDelegate: GpuDelegate = GpuDelegate() 18 | private var numberThreads = 4 19 | private var interpreter: Interpreter 20 | private var predictTime = 0L 21 | val note_names = mapOf( 22 | // musical notes 23 | // ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"] 24 | 0 to "C", 25 | 1 to "C#", 26 | 2 to "D", 27 | 3 to "D#", 28 | 4 to "E", 29 | 5 to "F", 30 | 6 to "F#", 31 | 7 to "G", 32 | 8 to "G#", 33 | 9 to "A", 34 | 10 to "A#", 35 | 11 to "B" 36 | ) 37 | 38 | init { 39 | interpreter = if (useGPU) { 40 | getInterpreter(context, PITCH_MODEL, true) 41 | } else { 42 | getInterpreter(context, PITCH_MODEL, false) 43 | } 44 | } 45 | 46 | // Values from original blog post colab notebook 47 | // https://colab.sandbox.google.com/github/tensorflow/hub/blob/master/examples/colab/spice.ipynb 48 | // https://blog.tensorflow.org/2020/06/estimating-pitch-with-spice-and-tensorflow-hub.html 49 | companion object { 50 | private const val PITCH_MODEL = "lite-model_spice_1.tflite" 51 | private const val PT_OFFSET = 25.58 52 | private const val PT_SLOPE = 63.07 53 | private const val FMIN = 10.0 54 | private const val BINS_PER_OCTAVE = 12.0 55 | private const val C0 = 16.351597831287414 56 | } 57 | 58 | fun execute(floatsInput: FloatArray): ArrayList { 59 | 60 | predictTime = System.currentTimeMillis() 61 | val inputSize = floatsInput.size // ~2 seconds of sound 62 | var outputSize = 0 63 | when (inputSize) { 64 | // 16.000 * 2 seconds recording 65 | 32000 -> outputSize = ceil(inputSize / 512.0).toInt() 66 | else -> outputSize = (ceil(inputSize / 512.0) + 1).toInt() 67 | } 68 | val inputValues = floatsInput//FloatArray(inputSize) 69 | 70 | val inputs = arrayOf(inputValues) 71 | val outputs = HashMap() 72 | 73 | val pitches = FloatArray(outputSize) 74 | val uncertainties = FloatArray(outputSize) 75 | 76 | outputs[0] = pitches 77 | outputs[1] = uncertainties 78 | 79 | try { 80 | interpreter.runForMultipleInputsOutputs(inputs, outputs) 81 | } catch (e: Exception) { 82 | Log.e("EXCEPTION", e.toString()) 83 | } 84 | 85 | Log.i("PITCHES", pitches.contentToString()) 86 | Log.i("PITCHES_SIZE", pitches.size.toString()) 87 | Log.i("UNCERTAIN", uncertainties.contentToString()) 88 | Log.i("UNCERTAIN_SIZE", uncertainties.size.toString()) 89 | 90 | // Calculate confidence over 90% 91 | // and store values inside an array list of floats 92 | // if confidence is lower than 90% then add 0F 93 | val arrayForConfidence = arrayListOf() 94 | for (i in uncertainties.indices) { 95 | if (1 - uncertainties[i] >= 0.9) { 96 | arrayForConfidence.add(pitches[i]) 97 | } else { 98 | arrayForConfidence.add(0F) 99 | } 100 | } 101 | 102 | // The pitch values returned by SPICE are in the range from 0 to 1. 103 | // Let's convert them to absolute pitch values in Hz. 104 | val hertzValues = DoubleArray(arrayForConfidence.size) 105 | for (i in 0 until arrayForConfidence.size) { 106 | hertzValues[i] = convertToAbsolutePitchValuesInHz(arrayForConfidence[i]) 107 | } 108 | 109 | // Calculate the offset during singing 110 | // When a person sings freely, the melody may have an offset to the absolute pitch values that notes can represent. 111 | // Hence, to convert predictions to notes, one needs to correct for this possible offset. 112 | val arrayForOffset = arrayListOf() 113 | for (i in hertzValues.indices) { 114 | if (hertzValues[i] > 0) 115 | arrayForOffset.add(hzToOffset(hertzValues[i].toFloat())) 116 | } 117 | 118 | val idealOffset = arrayForOffset.average() 119 | //Log.i("OFFSETS_AVERAGE", idealOffset.toString()) 120 | 121 | // We can now use some heuristics to try and estimate the most likely sequence of notes that were sung. 122 | // The ideal offset computed above is one ingredient - but we also need to know the speed 123 | // (how many predictions make, say, an eighth?), and the time offset to start quantizing. 124 | // To keep it simple, we'll just try different speeds and time offsets and measure the quantization error, 125 | // using in the end the values that minimize this error. 126 | 127 | // Code translation from python notebook from https://colab.sandbox.google.com/github/tensorflow/hub/blob/master/examples/colab/spice.ipynb 128 | var bestError = 10000000000000F//("+Inf").toFloat() 129 | var bestNotesAndRests = arrayListOf() 130 | var bestPredictionsPerNote = 0 131 | 132 | for (predictions_per_note in 20 until 65 step 1) { 133 | for (prediction_start_offset in 0 until predictions_per_note) { 134 | 135 | val (error, notes_and_rests) = getQuantizationAndError( 136 | hertzValues, predictions_per_note, 137 | prediction_start_offset, idealOffset.toFloat() 138 | ) 139 | 140 | if (error < bestError) { 141 | bestError = error 142 | bestNotesAndRests = notes_and_rests 143 | bestPredictionsPerNote = predictions_per_note 144 | } 145 | 146 | } 147 | } 148 | 149 | // BPM calculation 150 | val bpm = 60 * 60 / bestPredictionsPerNote 151 | Log.i("BPM", bpm.toString()) 152 | 153 | Log.i("BEST_ERROR", bestError.toString()) 154 | for (i in 0 until bestNotesAndRests.size) { 155 | Log.i("NOTES_AND_RESTS", bestNotesAndRests[i]) 156 | } 157 | 158 | // Remove rest at beginning and end of arrayList 159 | val noRestInBeginningAndEnd = arrayListOf() 160 | for (i in 0 until bestNotesAndRests.size) { 161 | if (i == 0 && bestNotesAndRests[0] != "Rest") { 162 | noRestInBeginningAndEnd.add(bestNotesAndRests[i]) 163 | } else if (i > 0 && i < bestNotesAndRests.size - 1) { 164 | noRestInBeginningAndEnd.add(bestNotesAndRests[i]) 165 | } else if (i == bestNotesAndRests.size - 1 && bestNotesAndRests[bestNotesAndRests.size - 1] != "Rest" 166 | ) { 167 | noRestInBeginningAndEnd.add(bestNotesAndRests[i]) 168 | } 169 | } 170 | 171 | predictTime = System.currentTimeMillis() - predictTime 172 | Log.i("PITCHES_TIME", predictTime.toString()) 173 | 174 | return noRestInBeginningAndEnd // ArrayList 175 | } 176 | 177 | private fun convertToAbsolutePitchValuesInHz(value: Float): Double { 178 | return if (value != 0F) { 179 | val cqtBin = value * PT_SLOPE + PT_OFFSET 180 | FMIN * (2.0.pow(cqtBin / BINS_PER_OCTAVE)) 181 | } else { 182 | 0.toDouble() 183 | } 184 | } 185 | 186 | private fun hzToOffset(hertzFloat: Float): Float { 187 | val h = (12 * log2(hertzFloat / C0)).roundToInt().toFloat() 188 | return (12 * log2(hertzFloat / C0) - h).toFloat() 189 | } 190 | 191 | private fun quantize_predictions(group: FloatArray, ideal_offset: Float): Pair { 192 | // Group values are either 0, or a pitch in Hz. 193 | val non_zero_values = arrayListOf() 194 | for (i in group.indices) { 195 | if (group[i] > 0) { 196 | non_zero_values.add(group[i]) 197 | } 198 | } 199 | //print(non_zero_values) 200 | val zero_values_count = group.size - non_zero_values.size 201 | 202 | // Create a rest if 80% is silent, otherwise create a note. 203 | if (zero_values_count > 0.8 * group.size) { 204 | // Interpret as a rest. Count each dropped note as an error, weighted a bit 205 | // worse than a badly sung note (which would 'cost' 0.5). 206 | return Pair(0.51 * non_zero_values.size, "Rest") 207 | } else { 208 | // Interpret as note, estimating as mean of non-rest predictions. 209 | val nonZeroAverageValues = arrayListOf() 210 | for (i in 0 until non_zero_values.size) { 211 | nonZeroAverageValues.add((12 * log2(non_zero_values[i] / C0) - ideal_offset).toFloat()) 212 | } 213 | 214 | val h = nonZeroAverageValues.average().roundToInt() 215 | val octave = h / 12 216 | //Log.i("OCTAVE",octave.toString()) 217 | val n = h.rem(12) 218 | //Log.i("NOTE",n.toString()) 219 | val note = note_names[n] + octave.toString() 220 | // Quantization error is the total difference from the quantized note. 221 | val nonZeroErrorValues = arrayListOf() 222 | for (i in 0 until non_zero_values.size) { 223 | nonZeroErrorValues.add( 224 | abs(12 * log2(non_zero_values[i] / C0) - ideal_offset - h) 225 | .toFloat() 226 | ) 227 | } 228 | val error = nonZeroErrorValues.sum() 229 | 230 | return Pair(error.toDouble(), note) 231 | } 232 | } 233 | 234 | private fun getQuantizationAndError( 235 | pitch_outputs_and_rests: DoubleArray, predictions_per_eighth: Int, 236 | prediction_start_offset: Int, ideal_offset: Float 237 | ): Pair> { 238 | 239 | val pitchOutputsAndRestsWithOffset = arrayListOf() 240 | for (i in 0 until prediction_start_offset) { 241 | pitchOutputsAndRestsWithOffset.add(0F) 242 | } 243 | 244 | for (i in pitch_outputs_and_rests.indices) { 245 | pitchOutputsAndRestsWithOffset.add(pitch_outputs_and_rests[i].toFloat()) 246 | } 247 | 248 | val groups = arrayListOf() 249 | for (i in 0 until pitchOutputsAndRestsWithOffset.size step predictions_per_eighth) { 250 | val firstArrayList = arrayListOf() 251 | try { 252 | for (k in i until i + predictions_per_eighth) { 253 | firstArrayList.add(pitchOutputsAndRestsWithOffset[k]) 254 | } 255 | 256 | } catch (e: Exception) { 257 | //Log.e("EXCEPTION", e.toString()) 258 | } 259 | 260 | 261 | val secondArray = FloatArray(firstArrayList.size) 262 | for (l in 0 until firstArrayList.size) { 263 | secondArray[l] = firstArrayList[l] 264 | } 265 | 266 | groups.add(secondArray) 267 | } 268 | 269 | //# Collect the predictions for each note (or rest). 270 | var quantizationError = 0.0 271 | val notesAndRests = arrayListOf() 272 | for (m in 0 until groups.size) { 273 | val (error, note) = quantize_predictions(groups[m], ideal_offset) 274 | quantizationError += error 275 | notesAndRests.add(note) 276 | } 277 | 278 | return Pair(quantizationError.toFloat(), notesAndRests) 279 | } 280 | 281 | // load tflite file from assets folder 282 | @Throws(IOException::class) 283 | private fun loadModelFile(context: Context, modelFile: String): MappedByteBuffer { 284 | val fileDescriptor = context.assets.openFd(modelFile) 285 | val inputStream = FileInputStream(fileDescriptor.fileDescriptor) 286 | val fileChannel = inputStream.channel 287 | val startOffset = fileDescriptor.startOffset 288 | val declaredLength = fileDescriptor.declaredLength 289 | val retFile = fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength) 290 | fileDescriptor.close() 291 | try { 292 | 293 | } catch (e: Exception) { 294 | Log.e("FIND_ERROR", e.toString()) 295 | } 296 | return retFile 297 | } 298 | 299 | @Throws(IOException::class) 300 | private fun getInterpreter( 301 | context: Context, 302 | modelName: String, 303 | useGpu: Boolean 304 | ): Interpreter { 305 | val tfliteOptions = Interpreter.Options() 306 | if (useGpu) { 307 | gpuDelegate = GpuDelegate() 308 | tfliteOptions.addDelegate(gpuDelegate) 309 | } 310 | 311 | tfliteOptions.setNumThreads(numberThreads) 312 | return Interpreter(loadModelFile(context, modelName), tfliteOptions) 313 | } 314 | 315 | fun close() { 316 | interpreter.close() 317 | } 318 | } -------------------------------------------------------------------------------- /app/src/main/java/com/george/pitch_estimator/SingRecorder.kt: -------------------------------------------------------------------------------- 1 | package com.george.pitch_estimator 2 | 3 | import android.content.Context 4 | import android.media.AudioFormat 5 | import android.media.AudioRecord 6 | import android.media.MediaRecorder 7 | import android.os.Environment 8 | import android.util.Log 9 | import org.json.JSONObject 10 | import org.tensorflow.lite.Interpreter 11 | import org.tensorflow.lite.gpu.GpuDelegate 12 | import java.io.* 13 | import java.nio.MappedByteBuffer 14 | import java.nio.channels.FileChannel 15 | 16 | // Class to collect sound in the form of short arraylist 17 | // to make it ready for inference 18 | 19 | class SingRecorder( 20 | private val mHotwordKey: String, 21 | numberRecordings: Int, 22 | context: Context 23 | //vad: Vad, 24 | //listener: HotwordSpeechListener 25 | ) { 26 | private val AUDIO_SOURCE = MediaRecorder.AudioSource.VOICE_RECOGNITION 27 | private val SAMPLE_RATE = 16000 28 | private val CHANNEL_MASK = AudioFormat.CHANNEL_IN_MONO 29 | private val ENCODING = AudioFormat.ENCODING_PCM_16BIT 30 | private val BUFFER_SIZE = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_MASK, ENCODING) 31 | private val AUDIO_FORMAT = 32 | AudioFormat.Builder().setEncoding(ENCODING) 33 | .setSampleRate(SAMPLE_RATE) 34 | .setChannelMask(CHANNEL_MASK) 35 | .build() 36 | var mPcmStream: ByteArrayOutputStream 37 | private var mRecorder: AudioRecord? = null 38 | 39 | //private var mRecorderVad: AudioRecord? = null 40 | private var mRecording: Boolean = true 41 | private var mThread: Thread? = null 42 | 43 | //private var mVadThread: Thread? = null 44 | private val mSampleLengths: DoubleArray 45 | private var mSamplesTaken: Int 46 | private val mContext: Context 47 | 48 | //Vad and silence 49 | //private val mVad: Vad 50 | private var done = false 51 | 52 | //private boolean cancelled; 53 | private val mMinimumVoice = 100 54 | private val mMaximumSilence = 700 55 | private val mUpperLimit = 100 56 | 57 | // From commands 58 | /*private var recordingThread: Thread? = null 59 | var shouldContinue = true 60 | private val SAMPLE_DURATION_MS = 1000 61 | private val RECORDING_LENGTH = (SAMPLE_RATE * SAMPLE_DURATION_MS / 1000) as Int 62 | var recordingBuffer = ShortArray(RECORDING_LENGTH) 63 | private val recordingBufferLock = ReentrantLock() 64 | var recordingOffset = 0 65 | private var recognitionThread: Thread? = null 66 | var shouldContinueRecognition = true 67 | var buffer = ShortArray(BUFFER_SIZE)*/ 68 | var buffer = ShortArray(BUFFER_SIZE) 69 | var bufferForInference: ArrayList//= arrayListOf() 70 | 71 | init { 72 | mPcmStream = ByteArrayOutputStream() 73 | bufferForInference = arrayListOf() 74 | mRecording = false 75 | mSampleLengths = DoubleArray(numberRecordings) 76 | mSamplesTaken = 0 77 | mContext = context 78 | //mVad = vad 79 | //mWordListener = listener 80 | 81 | } 82 | 83 | /** 84 | * Start the recording process. 85 | */ 86 | fun startRecording() { 87 | mRecorder = AudioRecord.Builder().setAudioSource(AUDIO_SOURCE) 88 | .setAudioFormat(AUDIO_FORMAT) 89 | .setBufferSizeInBytes(BUFFER_SIZE) 90 | .build() 91 | /*mRecorderVad = AudioRecord.Builder().setAudioSource(AUDIO_SOURCE) 92 | .setAudioFormat(AUDIO_FORMAT) 93 | .setBufferSizeInBytes(BUFFER_SIZE) 94 | .build()*/ 95 | 96 | //mVad.start(); 97 | done = false 98 | mRecording = true 99 | mRecorder?.startRecording() 100 | //mRecorderVad?.startRecording() 101 | mThread = Thread(readAudio) 102 | mThread!!.start() 103 | 104 | /*mVadThread = Thread(readVad) 105 | mVadThread!!.start()*/ 106 | } 107 | 108 | fun stopRecording(): ByteArrayOutputStream { 109 | if (mRecorder != null && mRecorder!!.state == AudioRecord.STATE_INITIALIZED) { 110 | mRecording = false 111 | mRecorder?.stop() 112 | mRecorder?.release(); 113 | mRecorder = null; 114 | 115 | //mVad.stop(); 116 | //mRecorderVad!!.stop() 117 | done = true 118 | } 119 | return mPcmStream 120 | } 121 | 122 | fun stopRecordingForInference(): ArrayList { 123 | return bufferForInference 124 | } 125 | 126 | /** 127 | * Read audio from the audio recorder stream. 128 | */ 129 | private val readAudio = Runnable { 130 | var readBytes: Int 131 | buffer = ShortArray(BUFFER_SIZE) 132 | while (mRecording) { 133 | readBytes = mRecorder!!.read(buffer, 0, BUFFER_SIZE) 134 | 135 | //Higher volume of microphone 136 | //https://stackoverflow.com/questions/25441166/how-to-adjust-microphone-sensitivity-while-recording-audio-in-android 137 | if (readBytes > 0) { 138 | for (i in 0 until readBytes) { 139 | buffer[i] = Math.min( 140 | (buffer[i] * 6.7).toInt(), 141 | Short.MAX_VALUE.toInt() 142 | ).toShort() 143 | } 144 | } 145 | if (readBytes != AudioRecord.ERROR_INVALID_OPERATION) { 146 | for (s in buffer) { 147 | 148 | // Add all values to arraylist 149 | bufferForInference.add(s) 150 | 151 | writeShort(mPcmStream, s) 152 | } 153 | } 154 | } 155 | //Log.e("READ_AUDIO_BUFFER", buffer.size.toString()) 156 | //Log.e("PCMSTREam", mPcmStream.size().toString()) 157 | //Log.e("BUFFER_FOR_INFER_SIZE", bufferForInference.size.toString()) 158 | //Log.e("BUFFER_FOR_INFER_VALUES", bufferForInference.takeLast(100).toString()) 159 | } 160 | 161 | /*fun startRecordingCommands() { 162 | if (recordingThread != null) { 163 | return 164 | } 165 | shouldContinue = true 166 | recordingThread = Thread( 167 | Runnable { recordCommands() }) 168 | recordingThread?.start() 169 | }*/ 170 | 171 | /* private fun recordCommands() { 172 | Process.setThreadPriority(Process.THREAD_PRIORITY_AUDIO) 173 | 174 | // Estimate the buffer size we'll need for this device. 175 | var bufferSize = AudioRecord.getMinBufferSize( 176 | SAMPLE_RATE, 177 | AudioFormat.CHANNEL_IN_MONO, 178 | AudioFormat.ENCODING_PCM_16BIT 179 | ) 180 | if (bufferSize == AudioRecord.ERROR || bufferSize == AudioRecord.ERROR_BAD_VALUE) { 181 | bufferSize = SAMPLE_RATE * 2 182 | } 183 | val audioBuffer = ShortArray(bufferSize / 2) 184 | val record = AudioRecord( 185 | MediaRecorder.AudioSource.DEFAULT, 186 | SAMPLE_RATE, 187 | AudioFormat.CHANNEL_IN_MONO, 188 | AudioFormat.ENCODING_PCM_16BIT, 189 | bufferSize 190 | ) 191 | if (record.state != AudioRecord.STATE_INITIALIZED) { 192 | Log.e( 193 | "AUDIO_RECORD", 194 | "Audio Record can't initialize!" 195 | ) 196 | return 197 | } 198 | record.startRecording() 199 | *//*Log.v( 200 | , 201 | "Start recording" 202 | )*//* 203 | 204 | // Loop, gathering audio data and copying it to a round-robin buffer. 205 | while (shouldContinue) { 206 | val numberRead = record.read(audioBuffer, 0, audioBuffer.size) 207 | val maxLength: Int = recordingBuffer.size 208 | val newRecordingOffset: Int = recordingOffset + numberRead 209 | val secondCopyLength = Math.max(0, newRecordingOffset - maxLength) 210 | val firstCopyLength = numberRead - secondCopyLength 211 | // We store off all the data for the recognition thread to access. The ML 212 | // thread will copy out of this buffer into its own, while holding the 213 | // lock, so this should be thread safe. 214 | recordingBufferLock.lock() 215 | try { 216 | System.arraycopy( 217 | audioBuffer, 218 | 0, 219 | recordingBuffer, 220 | recordingOffset, 221 | firstCopyLength 222 | ) 223 | System.arraycopy( 224 | audioBuffer, 225 | firstCopyLength, 226 | recordingBuffer, 227 | 0, 228 | secondCopyLength 229 | ) 230 | recordingOffset = newRecordingOffset % maxLength 231 | } finally { 232 | recordingBufferLock.unlock() 233 | } 234 | } 235 | record.stop() 236 | record.release() 237 | } 238 | 239 | @Synchronized 240 | fun startRecognitionCommands() { 241 | if (recognitionThread != null) { 242 | return 243 | } 244 | shouldContinueRecognition = true 245 | recognitionThread = Thread( 246 | Runnable { recognizeCommands() }) 247 | recognitionThread?.start() 248 | } 249 | 250 | private fun recognizeCommands() { 251 | 252 | val inputBuffer = 253 | ShortArray(RECORDING_LENGTH) 254 | val floatInputBuffer = Array( 255 | RECORDING_LENGTH 256 | ) { FloatArray(1) } 257 | val sampleRateList = 258 | intArrayOf(SAMPLE_RATE) 259 | 260 | // Loop, grabbing recorded data and running the recognition model on it. 261 | while (shouldContinueRecognition) { 262 | //val startTime = Date().time 263 | // The recording thread places data in this round-robin buffer, so lock to 264 | // make sure there's no writing happening and then copy it to our own 265 | // local version. 266 | recordingBufferLock.lock() 267 | try { 268 | val maxLength = recordingBuffer.size 269 | val firstCopyLength = maxLength - recordingOffset 270 | val secondCopyLength = recordingOffset 271 | System.arraycopy( 272 | recordingBuffer, 273 | recordingOffset, 274 | inputBuffer, 275 | 0, 276 | firstCopyLength 277 | ) 278 | System.arraycopy( 279 | recordingBuffer, 280 | 0, 281 | inputBuffer, 282 | firstCopyLength, 283 | secondCopyLength 284 | ) 285 | } finally { 286 | recordingBufferLock.unlock() 287 | } 288 | 289 | // We need to feed in float values between -1.0f and 1.0f, so divide the 290 | // signed 16-bit inputs. 291 | for (i in 0 until RECORDING_LENGTH) { 292 | floatInputBuffer[i][0] = inputBuffer[i] / 32767.0f 293 | } 294 | Log.e("FLOATINPUTBUFFER", inputBuffer.takeLast(100).toString()) 295 | } 296 | 297 | } 298 | 299 | @Synchronized 300 | fun stopRecordingCommands() { 301 | if (recordingThread == null) { 302 | return 303 | } 304 | shouldContinue = false 305 | recordingThread = null 306 | } 307 | 308 | @Synchronized 309 | fun stopRecognitionCommands() { 310 | if (recognitionThread == null) { 311 | return 312 | } 313 | shouldContinueRecognition = false 314 | recognitionThread = null 315 | }*/ 316 | 317 | /** 318 | * Stop the recording process. 319 | */ 320 | /*private val readVad = Runnable { 321 | try { 322 | var vad = 0 323 | var finishedvoice = false 324 | var touchedvoice = false 325 | var touchedsilence = false 326 | var samplesvoice: Long = 0 327 | var samplessilence: Long = 0 328 | val dtantes = System.currentTimeMillis() 329 | var dtantesmili = System.currentTimeMillis() 330 | var done = false 331 | Process.setThreadPriority(Process.THREAD_PRIORITY_URGENT_AUDIO) 332 | while (mRecording && !done) { 333 | var nshorts = 0 334 | val mBuftemp = 335 | ShortArray(FRAME_SIZE * 1 * 2) 336 | nshorts = mRecorderVad!!.read(mBuftemp, 0, mBuftemp.size) 337 | vad = mVad.feed(mBuftemp, nshorts) 338 | val dtdepois = System.currentTimeMillis() 339 | if (vad == 0) { 340 | Log.e("VAD", "0") 341 | mHotwordListener.onSpeechChange(6789) 342 | if (touchedvoice) { 343 | samplessilence += dtdepois - dtantesmili 344 | if (samplessilence > mMaximumSilence) touchedsilence = true 345 | } 346 | } else { // vad == 1 => Active voice 347 | Log.e("VAD", "1") 348 | samplesvoice += dtdepois - dtantesmili 349 | if (samplesvoice > mMinimumVoice) touchedvoice = true 350 | mHotwordListener.onSpeechChange(6789) 351 | } 352 | dtantesmili = dtdepois 353 | if (touchedvoice && touchedsilence) finishedvoice = true 354 | if (finishedvoice) { 355 | done = true 356 | Log.e("FINISHED_VOICE", "FINISHED_VOICE") 357 | mHotwordListener.onSpeechChange(1234) 358 | } 359 | 360 | //if voice is over mUpperlimit = .. seconds 361 | if ((dtdepois - dtantes) / 1000 > mUpperLimit) { 362 | Log.e("DTDEPOIS", "UPPER_LIMIT") 363 | done = true 364 | if (touchedvoice) { 365 | Log.e("TOUCHED_VOICE", "TOUCHED_VOICE") 366 | } else { 367 | Log.e("RAISED_NO_VOICE", "RAISED_NO_VOICE") 368 | } 369 | } 370 | if (nshorts <= 0) break 371 | } 372 | 373 | *//*mRecorderVad.release();*//* 374 | *//*mRecorderVad = null;*//* 375 | 376 | *//*if (cancelled) { 377 | cancelled = false; 378 | Log.e("CANCELED","CANCELED"); 379 | return; 380 | }*//* 381 | } catch (exc: Exception) { 382 | val error = 383 | String.format("General audio error %s", exc.message) 384 | Log.e("GENERAL_ERROR", "GENERAL_ERROR") 385 | exc.printStackTrace() 386 | } 387 | }*/ 388 | 389 | /** 390 | * Convert raw PCM data to a wav file. 391 | * 392 | * 393 | * See: https://stackoverflow.com/questions/43569304/android-how-can-i-write-byte-to-wav-file 394 | * 395 | * @return Byte array containing wav file data. 396 | */ 397 | @Throws(IOException::class) 398 | private fun pcmToWav(byteArrayOutputStream: ByteArrayOutputStream): ByteArray { 399 | val stream = ByteArrayOutputStream() 400 | val pcmAudio = byteArrayOutputStream.toByteArray() 401 | writeString(stream, "RIFF") // chunk id 402 | writeInt(stream, 36 + pcmAudio.size) // chunk size 403 | writeString(stream, "WAVE") // format 404 | writeString(stream, "fmt ") // subchunk 1 id 405 | writeInt(stream, 16) // subchunk 1 size 406 | writeShort(stream, 1.toShort()) // audio format (1 = PCM) 407 | writeShort(stream, 1.toShort()) // number of channels 408 | writeInt(stream, SAMPLE_RATE) // sample rate 409 | writeInt(stream, SAMPLE_RATE * 2) // byte rate 410 | writeShort(stream, 2.toShort()) // block align 411 | writeShort(stream, 16.toShort()) // bits per sample 412 | writeString(stream, "data") // subchunk 2 id 413 | writeInt(stream, pcmAudio.size) // subchunk 2 size 414 | stream.write(pcmAudio) 415 | return stream.toByteArray() 416 | } 417 | 418 | /** 419 | * Write a 32-bit integer to an output stream, in Little Endian format. 420 | * 421 | * @param output Output stream 422 | * @param value Integer value 423 | */ 424 | private fun writeInt(output: ByteArrayOutputStream, value: Int) { 425 | output.write(value) 426 | output.write(value shr 8) 427 | output.write(value shr 16) 428 | output.write(value shr 24) 429 | } 430 | 431 | /** 432 | * Write a 16-bit integer to an output stream, in Little Endian format. 433 | * 434 | * @param output Output stream 435 | * @param value Integer value 436 | */ 437 | private fun writeShort( 438 | output: ByteArrayOutputStream, 439 | value: Short 440 | ) { 441 | output.write(value.toInt()) 442 | output.write(value.toInt() shr 8) 443 | } 444 | 445 | /** 446 | * Write a string to an output stream. 447 | * 448 | * @param output Output stream 449 | * @param value String value 450 | */ 451 | private fun writeString( 452 | output: ByteArrayOutputStream, 453 | value: String 454 | ) { 455 | for (i in 0 until value.length) { 456 | output.write(value[i].toInt()) 457 | } 458 | } 459 | 460 | /** 461 | * Generate a JSON config for the hotword. 462 | * 463 | * @return JSONObject containing config. 464 | */ 465 | private fun generateConfig(): JSONObject { 466 | val obj = JSONObject() 467 | try { 468 | obj.put("hotword_key", mHotwordKey) 469 | obj.put("kind", "personal") 470 | obj.put("dtw_ref", 0.22) 471 | obj.put("from_mfcc", 1) 472 | obj.put("to_mfcc", 13) 473 | obj.put("band_radius", 10) 474 | obj.put("shift", 10) 475 | obj.put("window_size", 10) 476 | obj.put("sample_rate", SAMPLE_RATE) 477 | obj.put("frame_length_ms", 25.0) 478 | obj.put("frame_shift_ms", 10.0) 479 | obj.put("num_mfcc", 13) 480 | obj.put("num_mel_bins", 13) 481 | obj.put("mel_low_freq", 20) 482 | obj.put("cepstral_lifter", 22.0) 483 | obj.put("dither", 0.0) 484 | obj.put("window_type", "povey") 485 | obj.put("use_energy", false) 486 | obj.put("energy_floor", 0.0) 487 | obj.put("raw_energy", true) 488 | obj.put("preemphasis_coefficient", 0.97) 489 | } finally { 490 | return obj 491 | } 492 | } 493 | 494 | /** 495 | * Write a wav file from the current sample. 496 | * 497 | * @throws IOException 498 | */ 499 | fun writeWav(byteArrayOutputStream: ByteArrayOutputStream) { 500 | var wav = ByteArray(0) 501 | try { 502 | wav = pcmToWav(byteArrayOutputStream) 503 | //Log.e("WAV_size", wav.size.toString()) 504 | } catch (e: IOException) { 505 | e.printStackTrace() 506 | } 507 | var stream: FileOutputStream? = null 508 | try { 509 | try { 510 | stream = FileOutputStream( 511 | Environment.getExternalStorageDirectory().toString() + 512 | "/Pitch Estimator/soloupis.wav", false 513 | ) 514 | } catch (e: FileNotFoundException) { 515 | e.printStackTrace() 516 | } 517 | try { 518 | stream?.write(wav) 519 | } catch (e: Exception) { 520 | Log.e("PERMIS_STORAGE_DENIED","NOT ABLE TO WRITE .WAV TO SDCARD") 521 | } 522 | } finally { 523 | if (stream != null) { 524 | try { 525 | stream.close() 526 | } catch (e: IOException) { 527 | e.printStackTrace() 528 | } 529 | } 530 | } 531 | } 532 | 533 | companion object { 534 | const val FRAME_SIZE = 80 535 | } 536 | 537 | fun reInitializePcmStream() { 538 | mPcmStream = ByteArrayOutputStream() 539 | bufferForInference = arrayListOf() 540 | } 541 | 542 | } -------------------------------------------------------------------------------- /app/src/main/java/com/george/pitch_estimator/binding_adapters/BindingAdapters.kt: -------------------------------------------------------------------------------- 1 | package com.george.pitch_estimator.binding_adapters 2 | 3 | import android.os.Handler 4 | import android.webkit.WebView 5 | import android.webkit.WebViewClient 6 | import android.widget.TextView 7 | import androidx.databinding.BindingAdapter 8 | import kotlin.math.roundToInt 9 | 10 | @BindingAdapter("doubleArrayToString") 11 | fun bindDoubleArrayHertzToValues(textView: TextView, value: DoubleArray?) { 12 | 13 | if (value != null) { 14 | val roundedArray = DoubleArray(value.size) 15 | for (i in value.indices) { 16 | roundedArray[i] = value[i].roundToInt().toDouble() 17 | } 18 | 19 | textView.text = roundedArray.contentToString() 20 | } 21 | 22 | } 23 | 24 | // Binding adapter to display notes output on screen 25 | @BindingAdapter("noteArrayListToString") 26 | fun bindDoubleArrayHertzToValues(textView: TextView, value: ArrayList?) { 27 | 28 | textView.text = value.toString() 29 | 30 | } 31 | 32 | // this binding adapter helps load custom html from assets folder 33 | @BindingAdapter("htmlToScreen") 34 | fun bindTextViewHtml(webView: WebView, htmlValue: String) { 35 | 36 | webView.settings.javaScriptEnabled = true 37 | 38 | /*webView.webViewClient = object : WebViewClient() { 39 | override fun onPageFinished(view: WebView, url: String) { 40 | super.onPageFinished(view, url) 41 | val handler = Handler() 42 | handler.postDelayed( 43 | { 44 | //webView.loadUrl("javascript:globalVariable('" + 370 + "')") 45 | //webView.loadUrl("javascript:(function(){l=document.getElementById('music_sheet');e=document.createEvent('HTMLEvents');e.initEvent('click',true,true);l.dispatchEvent(e);})()") 46 | 47 | }, 48 | 10 49 | ) 50 | } 51 | }*/ 52 | 53 | webView.loadDataWithBaseURL("fake://not/needed", htmlValue, "text/html", "UTF-8", "") 54 | } -------------------------------------------------------------------------------- /app/src/main/java/com/george/pitch_estimator/di/SingingFragmentModule.kt: -------------------------------------------------------------------------------- 1 | package com.george.pitch_estimator.di 2 | 3 | import com.george.pitch_estimator.PitchModelExecutor 4 | import com.george.pitch_estimator.SingRecorder 5 | import com.george.pitch_estimator.singingFragment.SingingFragmentViewModel 6 | import org.koin.android.viewmodel.dsl.viewModel 7 | import org.koin.dsl.module 8 | 9 | val singingFragmentModule = module { 10 | 11 | single { SingRecorder("hotKey", 0, get()) } 12 | 13 | // Use factory instead of single when user presses back button... 14 | // to force execution of init block when interpreter is closed 15 | factory { PitchModelExecutor(get(), getKoin().getProperty("koinUseGpu")!!) } 16 | 17 | viewModel { 18 | SingingFragmentViewModel(get()) 19 | } 20 | } -------------------------------------------------------------------------------- /app/src/main/java/com/george/pitch_estimator/singingFragment/SingingFragment.kt: -------------------------------------------------------------------------------- 1 | package com.george.pitch_estimator.singingFragment 2 | 3 | 4 | /*Copyright 2016 The TensorFlow Authors. All Rights Reserved. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License.*/ 17 | 18 | 19 | import android.Manifest 20 | import android.content.Context 21 | import android.content.pm.PackageManager 22 | import android.os.Bundle 23 | import android.os.Environment 24 | import android.os.Handler 25 | import android.util.Log 26 | import android.view.LayoutInflater 27 | import android.view.View 28 | import android.view.ViewGroup 29 | import android.view.animation.AnimationUtils 30 | import android.widget.Toast 31 | import androidx.core.app.ActivityCompat 32 | import androidx.fragment.app.Fragment 33 | import androidx.lifecycle.Observer 34 | import com.george.pitch_estimator.PitchModelExecutor 35 | import com.george.pitch_estimator.R 36 | import com.george.pitch_estimator.SingRecorder 37 | import com.george.pitch_estimator.databinding.FragmentFirstBinding 38 | import org.koin.android.ext.android.get 39 | import org.koin.android.ext.android.getKoin 40 | import org.koin.android.viewmodel.ext.android.viewModel 41 | import java.io.File 42 | 43 | 44 | /** 45 | * A simple [Fragment] subclass as the default destination in the navigation. 46 | */ 47 | class SingingFragment : Fragment(), 48 | ActivityCompat.OnRequestPermissionsResultCallback { 49 | 50 | private lateinit var binding: FragmentFirstBinding 51 | 52 | // Koin ViewModel DI 53 | private val viewModel: SingingFragmentViewModel by viewModel() 54 | private lateinit var singRecorder: SingRecorder 55 | private lateinit var pitchModelExecutor: PitchModelExecutor 56 | 57 | // Permissions 58 | var PERMISSION_ALL = 123 59 | // App saves .wav audio file inside external storage of phone so anyone can compare 60 | // results with the colab notebook output. For that purpose this permission is mandatory 61 | var PERMISSIONS = arrayOf( 62 | Manifest.permission.RECORD_AUDIO, 63 | Manifest.permission.WRITE_EXTERNAL_STORAGE 64 | ) 65 | 66 | override fun onCreateView( 67 | inflater: LayoutInflater, container: ViewGroup?, 68 | savedInstanceState: Bundle? 69 | ): View? { 70 | binding = FragmentFirstBinding.inflate(inflater) 71 | binding.lifecycleOwner = this 72 | 73 | binding.viewModelxml = viewModel 74 | 75 | binding.buttonForSinging.setOnClickListener { 76 | 77 | if (viewModel.singingRunning) { 78 | singingStopped() 79 | } else { 80 | // Start animation 81 | animateSharkButton() 82 | // Start collecting sound and inferring immediately 83 | viewModel.setUpdateLoopSingingHandler() 84 | 85 | // Start karaoke 86 | viewModel.setUpdateKaraokeHandler() 87 | 88 | //Toast.makeText(activity, "Singing has started", Toast.LENGTH_LONG).show() 89 | 90 | } 91 | } 92 | 93 | //Check for permissions 94 | initialize() 95 | 96 | // Generate folder for saving .wav later 97 | generateFolder() 98 | 99 | // Observe notes as they come out of model and update webview respectively 100 | viewModel.noteValuesToDisplay.observe(viewLifecycleOwner, 101 | androidx.lifecycle.Observer { list -> 102 | 103 | if (list.size > 0) { 104 | var i = 0 105 | val handler = Handler() 106 | handler.post(object : Runnable { 107 | override fun run() { 108 | when (list[i]) { 109 | "C2" -> binding.webView.loadUrl("javascript:myMove('125')") 110 | "C#2" -> binding.webView.loadUrl("javascript:myMoveSharp('125')") 111 | "D2" -> binding.webView.loadUrl("javascript:myMove('130')") 112 | "D#2" -> binding.webView.loadUrl("javascript:myMoveSharp('130')") 113 | "E2" -> binding.webView.loadUrl("javascript:myMove('135')") 114 | "F2" -> binding.webView.loadUrl("javascript:myMove('140')") 115 | "F#2" -> binding.webView.loadUrl("javascript:myMoveSharp('140')") 116 | "G2" -> binding.webView.loadUrl("javascript:myMove('145')") 117 | "G#2" -> binding.webView.loadUrl("javascript:myMoveSharp('145')") 118 | "A2" -> binding.webView.loadUrl("javascript:myMove('150')") 119 | "A#2" -> binding.webView.loadUrl("javascript:myMoveSharp('150')") 120 | "B2" -> binding.webView.loadUrl("javascript:myMove('155')") 121 | 122 | "C3" -> binding.webView.loadUrl("javascript:myMove('160')") 123 | "C#3" -> binding.webView.loadUrl("javascript:myMoveSharp('160')") 124 | "D3" -> binding.webView.loadUrl("javascript:myMove('165')") 125 | "D#3" -> binding.webView.loadUrl("javascript:myMoveSharp('165')") 126 | "E3" -> binding.webView.loadUrl("javascript:myMove('170')") 127 | "F3" -> binding.webView.loadUrl("javascript:myMove('175')") 128 | "F#3" -> binding.webView.loadUrl("javascript:myMoveSharp('175')") 129 | "G3" -> binding.webView.loadUrl("javascript:myMove('180')") 130 | "G#3" -> binding.webView.loadUrl("javascript:myMoveSharp('180')") 131 | "A3" -> binding.webView.loadUrl("javascript:myMove('185')") 132 | "A#3" -> binding.webView.loadUrl("javascript:myMoveSharp('185')") 133 | "B3" -> binding.webView.loadUrl("javascript:myMove('190')") 134 | 135 | "C4" -> binding.webView.loadUrl("javascript:myMove('225')") 136 | "C#4" -> binding.webView.loadUrl("javascript:myMoveSharp('225')") 137 | "D4" -> binding.webView.loadUrl("javascript:myMove('230')") 138 | "D#4" -> binding.webView.loadUrl("javascript:myMoveSharp('230')") 139 | "E4" -> binding.webView.loadUrl("javascript:myMove('235')") 140 | "F4" -> binding.webView.loadUrl("javascript:myMove('240')") 141 | "F#4" -> binding.webView.loadUrl("javascript:myMoveSharp('240')") 142 | "G4" -> binding.webView.loadUrl("javascript:myMove('245')") 143 | "G#4" -> binding.webView.loadUrl("javascript:myMoveSharp('245')") 144 | "A4" -> binding.webView.loadUrl("javascript:myMove('250')") 145 | "A#4" -> binding.webView.loadUrl("javascript:myMoveSharp('250')") 146 | "B4" -> binding.webView.loadUrl("javascript:myMove('255')") 147 | 148 | "C5" -> binding.webView.loadUrl("javascript:myMove('260')") 149 | "C#5" -> binding.webView.loadUrl("javascript:myMoveSharp('260')") 150 | "D5" -> binding.webView.loadUrl("javascript:myMove('265')") 151 | "D#5" -> binding.webView.loadUrl("javascript:myMoveSharp('265')") 152 | "E5" -> binding.webView.loadUrl("javascript:myMove('270')") 153 | "F5" -> binding.webView.loadUrl("javascript:myMove('275')") 154 | "F#5" -> binding.webView.loadUrl("javascript:myMoveSharp('275')") 155 | "G5" -> binding.webView.loadUrl("javascript:myMove('280')") 156 | "G#5" -> binding.webView.loadUrl("javascript:myMoveSharp('280')") 157 | "A5" -> binding.webView.loadUrl("javascript:myMove('285')") 158 | "A#5" -> binding.webView.loadUrl("javascript:myMoveSharp('285')") 159 | "B5" -> binding.webView.loadUrl("javascript:myMove('290')") 160 | 161 | } 162 | i++ 163 | if (i < list.size) { 164 | handler.postDelayed(this, TIME_DELAY_FOR_NOTES) 165 | } 166 | } 167 | }) 168 | } 169 | }) 170 | 171 | // Observe viewmodel objects 172 | /*viewModel.spannableForKaraoke.observe( 173 | requireActivity(), 174 | Observer { karaokeString -> 175 | binding.textviewKaraoke.text = karaokeString 176 | } 177 | )*/ 178 | //binding.textviewKaraoke.text = getString(R.string.song_lyrics_baby) 179 | 180 | viewModel.singingEnd.observe( 181 | requireActivity(), 182 | Observer { end -> 183 | if (end) { 184 | // Clear animation 185 | binding.buttonAnimated.clearAnimation() 186 | } else { 187 | // Start animation 188 | animateSharkButton() 189 | } 190 | } 191 | ) 192 | 193 | return binding.root 194 | } 195 | 196 | fun singingStopped() { 197 | // Execute method to stop callbacks 198 | viewModel.stopAllSinging() 199 | 200 | // Clear animation 201 | binding.buttonAnimated.clearAnimation() 202 | 203 | //Toast.makeText(activity, "Singing has stopped", Toast.LENGTH_LONG).show() 204 | } 205 | 206 | private fun animateSharkButton() { 207 | val animation = AnimationUtils.loadAnimation(activity, R.anim.scale_anim) 208 | binding.buttonAnimated.startAnimation(animation) 209 | } 210 | 211 | private fun generateFolder() { 212 | val root = 213 | File(Environment.getExternalStorageDirectory(), "Pitch Estimator") 214 | if (!root.exists()) { 215 | root.mkdirs() 216 | } 217 | } 218 | 219 | private fun hasPermissions( 220 | context: Context?, 221 | vararg permissions: String? 222 | ): Boolean { 223 | if (context != null && permissions != null) { 224 | for (permission in permissions) { 225 | if (ActivityCompat.checkSelfPermission( 226 | context, 227 | permission!! 228 | ) != PackageManager.PERMISSION_GRANTED 229 | ) { 230 | return false 231 | } 232 | } 233 | } 234 | return true 235 | } 236 | 237 | private fun initialize() { 238 | if (!hasPermissions(activity, *PERMISSIONS)) { 239 | requestPermissions(PERMISSIONS, PERMISSION_ALL) 240 | } 241 | } 242 | 243 | override fun onRequestPermissionsResult( 244 | requestCode: Int, 245 | permissions: Array, 246 | grantResults: IntArray 247 | ) { 248 | 249 | if (requestCode == PERMISSION_ALL) { 250 | if (allPermissionsGranted(grantResults)) { 251 | 252 | Toast.makeText( 253 | activity, 254 | getString(R.string.allPermissionsGranted), 255 | Toast.LENGTH_LONG 256 | ).show() 257 | 258 | 259 | } else { 260 | 261 | Toast.makeText( 262 | activity, 263 | getString(R.string.permissionsNotGranted), 264 | Toast.LENGTH_LONG 265 | ).show() 266 | 267 | activity?.finish() 268 | 269 | } 270 | } else { 271 | super.onRequestPermissionsResult(requestCode, permissions, grantResults) 272 | } 273 | } 274 | 275 | private fun allPermissionsGranted(grantResults: IntArray) = grantResults.all { 276 | it == PackageManager.PERMISSION_GRANTED 277 | } 278 | 279 | override fun onResume() { 280 | super.onResume() 281 | //viewModel.setNotesOnStart() 282 | } 283 | 284 | companion object { 285 | private const val TIME_DELAY_FOR_NOTES = 555L 286 | 287 | // Update interval for widget 288 | const val UPDATE_INTERVAL_INFERENCE = 2048L 289 | const val UPDATE_INTERVAL_KARAOKE = 440L 290 | } 291 | } -------------------------------------------------------------------------------- /app/src/main/java/com/george/pitch_estimator/singingFragment/SingingFragmentViewModel.kt: -------------------------------------------------------------------------------- 1 | package com.george.pitch_estimator.singingFragment 2 | 3 | import android.app.Application 4 | import android.content.Context 5 | import android.graphics.Color 6 | import android.os.Handler 7 | import android.text.Spannable 8 | import android.text.SpannableString 9 | import android.text.style.ForegroundColorSpan 10 | import android.util.Log 11 | import androidx.lifecycle.AndroidViewModel 12 | import androidx.lifecycle.LiveData 13 | import androidx.lifecycle.MutableLiveData 14 | import androidx.lifecycle.viewModelScope 15 | import com.george.pitch_estimator.PitchModelExecutor 16 | import com.george.pitch_estimator.R 17 | import com.george.pitch_estimator.SingRecorder 18 | import kotlinx.coroutines.Dispatchers 19 | import kotlinx.coroutines.launch 20 | import kotlinx.coroutines.withContext 21 | import org.koin.android.ext.android.get 22 | import org.koin.android.ext.android.getKoin 23 | import org.koin.core.KoinComponent 24 | import org.koin.core.get 25 | import java.io.* 26 | 27 | class SingingFragmentViewModel(application: Application) : AndroidViewModel(application), 28 | KoinComponent { 29 | 30 | private var singRecorderObject: SingRecorder 31 | private var pitchModelExecutorObject: PitchModelExecutor 32 | 33 | lateinit var inputStringPentagram: String 34 | var singingRunning = false 35 | private var context: Context = application 36 | 37 | // Handlers for repeating sound collection and karaoke effect 38 | private val updateLoopSingingHandler = Handler() 39 | private val updateKaraokeHandler = Handler() 40 | private val handler = Handler() 41 | private val handlerMummy = Handler() 42 | 43 | private val _hertzValuesToDisplay = MutableLiveData() 44 | val hertzValuesToDisplay: LiveData 45 | get() = _hertzValuesToDisplay 46 | 47 | private val _noteValuesToDisplay = MutableLiveData>() 48 | val noteValuesToDisplay: LiveData> 49 | get() = _noteValuesToDisplay 50 | 51 | private val _inputTextFromAssets = MutableLiveData() 52 | 53 | val inputTextFromAssets: LiveData 54 | get() = _inputTextFromAssets 55 | 56 | private val _spannableForKaraoke = MutableLiveData() 57 | 58 | val spannableForKaraoke: LiveData 59 | get() = _spannableForKaraoke 60 | 61 | private val _inferenceDone = MutableLiveData() 62 | val inferenceDone: LiveData 63 | get() = _inferenceDone 64 | 65 | private val _singingEnd = MutableLiveData() 66 | val singingEnd: LiveData 67 | get() = _singingEnd 68 | 69 | init { 70 | 71 | // Koin DI 72 | getKoin().setProperty("koinUseGpu", false) 73 | singRecorderObject = get() 74 | // Here interpreter starts immediately 75 | pitchModelExecutorObject = get() 76 | 77 | // Start with loading musical pentagram from assets folder html code 78 | readHtmlFromAssets(application) 79 | 80 | // Initialize arraylist 81 | _noteValuesToDisplay.value = arrayListOf() 82 | 83 | // Init textview karaoke words 84 | _spannableForKaraoke.value = 85 | SpannableString(application.getString(R.string.song_lyrics_baby)) 86 | } 87 | 88 | fun setNotesOnStart() { 89 | _noteValuesToDisplay.value = arrayListOf() 90 | } 91 | 92 | // This should be replaced with DI 93 | fun setSingRecorderModule(singRecorder: SingRecorder, pitchModelExecutor: PitchModelExecutor) { 94 | //singRecorderObject = singRecorder 95 | //pitchModelExecutorObject = pitchModelExecutor 96 | } 97 | 98 | fun startSinging() { 99 | 100 | singingRunning = true 101 | 102 | singRecorderObject.startRecording() 103 | //singRecorderObject.startRecordingCommands() 104 | //singRecorderObject.startRecognitionCommands() 105 | } 106 | 107 | fun stopSinging() { 108 | 109 | val stream = singRecorderObject.stopRecording() 110 | val streamForInference = singRecorderObject.stopRecordingForInference() 111 | 112 | Log.i("VIEWMODEL_SIZE", streamForInference.size.toString()) 113 | Log.i("VIEWMODEL_VALUES", streamForInference.takeLast(100).toString()) 114 | 115 | singingRunning = false 116 | // Background thread to do inference with the generated short arraylist 117 | viewModelScope.launch { 118 | doInference(stream, streamForInference) 119 | } 120 | 121 | } 122 | 123 | private suspend fun doInference( 124 | stream: ByteArrayOutputStream, 125 | arrayListShorts: ArrayList 126 | ) = withContext(Dispatchers.IO) { 127 | // write .wav file to external directory 128 | singRecorderObject.writeWav(stream) 129 | // reset stream 130 | singRecorderObject.reInitializePcmStream() 131 | 132 | // The input must be normalized to floats between -1 and 1. 133 | // To normalize it, we just need to divide all the values by 2**16 or in our code, MAX_ABS_INT16 = 32768 134 | val floatsForInference = FloatArray(arrayListShorts.size) 135 | for ((index, value) in arrayListShorts.withIndex()) { 136 | floatsForInference[index] = (value / 32768F) 137 | } 138 | 139 | Log.i("FLOATS", floatsForInference.takeLast(100).toString()) 140 | 141 | // Inference 142 | _inferenceDone.postValue(false) 143 | _noteValuesToDisplay.postValue(pitchModelExecutorObject.execute(floatsForInference)) 144 | Log.i("HERTZ", hertzValuesToDisplay.toString()) 145 | _inferenceDone.postValue(true) 146 | 147 | //transcribe("/sdcard/Pitch Estimator/soloupis.wav") 148 | } 149 | 150 | private fun readHtmlFromAssets(application: Application) { 151 | try { 152 | val inputStreamPentagram: InputStream = application.assets.open("final1.txt") 153 | inputStringPentagram = inputStreamPentagram.bufferedReader().use { it.readText() } 154 | 155 | _inputTextFromAssets.value = inputStringPentagram 156 | Log.i("HTML", inputTextFromAssets.value) 157 | } catch (e: Exception) { 158 | Log.e("EXCEPTION_READ_HTML", e.toString()) 159 | } 160 | } 161 | 162 | fun setUpdateLoopSingingHandler() { 163 | // Start loop for collecting sound and inferring 164 | updateLoopSingingHandler.postDelayed(updateLoopSingingRunnable, 0) 165 | } 166 | 167 | fun setUpdateKaraokeHandler() { 168 | // Start loop for visualizing karaoke effect 169 | updateKaraokeHandler.postDelayed(updateKaraokeRunnable, 0) 170 | 171 | // Init textview karaoke words 172 | _spannableForKaraoke.value = 173 | SpannableString(context.getString(R.string.song_lyrics_baby)) 174 | 175 | // On start set white color to spannable words 176 | _spannableForKaraoke.value?.setSpan( 177 | ForegroundColorSpan(Color.WHITE), 178 | 0, 179 | _spannableForKaraoke.value!!.length, 180 | Spannable.SPAN_EXCLUSIVE_EXCLUSIVE 181 | ) 182 | } 183 | 184 | fun stopAllSinging() { 185 | // remove queue of callbacks when user presses stop before song stops 186 | updateLoopSingingHandler.removeCallbacks(updateLoopSingingRunnable) 187 | updateKaraokeHandler.removeCallbacks(updateKaraokeRunnable) 188 | handler.removeCallbacksAndMessages(null) 189 | handlerMummy.removeCallbacksAndMessages(null) 190 | 191 | _singingEnd.value = true 192 | } 193 | 194 | private var updateLoopSingingRunnable: Runnable = Runnable { 195 | run { 196 | 197 | // Start singing 198 | startSinging() 199 | _singingEnd.value = false 200 | 201 | // Stop after 2048 millis 202 | val handler = Handler() 203 | handler.postDelayed({ 204 | stopSinging() 205 | }, SingingFragment.UPDATE_INTERVAL_INFERENCE) 206 | 207 | // Re-run it after the update interval 208 | updateLoopSingingHandler.postDelayed( 209 | updateLoopSingingRunnable, 210 | SingingFragment.UPDATE_INTERVAL_INFERENCE 211 | ) 212 | 213 | } 214 | 215 | } 216 | 217 | // Runnable to loop every 2 seconds with writing sound and inferring 218 | private var updateKaraokeRunnable: Runnable = Runnable { 219 | run { 220 | 221 | for (i in 1..24) { 222 | 223 | handler.postDelayed({ 224 | // If statement if the user has stopped singing before end of song 225 | if (!_singingEnd.value!!) { 226 | _spannableForKaraoke.value = 227 | SpannableString(application.getString(R.string.song_lyrics_baby)) 228 | _spannableForKaraoke.value?.setSpan( 229 | ForegroundColorSpan(Color.BLUE), 230 | 0, 231 | 5 * i, 232 | Spannable.SPAN_EXCLUSIVE_EXCLUSIVE 233 | ) 234 | 235 | // Change words when first words stop 236 | if (i == 24) { 237 | 238 | val handlerRest = Handler() 239 | handlerRest.postDelayed({ 240 | 241 | for (k in 1..25) { 242 | 243 | handlerMummy.postDelayed({ 244 | // If statement if the user has stopped singing before end of song 245 | if (!_singingEnd.value!!) { 246 | _spannableForKaraoke.value = 247 | SpannableString(application.getString(R.string.song_lyrics_mummy)) 248 | _spannableForKaraoke.value?.setSpan( 249 | ForegroundColorSpan(Color.BLUE), 250 | 0, 251 | 5 * k, 252 | Spannable.SPAN_EXCLUSIVE_EXCLUSIVE 253 | ) 254 | 255 | // Stop everything after end of song 256 | if (k == 25) { 257 | stopAllSinging() 258 | _singingEnd.value = true 259 | } 260 | } 261 | 262 | }, SingingFragment.UPDATE_INTERVAL_KARAOKE * k) 263 | 264 | } 265 | 266 | }, 1400) 267 | 268 | 269 | } 270 | } 271 | 272 | }, SingingFragment.UPDATE_INTERVAL_KARAOKE * i) 273 | 274 | } 275 | } 276 | } 277 | 278 | /*private fun transcribe(audioFile: String) { 279 | val inferenceExecTime = longArrayOf(0) 280 | //Log.e("AUDIO_FORMAT", "audioFormat.toString()") 281 | try { 282 | val wave = RandomAccessFile(audioFile, "r") 283 | wave.seek(20) 284 | val audioFormat: Char = readLEChar(wave) 285 | //Log.e("AUDIO_FORMAT", (audioFormat.toInt() == 1).toString()) 286 | assert( 287 | audioFormat.toInt() == 1 // 1 is PCM 288 | ) 289 | // tv_audioFormat.setText("audioFormat=" + (audioFormat == 1 ? "PCM" : "!PCM")); 290 | wave.seek(22) 291 | val numChannels: Char = readLEChar(wave) 292 | //Log.e("NUMBER_CHANNEL", (numChannels.toInt()).toString()) 293 | assert( 294 | numChannels.toInt() == 1 // MONO 295 | ) 296 | // tv_numChannels.setText("numChannels=" + (numChannels == 1 ? "MONO" : "!MONO")); 297 | wave.seek(24) 298 | val sampleRate: Int = readLEInt(wave) 299 | //Log.e("SAMPLE_RATE", (sampleRate).toString()) 300 | assert( 301 | sampleRate == 16000// // desired sample rate 302 | ) 303 | // tv_sampleRate.setText("sampleRate=" + (sampleRate == 16000 ? "16kHz" : "!16kHz")); 304 | wave.seek(34) 305 | val bitsPerSample: Char = readLEChar(wave) 306 | //Log.e("BITS_PER_SAMPLE", (bitsPerSample.toInt() == 16).toString()) 307 | assert( 308 | bitsPerSample.toInt() == 16 // 16 bits per sample 309 | ) 310 | // tv_bitsPerSample.setText("bitsPerSample=" + (bitsPerSample == 16 ? "16-bits" : "!16-bits" )); 311 | wave.seek(40) 312 | val bufferSize: Int = readLEInt(wave) 313 | assert(bufferSize > 0) 314 | // tv_bufferSize.setText("bufferSize=" + bufferSize); 315 | wave.seek(44) 316 | val bytes = ByteArray(bufferSize) 317 | wave.readFully(bytes) 318 | 319 | 320 | *//*Log.i("BYTES", bytes.size.toString()) 321 | val shorts = ShortArray(bytes.size / 2) 322 | // to turn bytes to shorts as either big endian or little endian. 323 | ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer()[shorts] 324 | val inferenceStartTime = System.currentTimeMillis() 325 | Log.i("SHORTS", shorts.size.toString()) 326 | wholeSentence += _m.stt(shorts, shorts.size).toString() + ". " 327 | inferenceExecTime[0] = System.currentTimeMillis() - inferenceStartTime*//* 328 | } catch (ex: FileNotFoundException) { 329 | 330 | } catch (ex: IOException) { 331 | } 332 | } 333 | 334 | @Throws(IOException::class) 335 | private fun readLEChar(f: RandomAccessFile): Char { 336 | val b1 = f.readByte() 337 | val b2 = f.readByte() 338 | return (b2.toInt() shl 8 or b1.toInt()).toChar() 339 | } 340 | 341 | @Throws(IOException::class) 342 | private fun readLEInt(f: RandomAccessFile): Int { 343 | val b1 = f.readByte() 344 | val b2 = f.readByte() 345 | val b3 = f.readByte() 346 | val b4 = f.readByte() 347 | return ((b1 and 0xFF.toByte()).toInt() or ((b2 and 0xFF.toByte()).toInt() shl 8) 348 | or ((b3 and 0xFF.toByte()).toInt() shl 16) or ((b4 and 0xFF.toByte()).toInt() shl 24)) 349 | }*/ 350 | 351 | override fun onCleared() { 352 | super.onCleared() 353 | // Below stopAllSinging to execute when back button is used 354 | stopAllSinging() 355 | 356 | // Close interpreter here 357 | pitchModelExecutorObject.close() 358 | } 359 | } -------------------------------------------------------------------------------- /app/src/main/java/com/george/pitch_estimator/ui/MainActivity.kt: -------------------------------------------------------------------------------- 1 | package com.george.pitch_estimator.ui 2 | 3 | import android.os.Bundle 4 | import android.util.Log 5 | import android.view.Menu 6 | import android.view.MenuItem 7 | import androidx.appcompat.app.AppCompatActivity 8 | import com.george.pitch_estimator.R 9 | import com.george.pitch_estimator.singingFragment.SingingFragment 10 | import com.google.android.material.floatingactionbutton.FloatingActionButton 11 | import com.google.android.material.snackbar.Snackbar 12 | 13 | 14 | class MainActivity : AppCompatActivity() { 15 | 16 | override fun onCreate(savedInstanceState: Bundle?) { 17 | super.onCreate(savedInstanceState) 18 | setContentView(R.layout.activity_main) 19 | setSupportActionBar(findViewById(R.id.toolbar)) 20 | 21 | findViewById(R.id.fab).setOnClickListener { view -> 22 | Snackbar.make(view, "Replace with your own action", Snackbar.LENGTH_LONG) 23 | .setAction("Action", null).show() 24 | } 25 | } 26 | 27 | override fun onCreateOptionsMenu(menu: Menu): Boolean { 28 | // Inflate the menu; this adds items to the action bar if it is present. 29 | menuInflater.inflate(R.menu.menu_main, menu) 30 | return true 31 | } 32 | 33 | override fun onOptionsItemSelected(item: MenuItem): Boolean { 34 | // Handle action bar item clicks here. The action bar will 35 | // automatically handle clicks on the Home/Up button, so long 36 | // as you specify a parent activity in AndroidManifest.xml. 37 | return when (item.itemId) { 38 | R.id.action_settings -> true 39 | else -> super.onOptionsItemSelected(item) 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /app/src/main/java/com/george/pitch_estimator/ui/SecondFragment.kt: -------------------------------------------------------------------------------- 1 | package com.george.pitch_estimator.ui 2 | 3 | import android.os.Bundle 4 | import androidx.fragment.app.Fragment 5 | import android.view.LayoutInflater 6 | import android.view.View 7 | import android.view.ViewGroup 8 | import android.widget.Button 9 | import androidx.navigation.fragment.findNavController 10 | import com.george.pitch_estimator.R 11 | 12 | /** 13 | * A simple [Fragment] subclass as the second destination in the navigation. 14 | */ 15 | class SecondFragment : Fragment() { 16 | 17 | override fun onCreateView( 18 | inflater: LayoutInflater, container: ViewGroup?, 19 | savedInstanceState: Bundle? 20 | ): View? { 21 | // Inflate the layout for this fragment 22 | return inflater.inflate(R.layout.fragment_second, container, false) 23 | } 24 | 25 | override fun onViewCreated(view: View, savedInstanceState: Bundle?) { 26 | super.onViewCreated(view, savedInstanceState) 27 | 28 | /*view.findViewById