├── .gitignore ├── LICENSE ├── README.md └── model_hack_notebook.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Leon Fedden 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TensorFlow-Efficient-Neural-Audio-Synthesis 2 | 3 | This is a TensorFlow implementation of the paper Efficient Neural Audio Synthesis found at the [arxiv link here](https://arxiv.org/pdf/1802.08435.pdf). 4 | 5 | ### Road Map 6 | 7 | The first steps is to get an inefficient archetecture working. We wont be introducing custom GPU kernels or doing weight pruning until the model works well. Once we can sample the model with a degree of success, the objective is to implement the operations that make this model faster than real-time. 8 | 9 | ### Contributions 10 | 11 | Contributions are very welcome. Please leave issues or get in touch with me at leonfedden (at) gmail.com 12 | -------------------------------------------------------------------------------- /model_hack_notebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 17, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import tensorflow as tf\n", 10 | "\n", 11 | "from tensorflow.python.layers import base as base_layer\n", 12 | "from tensorflow.python.ops.rnn_cell_impl import _LayerRNNCell\n", 13 | "from tensorflow.python.ops.math_ops import tanh, sigmoid\n", 14 | "from tensorflow.python.ops.init_ops import constant_initializer, zeros_initializer" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 9, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 19, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "tf.reset_default_graph()\n", 31 | "\n", 32 | "class WaveRNN(_LayerRNNCell):\n", 33 | " \n", 34 | " def __init__(self, \n", 35 | " number_units,\n", 36 | " kernel_intialiser=None,\n", 37 | " bias_initialiser=None,\n", 38 | " reuse=None, \n", 39 | " name=None):\n", 40 | " \n", 41 | " super(WaveRNN, self).__init__(_reuse=reuse, \n", 42 | " name=name)\n", 43 | " \n", 44 | " # Inputs must be two dimensional.\n", 45 | " self.input_spec = base_layer.InputSpec(ndim=2)\n", 46 | " \n", 47 | " self.number_units = number_units\n", 48 | " self.kernel_initialiser = kernel_intialiser\n", 49 | " self.bias_initialiser = bias_initialiser\n", 50 | " \n", 51 | " @property\n", 52 | " def state_size(self):\n", 53 | " return self.number_units\n", 54 | " \n", 55 | " @property\n", 56 | " def output_size(self):\n", 57 | " return self.number_units\n", 58 | " \n", 59 | " def build(self, inputs_shape):\n", 60 | " input_depth = inputs_shape[1].value\n", 61 | " \n", 62 | " # TO DO: Change kernels and biases to match the vars needed in equation (2)\n", 63 | " # from the paper.\n", 64 | " \n", 65 | " # Gate kernel.\n", 66 | " gate_kernel_name = 'WaveRNN/gate/kernel'\n", 67 | " gate_kernel_shape = [input_depth + self.number_units, 2 * self.number_units]\n", 68 | " gate_kernel_initialiser = self.kernel_initialiser\n", 69 | " self.gate_kernel = self.add_variable(gate_kernel_name, \n", 70 | " shape=gate_kernel_shape, \n", 71 | " initializer=gate_kernel_initialiser)\n", 72 | " # Gate Bias.\n", 73 | " gate_bias_name = 'WaveRNN/gate/bias'\n", 74 | " gate_bias_shape = [2 * self.number_units]\n", 75 | " gate_bias_initialiser = self.bias_initialiser \\\n", 76 | " if self.bias_initialiser is not None \\\n", 77 | " else constant_initializer(1.0, dtype=self.dtype)\n", 78 | " \n", 79 | " self.gate_bias = self.add_variable(gate_bias_name, \n", 80 | " shape=gate_bias_shape, \n", 81 | " initializer=gate_bias_initialiser)\n", 82 | " # Candidate Kernel.\n", 83 | " candidate_kernel_name = 'WaveRNN/candidate/kernel'\n", 84 | " candidate_kernel_shape = [input_depth + self.number_units, self.number_units]\n", 85 | " candidate_kernel_initialiser = self.kernel_initialiser\n", 86 | " self.candidate_kernel = self.add_variable(candidate_kernel_name, \n", 87 | " shape=candidate_kernel_shape, \n", 88 | " initializer=candidate_kernel_initialiser)\n", 89 | " # Candidate Bias.\n", 90 | " candidate_bias_name = 'WaveRNN/candidate/bias'\n", 91 | " candidate_bias_shape = [self.number_units]\n", 92 | " candidate_bias_initialiser = self.bias_initialiser \\\n", 93 | " if self.bias_initialiser is not None \\\n", 94 | " else zeros_initializer(dtype=self.dtype)\n", 95 | " self.candidate_bias = self.add_variable(candidate_bias_name, \n", 96 | " shape=candidate_bias_shape, \n", 97 | " initializer=candidate_bias_initialiser)\n", 98 | " \n", 99 | " self.built = True\n", 100 | " \n", 101 | " \n", 102 | " def call(self, inputs, state):\n", 103 | " \n", 104 | " # Create Equation (2) from paper here.\n", 105 | " pass" 106 | ] 107 | } 108 | ], 109 | "metadata": { 110 | "kernelspec": { 111 | "display_name": "Python 3", 112 | "language": "python", 113 | "name": "python3" 114 | }, 115 | "language_info": { 116 | "codemirror_mode": { 117 | "name": "ipython", 118 | "version": 3 119 | }, 120 | "file_extension": ".py", 121 | "mimetype": "text/x-python", 122 | "name": "python", 123 | "nbconvert_exporter": "python", 124 | "pygments_lexer": "ipython3", 125 | "version": "3.5.2" 126 | } 127 | }, 128 | "nbformat": 4, 129 | "nbformat_minor": 2 130 | } 131 | --------------------------------------------------------------------------------