├── .gitignore ├── .vscode ├── c_cpp_properties.json ├── launch.json └── settings.json ├── README.md ├── config.yaml ├── ddsp ├── __init__.py ├── core.py ├── model.py └── utils.py ├── export.py ├── patchs ├── preprocess.pd └── timbre_transfer.pd ├── performance.py ├── preprocess.py ├── preprocess_from_sigmund.py ├── realtime ├── CMakeLists.txt └── ddsp_tilde │ ├── ddsp_model.cpp │ ├── ddsp_model.h │ ├── ddsp_tilde.cpp │ ├── m_pd.h │ └── signal_in_out_base.c ├── requirements.txt └── train.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.wav 2 | *.npy 3 | *.pyc 4 | *.csv 5 | *.opus 6 | *.pth 7 | *.torchscript 8 | runs/ 9 | temp/ 10 | notebooks/ 11 | !showcase/*.wav 12 | *build 13 | *tmp 14 | *.ts 15 | *debug.py 16 | *.tar.gz 17 | *DS_Store 18 | export/* -------------------------------------------------------------------------------- /.vscode/c_cpp_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "Mac", 5 | "includePath": [ 6 | "${workspaceFolder}/**", 7 | "/Users/caillon/Downloads/libtorch/include/**" 8 | 9 | ], 10 | "defines": [], 11 | "macFrameworkPath": [ 12 | "/System/Library/Frameworks", 13 | "/Library/Frameworks" 14 | ], 15 | "compilerPath": "/usr/bin/clang", 16 | "cStandard": "c11", 17 | "cppStandard": "c++98", 18 | "intelliSenseMode": "macos-clang-x64" 19 | } 20 | ], 21 | "version": 4 22 | } -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "(gdb) Launch", 9 | "type": "cppdbg", 10 | "request": "launch", 11 | "program": "/usr/bin/pd", 12 | "args": [ 13 | "-open", 14 | "/home/caillon/Documents/Pd/debug.pd", 15 | "-nogui" 16 | ], 17 | "stopAtEntry": false, 18 | "cwd": "${workspaceFolder}", 19 | "environment": [], 20 | "console": "externalTerminal", 21 | "MIMode": "gdb", 22 | "setupCommands": [ 23 | { 24 | "description": "Enable pretty-printing for gdb", 25 | "text": "-enable-pretty-printing", 26 | "ignoreFailures": true 27 | } 28 | ] 29 | }, 30 | { 31 | "name": "Python: Current File", 32 | "type": "python", 33 | "request": "launch", 34 | "program": "${file}", 35 | "console": "integratedTerminal" 36 | } 37 | ] 38 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.formatting.provider": "yapf", 3 | "editor.formatOnSave": true, 4 | "python.pythonPath": "/slow-2/antoine/miniconda3/bin/python", 5 | "files.associations": { 6 | "vector": "cpp" 7 | } 8 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Differentiable Digital Signal Processing 2 | 3 | ⚠️⚠️⚠️⚠️⚠️ **REPOSITORY TRANSFERED TO [https://github.com/acids-ircam/ddsp_pytorch](https://github.com/acids-ircam/ddsp_pytorch)** ⚠️⚠️⚠️⚠️⚠️ 4 | 5 | 6 | 7 | 8 | Implementation of the [DDSP model](https://github.com/magenta/ddsp) using PyTorch. This implementation can be exported to a torchscript model, ready to be used inside a realtime environment (see [this video](https://www.youtube.com/watch?v=_U6Bn-1FDHc)). 9 | 10 | ## Usage 11 | 12 | Edit the `config.yaml` file to fit your needs (audio location, preprocess folder, sampling rate, model parameters...), then preprocess your data using 13 | 14 | ```bash 15 | python preprocess.py 16 | ``` 17 | 18 | You can then train your model using 19 | 20 | ```bash 21 | python train.py --name mytraining --epochs 10000000 --batch 16 --lr .001 22 | ``` 23 | 24 | Once trained, export it using 25 | 26 | ```bash 27 | python export.py --run runs/mytraining/ 28 | ``` 29 | 30 | It will produce a file named `ddsp_pretrained_mytraining.ts`, that you can use inside a python environment like that 31 | 32 | ```python 33 | import torch 34 | 35 | model = torch.jit.load("ddsp_pretrained_mytraining.ts") 36 | 37 | pitch = torch.randn(1, 200, 1) 38 | loudness = torch.randn(1, 200, 1) 39 | 40 | audio = model(pitch, loudness) 41 | ``` 42 | 43 | ## Realtime usage 44 | 45 | If you want to use DDSP in realtime (yeah), we provide a pure data external wrapping the all thing. Simply pass the `--realtime true` option when exporting. This will disable the reverb and enable the use of the model in realtime. For now the external works on CPU, but you can enable GPU accelerated inference by changing `realtime/ddsp_tilde/ddsp_model.h` `DEVICE` to `torch::kCUDA`. 46 | Inside Pd, simply send `load your_model.ts` to the `ddsp~` object. The first inlet must be a pitch signal, the second a loudness signal. It can be directly plugged to the `sigmund~` object for real-time timbre transfer ;) 47 | 48 | ## Compilation 49 | 50 | You will need `cmake`, a C++ compiler, and `libtorch` somewhere on your computer. Then, run 51 | 52 | ```bash 53 | cd realtime 54 | mkdir build 55 | cd build 56 | cmake ../ -DCMAKE_PREFIX_PATH=/path/to/libtorch -DCMAKE_BUILD_TYPE=Release 57 | make 58 | ``` 59 | 60 | The Pd external still need some optimization, it will receive some updates very soon. 61 | -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- 1 | data: 2 | data_location: "./preprocessed.wav" 3 | extension: "wav" 4 | 5 | preprocess: 6 | sampling_rate: &samplingrate 48000 7 | signal_length: &signallength 192000 8 | block_size: &blocksize 512 9 | 10 | oneshot: false # crop every audio file to exactly signal length 11 | out_dir: ./preprocessed/ 12 | 13 | model: 14 | hidden_size: 512 15 | n_harmonic: 64 16 | n_bands: 65 17 | sampling_rate: *samplingrate 18 | block_size: *blocksize 19 | 20 | train: 21 | scales: [4096,2048, 1024, 512, 256, 128] 22 | overlap: .75 23 | -------------------------------------------------------------------------------- /ddsp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/caillonantoine/ddsp-recode/bea0a03f318de745876232d10da7726aea60288e/ddsp/__init__.py -------------------------------------------------------------------------------- /ddsp/core.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.fft as fft 4 | import numpy as np 5 | import librosa as li 6 | import crepe 7 | import math 8 | 9 | 10 | def safe_log(x): 11 | return torch.log(x + 1e-7) 12 | 13 | 14 | @torch.no_grad() 15 | def mean_std_loudness(dataset): 16 | mean = 0 17 | std = 0 18 | n = 0 19 | for _, _, l in dataset: 20 | n += 1 21 | mean += (l.mean().item() - mean) / n 22 | std += (l.std().item() - std) / n 23 | return mean, std 24 | 25 | 26 | def multiscale_fft(signal, scales, overlap): 27 | stfts = [] 28 | for s in scales: 29 | S = torch.stft( 30 | signal, 31 | s, 32 | int(s * (1 - overlap)), 33 | s, 34 | torch.hann_window(s).to(signal), 35 | True, 36 | normalized=True, 37 | return_complex=True, 38 | ).abs() 39 | stfts.append(S) 40 | return stfts 41 | 42 | 43 | def resample(x, factor: int): 44 | batch, frame, channel = x.shape 45 | x = x.permute(0, 2, 1).reshape(batch * channel, 1, frame) 46 | 47 | window = torch.hann_window( 48 | factor * 2, 49 | dtype=x.dtype, 50 | device=x.device, 51 | ).reshape(1, 1, -1) 52 | y = torch.zeros(x.shape[0], x.shape[1], factor * x.shape[2]).to(x) 53 | y[..., ::factor] = x 54 | y[..., -1:] = x[..., -1:] 55 | y = torch.nn.functional.pad(y, [factor, factor]) 56 | y = torch.nn.functional.conv1d(y, window)[..., :-1] 57 | 58 | y = y.reshape(batch, channel, factor * frame).permute(0, 2, 1) 59 | 60 | return y 61 | 62 | 63 | def upsample(signal, factor): 64 | signal = signal.permute(0, 2, 1) 65 | signal = nn.functional.interpolate(signal, scale_factor=factor) 66 | return signal.permute(0, 2, 1) 67 | 68 | 69 | def remove_above_nyquist(amplitudes, pitch, sampling_rate): 70 | n_harm = amplitudes.shape[-1] 71 | pitches = pitch * torch.arange(1, n_harm + 1).to(pitch) 72 | aa = (pitches < sampling_rate / 2).float() + 1e-4 73 | return amplitudes * aa 74 | 75 | 76 | def scale_function(x): 77 | return 2 * torch.sigmoid(x)**(math.log(10)) + 1e-7 78 | 79 | 80 | def extract_loudness(signal, sampling_rate, block_size, n_fft=2048): 81 | S = li.stft( 82 | signal, 83 | n_fft=n_fft, 84 | hop_length=block_size, 85 | win_length=n_fft, 86 | center=True, 87 | ) 88 | S = np.log(abs(S) + 1e-7) 89 | f = li.fft_frequencies(sampling_rate, n_fft) 90 | a_weight = li.A_weighting(f) 91 | 92 | S = S + a_weight.reshape(-1, 1) 93 | 94 | S = np.mean(S, 0)[..., :-1] 95 | 96 | return S 97 | 98 | 99 | def extract_pitch(signal, sampling_rate, block_size): 100 | f0 = crepe.predict( 101 | signal, 102 | sampling_rate, 103 | step_size=int(1000 * block_size / sampling_rate), 104 | verbose=1, 105 | center=True, 106 | viterbi=True, 107 | ) 108 | return f0[1].reshape(-1)[:-1] 109 | 110 | 111 | def mlp(in_size, hidden_size, n_layers): 112 | channels = [in_size] + (n_layers) * [hidden_size] 113 | net = [] 114 | for i in range(n_layers): 115 | net.append(nn.Linear(channels[i], channels[i + 1])) 116 | net.append(nn.LayerNorm(channels[i + 1])) 117 | net.append(nn.LeakyReLU()) 118 | return nn.Sequential(*net) 119 | 120 | 121 | def gru(n_input, hidden_size): 122 | return nn.GRU(n_input * hidden_size, hidden_size, batch_first=True) 123 | 124 | 125 | def harmonic_synth(pitch, amplitudes, sampling_rate): 126 | n_harmonic = amplitudes.shape[-1] 127 | omega = torch.cumsum(2 * math.pi * pitch / sampling_rate, 1) 128 | omegas = omega * torch.arange(1, n_harmonic + 1).to(omega) 129 | signal = (torch.sin(omegas) * amplitudes).sum(-1, keepdim=True) 130 | return signal 131 | 132 | 133 | def amp_to_impulse_response(amp, target_size): 134 | amp = torch.stack([amp, torch.zeros_like(amp)], -1) 135 | amp = torch.view_as_complex(amp) 136 | amp = fft.irfft(amp) 137 | 138 | filter_size = amp.shape[-1] 139 | 140 | amp = torch.roll(amp, filter_size // 2, -1) 141 | win = torch.hann_window(filter_size, dtype=amp.dtype, device=amp.device) 142 | 143 | amp = amp * win 144 | 145 | amp = nn.functional.pad(amp, (0, int(target_size) - int(filter_size))) 146 | amp = torch.roll(amp, -filter_size // 2, -1) 147 | 148 | return amp 149 | 150 | 151 | def fft_convolve(signal, kernel): 152 | signal = nn.functional.pad(signal, (0, signal.shape[-1])) 153 | kernel = nn.functional.pad(kernel, (kernel.shape[-1], 0)) 154 | 155 | output = fft.irfft(fft.rfft(signal) * fft.rfft(kernel)) 156 | output = output[..., output.shape[-1] // 2:] 157 | 158 | return output -------------------------------------------------------------------------------- /ddsp/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from .core import mlp, gru, scale_function, remove_above_nyquist, upsample 4 | from .core import harmonic_synth, amp_to_impulse_response, fft_convolve 5 | from .core import resample 6 | import math 7 | 8 | 9 | class Reverb(nn.Module): 10 | def __init__(self, length, sampling_rate, initial_wet=0, initial_decay=5): 11 | super().__init__() 12 | self.length = length 13 | self.sampling_rate = sampling_rate 14 | 15 | self.noise = nn.Parameter((torch.rand(length) * 2 - 1).unsqueeze(-1)) 16 | self.decay = nn.Parameter(torch.tensor(float(initial_decay))) 17 | self.wet = nn.Parameter(torch.tensor(float(initial_wet))) 18 | 19 | t = torch.arange(self.length) / self.sampling_rate 20 | t = t.reshape(1, -1, 1) 21 | self.register_buffer("t", t) 22 | 23 | def build_impulse(self): 24 | t = torch.exp(-nn.functional.softplus(-self.decay) * self.t * 500) 25 | noise = self.noise * t 26 | impulse = noise * torch.sigmoid(self.wet) 27 | impulse[:, 0] = 1 28 | return impulse 29 | 30 | def forward(self, x): 31 | lenx = x.shape[1] 32 | impulse = self.build_impulse() 33 | impulse = nn.functional.pad(impulse, (0, 0, 0, lenx - self.length)) 34 | 35 | x = fft_convolve(x.squeeze(-1), impulse.squeeze(-1)).unsqueeze(-1) 36 | 37 | return x 38 | 39 | 40 | class DDSP(nn.Module): 41 | def __init__(self, hidden_size, n_harmonic, n_bands, sampling_rate, 42 | block_size): 43 | super().__init__() 44 | self.register_buffer("sampling_rate", torch.tensor(sampling_rate)) 45 | self.register_buffer("block_size", torch.tensor(block_size)) 46 | 47 | self.in_mlps = nn.ModuleList([mlp(1, hidden_size, 3)] * 2) 48 | self.gru = gru(2, hidden_size) 49 | self.out_mlp = mlp(hidden_size + 2, hidden_size, 3) 50 | 51 | self.proj_matrices = nn.ModuleList([ 52 | nn.Linear(hidden_size, n_harmonic + 1), 53 | nn.Linear(hidden_size, n_bands), 54 | ]) 55 | 56 | self.reverb = Reverb(sampling_rate, sampling_rate) 57 | 58 | self.register_buffer("cache_gru", torch.zeros(1, 1, hidden_size)) 59 | self.register_buffer("phase", torch.zeros(1)) 60 | 61 | def forward(self, pitch, loudness): 62 | hidden = torch.cat([ 63 | self.in_mlps[0](pitch), 64 | self.in_mlps[1](loudness), 65 | ], -1) 66 | hidden = torch.cat([self.gru(hidden)[0], pitch, loudness], -1) 67 | hidden = self.out_mlp(hidden) 68 | 69 | # harmonic part 70 | param = scale_function(self.proj_matrices[0](hidden)) 71 | 72 | total_amp = param[..., :1] 73 | amplitudes = param[..., 1:] 74 | 75 | amplitudes = remove_above_nyquist( 76 | amplitudes, 77 | pitch, 78 | self.sampling_rate, 79 | ) 80 | amplitudes /= amplitudes.sum(-1, keepdim=True) 81 | amplitudes *= total_amp 82 | 83 | amplitudes = upsample(amplitudes, self.block_size) 84 | pitch = upsample(pitch, self.block_size) 85 | 86 | harmonic = harmonic_synth(pitch, amplitudes, self.sampling_rate) 87 | 88 | # noise part 89 | param = scale_function(self.proj_matrices[1](hidden) - 5) 90 | 91 | impulse = amp_to_impulse_response(param, self.block_size) 92 | noise = torch.rand( 93 | impulse.shape[0], 94 | impulse.shape[1], 95 | self.block_size, 96 | ).to(impulse) * 2 - 1 97 | 98 | noise = fft_convolve(noise, impulse).contiguous() 99 | noise = noise.reshape(noise.shape[0], -1, 1) 100 | 101 | signal = harmonic + noise 102 | 103 | #reverb part 104 | signal = self.reverb(signal) 105 | 106 | return signal 107 | 108 | def realtime_forward(self, pitch, loudness): 109 | hidden = torch.cat([ 110 | self.in_mlps[0](pitch), 111 | self.in_mlps[1](loudness), 112 | ], -1) 113 | 114 | gru_out, cache = self.gru(hidden, self.cache_gru) 115 | self.cache_gru.copy_(cache) 116 | 117 | hidden = torch.cat([gru_out, pitch, loudness], -1) 118 | hidden = self.out_mlp(hidden) 119 | 120 | # harmonic part 121 | param = scale_function(self.proj_matrices[0](hidden)) 122 | 123 | total_amp = param[..., :1] 124 | amplitudes = param[..., 1:] 125 | 126 | amplitudes = remove_above_nyquist( 127 | amplitudes, 128 | pitch, 129 | self.sampling_rate, 130 | ) 131 | amplitudes /= amplitudes.sum(-1, keepdim=True) 132 | amplitudes *= total_amp 133 | 134 | amplitudes = upsample(amplitudes, self.block_size) 135 | pitch = upsample(pitch, self.block_size) 136 | 137 | n_harmonic = amplitudes.shape[-1] 138 | omega = torch.cumsum(2 * math.pi * pitch / self.sampling_rate, 1) 139 | 140 | omega = omega + self.phase 141 | self.phase.copy_(omega[0, -1, 0] % (2 * math.pi)) 142 | 143 | omegas = omega * torch.arange(1, n_harmonic + 1).to(omega) 144 | 145 | harmonic = (torch.sin(omegas) * amplitudes).sum(-1, keepdim=True) 146 | 147 | # noise part 148 | param = scale_function(self.proj_matrices[1](hidden) - 5) 149 | 150 | impulse = amp_to_impulse_response(param, self.block_size) 151 | noise = torch.rand( 152 | impulse.shape[0], 153 | impulse.shape[1], 154 | self.block_size, 155 | ).to(impulse) * 2 - 1 156 | 157 | noise = fft_convolve(noise, impulse).contiguous() 158 | noise = noise.reshape(noise.shape[0], -1, 1) 159 | 160 | signal = harmonic + noise 161 | 162 | return signal -------------------------------------------------------------------------------- /ddsp/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def get_scheduler(len_dataset, start_lr, stop_lr, length): 5 | def schedule(epoch): 6 | step = epoch * len_dataset 7 | if step < length: 8 | t = step / length 9 | return start_lr * (1 - t) + stop_lr * t 10 | else: 11 | return stop_lr 12 | 13 | return schedule -------------------------------------------------------------------------------- /export.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import yaml 3 | from effortless_config import Config 4 | from os import path, makedirs, system 5 | from ddsp.model import DDSP 6 | import soundfile as sf 7 | from preprocess import get_files 8 | 9 | torch.set_grad_enabled(False) 10 | 11 | 12 | class args(Config): 13 | RUN = None 14 | DATA = False 15 | OUT_DIR = "export" 16 | REALTIME = False 17 | 18 | 19 | args.parse_args() 20 | makedirs(args.OUT_DIR, exist_ok=True) 21 | 22 | 23 | class ScriptDDSP(torch.nn.Module): 24 | def __init__(self, ddsp, mean_loudness, std_loudness, realtime): 25 | super().__init__() 26 | self.ddsp = ddsp 27 | self.ddsp.gru.flatten_parameters() 28 | 29 | self.register_buffer("mean_loudness", torch.tensor(mean_loudness)) 30 | self.register_buffer("std_loudness", torch.tensor(std_loudness)) 31 | self.realtime = realtime 32 | 33 | def forward(self, pitch, loudness): 34 | loudness = (loudness - self.mean_loudness) / self.std_loudness 35 | if self.realtime: 36 | pitch = pitch[:, ::self.ddsp.block_size] 37 | loudness = loudness[:, ::self.ddsp.block_size] 38 | return self.ddsp.realtime_forward(pitch, loudness) 39 | else: 40 | return self.ddsp(pitch, loudness) 41 | 42 | 43 | with open(path.join(args.RUN, "config.yaml"), "r") as config: 44 | config = yaml.safe_load(config) 45 | 46 | ddsp = DDSP(**config["model"]) 47 | 48 | state = ddsp.state_dict() 49 | pretrained = torch.load(path.join(args.RUN, "state.pth"), map_location="cpu") 50 | state.update(pretrained) 51 | ddsp.load_state_dict(state) 52 | 53 | name = path.basename(path.normpath(args.RUN)) 54 | 55 | scripted_model = torch.jit.script( 56 | ScriptDDSP( 57 | ddsp, 58 | config["data"]["mean_loudness"], 59 | config["data"]["std_loudness"], 60 | args.REALTIME, 61 | )) 62 | torch.jit.save( 63 | scripted_model, 64 | path.join(args.OUT_DIR, f"ddsp_{name}_pretrained.ts"), 65 | ) 66 | 67 | impulse = ddsp.reverb.build_impulse().reshape(-1).numpy() 68 | sf.write( 69 | path.join(args.OUT_DIR, f"ddsp_{name}_impulse.wav"), 70 | impulse, 71 | config["preprocess"]["sampling_rate"], 72 | ) 73 | 74 | with open( 75 | path.join(args.OUT_DIR, f"ddsp_{name}_config.yaml"), 76 | "w", 77 | ) as config_out: 78 | yaml.safe_dump(config, config_out) 79 | 80 | if args.DATA: 81 | makedirs(path.join(args.OUT_DIR, "data"), exist_ok=True) 82 | file_list = get_files(**config["data"]) 83 | file_list = [str(f).replace(" ", "\\ ") for f in file_list] 84 | system(f"cp {' '.join(file_list)} {path.normpath(args.OUT_DIR)}/data/") -------------------------------------------------------------------------------- /patchs/preprocess.pd: -------------------------------------------------------------------------------- 1 | #N canvas 1055 22 539 310 12; 2 | #X obj 177 103 inlet~; 3 | #X obj 177 132 sigmund~ -hop 512; 4 | #X obj 177 161 mtof; 5 | #X obj 177 190 sig~; 6 | #X obj 177 248 outlet~; 7 | #X obj 293 190 sig~; 8 | #X obj 293 248 outlet~; 9 | #X obj 50 132 r tab_bang; 10 | #X obj 67 183 clip 0 127; 11 | #X obj 404 105 inlet; 12 | #X obj 373 248 tabwrite~ loudness; 13 | #X obj 404 134 metro 100; 14 | #X obj 40 235 tabwrite~ pitch; 15 | #X obj 263 106 tabwrite~ oscillo; 16 | #X obj 177 219 lop~ 1000; 17 | #X text 44 29 pitch and loudness extraction using sigmund~. Both signals 18 | are low-passed at 1000Hz to prevent quantization artifacts., f 52 19 | ; 20 | #X obj 293 219 lop~ 1000; 21 | #X connect 0 0 1 0; 22 | #X connect 0 0 13 0; 23 | #X connect 1 0 2 0; 24 | #X connect 1 0 8 0; 25 | #X connect 1 1 5 0; 26 | #X connect 2 0 3 0; 27 | #X connect 3 0 14 0; 28 | #X connect 5 0 16 0; 29 | #X connect 7 0 12 0; 30 | #X connect 8 0 12 0; 31 | #X connect 9 0 11 0; 32 | #X connect 11 0 10 0; 33 | #X connect 11 0 12 0; 34 | #X connect 11 0 13 0; 35 | #X connect 14 0 4 0; 36 | #X connect 16 0 6 0; 37 | #X connect 16 0 10 0; 38 | -------------------------------------------------------------------------------- /patchs/timbre_transfer.pd: -------------------------------------------------------------------------------- 1 | #N canvas 625 600 1453 370 12; 2 | #X obj 339 58 bng 15 250 50 0 empty empty empty 17 7 0 10 #fcfcfc #000000 3 | #000000; 4 | #X obj 328 93 openpanel; 5 | #X msg 328 122 open \$1 \, start; 6 | #X obj 328 151 readsf~; 7 | #X obj 333 265 writesf~ 3; 8 | #X obj 141 174 t b; 9 | #X obj 274 154 dac~; 10 | #X msg 470 188 stop; 11 | #X msg 69 231 open -bytes 4 preprocessed.wav \, start; 12 | #X text 16 26 PREPROCESS AUDIO Press the button and select your source 13 | audio file to start preprocessing. The resulting preprocessed file 14 | will be saved alongside this script, f 33; 15 | #X obj 811 34 bng 15 250 50 0 empty empty empty 17 7 0 10 #fcfcfc #000000 16 | #000000; 17 | #X obj 811 58 openpanel; 18 | #X msg 811 87 open \$1 \, start; 19 | #X obj 811 116 readsf~; 20 | #X obj 811 201 ddsp~; 21 | #X obj 930 250 table impulse; 22 | #X obj 811 230 bsaylor/partconv~ impulse 2048; 23 | #X obj 657 35 bng 15 250 50 0 empty empty empty 17 7 0 10 #fcfcfc #000000 24 | #000000; 25 | #X obj 657 59 openpanel; 26 | #X msg 657 88 load \$1; 27 | #X obj 1021 33 bng 15 250 50 0 empty empty empty 17 7 0 10 #fcfcfc 28 | #000000 #000000; 29 | #X obj 1021 57 openpanel; 30 | #X msg 1021 86 read -resize \$1 impulse; 31 | #X obj 1021 115 soundfiler; 32 | #X obj 1021 144 t b; 33 | #X msg 1021 173 set impulse; 34 | #X text 653 10 load pretrained state; 35 | #X text 809 10 load input audio; 36 | #X text 1020 9 load impulse response; 37 | #N canvas 0 22 450 300 (subpatch) 0; 38 | #X array oscillo 480 float 0 black black; 39 | #X coords 0 1 479 -1 200 100 1; 40 | #X restore 1211 33 graph; 41 | #X obj 811 259 clip~ -1 1; 42 | #X obj 726 184 vsl 15 128 0 100 0 0 empty empty empty 0 -9 0 10 #f8f8f6 43 | #000000 #000000 0 1; 44 | #X obj 726 326 dbtorms; 45 | #X obj 811 317 dac~; 46 | #X obj 811 288 *~ 0; 47 | #N canvas 0 22 450 300 (subpatch) 0; 48 | #X array pitch 480 float 2 black black; 49 | #X coords 0 127 480 0 200 100 1; 50 | #X restore 1211 142 graph; 51 | #N canvas 0 22 450 300 (subpatch) 0; 52 | #X array loudness 480 float 2 black black; 53 | #X coords 0 100 480 0 200 100 1; 54 | #X restore 1211 248 graph; 55 | #X obj 328 185 preprocess; 56 | #X obj 811 145 preprocess; 57 | #X obj 874 118 tgl 15 0 empty empty empty 17 7 0 10 #fcfcfc #000000 58 | #000000 0 1; 59 | #X obj 395 155 tgl 15 0 empty empty empty 17 7 0 10 #fcfcfc #000000 60 | #000000 0 1; 61 | #X text 412 153 display; 62 | #X text 890 115 display; 63 | #X obj 515 61 loadbang; 64 | #X msg 515 119 1024; 65 | #X obj 554 119 samplerate~; 66 | #X obj 515 148 /; 67 | #X obj 515 90 t b b; 68 | #X obj 514 234 delread~ delay; 69 | #X obj 515 177 * 1000; 70 | #X obj 515 206 / 2; 71 | #X obj 177 193 delwrite~ delay 1000; 72 | #X connect 0 0 1 0; 73 | #X connect 1 0 2 0; 74 | #X connect 1 0 5 0; 75 | #X connect 2 0 3 0; 76 | #X connect 3 0 37 0; 77 | #X connect 3 0 51 0; 78 | #X connect 3 0 6 0; 79 | #X connect 3 0 6 1; 80 | #X connect 3 1 7 0; 81 | #X connect 5 0 8 0; 82 | #X connect 7 0 3 0; 83 | #X connect 7 0 4 0; 84 | #X connect 8 0 4 0; 85 | #X connect 10 0 11 0; 86 | #X connect 11 0 12 0; 87 | #X connect 12 0 13 0; 88 | #X connect 13 0 38 0; 89 | #X connect 14 0 16 0; 90 | #X connect 16 0 30 0; 91 | #X connect 17 0 18 0; 92 | #X connect 18 0 19 0; 93 | #X connect 19 0 14 0; 94 | #X connect 20 0 21 0; 95 | #X connect 21 0 22 0; 96 | #X connect 22 0 23 0; 97 | #X connect 23 0 24 0; 98 | #X connect 24 0 25 0; 99 | #X connect 25 0 16 0; 100 | #X connect 30 0 34 0; 101 | #X connect 31 0 32 0; 102 | #X connect 32 0 34 1; 103 | #X connect 34 0 33 0; 104 | #X connect 34 0 33 1; 105 | #X connect 37 0 4 1; 106 | #X connect 37 1 4 2; 107 | #X connect 38 0 14 0; 108 | #X connect 38 1 14 1; 109 | #X connect 39 0 38 1; 110 | #X connect 40 0 37 1; 111 | #X connect 43 0 47 0; 112 | #X connect 44 0 46 0; 113 | #X connect 45 0 46 1; 114 | #X connect 46 0 49 0; 115 | #X connect 47 0 44 0; 116 | #X connect 47 1 45 0; 117 | #X connect 48 0 4 0; 118 | #X connect 49 0 50 0; 119 | #X connect 50 0 48 0; 120 | -------------------------------------------------------------------------------- /performance.py: -------------------------------------------------------------------------------- 1 | #%% 2 | import torch 3 | torch.set_grad_enabled(False) 4 | from time import time 5 | import math 6 | from tqdm import tqdm 7 | from effortless_config import Config 8 | 9 | 10 | class args(Config): 11 | MODEL = None 12 | N_RUN = 10 13 | 14 | 15 | args.parse_args() 16 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 17 | 18 | model = torch.jit.load(args.MODEL).eval().to(device) 19 | sr = model.ddsp.sampling_rate 20 | 21 | N = 2**(math.ceil(math.log2(sr))) 22 | x = torch.randn(1, N, 1).to(device) 23 | 24 | n_run = args.N_RUN 25 | mean = 0 26 | nel = 0 27 | 28 | for i in tqdm(range(n_run), desc="testing..."): 29 | st = time() 30 | y = model(x, x) 31 | nel += 1 32 | mean += (time() - st - mean) / nel 33 | 34 | realtime = N / (mean * sr) 35 | smiley = ":)" if realtime >= 1 else ":(" 36 | 37 | print("\n") 38 | print( 39 | f"average of {1000*mean:.2f}ms to generate {1000*N/sr:.2f}ms over {n_run} trials on device {device}" 40 | ) 41 | print(f"generation is {realtime:.2f}x realtime {smiley}") 42 | print(80 * "-") 43 | # %% 44 | -------------------------------------------------------------------------------- /preprocess.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | import pathlib 3 | import librosa as li 4 | from ddsp.core import extract_loudness, extract_pitch 5 | from effortless_config import Config 6 | import numpy as np 7 | from tqdm import tqdm 8 | import numpy as np 9 | from os import makedirs, path 10 | import torch 11 | from scipy.io import wavfile 12 | 13 | 14 | def get_files(data_location, extension, **kwargs): 15 | return list(pathlib.Path(data_location).rglob(f"*.{extension}")) 16 | 17 | 18 | def preprocess(f, sampling_rate, block_size, signal_length, oneshot, **kwargs): 19 | x, sr = li.load(f, sampling_rate) 20 | N = (signal_length - len(x) % signal_length) % signal_length 21 | x = np.pad(x, (0, N)) 22 | 23 | if oneshot: 24 | x = x[..., :signal_length] 25 | 26 | pitch = extract_pitch(x, sampling_rate, block_size) 27 | loudness = extract_loudness(x, sampling_rate, block_size) 28 | 29 | x = x.reshape(-1, signal_length) 30 | pitch = pitch.reshape(x.shape[0], -1) 31 | loudness = loudness.reshape(x.shape[0], -1) 32 | 33 | return x, pitch, loudness 34 | 35 | 36 | class Dataset(torch.utils.data.Dataset): 37 | def __init__(self, out_dir): 38 | super().__init__() 39 | self.signals = np.load(path.join(out_dir, "signals.npy")) 40 | self.pitchs = np.load(path.join(out_dir, "pitchs.npy")) 41 | self.loudness = np.load(path.join(out_dir, "loudness.npy")) 42 | 43 | def __len__(self): 44 | return self.signals.shape[0] 45 | 46 | def __getitem__(self, idx): 47 | s = torch.from_numpy(self.signals[idx]) 48 | p = torch.from_numpy(self.pitchs[idx]) 49 | l = torch.from_numpy(self.loudness[idx]) 50 | return s, p, l 51 | 52 | 53 | def main(): 54 | class args(Config): 55 | CONFIG = "config.yaml" 56 | 57 | args.parse_args() 58 | with open(args.CONFIG, "r") as config: 59 | config = yaml.safe_load(config) 60 | 61 | files = get_files(**config["data"]) 62 | pb = tqdm(files) 63 | 64 | signals = [] 65 | pitchs = [] 66 | loudness = [] 67 | 68 | for f in pb: 69 | pb.set_description(str(f)) 70 | x, p, l = preprocess(f, **config["preprocess"]) 71 | signals.append(x) 72 | pitchs.append(p) 73 | loudness.append(l) 74 | 75 | signals = np.concatenate(signals, 0).astype(np.float32) 76 | pitchs = np.concatenate(pitchs, 0).astype(np.float32) 77 | loudness = np.concatenate(loudness, 0).astype(np.float32) 78 | 79 | out_dir = config["preprocess"]["out_dir"] 80 | makedirs(out_dir, exist_ok=True) 81 | 82 | np.save(path.join(out_dir, "signals.npy"), signals) 83 | np.save(path.join(out_dir, "pitchs.npy"), pitchs) 84 | np.save(path.join(out_dir, "loudness.npy"), loudness) 85 | 86 | 87 | if __name__ == "__main__": 88 | main() -------------------------------------------------------------------------------- /preprocess_from_sigmund.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | from scipy.io import wavfile 3 | import numpy as np 4 | from effortless_config import Config 5 | from types import SimpleNamespace 6 | from einops import rearrange 7 | from os import path, makedirs 8 | from ddsp.core import extract_pitch 9 | 10 | with open("config.yaml", "r") as config: 11 | config = SimpleNamespace(**yaml.safe_load(config)) 12 | 13 | sr, x = wavfile.read(config.data["data_location"]) 14 | assert sr == config.preprocess["sampling_rate"] 15 | 16 | n_signal = config.preprocess["signal_length"] 17 | pad = (n_signal - (x.shape[0] % n_signal)) % n_signal 18 | 19 | x = np.pad(x, ((0, pad), (0, 0))) 20 | 21 | # pitch = extract_pitch(x[:, 0], sr, config.preprocess["block_size"]) 22 | 23 | x = rearrange( 24 | x, 25 | "(batch n_signal) channel -> channel batch n_signal", 26 | n_signal=n_signal, 27 | ) 28 | 29 | # pitch = pitch.reshape(x.shape[1], -1).astype(np.float32) 30 | 31 | out_dir = config.preprocess["out_dir"] 32 | makedirs(out_dir, exist_ok=True) 33 | 34 | np.save(path.join(out_dir, "signals.npy"), x[0]) 35 | 36 | np.save( 37 | path.join(out_dir, "pitchs.npy"), 38 | x[1, :, ::config.preprocess["block_size"]], 39 | ) 40 | 41 | np.save( 42 | path.join(out_dir, "loudness.npy"), 43 | x[2, :, ::config.preprocess["block_size"]], 44 | ) 45 | -------------------------------------------------------------------------------- /realtime/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0) 2 | project(ddsp_external) 3 | 4 | find_package(Torch REQUIRED) 5 | 6 | if (APPLE) 7 | set(CMAKE_CXX_COMPILER "clang++") 8 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -undefined dynamic_lookup -stdlib=libc++") 9 | set(EXTERNAL_SUFFIX ".pd_darwin") 10 | endif() 11 | 12 | if (UNIX AND NOT APPLE) 13 | set(EXTERNAL_SUFFIX ".pd_linux") 14 | endif() 15 | 16 | set(CMAKE_CXX_FLAGS_RELEASE "-O3") 17 | 18 | add_library(ddsp_tilde SHARED ddsp_tilde/ddsp_tilde.cpp ddsp_tilde/ddsp_model.cpp) 19 | target_link_libraries(ddsp_tilde "${TORCH_LIBRARIES}") 20 | set_target_properties(ddsp_tilde PROPERTIES PREFIX "" OUTPUT_NAME "ddsp~" SUFFIX "${EXTERNAL_SUFFIX}") 21 | set_property(TARGET ddsp_tilde PROPERTY CXX_STANDARD 14) -------------------------------------------------------------------------------- /realtime/ddsp_tilde/ddsp_model.cpp: -------------------------------------------------------------------------------- 1 | #include "ddsp_model.h" 2 | 3 | DDSPModel::DDSPModel() : m_loaded(0) 4 | { 5 | at::init_num_threads(); 6 | } 7 | 8 | int DDSPModel::load(std::string path) 9 | { 10 | try 11 | { 12 | m_scripted_model = torch::jit::load(path); 13 | m_scripted_model.eval(); 14 | m_scripted_model.to(DEVICE); 15 | m_loaded = 1; 16 | return 0; 17 | } 18 | catch (const std::exception &e) 19 | { 20 | std::cerr << e.what() << '\n'; 21 | m_loaded = 0; 22 | return 1; 23 | } 24 | } 25 | 26 | void DDSPModel::perform(float *pitch, float *loudness, float *out_buffer, int buffer_size) 27 | { 28 | torch::NoGradGuard no_grad; 29 | if (m_loaded) 30 | { 31 | auto pitch_tensor = torch::from_blob(pitch, {1, buffer_size, 1}); 32 | auto loudness_tensor = torch::from_blob(loudness, {1, buffer_size, 1}); 33 | 34 | pitch_tensor = pitch_tensor.to(DEVICE); 35 | loudness_tensor = loudness_tensor.to(DEVICE); 36 | 37 | std::vector inputs = {pitch_tensor, loudness_tensor}; 38 | 39 | auto out_tensor = m_scripted_model.forward(inputs).toTensor(); 40 | out_tensor = out_tensor.to(CPU); 41 | 42 | auto out = out_tensor.contiguous().data_ptr(); 43 | 44 | memcpy(out_buffer, out, buffer_size * sizeof(float)); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /realtime/ddsp_tilde/ddsp_model.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #define DEVICE torch::kCPU 10 | #define CPU torch::kCPU 11 | 12 | class DDSPModel 13 | { 14 | private: 15 | torch::jit::script::Module m_scripted_model; 16 | int m_loaded; 17 | 18 | public: 19 | DDSPModel(); 20 | int load(std::string path); 21 | void perform(float *pitch, float *loudness, float *out_buffer, int buffer_size); 22 | }; -------------------------------------------------------------------------------- /realtime/ddsp_tilde/ddsp_tilde.cpp: -------------------------------------------------------------------------------- 1 | #include "m_pd.h" 2 | #include "ddsp_model.h" 3 | #include "dlfcn.h" 4 | #include 5 | #include 6 | 7 | #define B_SIZE 1024 8 | 9 | static t_class *ddsp_tilde_class; 10 | 11 | struct ddsp_tilde 12 | { 13 | t_object x_obj; 14 | t_float f; 15 | 16 | t_inlet *x_in2; 17 | t_outlet *x_out; 18 | 19 | float pitch_buffer[2 * B_SIZE]; 20 | float loudness_buffer[2 * B_SIZE]; 21 | float out_buffer[2 * B_SIZE]; 22 | 23 | int head; 24 | 25 | std::thread *compute_thread; 26 | 27 | DDSPModel *model; 28 | }; 29 | 30 | void thread_perform(ddsp_tilde *x, float *pitch, float *loudness, 31 | float *out_buffer, int buffer_size) 32 | { 33 | x->model->perform(pitch, loudness, out_buffer, buffer_size); 34 | } 35 | 36 | void *ddsp_tilde_new() 37 | { 38 | ddsp_tilde *x = (ddsp_tilde *)pd_new(ddsp_tilde_class); 39 | x->x_in2 = inlet_new(&x->x_obj, &x->x_obj.ob_pd, &s_signal, &s_signal); 40 | x->x_out = outlet_new(&x->x_obj, &s_signal); 41 | x->model = new DDSPModel; 42 | 43 | x->head = 0; 44 | 45 | return (void *)x; 46 | } 47 | 48 | void ddsp_tilde_free(ddsp_tilde *x) 49 | { 50 | inlet_free(x->x_in2); 51 | outlet_free(x->x_out); 52 | } 53 | 54 | void ddsp_tilde_load(ddsp_tilde *x, t_symbol *sym) 55 | { 56 | int status = x->model->load(sym->s_name); 57 | if (!status) 58 | { 59 | post("successfully loaded model"); 60 | } 61 | else 62 | { 63 | post("error loading model"); 64 | } 65 | } 66 | 67 | t_int *ddsp_tilde_perform(t_int *w) 68 | { 69 | ddsp_tilde *x = (ddsp_tilde *)(w[1]); 70 | t_sample *in1 = (t_sample *)(w[2]); 71 | t_sample *in2 = (t_sample *)(w[3]); 72 | t_sample *out = (t_sample *)(w[4]); 73 | int n = (int)(w[5]); 74 | 75 | memcpy(x->pitch_buffer + x->head, in1, n * sizeof(float)); 76 | memcpy(x->loudness_buffer + x->head, in2, n * sizeof(float)); 77 | memcpy(out, x->out_buffer + x->head, n * sizeof(float)); 78 | 79 | x->head += n; 80 | 81 | if (!(x->head % B_SIZE)) 82 | { 83 | if (x->compute_thread) 84 | { 85 | x->compute_thread->join(); 86 | } 87 | int model_head = ((x->head + B_SIZE) % (2 * B_SIZE)); 88 | x->compute_thread = new std::thread(thread_perform, x, 89 | x->pitch_buffer + model_head, 90 | x->loudness_buffer + model_head, 91 | x->out_buffer + model_head, 92 | B_SIZE); 93 | 94 | x->head = x->head % (2 * B_SIZE); 95 | } 96 | 97 | return (w + 6); 98 | } 99 | 100 | void ddsp_tilde_dsp(ddsp_tilde *x, t_signal **sp) 101 | { 102 | dsp_add(ddsp_tilde_perform, 5, x, 103 | sp[0]->s_vec, sp[1]->s_vec, sp[2]->s_vec, sp[0]->s_n); 104 | } 105 | 106 | extern "C" 107 | { 108 | void ddsp_tilde_setup(void) 109 | { 110 | ddsp_tilde_class = class_new(gensym("ddsp~"), 111 | (t_newmethod)ddsp_tilde_new, 112 | (t_method)ddsp_tilde_free, 113 | sizeof(ddsp_tilde), 114 | CLASS_DEFAULT, 115 | A_DEFFLOAT, 0); 116 | CLASS_MAINSIGNALIN(ddsp_tilde_class, ddsp_tilde, f); 117 | 118 | class_addmethod(ddsp_tilde_class, 119 | (t_method)ddsp_tilde_load, 120 | gensym("load"), 121 | A_SYMBOL, 0); 122 | 123 | class_addmethod(ddsp_tilde_class, 124 | (t_method)ddsp_tilde_dsp, 125 | gensym("dsp"), A_CANT, 0); 126 | } 127 | } -------------------------------------------------------------------------------- /realtime/ddsp_tilde/m_pd.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 1997-1999 Miller Puckette. 2 | * For information on usage and redistribution, and for a DISCLAIMER OF ALL 3 | * WARRANTIES, see the file, "LICENSE.txt," in this distribution. */ 4 | 5 | #ifndef __m_pd_h_ 6 | 7 | #if defined(_LANGUAGE_C_PLUS_PLUS) || defined(__cplusplus) 8 | extern "C" { 9 | #endif 10 | 11 | #define PD_MAJOR_VERSION 0 12 | #define PD_MINOR_VERSION 51 13 | #define PD_BUGFIX_VERSION 4 14 | #define PD_TEST_VERSION "" 15 | extern int pd_compatibilitylevel; /* e.g., 43 for pd 0.43 compatibility */ 16 | 17 | /* old name for "MSW" flag -- we have to take it for the sake of many old 18 | "nmakefiles" for externs, which will define NT and not MSW */ 19 | #if defined(NT) && !defined(MSW) 20 | #define MSW 21 | #endif 22 | 23 | /* These pragmas are only used for MSVC, not MinGW or Cygwin */ 24 | #ifdef _MSC_VER 25 | /* #pragma warning( disable : 4091 ) */ 26 | #pragma warning( disable : 4305 ) /* uncast const double to float */ 27 | #pragma warning( disable : 4244 ) /* uncast float/int conversion etc. */ 28 | #pragma warning( disable : 4101 ) /* unused automatic variables */ 29 | #endif /* _MSC_VER */ 30 | 31 | /* the external storage class is "extern" in UNIX; in MSW it's ugly. */ 32 | #ifdef _WIN32 33 | #ifdef PD_INTERNAL 34 | #define EXTERN __declspec(dllexport) extern 35 | #else 36 | #define EXTERN __declspec(dllimport) extern 37 | #endif /* PD_INTERNAL */ 38 | #else 39 | #define EXTERN extern 40 | #endif /* _WIN32 */ 41 | 42 | /* On most c compilers, you can just say "struct foo;" to declare a 43 | structure whose elements are defined elsewhere. On MSVC, when compiling 44 | C (but not C++) code, you have to say "extern struct foo;". So we make 45 | a stupid macro: */ 46 | #if defined(_MSC_VER) && !defined(_LANGUAGE_C_PLUS_PLUS) \ 47 | && !defined(__cplusplus) 48 | #define EXTERN_STRUCT extern struct 49 | #else 50 | #define EXTERN_STRUCT struct 51 | #endif 52 | 53 | /* Define some attributes, specific to the compiler */ 54 | #if defined(__GNUC__) 55 | #define ATTRIBUTE_FORMAT_PRINTF(a, b) __attribute__ ((format (printf, a, b))) 56 | #else 57 | #define ATTRIBUTE_FORMAT_PRINTF(a, b) 58 | #endif 59 | 60 | #if !defined(_SIZE_T) && !defined(_SIZE_T_) 61 | #include /* just for size_t -- how lame! */ 62 | #endif 63 | 64 | /* Microsoft Visual Studio is not C99, but since VS2015 has included most C99 headers: 65 | https://docs.microsoft.com/en-us/previous-versions/hh409293(v=vs.140)#c-runtime-library 66 | These definitions recreate stdint.h types, but only in pre-2015 Visual Studio: */ 67 | #if defined(_MSC_VER) && _MSC_VER < 1900 68 | typedef signed __int8 int8_t; 69 | typedef signed __int16 int16_t; 70 | typedef signed __int32 int32_t; 71 | typedef signed __int64 int64_t; 72 | typedef unsigned __int8 uint8_t; 73 | typedef unsigned __int16 uint16_t; 74 | typedef unsigned __int32 uint32_t; 75 | typedef unsigned __int64 uint64_t; 76 | #else 77 | # include 78 | #endif 79 | 80 | /* for FILE, needed by sys_fopen() and sys_fclose() only */ 81 | #include 82 | 83 | #define MAXPDSTRING 1000 /* use this for anything you want */ 84 | #define MAXPDARG 5 /* max number of args we can typecheck today */ 85 | 86 | /* signed and unsigned integer types the size of a pointer: */ 87 | #if !defined(PD_LONGINTTYPE) 88 | #if defined(_WIN32) && defined(_WIN64) 89 | #define PD_LONGINTTYPE long long 90 | #else 91 | #define PD_LONGINTTYPE long 92 | #endif 93 | #endif 94 | 95 | #if !defined(PD_FLOATSIZE) 96 | /* normally, our floats (t_float, t_sample,...) are 32bit */ 97 | # define PD_FLOATSIZE 32 98 | #endif 99 | 100 | #if PD_FLOATSIZE == 32 101 | # define PD_FLOATTYPE float 102 | /* an unsigned int of the same size as FLOATTYPE: */ 103 | # define PD_FLOATUINTTYPE uint32_t 104 | 105 | #elif PD_FLOATSIZE == 64 106 | # define PD_FLOATTYPE double 107 | # define PD_FLOATUINTTYPE uint64_t 108 | #else 109 | # error invalid FLOATSIZE: must be 32 or 64 110 | #endif 111 | 112 | typedef PD_LONGINTTYPE t_int; /* pointer-size integer */ 113 | typedef PD_FLOATTYPE t_float; /* a float type at most the same size */ 114 | typedef PD_FLOATTYPE t_floatarg; /* float type for function calls */ 115 | 116 | typedef struct _symbol 117 | { 118 | const char *s_name; 119 | struct _class **s_thing; 120 | struct _symbol *s_next; 121 | } t_symbol; 122 | 123 | EXTERN_STRUCT _array; 124 | #define t_array struct _array /* g_canvas.h */ 125 | 126 | /* pointers to glist and array elements go through a "stub" which sticks 127 | around after the glist or array is freed. The stub itself is deleted when 128 | both the glist/array is gone and the refcount is zero, ensuring that no 129 | gpointers are pointing here. */ 130 | 131 | #define GP_NONE 0 /* the stub points nowhere (has been cut off) */ 132 | #define GP_GLIST 1 /* the stub points to a glist element */ 133 | #define GP_ARRAY 2 /* ... or array */ 134 | 135 | typedef struct _gstub 136 | { 137 | union 138 | { 139 | struct _glist *gs_glist; /* glist we're in */ 140 | struct _array *gs_array; /* array we're in */ 141 | } gs_un; 142 | int gs_which; /* GP_GLIST/GP_ARRAY */ 143 | int gs_refcount; /* number of gpointers pointing here */ 144 | } t_gstub; 145 | 146 | typedef struct _gpointer /* pointer to a gobj in a glist */ 147 | { 148 | union 149 | { 150 | struct _scalar *gp_scalar; /* scalar we're in (if glist) */ 151 | union word *gp_w; /* raw data (if array) */ 152 | } gp_un; 153 | int gp_valid; /* number which must match gpointee */ 154 | t_gstub *gp_stub; /* stub which points to glist/array */ 155 | } t_gpointer; 156 | 157 | typedef union word 158 | { 159 | t_float w_float; 160 | t_symbol *w_symbol; 161 | t_gpointer *w_gpointer; 162 | t_array *w_array; 163 | struct _binbuf *w_binbuf; 164 | int w_index; 165 | } t_word; 166 | 167 | typedef enum 168 | { 169 | A_NULL, 170 | A_FLOAT, 171 | A_SYMBOL, 172 | A_POINTER, 173 | A_SEMI, 174 | A_COMMA, 175 | A_DEFFLOAT, 176 | A_DEFSYM, 177 | A_DOLLAR, 178 | A_DOLLSYM, 179 | A_GIMME, 180 | A_CANT 181 | } t_atomtype; 182 | 183 | #define A_DEFSYMBOL A_DEFSYM /* better name for this */ 184 | 185 | typedef struct _atom 186 | { 187 | t_atomtype a_type; 188 | union word a_w; 189 | } t_atom; 190 | 191 | EXTERN_STRUCT _class; 192 | #define t_class struct _class 193 | 194 | EXTERN_STRUCT _outlet; 195 | #define t_outlet struct _outlet 196 | 197 | EXTERN_STRUCT _inlet; 198 | #define t_inlet struct _inlet 199 | 200 | EXTERN_STRUCT _binbuf; 201 | #define t_binbuf struct _binbuf 202 | 203 | EXTERN_STRUCT _clock; 204 | #define t_clock struct _clock 205 | 206 | EXTERN_STRUCT _outconnect; 207 | #define t_outconnect struct _outconnect 208 | 209 | EXTERN_STRUCT _glist; 210 | #define t_glist struct _glist 211 | #define t_canvas struct _glist /* LATER lose this */ 212 | 213 | EXTERN_STRUCT _template; 214 | 215 | typedef t_class *t_pd; /* pure datum: nothing but a class pointer */ 216 | 217 | typedef struct _gobj /* a graphical object */ 218 | { 219 | t_pd g_pd; /* pure datum header (class) */ 220 | struct _gobj *g_next; /* next in list */ 221 | } t_gobj; 222 | 223 | typedef struct _scalar /* a graphical object holding data */ 224 | { 225 | t_gobj sc_gobj; /* header for graphical object */ 226 | t_symbol *sc_template; /* template name (LATER replace with pointer) */ 227 | t_word sc_vec[1]; /* indeterminate-length array of words */ 228 | } t_scalar; 229 | 230 | typedef struct _text /* patchable object - graphical, with text */ 231 | { 232 | t_gobj te_g; /* header for graphical object */ 233 | t_binbuf *te_binbuf; /* holder for the text */ 234 | t_outlet *te_outlet; /* linked list of outlets */ 235 | t_inlet *te_inlet; /* linked list of inlets */ 236 | short te_xpix; /* x&y location (within the toplevel) */ 237 | short te_ypix; 238 | short te_width; /* requested width in chars, 0 if auto */ 239 | unsigned int te_type:2; /* from defs below */ 240 | } t_text; 241 | 242 | #define T_TEXT 0 /* just a textual comment */ 243 | #define T_OBJECT 1 /* a MAX style patchable object */ 244 | #define T_MESSAGE 2 /* a MAX type message */ 245 | #define T_ATOM 3 /* a cell to display a number or symbol */ 246 | 247 | #define te_pd te_g.g_pd 248 | 249 | /* t_object is synonym for t_text (LATER unify them) */ 250 | 251 | typedef struct _text t_object; 252 | 253 | #define ob_outlet te_outlet 254 | #define ob_inlet te_inlet 255 | #define ob_binbuf te_binbuf 256 | #define ob_pd te_g.g_pd 257 | #define ob_g te_g 258 | 259 | typedef void (*t_method)(void); 260 | typedef void *(*t_newmethod)(void); 261 | 262 | /* in ARM 64 a varargs prototype generates a different function call sequence 263 | from a fixed one, so in that special case we make a more restrictive 264 | definition for t_gotfn. This will break some code in the "chaos" package 265 | in Pd extended. (that code will run incorrectly anyhow so why not catch it 266 | at compile time anyhow.) */ 267 | #if defined(__APPLE__) && defined(__aarch64__) 268 | typedef void (*t_gotfn)(void *x); 269 | #else 270 | typedef void (*t_gotfn)(void *x, ...); 271 | #endif 272 | 273 | /* ---------------- pre-defined objects and symbols --------------*/ 274 | EXTERN t_pd pd_objectmaker; /* factory for creating "object" boxes */ 275 | EXTERN t_pd pd_canvasmaker; /* factory for creating canvases */ 276 | 277 | /* --------- prototypes from the central message system ----------- */ 278 | EXTERN void pd_typedmess(t_pd *x, t_symbol *s, int argc, t_atom *argv); 279 | EXTERN void pd_forwardmess(t_pd *x, int argc, t_atom *argv); 280 | EXTERN t_symbol *gensym(const char *s); 281 | EXTERN t_gotfn getfn(const t_pd *x, t_symbol *s); 282 | EXTERN t_gotfn zgetfn(const t_pd *x, t_symbol *s); 283 | EXTERN void nullfn(void); 284 | EXTERN void pd_vmess(t_pd *x, t_symbol *s, const char *fmt, ...); 285 | 286 | /* the following macros are for sending non-type-checkable messages, i.e., 287 | using function lookup but circumventing type checking on arguments. Only 288 | use for internal messaging protected by A_CANT so that the message can't 289 | be generated at patch level. */ 290 | #define mess0(x, s) ((*getfn((x), (s)))((x))) 291 | typedef void (*t_gotfn1)(void *x, void *arg1); 292 | #define mess1(x, s, a) ((*(t_gotfn1)getfn((x), (s)))((x), (a))) 293 | typedef void (*t_gotfn2)(void *x, void *arg1, void *arg2); 294 | #define mess2(x, s, a,b) ((*(t_gotfn2)getfn((x), (s)))((x), (a),(b))) 295 | typedef void (*t_gotfn3)(void *x, void *arg1, void *arg2, void *arg3); 296 | #define mess3(x, s, a,b,c) ((*(t_gotfn3)getfn((x), (s)))((x), (a),(b),(c))) 297 | typedef void (*t_gotfn4)(void *x, 298 | void *arg1, void *arg2, void *arg3, void *arg4); 299 | #define mess4(x, s, a,b,c,d) \ 300 | ((*(t_gotfn4)getfn((x), (s)))((x), (a),(b),(c),(d))) 301 | typedef void (*t_gotfn5)(void *x, 302 | void *arg1, void *arg2, void *arg3, void *arg4, void *arg5); 303 | #define mess5(x, s, a,b,c,d,e) \ 304 | ((*(t_gotfn5)getfn((x), (s)))((x), (a),(b),(c),(d),(e))) 305 | 306 | EXTERN void obj_list(t_object *x, t_symbol *s, int argc, t_atom *argv); 307 | EXTERN t_pd *pd_newest(void); 308 | 309 | /* --------------- memory management -------------------- */ 310 | EXTERN void *getbytes(size_t nbytes); 311 | EXTERN void *getzbytes(size_t nbytes); 312 | EXTERN void *copybytes(const void *src, size_t nbytes); 313 | EXTERN void freebytes(void *x, size_t nbytes); 314 | EXTERN void *resizebytes(void *x, size_t oldsize, size_t newsize); 315 | 316 | /* -------------------- atoms ----------------------------- */ 317 | 318 | #define SETSEMI(atom) ((atom)->a_type = A_SEMI, (atom)->a_w.w_index = 0) 319 | #define SETCOMMA(atom) ((atom)->a_type = A_COMMA, (atom)->a_w.w_index = 0) 320 | #define SETPOINTER(atom, gp) ((atom)->a_type = A_POINTER, \ 321 | (atom)->a_w.w_gpointer = (gp)) 322 | #define SETFLOAT(atom, f) ((atom)->a_type = A_FLOAT, (atom)->a_w.w_float = (f)) 323 | #define SETSYMBOL(atom, s) ((atom)->a_type = A_SYMBOL, \ 324 | (atom)->a_w.w_symbol = (s)) 325 | #define SETDOLLAR(atom, n) ((atom)->a_type = A_DOLLAR, \ 326 | (atom)->a_w.w_index = (n)) 327 | #define SETDOLLSYM(atom, s) ((atom)->a_type = A_DOLLSYM, \ 328 | (atom)->a_w.w_symbol= (s)) 329 | 330 | EXTERN t_float atom_getfloat(const t_atom *a); 331 | EXTERN t_int atom_getint(const t_atom *a); 332 | EXTERN t_symbol *atom_getsymbol(const t_atom *a); 333 | EXTERN t_symbol *atom_gensym(const t_atom *a); 334 | EXTERN t_float atom_getfloatarg(int which, int argc, const t_atom *argv); 335 | EXTERN t_int atom_getintarg(int which, int argc, const t_atom *argv); 336 | EXTERN t_symbol *atom_getsymbolarg(int which, int argc, const t_atom *argv); 337 | 338 | EXTERN void atom_string(const t_atom *a, char *buf, unsigned int bufsize); 339 | 340 | /* ------------------ binbufs --------------- */ 341 | 342 | EXTERN t_binbuf *binbuf_new(void); 343 | EXTERN void binbuf_free(t_binbuf *x); 344 | EXTERN t_binbuf *binbuf_duplicate(const t_binbuf *y); 345 | 346 | EXTERN void binbuf_text(t_binbuf *x, const char *text, size_t size); 347 | EXTERN void binbuf_gettext(const t_binbuf *x, char **bufp, int *lengthp); 348 | EXTERN void binbuf_clear(t_binbuf *x); 349 | EXTERN void binbuf_add(t_binbuf *x, int argc, const t_atom *argv); 350 | EXTERN void binbuf_addv(t_binbuf *x, const char *fmt, ...); 351 | EXTERN void binbuf_addbinbuf(t_binbuf *x, const t_binbuf *y); 352 | EXTERN void binbuf_addsemi(t_binbuf *x); 353 | EXTERN void binbuf_restore(t_binbuf *x, int argc, const t_atom *argv); 354 | EXTERN void binbuf_print(const t_binbuf *x); 355 | EXTERN int binbuf_getnatom(const t_binbuf *x); 356 | EXTERN t_atom *binbuf_getvec(const t_binbuf *x); 357 | EXTERN int binbuf_resize(t_binbuf *x, int newsize); 358 | EXTERN void binbuf_eval(const t_binbuf *x, t_pd *target, int argc, const t_atom *argv); 359 | EXTERN int binbuf_read(t_binbuf *b, const char *filename, const char *dirname, 360 | int crflag); 361 | EXTERN int binbuf_read_via_canvas(t_binbuf *b, const char *filename, const t_canvas *canvas, 362 | int crflag); 363 | EXTERN int binbuf_read_via_path(t_binbuf *b, const char *filename, const char *dirname, 364 | int crflag); 365 | EXTERN int binbuf_write(const t_binbuf *x, const char *filename, const char *dir, 366 | int crflag); 367 | EXTERN void binbuf_evalfile(t_symbol *name, t_symbol *dir); 368 | EXTERN t_symbol *binbuf_realizedollsym(t_symbol *s, int ac, const t_atom *av, 369 | int tonew); 370 | 371 | /* ------------------ clocks --------------- */ 372 | 373 | EXTERN t_clock *clock_new(void *owner, t_method fn); 374 | EXTERN void clock_set(t_clock *x, double systime); 375 | EXTERN void clock_delay(t_clock *x, double delaytime); 376 | EXTERN void clock_unset(t_clock *x); 377 | EXTERN void clock_setunit(t_clock *x, double timeunit, int sampflag); 378 | EXTERN double clock_getlogicaltime(void); 379 | EXTERN double clock_getsystime(void); /* OBSOLETE; use clock_getlogicaltime() */ 380 | EXTERN double clock_gettimesince(double prevsystime); 381 | EXTERN double clock_gettimesincewithunits(double prevsystime, 382 | double units, int sampflag); 383 | EXTERN double clock_getsystimeafter(double delaytime); 384 | EXTERN void clock_free(t_clock *x); 385 | 386 | /* ----------------- pure data ---------------- */ 387 | EXTERN t_pd *pd_new(t_class *cls); 388 | EXTERN void pd_free(t_pd *x); 389 | EXTERN void pd_bind(t_pd *x, t_symbol *s); 390 | EXTERN void pd_unbind(t_pd *x, t_symbol *s); 391 | EXTERN t_pd *pd_findbyclass(t_symbol *s, const t_class *c); 392 | EXTERN void pd_pushsym(t_pd *x); 393 | EXTERN void pd_popsym(t_pd *x); 394 | EXTERN void pd_bang(t_pd *x); 395 | EXTERN void pd_pointer(t_pd *x, t_gpointer *gp); 396 | EXTERN void pd_float(t_pd *x, t_float f); 397 | EXTERN void pd_symbol(t_pd *x, t_symbol *s); 398 | EXTERN void pd_list(t_pd *x, t_symbol *s, int argc, t_atom *argv); 399 | EXTERN void pd_anything(t_pd *x, t_symbol *s, int argc, t_atom *argv); 400 | #define pd_class(x) (*(x)) 401 | 402 | /* ----------------- pointers ---------------- */ 403 | EXTERN void gpointer_init(t_gpointer *gp); 404 | EXTERN void gpointer_copy(const t_gpointer *gpfrom, t_gpointer *gpto); 405 | EXTERN void gpointer_unset(t_gpointer *gp); 406 | EXTERN int gpointer_check(const t_gpointer *gp, int headok); 407 | 408 | /* ----------------- patchable "objects" -------------- */ 409 | EXTERN t_inlet *inlet_new(t_object *owner, t_pd *dest, t_symbol *s1, 410 | t_symbol *s2); 411 | EXTERN t_inlet *pointerinlet_new(t_object *owner, t_gpointer *gp); 412 | EXTERN t_inlet *floatinlet_new(t_object *owner, t_float *fp); 413 | EXTERN t_inlet *symbolinlet_new(t_object *owner, t_symbol **sp); 414 | EXTERN t_inlet *signalinlet_new(t_object *owner, t_float f); 415 | EXTERN void inlet_free(t_inlet *x); 416 | 417 | EXTERN t_outlet *outlet_new(t_object *owner, t_symbol *s); 418 | EXTERN void outlet_bang(t_outlet *x); 419 | EXTERN void outlet_pointer(t_outlet *x, t_gpointer *gp); 420 | EXTERN void outlet_float(t_outlet *x, t_float f); 421 | EXTERN void outlet_symbol(t_outlet *x, t_symbol *s); 422 | EXTERN void outlet_list(t_outlet *x, t_symbol *s, int argc, t_atom *argv); 423 | EXTERN void outlet_anything(t_outlet *x, t_symbol *s, int argc, t_atom *argv); 424 | EXTERN t_symbol *outlet_getsymbol(t_outlet *x); 425 | EXTERN void outlet_free(t_outlet *x); 426 | EXTERN t_object *pd_checkobject(t_pd *x); 427 | 428 | 429 | /* -------------------- canvases -------------- */ 430 | 431 | EXTERN void glob_setfilename(void *dummy, t_symbol *name, t_symbol *dir); 432 | 433 | EXTERN void canvas_setargs(int argc, const t_atom *argv); 434 | EXTERN void canvas_getargs(int *argcp, t_atom **argvp); 435 | EXTERN t_symbol *canvas_getcurrentdir(void); 436 | EXTERN t_glist *canvas_getcurrent(void); 437 | EXTERN void canvas_makefilename(const t_glist *c, const char *file, 438 | char *result, int resultsize); 439 | EXTERN t_symbol *canvas_getdir(const t_glist *x); 440 | EXTERN char sys_font[]; /* default typeface set in s_main.c */ 441 | EXTERN char sys_fontweight[]; /* default font weight set in s_main.c */ 442 | EXTERN int sys_hostfontsize(int fontsize, int zoom); 443 | EXTERN int sys_zoomfontwidth(int fontsize, int zoom, int worstcase); 444 | EXTERN int sys_zoomfontheight(int fontsize, int zoom, int worstcase); 445 | EXTERN int sys_fontwidth(int fontsize); 446 | EXTERN int sys_fontheight(int fontsize); 447 | EXTERN void canvas_dataproperties(t_glist *x, t_scalar *sc, t_binbuf *b); 448 | EXTERN int canvas_open(const t_canvas *x, const char *name, const char *ext, 449 | char *dirresult, char **nameresult, unsigned int size, int bin); 450 | 451 | /* ---------------- widget behaviors ---------------------- */ 452 | 453 | EXTERN_STRUCT _widgetbehavior; 454 | #define t_widgetbehavior struct _widgetbehavior 455 | 456 | EXTERN_STRUCT _parentwidgetbehavior; 457 | #define t_parentwidgetbehavior struct _parentwidgetbehavior 458 | EXTERN const t_parentwidgetbehavior *pd_getparentwidget(t_pd *x); 459 | 460 | /* -------------------- classes -------------- */ 461 | 462 | #define CLASS_DEFAULT 0 /* flags for new classes below */ 463 | #define CLASS_PD 1 464 | #define CLASS_GOBJ 2 465 | #define CLASS_PATCHABLE 3 466 | #define CLASS_NOINLET 8 467 | 468 | #define CLASS_TYPEMASK 3 469 | 470 | EXTERN t_class *class_new(t_symbol *name, t_newmethod newmethod, 471 | t_method freemethod, size_t size, int flags, t_atomtype arg1, ...); 472 | 473 | EXTERN t_class *class_new64(t_symbol *name, t_newmethod newmethod, 474 | t_method freemethod, size_t size, int flags, t_atomtype arg1, ...); 475 | 476 | EXTERN void class_free(t_class *c); 477 | 478 | #ifdef PDINSTANCE 479 | EXTERN t_class *class_getfirst(void); 480 | #endif 481 | 482 | EXTERN void class_addcreator(t_newmethod newmethod, t_symbol *s, 483 | t_atomtype type1, ...); 484 | EXTERN void class_addmethod(t_class *c, t_method fn, t_symbol *sel, 485 | t_atomtype arg1, ...); 486 | EXTERN void class_addbang(t_class *c, t_method fn); 487 | EXTERN void class_addpointer(t_class *c, t_method fn); 488 | EXTERN void class_doaddfloat(t_class *c, t_method fn); 489 | EXTERN void class_addsymbol(t_class *c, t_method fn); 490 | EXTERN void class_addlist(t_class *c, t_method fn); 491 | EXTERN void class_addanything(t_class *c, t_method fn); 492 | EXTERN void class_sethelpsymbol(t_class *c, t_symbol *s); 493 | EXTERN void class_setwidget(t_class *c, const t_widgetbehavior *w); 494 | EXTERN void class_setparentwidget(t_class *c, const t_parentwidgetbehavior *w); 495 | EXTERN const char *class_getname(const t_class *c); 496 | EXTERN const char *class_gethelpname(const t_class *c); 497 | EXTERN const char *class_gethelpdir(const t_class *c); 498 | EXTERN void class_setdrawcommand(t_class *c); 499 | EXTERN int class_isdrawcommand(const t_class *c); 500 | EXTERN void class_domainsignalin(t_class *c, int onset); 501 | EXTERN void class_set_extern_dir(t_symbol *s); 502 | #define CLASS_MAINSIGNALIN(c, type, field) \ 503 | class_domainsignalin(c, (char *)(&((type *)0)->field) - (char *)0) 504 | 505 | /* prototype for functions to save Pd's to a binbuf */ 506 | typedef void (*t_savefn)(t_gobj *x, t_binbuf *b); 507 | EXTERN void class_setsavefn(t_class *c, t_savefn f); 508 | EXTERN t_savefn class_getsavefn(const t_class *c); 509 | EXTERN void obj_saveformat(const t_object *x, t_binbuf *bb); /* add format to bb */ 510 | 511 | /* prototype for functions to open properties dialogs */ 512 | typedef void (*t_propertiesfn)(t_gobj *x, struct _glist *glist); 513 | EXTERN void class_setpropertiesfn(t_class *c, t_propertiesfn f); 514 | EXTERN t_propertiesfn class_getpropertiesfn(const t_class *c); 515 | 516 | typedef void (*t_classfreefn)(t_class *); 517 | EXTERN void class_setfreefn(t_class *c, t_classfreefn fn); 518 | 519 | #ifndef PD_CLASS_DEF 520 | #define class_addbang(x, y) class_addbang((x), (t_method)(y)) 521 | #define class_addpointer(x, y) class_addpointer((x), (t_method)(y)) 522 | #define class_addfloat(x, y) class_doaddfloat((x), (t_method)(y)) 523 | #define class_addsymbol(x, y) class_addsymbol((x), (t_method)(y)) 524 | #define class_addlist(x, y) class_addlist((x), (t_method)(y)) 525 | #define class_addanything(x, y) class_addanything((x), (t_method)(y)) 526 | #endif 527 | 528 | #if PD_FLOATSIZE == 64 529 | # define class_new class_new64 530 | #endif 531 | 532 | /* ------------ printing --------------------------------- */ 533 | EXTERN void post(const char *fmt, ...); 534 | EXTERN void startpost(const char *fmt, ...); 535 | EXTERN void poststring(const char *s); 536 | EXTERN void postfloat(t_floatarg f); 537 | EXTERN void postatom(int argc, const t_atom *argv); 538 | EXTERN void endpost(void); 539 | EXTERN void error(const char *fmt, ...) ATTRIBUTE_FORMAT_PRINTF(1, 2); 540 | EXTERN void verbose(int level, const char *fmt, ...) ATTRIBUTE_FORMAT_PRINTF(2, 3); 541 | EXTERN void bug(const char *fmt, ...) ATTRIBUTE_FORMAT_PRINTF(1, 2); 542 | EXTERN void pd_error(const void *object, const char *fmt, ...) ATTRIBUTE_FORMAT_PRINTF(2, 3); 543 | EXTERN void logpost(const void *object, const int level, const char *fmt, ...) 544 | ATTRIBUTE_FORMAT_PRINTF(3, 4); 545 | 546 | /* ------------ system interface routines ------------------- */ 547 | EXTERN int sys_isabsolutepath(const char *dir); 548 | EXTERN void sys_bashfilename(const char *from, char *to); 549 | EXTERN void sys_unbashfilename(const char *from, char *to); 550 | EXTERN int open_via_path(const char *dir, const char *name, const char *ext, 551 | char *dirresult, char **nameresult, unsigned int size, int bin); 552 | EXTERN int sched_geteventno(void); 553 | EXTERN double sys_getrealtime(void); 554 | EXTERN int (*sys_idlehook)(void); /* hook to add idle time computation */ 555 | 556 | /* Win32's open()/fopen() do not handle UTF-8 filenames so we need 557 | * these internal versions that handle UTF-8 filenames the same across 558 | * all platforms. They are recommended for use in external 559 | * objectclasses as well so they work with Unicode filenames on Windows */ 560 | EXTERN int sys_open(const char *path, int oflag, ...); 561 | EXTERN int sys_close(int fd); 562 | EXTERN FILE *sys_fopen(const char *filename, const char *mode); 563 | EXTERN int sys_fclose(FILE *stream); 564 | 565 | /* ------------ threading ------------------- */ 566 | EXTERN void sys_lock(void); 567 | EXTERN void sys_unlock(void); 568 | EXTERN int sys_trylock(void); 569 | 570 | 571 | /* --------------- signals ----------------------------------- */ 572 | 573 | typedef PD_FLOATTYPE t_sample; 574 | typedef union _sampleint_union { 575 | t_sample f; 576 | PD_FLOATUINTTYPE i; 577 | } t_sampleint_union; 578 | #define MAXLOGSIG 32 579 | #define MAXSIGSIZE (1 << MAXLOGSIG) 580 | 581 | typedef struct _signal 582 | { 583 | int s_n; /* number of points in the array */ 584 | t_sample *s_vec; /* the array */ 585 | t_float s_sr; /* sample rate */ 586 | int s_refcount; /* number of times used */ 587 | int s_isborrowed; /* whether we're going to borrow our array */ 588 | struct _signal *s_borrowedfrom; /* signal to borrow it from */ 589 | struct _signal *s_nextfree; /* next in freelist */ 590 | struct _signal *s_nextused; /* next in used list */ 591 | int s_vecsize; /* allocated size of array in points */ 592 | } t_signal; 593 | 594 | typedef t_int *(*t_perfroutine)(t_int *args); 595 | 596 | EXTERN t_int *plus_perform(t_int *args); 597 | EXTERN t_int *zero_perform(t_int *args); 598 | EXTERN t_int *copy_perform(t_int *args); 599 | 600 | EXTERN void dsp_add_plus(t_sample *in1, t_sample *in2, t_sample *out, int n); 601 | EXTERN void dsp_add_copy(t_sample *in, t_sample *out, int n); 602 | EXTERN void dsp_add_scalarcopy(t_float *in, t_sample *out, int n); 603 | EXTERN void dsp_add_zero(t_sample *out, int n); 604 | 605 | EXTERN int sys_getblksize(void); 606 | EXTERN t_float sys_getsr(void); 607 | EXTERN int sys_get_inchannels(void); 608 | EXTERN int sys_get_outchannels(void); 609 | 610 | EXTERN void dsp_add(t_perfroutine f, int n, ...); 611 | EXTERN void dsp_addv(t_perfroutine f, int n, t_int *vec); 612 | EXTERN void pd_fft(t_float *buf, int npoints, int inverse); 613 | EXTERN int ilog2(int n); 614 | 615 | EXTERN void mayer_fht(t_sample *fz, int n); 616 | EXTERN void mayer_fft(int n, t_sample *real, t_sample *imag); 617 | EXTERN void mayer_ifft(int n, t_sample *real, t_sample *imag); 618 | EXTERN void mayer_realfft(int n, t_sample *real); 619 | EXTERN void mayer_realifft(int n, t_sample *real); 620 | 621 | EXTERN float *cos_table; 622 | #define LOGCOSTABSIZE 9 623 | #define COSTABSIZE (1<> 1) & 0x20000000)); 755 | } 756 | 757 | #elif PD_FLOATSIZE == 64 758 | 759 | typedef union 760 | { 761 | t_float f; 762 | unsigned int ui[2]; 763 | }t_bigorsmall64; 764 | 765 | static inline int PD_BADFLOAT(t_float f) /* malformed double */ 766 | { 767 | t_bigorsmall64 pun; 768 | pun.f = f; 769 | pun.ui[1] &= 0x7ff00000; 770 | return((pun.ui[1] == 0) | (pun.ui[1] == 0x7ff00000)); 771 | } 772 | 773 | static inline int PD_BIGORSMALL(t_float f) /* exponent outside (-512,512) */ 774 | { 775 | t_bigorsmall64 pun; 776 | pun.f = f; 777 | return((pun.ui[1] & 0x20000000) == ((pun.ui[1] >> 1) & 0x20000000)); 778 | } 779 | 780 | #endif /* PD_FLOATSIZE */ 781 | #else /* not INTEL or ARM */ 782 | #define PD_BADFLOAT(f) 0 783 | #define PD_BIGORSMALL(f) 0 784 | #endif 785 | 786 | #else /* _MSC_VER */ 787 | #if PD_FLOATSIZE == 32 788 | #define PD_BADFLOAT(f) ((((*(unsigned int*)&(f))&0x7f800000)==0) || \ 789 | (((*(unsigned int*)&(f))&0x7f800000)==0x7f800000)) 790 | /* more stringent test: anything not between 1e-19 and 1e19 in absolute val */ 791 | #define PD_BIGORSMALL(f) ((((*(unsigned int*)&(f))&0x60000000)==0) || \ 792 | (((*(unsigned int*)&(f))&0x60000000)==0x60000000)) 793 | #else /* 64 bits... don't know what to do here */ 794 | #define PD_BADFLOAT(f) (!(((f) >= 0) || ((f) <= 0))) 795 | #define PD_BIGORSMALL(f) ((f) > 1e150 || (f) < -1e150 \ 796 | || (f) > -1e-150 && (f) < 1e-150 ) 797 | #endif 798 | #endif /* _MSC_VER */ 799 | /* get version number at run time */ 800 | EXTERN void sys_getversion(int *major, int *minor, int *bugfix); 801 | 802 | EXTERN_STRUCT _instancemidi; 803 | #define t_instancemidi struct _instancemidi 804 | 805 | EXTERN_STRUCT _instanceinter; 806 | #define t_instanceinter struct _instanceinter 807 | 808 | EXTERN_STRUCT _instancecanvas; 809 | #define t_instancecanvas struct _instancecanvas 810 | 811 | EXTERN_STRUCT _instanceugen; 812 | #define t_instanceugen struct _instanceugen 813 | 814 | EXTERN_STRUCT _instancestuff; 815 | #define t_instancestuff struct _instancestuff 816 | 817 | #ifndef PDTHREADS 818 | #define PDTHREADS 1 819 | #endif 820 | 821 | struct _pdinstance 822 | { 823 | double pd_systime; /* global time in Pd ticks */ 824 | t_clock *pd_clock_setlist; /* list of set clocks */ 825 | t_canvas *pd_canvaslist; /* list of all root canvases */ 826 | struct _template *pd_templatelist; /* list of all templates */ 827 | int pd_instanceno; /* ordinal number of this instance */ 828 | t_symbol **pd_symhash; /* symbol table hash table */ 829 | t_instancemidi *pd_midi; /* private stuff for x_midi.c */ 830 | t_instanceinter *pd_inter; /* private stuff for s_inter.c */ 831 | t_instanceugen *pd_ugen; /* private stuff for d_ugen.c */ 832 | t_instancecanvas *pd_gui; /* semi-private stuff in g_canvas.h */ 833 | t_instancestuff *pd_stuff; /* semi-private stuff in s_stuff.h */ 834 | t_pd *pd_newest; /* most recently created object */ 835 | #ifdef PDINSTANCE 836 | t_symbol pd_s_pointer; 837 | t_symbol pd_s_float; 838 | t_symbol pd_s_symbol; 839 | t_symbol pd_s_bang; 840 | t_symbol pd_s_list; 841 | t_symbol pd_s_anything; 842 | t_symbol pd_s_signal; 843 | t_symbol pd_s__N; 844 | t_symbol pd_s__X; 845 | t_symbol pd_s_x; 846 | t_symbol pd_s_y; 847 | t_symbol pd_s_; 848 | #endif 849 | #if PDTHREADS 850 | int pd_islocked; 851 | #endif 852 | }; 853 | #define t_pdinstance struct _pdinstance 854 | EXTERN t_pdinstance pd_maininstance; 855 | 856 | /* m_pd.c */ 857 | #ifdef PDINSTANCE 858 | EXTERN t_pdinstance *pdinstance_new(void); 859 | EXTERN void pd_setinstance(t_pdinstance *x); 860 | EXTERN void pdinstance_free(t_pdinstance *x); 861 | #endif /* PDINSTANCE */ 862 | 863 | #if defined(PDTHREADS) && defined(PDINSTANCE) 864 | #ifdef _MSC_VER 865 | #define PERTHREAD __declspec(thread) 866 | #else 867 | #define PERTHREAD __thread 868 | #endif /* _MSC_VER */ 869 | #else 870 | #define PERTHREAD 871 | #endif 872 | 873 | #ifdef PDINSTANCE 874 | extern PERTHREAD t_pdinstance *pd_this; 875 | EXTERN t_pdinstance **pd_instances; 876 | EXTERN int pd_ninstances; 877 | #else 878 | #define pd_this (&pd_maininstance) 879 | #endif /* PDINSTANCE */ 880 | 881 | #ifdef PDINSTANCE 882 | #define s_pointer (pd_this->pd_s_pointer) 883 | #define s_float (pd_this->pd_s_float) 884 | #define s_symbol (pd_this->pd_s_symbol) 885 | #define s_bang (pd_this->pd_s_bang) 886 | #define s_list (pd_this->pd_s_list) 887 | #define s_anything (pd_this->pd_s_anything) 888 | #define s_signal (pd_this->pd_s_signal) 889 | #define s__N (pd_this->pd_s__N) 890 | #define s__X (pd_this->pd_s__X) 891 | #define s_x (pd_this->pd_s_x) 892 | #define s_y (pd_this->pd_s_y) 893 | #define s_ (pd_this->pd_s_) 894 | #else 895 | EXTERN t_symbol s_pointer, s_float, s_symbol, s_bang, s_list, s_anything, 896 | s_signal, s__N, s__X, s_x, s_y, s_; 897 | #endif 898 | 899 | EXTERN t_canvas *pd_getcanvaslist(void); 900 | EXTERN int pd_getdspstate(void); 901 | 902 | /* x_text.c */ 903 | EXTERN t_binbuf *text_getbufbyname(t_symbol *s); /* get binbuf from text obj */ 904 | EXTERN void text_notifybyname(t_symbol *s); /* notify it was modified */ 905 | 906 | #if defined(_LANGUAGE_C_PLUS_PLUS) || defined(__cplusplus) 907 | } 908 | #endif 909 | 910 | #define __m_pd_h_ 911 | #endif /* __m_pd_h_ */ -------------------------------------------------------------------------------- /realtime/ddsp_tilde/signal_in_out_base.c: -------------------------------------------------------------------------------- 1 | #include "m_pd.h" 2 | 3 | static t_class *pan_tilde_class; 4 | 5 | typedef struct _pan_tilde 6 | { 7 | t_object x_obj; 8 | t_sample f_pan; 9 | t_sample f; 10 | 11 | t_inlet *x_in2; 12 | t_inlet *x_in3; 13 | t_outlet *x_out; 14 | } t_pan_tilde; 15 | 16 | t_int *pan_tilde_perform(t_int *w) 17 | { 18 | t_pan_tilde *x = (t_pan_tilde *)(w[1]); 19 | t_sample *in1 = (t_sample *)(w[2]); 20 | t_sample *in2 = (t_sample *)(w[3]); 21 | t_sample *out = (t_sample *)(w[4]); 22 | int n = (int)(w[5]); 23 | t_sample f_pan = (x->f_pan < 0) ? 0.0 : (x->f_pan > 1) ? 1.0 : x->f_pan; 24 | 25 | while (n--) 26 | *out++ = (*in1++) * (1 - f_pan) + (*in2++) * f_pan; 27 | 28 | return (w + 6); 29 | } 30 | 31 | void pan_tilde_dsp(t_pan_tilde *x, t_signal **sp) 32 | { 33 | dsp_add(pan_tilde_perform, 5, x, 34 | sp[0]->s_vec, sp[1]->s_vec, sp[2]->s_vec, sp[0]->s_n); 35 | } 36 | 37 | void pan_tilde_free(t_pan_tilde *x) 38 | { 39 | inlet_free(x->x_in2); 40 | inlet_free(x->x_in3); 41 | outlet_free(x->x_out); 42 | } 43 | 44 | void *pan_tilde_new(t_floatarg f) 45 | { 46 | t_pan_tilde *x = (t_pan_tilde *)pd_new(pan_tilde_class); 47 | 48 | x->f_pan = f; 49 | 50 | x->x_in2 = inlet_new(&x->x_obj, &x->x_obj.ob_pd, &s_signal, &s_signal); 51 | x->x_in3 = floatinlet_new(&x->x_obj, &x->f_pan); 52 | x->x_out = outlet_new(&x->x_obj, &s_signal); 53 | 54 | return (void *)x; 55 | } 56 | 57 | void pan_tilde_setup(void) 58 | { 59 | pan_tilde_class = class_new(gensym("pan~"), 60 | (t_newmethod)pan_tilde_new, 61 | 0, sizeof(t_pan_tilde), 62 | CLASS_DEFAULT, 63 | A_DEFFLOAT, 0); 64 | 65 | class_addmethod(pan_tilde_class, 66 | (t_method)pan_tilde_dsp, gensym("dsp"), A_CANT, 0); 67 | CLASS_MAINSIGNALIN(pan_tilde_class, t_pan_tilde, f); 68 | } 69 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.19.4 2 | crepe>=0.0.11 3 | librosa>=0.8.0 4 | einops>=0.3.0 5 | tqdm>=4.46.0 6 | torch>=1.7.0 7 | effortless_config>=0.7.0 8 | SoundFile>=0.10.3.post1 9 | Flask>=1.1.2 10 | PyYAML>=5.3.1 11 | tensorflow 12 | tensorboard -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.utils.tensorboard import SummaryWriter 3 | import yaml 4 | from ddsp.model import DDSP 5 | from effortless_config import Config 6 | from os import path 7 | from preprocess import Dataset 8 | from tqdm import tqdm 9 | from ddsp.core import multiscale_fft, safe_log, mean_std_loudness 10 | import soundfile as sf 11 | from einops import rearrange 12 | from ddsp.utils import get_scheduler 13 | import numpy as np 14 | 15 | 16 | class args(Config): 17 | CONFIG = "config.yaml" 18 | NAME = "debug" 19 | ROOT = "runs" 20 | STEPS = 500000 21 | BATCH = 16 22 | START_LR = 1e-3 23 | STOP_LR = 1e-4 24 | DECAY_OVER = 400000 25 | 26 | 27 | args.parse_args() 28 | 29 | with open(args.CONFIG, "r") as config: 30 | config = yaml.safe_load(config) 31 | 32 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 33 | 34 | model = DDSP(**config["model"]).to(device) 35 | 36 | dataset = Dataset(config["preprocess"]["out_dir"]) 37 | 38 | dataloader = torch.utils.data.DataLoader( 39 | dataset, 40 | args.BATCH, 41 | True, 42 | drop_last=True, 43 | ) 44 | 45 | mean_loudness, std_loudness = mean_std_loudness(dataloader) 46 | config["data"]["mean_loudness"] = mean_loudness 47 | config["data"]["std_loudness"] = std_loudness 48 | 49 | writer = SummaryWriter(path.join(args.ROOT, args.NAME), flush_secs=20) 50 | 51 | with open(path.join(args.ROOT, args.NAME, "config.yaml"), "w") as out_config: 52 | yaml.safe_dump(config, out_config) 53 | 54 | opt = torch.optim.Adam(model.parameters(), lr=args.START_LR) 55 | 56 | schedule = get_scheduler( 57 | len(dataloader), 58 | args.START_LR, 59 | args.STOP_LR, 60 | args.DECAY_OVER, 61 | ) 62 | 63 | # scheduler = torch.optim.lr_scheduler.LambdaLR(opt, schedule) 64 | 65 | best_loss = float("inf") 66 | mean_loss = 0 67 | n_element = 0 68 | step = 0 69 | epochs = int(np.ceil(args.STEPS / len(dataloader))) 70 | 71 | for e in tqdm(range(epochs)): 72 | for s, p, l in dataloader: 73 | s = s.to(device) 74 | p = p.unsqueeze(-1).to(device) 75 | l = l.unsqueeze(-1).to(device) 76 | 77 | l = (l - mean_loudness) / std_loudness 78 | 79 | y = model(p, l).squeeze(-1) 80 | 81 | ori_stft = multiscale_fft( 82 | s, 83 | config["train"]["scales"], 84 | config["train"]["overlap"], 85 | ) 86 | rec_stft = multiscale_fft( 87 | y, 88 | config["train"]["scales"], 89 | config["train"]["overlap"], 90 | ) 91 | 92 | loss = 0 93 | for s_x, s_y in zip(ori_stft, rec_stft): 94 | lin_loss = (s_x - s_y).abs().mean() 95 | log_loss = (safe_log(s_x) - safe_log(s_y)).abs().mean() 96 | loss = loss + lin_loss + log_loss 97 | 98 | opt.zero_grad() 99 | loss.backward() 100 | opt.step() 101 | 102 | writer.add_scalar("loss", loss.item(), step) 103 | 104 | step += 1 105 | 106 | n_element += 1 107 | mean_loss += (loss.item() - mean_loss) / n_element 108 | 109 | if not e % 1000: 110 | writer.add_scalar("lr", schedule(e), e) 111 | writer.add_scalar("reverb_decay", model.reverb.decay.item(), e) 112 | writer.add_scalar("reverb_wet", model.reverb.wet.item(), e) 113 | # scheduler.step() 114 | if mean_loss < best_loss: 115 | best_loss = mean_loss 116 | torch.save( 117 | model.state_dict(), 118 | path.join(args.ROOT, args.NAME, "state.pth"), 119 | ) 120 | 121 | mean_loss = 0 122 | n_element = 0 123 | 124 | audio = torch.cat([s, y], -1).reshape(-1).detach().cpu().numpy() 125 | 126 | sf.write( 127 | path.join(args.ROOT, args.NAME, f"eval_{e:06d}.wav"), 128 | audio, 129 | config["preprocess"]["sampling_rate"], 130 | ) 131 | --------------------------------------------------------------------------------