├── matlab_ir.wav ├── rirs ├── h001_Bedroom_65txts_48000.wav ├── h027_Classroom_8txts_48000.wav ├── h163_Bathroom_1txts_48000.wav ├── h025_Diningroom_8txts_48000.wav ├── h229_Office_Lobby_1txts_48000.wav ├── h110_Office_MeetingRoom_1txts_48000.wav ├── edc-params │ ├── h001_Bedroom_65txts_48000_est.mat │ ├── h027_Classroom_8txts_48000_est.mat │ ├── h163_Bathroom_1txts_48000_est.mat │ ├── h025_Diningroom_8txts_48000_est.mat │ ├── h229_Office_Lobby_1txts_48000_est.mat │ ├── h110_Office_MeetingRoom_1txts_48000_est.mat │ └── h042_Hallway_ElementarySchool_4txts_48000_est.mat └── h042_Hallway_ElementarySchool_4txts_48000.wav ├── .gitignore ├── .gitmodules ├── README.md ├── inference.m └── get_filters.m /matlab_ir.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/matlab_ir.wav -------------------------------------------------------------------------------- /rirs/h001_Bedroom_65txts_48000.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/rirs/h001_Bedroom_65txts_48000.wav -------------------------------------------------------------------------------- /rirs/h027_Classroom_8txts_48000.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/rirs/h027_Classroom_8txts_48000.wav -------------------------------------------------------------------------------- /rirs/h163_Bathroom_1txts_48000.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/rirs/h163_Bathroom_1txts_48000.wav -------------------------------------------------------------------------------- /rirs/h025_Diningroom_8txts_48000.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/rirs/h025_Diningroom_8txts_48000.wav -------------------------------------------------------------------------------- /rirs/h229_Office_Lobby_1txts_48000.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/rirs/h229_Office_Lobby_1txts_48000.wav -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | .DS_Store 3 | *.asv 4 | /output 5 | informed/.DS_Store 6 | .DS_Store 7 | .vscode/launch.json 8 | *.json 9 | -------------------------------------------------------------------------------- /rirs/h110_Office_MeetingRoom_1txts_48000.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/rirs/h110_Office_MeetingRoom_1txts_48000.wav -------------------------------------------------------------------------------- /rirs/edc-params/h001_Bedroom_65txts_48000_est.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/rirs/edc-params/h001_Bedroom_65txts_48000_est.mat -------------------------------------------------------------------------------- /rirs/edc-params/h027_Classroom_8txts_48000_est.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/rirs/edc-params/h027_Classroom_8txts_48000_est.mat -------------------------------------------------------------------------------- /rirs/edc-params/h163_Bathroom_1txts_48000_est.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/rirs/edc-params/h163_Bathroom_1txts_48000_est.mat -------------------------------------------------------------------------------- /rirs/h042_Hallway_ElementarySchool_4txts_48000.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/rirs/h042_Hallway_ElementarySchool_4txts_48000.wav -------------------------------------------------------------------------------- /rirs/edc-params/h025_Diningroom_8txts_48000_est.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/rirs/edc-params/h025_Diningroom_8txts_48000_est.mat -------------------------------------------------------------------------------- /rirs/edc-params/h229_Office_Lobby_1txts_48000_est.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/rirs/edc-params/h229_Office_Lobby_1txts_48000_est.mat -------------------------------------------------------------------------------- /rirs/edc-params/h110_Office_MeetingRoom_1txts_48000_est.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/rirs/edc-params/h110_Office_MeetingRoom_1txts_48000_est.mat -------------------------------------------------------------------------------- /rirs/edc-params/h042_Hallway_ElementarySchool_4txts_48000_est.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gdalsanto/rir2fdn/HEAD/rirs/edc-params/h042_Hallway_ElementarySchool_4txts_48000_est.mat -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "informed/Two_stage_filter"] 2 | path = informed/Two_stage_filter 3 | url = git@github.com:gdalsanto/Two_stage_filter.git 4 | [submodule "informed/diff-fdn-colorless"] 5 | path = informed/diff-fdn-colorless 6 | url = git@github.com:gdalsanto/diff-fdn-colorless.git 7 | [submodule "blind/diff-delay-net"] 8 | path = blind/diff-delay-net 9 | url = git@github.com:gdalsanto/diff-delay-net.git 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RIR2FDN 2 | This is the companion code to the **RIR2FDN: An improved room impulse response analysis and synthesis** paper submitted to the 27th International Conference on Audio Effects (DAFx24), Guilford, UK, 3-7 September 2024. 3 | 4 | 5 | 6 | **RIR2FDN**: a mapping from RIR to Feedback Delay Network (FDN) parameters such that the impulse response of the FDN is perceptually similar to the target RIR. 7 | To do so, we use an informed method incorporating improved energy decay estimation and synthesis within an optimized feedback delay network. 8 | 9 | This repository relies on 4 submodules: 10 | - [diff-fdn-colorless](https://github.com/gdalsanto/diff-fdn-colorless): optimization code to tune a set of FDN parameters (feedback matrix, input gains, and output gains) to achieve a smoother and less colored reverberation [1, 2]. 11 | In the submitted paper we use the scattering feedback matrix (arg `--scattering`), 6 delay lines with lengths [593, 743, 929, 1153, 1399, 1699]. 12 | - [Two_stage_filter](https://github.com/gdalsanto/Two_stage_filter): two stage attenuation filter design by Vesa Välimäki et al. [3] 13 | - [DecayFitNet](https://github.com/georg-goetz/DecayFitNet/tree/01daf3e7bbfd637aa1269bbca0cab7f445db0d5d): neural-network-based approach to estimate RIR decay parameters from energy decay curves (EDCs), which are modeled as a combination of multiple exponential decays, each characterized by an amplitude, decay time, and a noise term. We use these values to design the prototype attenuation and tone corrector filters. 14 | - [fdnToolbox](https://github.com/SebastianJiroSchlecht/fdnToolbox): Matlab toolbox for FDN, used to generate the impulse response at inference (by the `inference.m` script) from the estimated FDN parameters. 15 | - [diff-delay-net](https://github.com/gdalsanto/diff-delay-net): implementation of the baseline, i.e. the differentiable delay network presented by S. Lee et al. in [4]. **Please note that this is not the official implementation.** 16 | 17 | 18 | ## Getting started 19 | When cloning this repository, make sure to clone all the submodules, by running 20 | ``` 21 | git clone --recurse-submodules https://github.com/gdalsanto/rir2fdn 22 | ``` 23 | FDN colorless optimization: to run colorless optimization run `./diff-fdn-colorless/solver.py`. The output parameters will be saved in .mat format. 24 | 25 | Decay parameters estimation: run `./diff-fdn-colorless/rir_analysis.py` to run the energy decay analysis and the EDC parameters estimation of the RIR you want to synthesize. The solver can do this already by setting the argument `--reference_ir`. 26 | 27 | RIR2FDN: run the `.inference.m` script to build an FDN with the optimized FDN parameters. The script uses the two-stage attenuation filter and a graphic equalizer for the tone correction filter. The prototype filters are derived from the pre-estimated EDC parameters. 28 | 29 | 30 | In `./rir` you can already find a set of 7 rirs and their EDC parameters. These RIR were used for the paper to evaluate the presented model. 31 | 32 | **Note**: as it is, this code doesn't take into account the early reflections and, as such, should be used to synthesize only the late reverberation part. 33 | 34 | ## References 35 | Audio demos are published in [RIR2FDN](http://research.spa.aalto.fi/publications/papers/dafx24-rir2fdn/). 36 | The paper is not yet available, but you can check related work: 37 | ``` 38 | [1] Dal Santo G., Prawda K., Schlecht S. J., and Välimäki V., "Efficient Optimization of Feedback Delay Networks for Smooth Reverberation" in EURASIP Journal on Audio, Speech, and Music Processing - submitted for reviews on 31.01.2024 39 | [2] Dal Santo G., Prawda K., Schlecht S. J., and Välimäki V., "Differentiable Feedback Delay Network for colorless reverberation." in the 26th International Conference on Digital Audio Effects (DAFx23), Copenhagen, Denmark, Sept. 4-7 2023 40 | [3] Välimäki, V., Prawda, K., & Schlecht, S. J., "Two-stage attenuation filter for artificial reverberation." IEEE Signal Processing Letters, 2024 41 | [4] Lee, S., Choi, H. S., & Lee, K., "Differentiable artificial reverberation." in IEEE/ACM Transactions on Audio, Speech, and Language Processing, 30, 2541-2556, 2022. 42 | ``` 43 | -------------------------------------------------------------------------------- /inference.m: -------------------------------------------------------------------------------- 1 | % Differentiable FDN for Colorless Reverberation 2 | % 3 | % Gloria Dal Santo, last modified on 17.05.2024, Espoo (FI) 4 | 5 | clear; clc; close all; 6 | 7 | %% PREPARE FILEPATHS 8 | 9 | disp('Adding toolboxes to the path ...') 10 | addpath(genpath('./informed/diff-fdn-colorless/fdnToolbox')) 11 | addpath(genpath('./informed/diff-fdn-colorless/DecayFitNet')) 12 | addpath(genpath('./informed/Two_stage_filter')) 13 | disp('ok!') 14 | 15 | % initializing filepaths 16 | results_dir = "./output/20240521-092940"; % directory containing the solver's output 17 | output_dir = fullfile(results_dir, 'inference-matlab'); % output directory 18 | if ~exist(output_dir, 'dir') 19 | % create output directory 20 | mkdir(output_dir) 21 | end 22 | rirs_dir = './rirs'; % directory containing rirs to synthesize 23 | edc_est_dir = './rirs/edc-params'; % directory containing EDC analysis results for the rirs to synthesize 24 | [RIRfilepaths, RIRfilenames] = getFilepaths(rirs_dir, 'wav'); % filepaths of rirs to synthesize 25 | 26 | 27 | %% VARIABLES INIT 28 | 29 | % detect whether it uses a scattering feedback matrix 30 | if isfile(fullfile(results_dir, 'scat_parameters.mat')) 31 | type = 'SCAT'; 32 | else 33 | type = 'DFDN'; 34 | end 35 | 36 | fs = 48000; % sampling frequnecy 37 | octaveBand = 3; % frequnecy resolution of the filters 38 | isTranspose = 1; % relative position of the feedback matrix and delays (0/1) 39 | %% BUILD FDNs AND SAVE RIRS 40 | 41 | for iRIR = 1:length(RIRfilepaths) 42 | RIRfilename = char(RIRfilenames(iRIR)); 43 | % read EDC estimations (ideally I would compute them here, but 44 | % DecayFitNet dependencies don't support M2 chip 45 | edc_est_path = fullfile(edc_est_dir, RIRfilename(1:end-4) + "_est.mat"); 46 | % read RIR 47 | rir = audioread(RIRfilepaths(iRIR)); 48 | irLen = length(rir); 49 | 50 | switch type 51 | case 'DFDN' 52 | % load parameters 53 | load(fullfile(results_dir,'parameters.mat')) 54 | B = B(:); % input gains as column 55 | D = zeros(1:1); % direct gains 56 | % make matrix A orthogonal 57 | A = double(expm(skew(A))); 58 | 59 | % get filters 60 | [attenuationSOS, equalizationSOS] = get_filters('TwoStage', m, octaveBand, fs, irLen, edc_est_path=edc_est_path ); 61 | zAbsorption = zSOS(attenuationSOS,'isDiagonal',true); 62 | 63 | % apply tone corrector filter 64 | C = zSOS(permute(equalizationSOS,[3 4 1 2]) .* C); 65 | if isTranspose 66 | ir = dss2impzTransposed(... 67 | irLen, m, A, B, C, D, 'absorptionFilters', zAbsorption); 68 | else 69 | ir = dss2impz(... 70 | irLen, m, A, B, C, D, 'absorptionFilters', zAbsorption); 71 | end 72 | case 'SCAT' 73 | load(fullfile(results_dir, 'scat_parameters.mat')) 74 | Adl = size(feedbackMatrix, 3); % delays introduced by the scattering feedback matrix 75 | B = inputGain(:); % input gains 76 | C = outputGain(:)'; % output gains 77 | D = zeros(1:1); % direct gains 78 | 79 | % get filters 80 | [attenuationSOS, equalizationSOS] = get_filters('TwoStage', double(delays+Adl/2), octaveBand, fs, irLen, edc_est_path=edc_est_path); 81 | zAbsorption = zSOS(attenuationSOS,'isDiagonal',true); 82 | 83 | % apply tone corrector filter 84 | C = zSOS(permute(equalizationSOS,[3 4 1 2]) .* C); 85 | % build scattering matrix 86 | A = shiftMatrix(double(feedbackMatrix), double(delayLeft), 'left'); 87 | A = shiftMatrix(A, double(delayRight), 'right'); 88 | m = double(delays - double(delayLeft) - double(delayRight)); 89 | A = zFIR(double(A)); 90 | if isTranspose 91 | ir = dss2impzTransposed(... 92 | irLen, m, A, B, C, D, 'absorptionFilters', zAbsorption); 93 | else 94 | ir = dss2impz(... 95 | irLen, m, A, B, C, D, 'absorptionFilters', zAbsorption); 96 | end 97 | end 98 | % save impulse response 99 | name = RIRfilenames(iRIR) + "_" + type + "_est.wav"; 100 | filename = fullfile(output_dir,name); 101 | audiowrite(filename, ir, fs); 102 | end 103 | 104 | %% AUXILIARY FUNCTIONS 105 | 106 | function Y = skew(X) 107 | % orthogonal mapping 108 | X = triu(X,1); 109 | Y = X - transpose(X); 110 | end 111 | 112 | function [filePaths, baseFilename] = getFilepaths(folder, extension) 113 | % retrieves the full file paths of all files with a specified 114 | % extension in a given folder. 115 | 116 | % get a list of all .extension files in the folder 117 | waveFiles = dir(fullfile(folder, "*."+ extension)); 118 | 119 | % initialize a cell array to store the full file paths 120 | filePaths = cell(length(waveFiles), 1); 121 | baseFilename = cell(length(waveFiles), 1); 122 | % loop through the list of files to construct full file paths 123 | for k = 1:length(waveFiles) 124 | filePaths{k} = fullfile(folder, waveFiles(k).name); 125 | baseFilename{k} = waveFiles(k).name; 126 | end 127 | 128 | % convert cell array to a string array if needed 129 | filePaths = string(filePaths); 130 | baseFilename = string(baseFilename); 131 | end 132 | -------------------------------------------------------------------------------- /get_filters.m: -------------------------------------------------------------------------------- 1 | function [attenuationSOS, equalizationSOS] = get_filters(method, delays, octaveBand, fs, rirLen, varargin) 2 | % GET_FILTERS - Obtain attenuation and tone correction filters for RIR synthesis using FDN. 3 | % 4 | % Syntax: 5 | % [attenuationSOS, equalizationSOS] = get_filters(method, delays, octaveBand, fs, rirLen, varargin) 6 | % 7 | % Input Arguments: 8 | % method - Method for obtaining filters. Choices are: 9 | % * 'matlabGEQ': Use pre-estimated Energy Decay Curve (EDC) parameters values to design a Graphic Equalizer (GEQ) [1]. 10 | % * 'TwoStage': Use pre-estimated EDC parameters values to design a two-stage attenuation filter [2]. 11 | % note that the cutoff frequency of the prefilter wc be tuned manually 12 | % * 'pythonGEQ': Use pre-computed Second Order Sections (SOS) filter parameters computed in Python. The 13 | % filters can be generated using the FilterDesigner in diff-fdn-colorless/eq_design.py and are automatically generated when 14 | % running solver.py with --reference_ir argument 15 | % delays - Vector of FDN delay line lengths. 16 | % octaveBand - Octave band division. Specify 1 for 1 octave bands or 3 for 1/3 octave bands. 17 | % fs - Sampling frequency in Hz. 18 | % rirLen - Length of the room impulse response in samples. 19 | % varargin - Optional arguments depending on the chosen method: 20 | % * For 'matlabGEQ' and 'TwoStage': Provide the path to the .mat file containing EDC parameter estimations. 21 | % * For 'pythonGEQ': Provide the path to the .mat file containing pre-computed SOS filter parameters. 22 | % 23 | % Output Arguments: 24 | % attenuationSOS - Attenuation filters represented as SOS in a 4D array. 25 | % equalizationSOS - Equalization filters represented as SOS in a 2D array. 26 | % 27 | % References: 28 | % [1] J. Liski and V. Välimäki, “The quest for the best graphic equalizer,” 29 | % in Proc. DAFx, Edinburgh, UK, Sep. 2017, pp. 95–102. 30 | % [2] V. Välimäki, K. Prawda, and S. J. Schlecht, “Two-stage attenuation 31 | % filter for artificial reverberation,” IEEE Signal Process. Lett., Jan. 2024. 32 | % 33 | % Gloria Dal Santo, last modified on 17.05.2024, Espoo (FI) 34 | 35 | 36 | p = inputParser; 37 | addRequired(p, 'method', @(x) any(validatestring(x, {'matlabGEQ', 'pythonGEQ', 'TwoStage'}))); 38 | addRequired(p, 'delays', @(x) isvector(x) ) % FDN delay lines lengths 39 | addRequired(p, 'octaveBand', @(x) x == 1 || x == 3 ) % octave band division 40 | addRequired(p, 'fs' , @(x) isnumeric(x)) % sampling frequency 41 | addRequired(p, 'rirLen' , @(x) isnumeric(x)) % sampling frequency 42 | 43 | addOptional(p, 'edc_est_path', @(x) endsWith(x,".mat") ) % path to the EDC parameters estimations 44 | addOptional(p, 'filters_path', @(x) endsWith(x,".mat") ) % path to the SOS filters parameters computed in python 45 | 46 | parse(p,method, delays, octaveBand, fs, rirLen, varargin{:}); 47 | 48 | % get center frequencies 49 | if octaveBand == 1 50 | % hardcoded 1/1 octave bands 51 | fBands = [63, 125, 250, 500, 1000, 2000, 4000, 8000, 16000]; 52 | correction_T = [0.9, 0.9, 1, 1, 1, 1, 1, 0.9, 0.5]; 53 | correction_L = [5, 0, 0, 0, 0, 0, 0, 5, 5]; 54 | nBands = 10; % total number of bands expected by the fitler desgin functions 55 | elseif octaveBand == 3 56 | % hardcoded 1/3 octave bands 57 | fBands = [63, 80, 100, 125, 160, 200, 250, 315, 400, 500, 630, 800, 1000, 1250, 1600, 2000, 2500, 3150, 4000, 5000, 6300, 8000, 10000, 12500, 16000]; 58 | correction_T = [0.9, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.9]; 59 | correction_L = [15, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 15]; 60 | nBands = 31; % total number of bands expected by the fitler desgin functions 61 | end 62 | 63 | if strcmp(method, 'matlabGEQ') || strcmp(method, 'TwoStage') 64 | % Unless you have the >=M2 chip, you should be able to get the EDC 65 | % parameters from the DecayFitNet Toolbox directly in Matlab. 66 | % This code uploads the values pre-estimated in Python 67 | load(varargin{2}); 68 | est.T = double(T); % decay time 69 | est.A = double(A); % amplitude 70 | est.N = double(N); % noise level 71 | est.norm = double(norm_vals)'; % level normalization term 72 | est = transposeAllFields(est); 73 | [est.L, est.A, est.N] = decayFitNet2InitialLevel(est.T, est.A, est.N, est.norm, fs, rirLen, fBands); 74 | est.T = est.T.*correction_T; 75 | targetLevel = mag2db(est.L) - correction_L; 76 | 77 | if strcmp(method, 'matlabGEQ') 78 | % 1/1 octave band is expected 79 | targetT60 = est.T([1 1:end end]); % seconds 80 | attenuationSOS = absorptionGEQ(targetT60, delays, fs); 81 | elseif strcmp(method, 'TwoStage') 82 | % uses shelving prefilter and GEQ 83 | targetT60 = est.T; 84 | % pad RT values 85 | if octaveBand == 1 86 | targetT60 = [targetT60(1)*ones(1), targetT60]; 87 | elseif octaveBand == 3 88 | targetT60 = [targetT60(1)*ones(1, 5), targetT60, targetT60(end)*ones(1)]; 89 | end 90 | wc = 8000; % cut off frequency for the prefilter TODO: make wc detection automatic 91 | % get the frequency response of the attenuation filter 92 | attenuationSOS = zeros(length(delays), 1, nBands+1, 6); 93 | for i = 1:length(delays) 94 | [~, ~, ~, ~, iSOS] = twoFilters(targetT60, delays(i), fs, 'shelf', wc); 95 | if ~all(iSOS(end,:)) 96 | % TODO Last row is all zero, this is a bug that needs to be fixed 97 | iSOS(end,:) = []; 98 | end 99 | attenuationSOS(i, 1, :, :) = iSOS; 100 | end 101 | end 102 | % Tone correction filter 103 | if octaveBand == 1 104 | targetLevel = [targetLevel(1)*ones(1), targetLevel]; 105 | [nums,dens, ~] = aceq(targetLevel, 0, fs); 106 | nums(:, end) = []; 107 | dens(:, end) = []; 108 | elseif octaveBand == 3 109 | targetLevel = [targetLevel(1)*ones(1, 5), targetLevel, targetLevel(end)*ones(1)]; 110 | [nums,dens, ~] = acge3(targetLevel, 0, fs); 111 | end 112 | % get the frequency response of the tone correction filter 113 | equalizationSOS = horzcat(nums', dens'); 114 | 115 | elseif strcmp(method, 'pythonGEQ') 116 | % load filters computed in python during training (solver.py) 117 | load(varargin{2}) 118 | attenuationSOS = G_SOS; 119 | equalizationSOS = TC_SOS; 120 | end 121 | end --------------------------------------------------------------------------------