├── .gitignore ├── LICENSE ├── README.md ├── assets ├── example_results.csv ├── logo-sheet-only.png └── mos-bench-table.png ├── egs ├── BENCHMARKS │ ├── READMD.md │ ├── get_all_bencmarks.sh │ ├── get_all_bencmarks_installation_free.sh │ ├── run_all_bencmarks.sh │ ├── run_all_bencmarks_np.sh │ ├── run_bc19_test.sh │ ├── run_bvcc_test.sh │ ├── run_nisqa_test.sh │ ├── run_singmos_test.sh │ ├── run_somos_test.sh │ ├── run_tmhint_qi_test.sh │ ├── run_vmc23_test.sh │ └── utils ├── README.md ├── TEMPLATE │ ├── cmd.sh │ ├── conf │ │ ├── ldnet-ml.yaml │ │ └── ssl-mos-wav2vec2.yaml │ ├── local │ │ └── data_prep.py │ ├── path.sh │ ├── run.sh │ └── utils ├── bc19 │ └── local │ │ ├── data_download.sh │ │ └── data_prep.py ├── bvcc+nisqa+pstn+singmos+somos+tencent+tmhint-qi │ ├── cmd.sh │ ├── conf │ │ ├── alignnet-wav2vec2-mdf.yaml │ │ ├── alignnet-wav2vec2.yaml │ │ ├── ssl-mos-wav2vec2-mdf.yaml │ │ └── ssl-mos-wav2vec2.yaml │ ├── local │ │ └── data_prep.py │ ├── path.sh │ ├── run.sh │ └── utils ├── bvcc │ ├── README.md │ ├── cmd.sh │ ├── conf │ │ ├── ldnet-ml.yaml │ │ ├── ssl-mos-wav2vec2-categorical.yaml │ │ ├── ssl-mos-wav2vec2.yaml │ │ ├── stacking_ridge.yaml │ │ └── utmos-strong.yaml │ ├── local │ │ ├── data_download.sh │ │ └── data_prep.py │ ├── path.sh │ ├── run.sh │ └── utils ├── nisqa │ ├── README.md │ ├── cmd.sh │ ├── conf │ │ ├── alignnet-wav2vec2.yaml │ │ └── ssl-mos-wav2vec2.yaml │ ├── local │ │ ├── data_download.sh │ │ └── data_prep.py │ ├── path.sh │ ├── run.sh │ └── utils ├── pstn │ ├── README.md │ ├── cmd.sh │ ├── conf │ │ ├── ldnet-ml.yaml │ │ └── ssl-mos-wav2vec2.yaml │ ├── local │ │ ├── data_download.sh │ │ └── data_prep.py │ ├── path.sh │ ├── run.sh │ └── utils ├── singmos │ ├── README.md │ ├── cmd.sh │ ├── conf │ │ ├── ldnet-ml.yaml │ │ └── ssl-mos-wav2vec2.yaml │ ├── local │ │ ├── data_download.sh │ │ └── data_prep.py │ ├── path.sh │ ├── run.sh │ └── utils ├── somos │ ├── README.md │ ├── cmd.sh │ ├── conf │ │ └── ssl-mos-wav2vec2.yaml │ ├── local │ │ ├── data_download.sh │ │ └── data_prep.py │ ├── path.sh │ ├── run.sh │ └── utils ├── tencent │ ├── README.md │ ├── cmd.sh │ ├── conf │ │ ├── ldnet-ml.yaml │ │ └── ssl-mos-wav2vec2.yaml │ ├── local │ │ └── data_prep.py │ ├── path.sh │ ├── run.sh │ └── utils ├── tmhint-qi │ ├── README.md │ ├── cmd.sh │ ├── conf │ │ ├── ldnet-ml.yaml │ │ └── ssl-mos-wav2vec2.yaml │ ├── local │ │ ├── data_download.sh │ │ └── data_prep.py │ ├── path.sh │ ├── run.sh │ └── utils └── vmc23 │ ├── answers │ ├── track1_answer.txt │ ├── track2_answer.txt │ └── track3_answer.txt │ └── local │ ├── data_download.sh │ └── data_prep.py ├── hubconf.py ├── pyproject.toml ├── setup.cfg ├── sheet ├── __init__.py ├── bin │ ├── construct_datastore.py │ ├── inference.py │ ├── nonparametric_inference.py │ ├── train.py │ └── train_stack.py ├── collaters │ ├── __init__.py │ └── non_intrusive.py ├── datasets │ ├── __init__.py │ └── non_intrusive.py ├── evaluation │ ├── metrics.py │ └── plot.py ├── losses │ ├── __init__.py │ ├── basic_losses.py │ └── contrastive_loss.py ├── models │ ├── __init__.py │ ├── alignnet.py │ ├── ldnet.py │ ├── sslmos.py │ └── utmos.py ├── modules │ ├── ldnet │ │ ├── mobilenetv2.py │ │ ├── mobilenetv3.py │ │ └── modules.py │ └── utils.py ├── nonparametric │ └── datastore.py ├── schedulers │ ├── __init__.py │ └── schedulers.py ├── trainers │ ├── __init__.py │ ├── base.py │ └── non_intrusive.py ├── utils │ ├── __init__.py │ ├── download.py │ ├── model_io.py │ ├── types.py │ └── utils.py └── warmup_lr.py ├── tools └── Makefile └── utils ├── BENCHMARKS ├── calculate_metrics.py ├── combine_datasets.py ├── hf_download.py ├── parse_options.sh ├── queue.pl ├── run.pl └── subsample.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | exp/ 132 | downloads/ 133 | data/ 134 | *.done 135 | *.wav 136 | *.txt 137 | egs/playground/ 138 | egs/visualize_for_mos_bench_journal/ 139 | egs/bvcc+nisqa/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Wen-Chin Huang (unilight) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /assets/logo-sheet-only.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unilight/sheet/b0d1465b1b7d3c122f6e63d26c20873cd70f2e42/assets/logo-sheet-only.png -------------------------------------------------------------------------------- /assets/mos-bench-table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unilight/sheet/b0d1465b1b7d3c122f6e63d26c20873cd70f2e42/assets/mos-bench-table.png -------------------------------------------------------------------------------- /egs/BENCHMARKS/READMD.md: -------------------------------------------------------------------------------- 1 | # Zero-shot evaluation on benchmarks 2 | 3 | **NOTE**: Please do NOT run these recipes in this folder. 4 | 5 | ## Usage 6 | 7 | Let's say you want to benchmark on `vmc23` (which stands for the VoiceMOS Challenge 2023). 8 | 9 | 1. You need to have a trained model in anothe recipe (ex., `egs/bvcc`). 10 | 11 | 2. Then, **IN THAT FOLDER**, execute the following: 12 | ``` 13 | utils/BENCHMARKS/run_vmc23_test.sh --conf XXX.yaml --checkpoint YYY.ckpt 14 | ``` 15 | 16 | ## Recipe structure 17 | 18 | All the scripts in this folder share the following stage structure: 19 | 20 | - Stage -1: Dataset download. Please modify the `db_root` variable in each script to specify where to download the dataset. 21 | - Stage 0: Dataset preparation and csv file generation. They will be stored in `..//data`. (Ex. `../vmc23/data`). 22 | - Stage 1: Inference. -------------------------------------------------------------------------------- /egs/BENCHMARKS/get_all_bencmarks.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2024 Wen-Chin Huang (Nagoya University) 4 | # MIT License (https://opensource.org/licenses/MIT) 5 | 6 | stage=-1 # stage to start 7 | stop_stage=0 # stage to stop 8 | 9 | # shellcheck disable=SC1091 10 | . utils/parse_options.sh || exit 1; 11 | 12 | set -euo pipefail 13 | 14 | if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then 15 | echo "stage -1: Download data for all benchmark sets" 16 | 17 | _opts+="--stage -1 --stop_stage -1 " 18 | 19 | utils/BENCHMARKS/run_bvcc_test.sh ${_opts} 20 | utils/BENCHMARKS/run_bc19_test.sh ${_opts} 21 | utils/BENCHMARKS/run_somos_test.sh ${_opts} 22 | utils/BENCHMARKS/run_singmos_test.sh ${_opts} 23 | utils/BENCHMARKS/run_nisqa_test.sh ${_opts} 24 | utils/BENCHMARKS/run_tmhint_qi_test.sh ${_opts} 25 | utils/BENCHMARKS/run_vmc23_test.sh ${_opts} 26 | 27 | echo "Please follow instructions in bvcc, bc19 to finish the download process." 28 | fi 29 | 30 | 31 | if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then 32 | echo "stage 0: Data preparation for all benchmark sets" 33 | 34 | _opts+="--stage 0 --stop_stage 0 " 35 | 36 | utils/BENCHMARKS/run_bvcc_test.sh ${_opts} 37 | utils/BENCHMARKS/run_bc19_test.sh ${_opts} 38 | utils/BENCHMARKS/run_somos_test.sh ${_opts} 39 | utils/BENCHMARKS/run_singmos_test.sh ${_opts} 40 | utils/BENCHMARKS/run_nisqa_test.sh ${_opts} 41 | utils/BENCHMARKS/run_tmhint_qi_test.sh ${_opts} 42 | utils/BENCHMARKS/run_vmc23_test.sh ${_opts} 43 | fi 44 | -------------------------------------------------------------------------------- /egs/BENCHMARKS/get_all_bencmarks_installation_free.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2024 Wen-Chin Huang (Nagoya University) 4 | # MIT License (https://opensource.org/licenses/MIT) 5 | 6 | db_root= 7 | datadir=data 8 | 9 | stage=-1 # stage to start 10 | stop_stage=0 # stage to stop 11 | 12 | # shellcheck disable=SC1091 13 | . utils/parse_options.sh || exit 1; 14 | 15 | set -euo pipefail 16 | 17 | if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then 18 | echo "stage -1: Download data for all benchmark sets" 19 | 20 | ../bvcc/local/data_download.sh ${db_root}/bvcc 21 | ../bc19/local/data_download.sh ${db_root}/bc19 22 | ../somos/local/data_download.sh ${db_root}/somos 23 | ../singmos/local/data_download.sh ${db_root}/singmos 24 | ../nisqa/local/data_download.sh ${db_root}/nisqa 25 | ../tmhint-qi/local/data_download.sh ${db_root}/tmhint_qi 26 | ../vmc23/local/data_download.sh ${db_root}/vmc23 27 | 28 | echo "Please follow instructions in bvcc, bc19 to finish the download process." 29 | fi 30 | 31 | if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then 32 | echo "stage 0: Data preparation for all benchmark sets" 33 | 34 | mkdir -p "${datadir}" 35 | 36 | # bvcc 37 | echo "=== Data preparation for BVCC test ===" 38 | ../bvcc/local/data_prep.py --avg-score-only \ 39 | --original-path "${db_root}/bvcc/main/DATA/sets/TESTSET" --wavdir "${db_root}/main/DATA/wav" --out "${datadir}/bvcc_test.csv" 40 | echo 41 | 42 | # bc19 43 | echo "=== Data preparation for VMC'22 OOD track (bc'19) ===" 44 | ../bc19/local/data_prep.py --avg-score-only \ 45 | --original-path "${db_root}/bc19/ood/DATA/sets/TESTSET" --wavdir "${db_root}/bc19/ood/DATA/wav" --out "${datadir}/bc19_test.csv" 46 | echo 47 | 48 | # somos 49 | echo "=== Data preparation for SOMOS test ===" 50 | ../somos/local/data_prep.py --avg-score-only \ 51 | --original-path "${db_root}/somos/training_files/split1/clean/TESTSET" --wavdir "${db_root}/somos/audios" --out "${datadir}/somos_test.csv" 52 | echo 53 | 54 | # singmos 55 | echo "=== Data preparation for VMC'24 track 2 (SingMOS test) ===" 56 | ../singmos/local/data_prep.py --avg-score-only \ 57 | --original-path "${db_root}/singmos/DATA/sets/eval_mos_list.txt" --wavdir "${db_root}/DATA/wav" --out "${datadir}/singmos_test.csv" 58 | echo 59 | 60 | # nisqa 61 | for test_set in LIVETALK FOR P501; do 62 | echo "=== Data preparation for NISQA TEST ${test_set} ===" 63 | ../nisqa/local/data_prep.py --avg-score-only \ 64 | --original-path "${db_root}/nisqa/NISQA_TEST_${test_set}/NISQA_TEST_${test_set}_file.csv" \ 65 | --wavdir "${db_root}/nisqa/NISQA_TEST_${test_set}/deg" \ 66 | --out "${datadir}/nisqa_${test_set}.csv" 67 | echo 68 | done 69 | 70 | # tmhint-qi 71 | echo "=== Data preparation for TMHINT-QI test ===" 72 | ../tmhint-qi/local/data_prep.py --avg-score-only \ 73 | --original-path "${db_root}/tmhint_qi/raw_data.csv" --wavdir "${db_root}/tmhint_qi/test" --setname "test" --out "${datadir}/tmhintqi_test.csv" 74 | echo 75 | 76 | # vmc23 77 | for track in track1a track1b track2 track3; do 78 | echo "=== Data preparation for VMC'22 ${track} ===" 79 | if [ "${track}" = "track1a" ] || [ "${track}" = "track1b" ]; then 80 | _track=track1 81 | else 82 | _track="${track}" 83 | fi 84 | ../vmc23/local/data_prep.py --avg-score-only \ 85 | --original-path "${db_root}/vmc23/${_track}" \ 86 | --wavdir "${db_root}/vmc23/${_track}" \ 87 | --answer_path "../vmc23/answers/${_track}_answer.txt" \ 88 | --track "${track}" \ 89 | --out "${datadir}/vmc23_${track}_test.csv" 90 | echo 91 | done 92 | fi 93 | -------------------------------------------------------------------------------- /egs/BENCHMARKS/run_all_bencmarks.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2024 Wen-Chin Huang (Nagoya University) 4 | # MIT License (https://opensource.org/licenses/MIT) 5 | 6 | conf= 7 | tag= 8 | np_inference_mode= 9 | seed= 10 | checkpoint= 11 | 12 | # shellcheck disable=SC1091 13 | . utils/parse_options.sh || exit 1; 14 | 15 | set -euo pipefail 16 | 17 | _opts= 18 | if [ ! -z ${np_inference_mode} ]; then 19 | _opts+="--stage 3 --stop_stage 3 --np-inference-mode ${np_inference_mode} " 20 | else 21 | _opts+="--stage 1 --stop_stage 1 " 22 | fi 23 | if [ ! -z ${tag} ]; then 24 | _opts+="--tag ${tag} " 25 | fi 26 | 27 | utils/BENCHMARKS/run_bvcc_test.sh --conf ${conf} --seed ${seed} --checkpoint ${checkpoint} ${_opts} 28 | utils/BENCHMARKS/run_bc19_test.sh --conf ${conf} --seed ${seed} --checkpoint ${checkpoint} ${_opts} 29 | utils/BENCHMARKS/run_somos_test.sh --conf ${conf} --seed ${seed} --checkpoint ${checkpoint} ${_opts} 30 | utils/BENCHMARKS/run_singmos_test.sh --conf ${conf} --seed ${seed} --checkpoint ${checkpoint} ${_opts} 31 | utils/BENCHMARKS/run_nisqa_test.sh --conf ${conf} --seed ${seed} --checkpoint ${checkpoint} ${_opts} 32 | utils/BENCHMARKS/run_tmhint_qi_test.sh --conf ${conf} --seed ${seed} --checkpoint ${checkpoint} ${_opts} 33 | utils/BENCHMARKS/run_vmc23_test.sh --conf ${conf} --seed ${seed} --checkpoint ${checkpoint} ${_opts} -------------------------------------------------------------------------------- /egs/BENCHMARKS/run_all_bencmarks_np.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2024 Wen-Chin Huang (Nagoya University) 4 | # MIT License (https://opensource.org/licenses/MIT) 5 | 6 | conf= 7 | datadir= 8 | 9 | # shellcheck disable=SC1091 10 | . utils/parse_options.sh || exit 1; 11 | 12 | set -euo pipefail 13 | 14 | if [ ! -z ${datadir} ]; then 15 | utils/BENCHMARKS/run_bvcc_test.sh --stage 1 --stop_stage 1 --conf ${conf} --datadir ${datadir} 16 | utils/BENCHMARKS/run_somos_test.sh --stage 1 --stop_stage 1 --conf ${conf} --datadir ${datadir} 17 | utils/BENCHMARKS/run_singmos_test.sh --stage 1 --stop_stage 1 --conf ${conf} --datadir ${datadir} 18 | utils/BENCHMARKS/run_nisqa_test.sh --stage 1 --stop_stage 1 --conf ${conf} --datadir ${datadir} 19 | utils/BENCHMARKS/run_tmhint_qi_test.sh --stage 1 --stop_stage 1 --conf ${conf} --datadir ${datadir} 20 | utils/BENCHMARKS/run_vmc23_test.sh --stage 2 --stop_stage 2 --conf ${conf} 21 | else 22 | utils/BENCHMARKS/run_bvcc_test.sh --stage 1 --stop_stage 1 --conf ${conf} 23 | utils/BENCHMARKS/run_somos_test.sh --stage 1 --stop_stage 1 --conf ${conf} 24 | utils/BENCHMARKS/run_singmos_test.sh --stage 1 --stop_stage 1 --conf ${conf} 25 | utils/BENCHMARKS/run_nisqa_test.sh --stage 1 --stop_stage 1 --conf ${conf} 26 | utils/BENCHMARKS/run_tmhint_qi_test.sh --stage 1 --stop_stage 1 --conf ${conf} 27 | utils/BENCHMARKS/run_vmc23_test.sh --stage 1 --stop_stage 1 --conf ${conf} 28 | fi -------------------------------------------------------------------------------- /egs/BENCHMARKS/run_bc19_test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2024 Wen-Chin Huang (Nagoya University) 4 | # MIT License (https://opensource.org/licenses/MIT) 5 | 6 | . ./path.sh || exit 1; 7 | . ./cmd.sh || exit 1; 8 | 9 | # basic settings 10 | stage=-1 # stage to start 11 | stop_stage=100 # stage to stop 12 | verbose=1 # verbosity level (lower is less info) 13 | n_gpus=1 # number of gpus in training 14 | seed=1337 15 | 16 | conf=conf/ssl-mos-wav2vec2.yaml 17 | meta_model_conf=conf/stacking_ridge.yaml 18 | 19 | # dataset configuration 20 | bc19_db_root=/data/group1/z44476r/Corpora/bc19/ood/DATA # change this to your dataset folder 21 | datadir="../bc19/data" 22 | domain_idx=0 23 | target_sampling_rate=16000 24 | 25 | # training related setting 26 | tag="" # tag for directory to save model 27 | 28 | # decoding related setting 29 | test_sets="bc19_test" 30 | checkpoint="" # checkpoint path to be used for decoding 31 | # if not provided, the latest one will be used 32 | # (e.g. //checkpoint-400000steps.pkl) 33 | model_averaging="False" 34 | use_stacking="False" 35 | meta_model_checkpoint="" 36 | np_inference_mode= 37 | 38 | # shellcheck disable=SC1091 39 | . utils/parse_options.sh || exit 1; 40 | 41 | set -euo pipefail 42 | 43 | if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then 44 | echo "stage -1: Data and Pretrained Model Download" 45 | 46 | ../bc19/local/data_download.sh ${bc19_db_root} 47 | fi 48 | 49 | 50 | mkdir -p "${datadir}" 51 | if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then 52 | echo "stage 0: Data preparation" 53 | 54 | ../bc19/local/data_prep.py \ 55 | --original-path "${bc19_db_root}/sets/TESTSET" --wavdir "${bc19_db_root}/wav" --out "${datadir}/bc19_test.csv" \ 56 | --resample --target-sampling-rate "${target_sampling_rate}" --target-wavdir "${bc19_db_root}/wav_${target_sampling_rate}" 57 | 58 | fi 59 | 60 | if [ "${stage}" -le 1 ] && [ "${stop_stage}" -ge 1 ]; then 61 | echo "Stage 1: Inference" 62 | # shellcheck disable=SC2012 63 | 64 | if [ -z ${tag} ]; then 65 | expname="$(basename ${conf%.*})-${seed}" 66 | else 67 | expname="${tag}-${seed}" 68 | fi 69 | expdir=exp/${expname} 70 | 71 | if [ "${use_stacking}" = "True" ]; then 72 | [ -z "${meta_model_checkpoint}" ] && meta_model_checkpoint="${expdir}/meta_model.pkl" 73 | outdir="${expdir}/results/stacking-model" 74 | elif [ "${model_averaging}" = "True" ]; then 75 | outdir="${expdir}/results/model-averaging" 76 | else 77 | [ -z "${checkpoint}" ] && checkpoint="${expdir}/checkpoint-best.pkl" 78 | outdir="${expdir}/results/$(basename "${checkpoint}" .pkl)" 79 | fi 80 | 81 | for name in ${test_sets}; do 82 | [ ! -e "${outdir}/${name}" ] && mkdir -p "${outdir}/${name}" 83 | [ "${n_gpus}" -gt 1 ] && n_gpus=1 84 | echo "Inference start. See the progress via ${outdir}/${name}/inference.log." 85 | ${cuda_cmd} --gpu "${n_gpus}" "${outdir}/${name}/inference.log" \ 86 | inference.py \ 87 | --config "${expdir}/config.yml" \ 88 | --csv-path "${datadir}/${name}.csv" \ 89 | --checkpoint "${checkpoint}" \ 90 | --outdir "${outdir}/${name}" \ 91 | --model-averaging "${model_averaging}" \ 92 | --use-stacking "${use_stacking}" \ 93 | --meta-model-checkpoint "${meta_model_checkpoint}" \ 94 | --verbose "${verbose}" 95 | echo "Successfully finished inference of ${name} set." 96 | grep "UTT" "${outdir}/${name}/inference.log" 97 | done 98 | echo "Successfully finished inference." 99 | fi 100 | 101 | if [ "${stage}" -le 3 ] && [ "${stop_stage}" -ge 3 ]; then 102 | echo "Stage 3: Non-parametric inference" 103 | # shellcheck disable=SC2012 104 | 105 | if [ -z ${tag} ]; then 106 | expname="$(basename ${conf%.*})-${seed}" 107 | else 108 | expname="${tag}-${seed}" 109 | fi 110 | expdir=exp/${expname} 111 | 112 | [ -z "${checkpoint}" ] && checkpoint="${expdir}/checkpoint-best.pkl" 113 | outdir="${expdir}/results/np_$(basename "${checkpoint}" .pkl)/${np_inference_mode}" 114 | 115 | for name in ${test_sets}; do 116 | [ ! -e "${outdir}/${name}" ] && mkdir -p "${outdir}/${name}" 117 | [ "${n_gpus}" -gt 1 ] && n_gpus=1 118 | echo "Inference start. See the progress via ${outdir}/${name}/inference.log." 119 | ${cuda_cmd} --gpu "${n_gpus}" "${outdir}/${name}/inference.log" \ 120 | nonparametric_inference.py \ 121 | --config "${expdir}/config.yml" \ 122 | --datastore "${expdir}/datastore/$(basename "${checkpoint}" .pkl)/datastore.h5" \ 123 | --csv-path "${datadir}/${name}.csv" \ 124 | --checkpoint "${checkpoint}" \ 125 | --outdir "${outdir}/${name}" \ 126 | --np-inference-mode "${np_inference_mode}" \ 127 | --verbose "${verbose}" 128 | echo "Successfully finished inference of ${name} set." 129 | grep "UTT" "${outdir}/${name}/inference.log" 130 | done 131 | echo "Successfully finished inference." 132 | fi -------------------------------------------------------------------------------- /egs/BENCHMARKS/utils: -------------------------------------------------------------------------------- 1 | ../../utils -------------------------------------------------------------------------------- /egs/TEMPLATE/cmd.sh: -------------------------------------------------------------------------------- 1 | # ====== About run.pl, queue.pl, slurm.pl, and ssh.pl ====== 2 | # Usage: .pl [options] JOB=1: 3 | # e.g. 4 | # run.pl --mem 4G JOB=1:10 echo.JOB.log echo JOB 5 | # 6 | # Options: 7 | # --time