"
53 | commit_subject = "Release {version}"
54 | commit_message = "" # commit message
55 |
56 | [tool.ruff]
57 | # Never enforce `E501` (line length violations).
58 | ignore = ["C901", "E501", "E741", "F402", "F823"]
59 | select = ["C", "E", "F", "I", "W"]
60 | line-length = 119
61 |
62 | # Ignore import violations in all `__init__.py` files.
63 | [tool.ruff.per-file-ignores]
64 | "__init__.py" = ["E402", "F401", "F403", "F811"]
65 |
66 | [tool.ruff.isort]
67 | lines-after-imports = 2
68 | known-first-party = ["diarizen"]
69 |
70 | [tool.ruff.format]
71 | # Like Black, use double quotes for strings.
72 | quote-style = "double"
73 | # Like Black, indent with spaces, rather than tabs.
74 | indent-style = "space"
75 | # Like Black, respect magic trailing commas.
76 | skip-magic-trailing-comma = false
77 | # Like Black, automatically detect the appropriate line ending.
78 | line-ending = "auto"
79 | docstring-code-format = true
80 | docstring-code-line-length = 119
81 |
82 | [tool.ruff.lint.pydocstyle]
83 | convention = "google"
84 |
--------------------------------------------------------------------------------
/diarizen/logger.py:
--------------------------------------------------------------------------------
1 | # Licensed under the MIT license.
2 | # Copy from https://github.com/haoxiangsnr/spiking-fullsubnet/blob/main/audiozen/logger.py
3 | # Copyright 2024 Hong Kong Polytechnic University (author: Xiang Hao, haoxiangsnr@gmail.com)
4 |
5 | import logging
6 | import os
7 | import time
8 | from pathlib import Path
9 |
10 | import toml
11 | from torch.utils.tensorboard import SummaryWriter # type: ignore
12 |
13 |
14 | class TensorboardLogger(SummaryWriter):
15 | def __init__(self, log_dir: str = "") -> None:
16 | super().__init__(log_dir=log_dir, max_queue=5, flush_secs=30)
17 |
18 | def log_config(self, config: dict) -> None:
19 | self.add_text(
20 | tag="Configuration",
21 | text_string=f" \n{toml.dumps(config)} \n",
22 | global_step=1,
23 | )
24 |
25 |
26 | def init_logging_logger(config):
27 | """Initialize logging logger with handlers.
28 |
29 | Args:
30 | log_fpath: Path to save log file.
31 |
32 | Examples:
33 | >>> # Call this function at the beginning of main file.
34 | >>> init_logger(log_fpath="log_path")
35 | >>> # Use this logger in other modules.
36 | >>> import logging
37 | >>> logger = logging.getLogger(__name__)
38 | >>> logger.info("info message")
39 | """
40 | # Parse log_fpath
41 | log_dir: Path = Path(config["meta"]["save_dir"]).expanduser().absolute() / config["meta"]["exp_id"]
42 | log_dir.mkdir(parents=True, exist_ok=True)
43 |
44 | # disable logging for libraries that use standard logging module
45 | logging.getLogger("matplotlib").setLevel(logging.WARNING)
46 | logging.getLogger("numba").setLevel(logging.ERROR)
47 |
48 | # Create logger
49 | logger = logging.getLogger()
50 |
51 | # Set the lowest level of root logger and controls logging via handlers' level
52 | logger.setLevel(logging.DEBUG)
53 |
54 | # Get log level from environment variable
55 | log_level = os.environ.get("LOG_LEVEL", "INFO").upper()
56 |
57 | # Create a console handler and set level to info
58 | console_handler = logging.StreamHandler()
59 | console_handler.setLevel(level=log_level)
60 |
61 | # Create a file handler and set the logger level to debug
62 | time_now = time.strftime("%Y_%m_%d--%H_%M_%S")
63 | file_handler = logging.FileHandler(str(log_dir / f"{config['meta']['exp_id']}_{time_now}.log"))
64 | file_handler.setLevel(level=log_level)
65 |
66 | # Create formatters (file logger have more info)
67 | console_formatter = logging.Formatter(
68 | "%(asctime)s: %(message)s",
69 | datefmt="%m-%d %H:%M:%S",
70 | )
71 | file_formatter = logging.Formatter(
72 | "%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d]: %(message)s",
73 | datefmt="%m-%d %H:%M:%S",
74 | )
75 |
76 | # Add formatter to ch
77 | console_handler.setFormatter(console_formatter)
78 | file_handler.setFormatter(file_formatter)
79 |
80 | # Add ch to logger
81 | logger.addHandler(console_handler)
82 | logger.addHandler(file_handler)
83 |
84 | logger.info(f"Initialized logger with log file in {log_dir.as_posix()}.")
85 |
--------------------------------------------------------------------------------
/pyannote-audio/tests/inference_test.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pytest
3 | import pytorch_lightning as pl
4 | from pyannote.core import SlidingWindowFeature
5 | from pyannote.database import FileFinder, get_protocol
6 |
7 | from pyannote.audio import Inference, Model
8 | from pyannote.audio.core.task import Resolution
9 | from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel
10 | from pyannote.audio.tasks import VoiceActivityDetection
11 |
12 | HF_SAMPLE_MODEL_ID = "pyannote/ci-segmentation"
13 |
14 |
15 | def test_hf_download_inference():
16 | inference = Inference(HF_SAMPLE_MODEL_ID, device="cpu")
17 | assert isinstance(inference, Inference)
18 |
19 |
20 | def test_hf_download_model():
21 | model = Model.from_pretrained(HF_SAMPLE_MODEL_ID)
22 | assert isinstance(model, Model)
23 |
24 |
25 | @pytest.fixture()
26 | def trained():
27 | protocol = get_protocol(
28 | "Debug.SpeakerDiarization.Debug", preprocessors={"audio": FileFinder()}
29 | )
30 | vad = VoiceActivityDetection(protocol, duration=2.0, batch_size=16, num_workers=4)
31 | model = SimpleSegmentationModel(task=vad)
32 | trainer = pl.Trainer(fast_dev_run=True, accelerator="cpu")
33 | trainer.fit(model)
34 | return protocol, model
35 |
36 |
37 | @pytest.fixture()
38 | def pretrained_model():
39 | return Model.from_pretrained(HF_SAMPLE_MODEL_ID)
40 |
41 |
42 | @pytest.fixture()
43 | def dev_file():
44 | protocol = get_protocol(
45 | "Debug.SpeakerDiarization.Debug", preprocessors={"audio": FileFinder()}
46 | )
47 | return next(protocol.development())
48 |
49 |
50 | def test_duration_warning(trained):
51 | protocol, model = trained
52 | with pytest.warns(UserWarning):
53 | duration = model.specifications.duration
54 | new_duration = duration + 1
55 | Inference(model, duration=new_duration, step=0.1, batch_size=128)
56 |
57 |
58 | def test_step_check_warning(trained):
59 | protocol, model = trained
60 | with pytest.raises(ValueError):
61 | duration = model.specifications.duration
62 | Inference(model, step=duration + 1, batch_size=128)
63 |
64 |
65 | def test_invalid_window_fails(trained):
66 | protocol, model = trained
67 | with pytest.raises(ValueError):
68 | Inference(model, window="unknown")
69 |
70 |
71 | def test_invalid_resolution_fails(trained):
72 | protocol, model = trained
73 | with pytest.warns(UserWarning):
74 | model.specifications.resolution = Resolution.FRAME
75 | Inference(model, window="whole", batch_size=128)
76 |
77 |
78 | def test_whole_window_slide(trained):
79 | protocol, model = trained
80 | inference = Inference(model, window="whole", batch_size=128)
81 | dev_file = next(protocol.development())
82 | output = inference(dev_file)
83 | assert isinstance(output, np.ndarray)
84 |
85 |
86 | def test_on_file_path(trained):
87 | protocol, model = trained
88 | inference = Inference(model, batch_size=128)
89 | output = inference("tests/data/dev00.wav")
90 | assert isinstance(output, SlidingWindowFeature)
91 |
92 |
93 | def test_skip_aggregation(pretrained_model, dev_file):
94 | inference = Inference(pretrained_model, skip_aggregation=True)
95 | scores = inference(dev_file)
96 | assert len(scores.data.shape) == 3
97 |
--------------------------------------------------------------------------------
/pyannote-audio/tests/utils/test_permutation.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import torch
3 |
4 | from pyannote.audio.utils.permutation import permutate
5 |
6 |
7 | def test_permutate_torch():
8 |
9 | num_frames, num_speakers = 10, 3
10 |
11 | actual_permutations = [
12 | (0, 1, 2),
13 | (0, 2, 1),
14 | (1, 0, 2),
15 | (1, 2, 0),
16 | (2, 0, 1),
17 | (2, 1, 0),
18 | ]
19 | batch_size = len(actual_permutations)
20 |
21 | y2 = torch.randn((num_frames, num_speakers))
22 | y1 = torch.zeros((batch_size, num_frames, num_speakers))
23 |
24 | for p, permutation in enumerate(actual_permutations):
25 | y1[p] = y2[:, permutation]
26 |
27 | permutated_y2, permutations = permutate(y1, y2)
28 | assert actual_permutations == permutations
29 |
30 | for p, permutation in enumerate(actual_permutations):
31 | np.testing.assert_allclose(permutated_y2[p], y2[:, permutation])
32 |
33 |
34 | def test_permutate_numpy():
35 |
36 | num_frames, num_speakers = 10, 3
37 |
38 | actual_permutations = [
39 | (0, 1, 2),
40 | (0, 2, 1),
41 | (1, 0, 2),
42 | (1, 2, 0),
43 | (2, 0, 1),
44 | (2, 1, 0),
45 | ]
46 | batch_size = len(actual_permutations)
47 |
48 | y2 = np.random.randn(num_frames, num_speakers)
49 | y1 = np.zeros((batch_size, num_frames, num_speakers))
50 |
51 | for p, permutation in enumerate(actual_permutations):
52 | y1[p] = y2[:, permutation]
53 |
54 | permutated_y2, permutations = permutate(y1, y2)
55 | assert actual_permutations == permutations
56 |
57 | for p, permutation in enumerate(actual_permutations):
58 | np.testing.assert_allclose(permutated_y2[p], y2[:, permutation])
59 |
60 |
61 | def test_permutate_less_speakers():
62 |
63 | num_frames = 10
64 |
65 | actual_permutations = [
66 | (0, 1, None),
67 | (0, None, 1),
68 | (1, 0, None),
69 | (1, None, 0),
70 | (None, 0, 1),
71 | (None, 1, 0),
72 | ]
73 | batch_size = len(actual_permutations)
74 |
75 | y2 = np.random.randn(num_frames, 2)
76 | y1 = np.zeros((batch_size, num_frames, 3))
77 |
78 | for p, permutation in enumerate(actual_permutations):
79 | for i, j in enumerate(permutation):
80 | if j is not None:
81 | y1[p, :, i] = y2[:, j]
82 |
83 | permutated_y2, permutations = permutate(y1, y2)
84 |
85 | assert permutations == actual_permutations
86 |
87 |
88 | def test_permutate_more_speakers():
89 |
90 | num_frames = 10
91 |
92 | actual_permutations = [
93 | (0, 1),
94 | (0, 2),
95 | (1, 0),
96 | (1, 2),
97 | (2, 0),
98 | (2, 1),
99 | ]
100 | batch_size = len(actual_permutations)
101 |
102 | y2 = np.random.randn(num_frames, 3)
103 | y1 = np.zeros((batch_size, num_frames, 2))
104 |
105 | for p, permutation in enumerate(actual_permutations):
106 | for i, j in enumerate(permutation):
107 | y1[p, :, i] = y2[:, j]
108 |
109 | permutated_y2, permutations = permutate(y1, y2)
110 |
111 | assert permutations == actual_permutations
112 | np.testing.assert_allclose(permutated_y2, y1)
113 |
--------------------------------------------------------------------------------
/pyannote-audio/notebook/augmentation.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "# gett a 5s excerpt of first test file\n",
10 | "from pyannote.database import get_protocol, FileFinder\n",
11 | "protocol = get_protocol('Debug.SpeakerDiarization.Debug', \n",
12 | " preprocessors={\"audio\": FileFinder()})\n",
13 | "\n",
14 | "from pyannote.audio.core.io import Audio\n",
15 | "audio = Audio(sample_rate=16000, mono=\"downmix\")\n",
16 | "file = next(protocol.test())\n",
17 | "\n",
18 | "from pyannote.core import Segment\n",
19 | "waveform, sample_rate = audio.crop(file, Segment(5, 10))\n",
20 | "\n",
21 | "import torch\n",
22 | "waveforms = torch.tensor(waveform)[None, :]"
23 | ]
24 | },
25 | {
26 | "cell_type": "code",
27 | "execution_count": null,
28 | "metadata": {},
29 | "outputs": [],
30 | "source": [
31 | "# play the excerpt\n",
32 | "from IPython.display import Audio as Play\n",
33 | "Play(waveforms.squeeze(), rate=sample_rate, normalize=False, autoplay=True)"
34 | ]
35 | },
36 | {
37 | "cell_type": "code",
38 | "execution_count": null,
39 | "metadata": {},
40 | "outputs": [],
41 | "source": [
42 | "# define a model that simply returns the waveform\n",
43 | "from pyannote.audio.core.model import Model\n",
44 | "class Passthrough(Model):\n",
45 | " def forward(self, waveforms):\n",
46 | " return waveforms\n",
47 | " \n",
48 | "identity = Passthrough()"
49 | ]
50 | },
51 | {
52 | "cell_type": "code",
53 | "execution_count": null,
54 | "metadata": {},
55 | "outputs": [],
56 | "source": [
57 | "# pass the waveform through this \"identity\" model\n",
58 | "Play(identity(waveforms).squeeze(), rate=sample_rate, normalize=False, autoplay=True)"
59 | ]
60 | },
61 | {
62 | "cell_type": "code",
63 | "execution_count": null,
64 | "metadata": {},
65 | "outputs": [],
66 | "source": [
67 | "# add one torch_audiomentations waveform transform to the model\n",
68 | "from pyannote.audio.augmentation.registry import register_augmentation\n",
69 | "from torch_audiomentations import Gain\n",
70 | "gain = Gain(\n",
71 | " min_gain_in_db=-15.0,\n",
72 | " max_gain_in_db=5.0,\n",
73 | " p=0.5)\n",
74 | "register_augmentation(gain, identity, when='input')"
75 | ]
76 | },
77 | {
78 | "cell_type": "code",
79 | "execution_count": null,
80 | "metadata": {},
81 | "outputs": [],
82 | "source": [
83 | "# pass the waveform through the \"augmented\" model\n",
84 | "Play(identity(waveforms).squeeze(), rate=sample_rate, normalize=False, autoplay=True)"
85 | ]
86 | }
87 | ],
88 | "metadata": {
89 | "kernelspec": {
90 | "display_name": "Python 3",
91 | "language": "python",
92 | "name": "python3"
93 | },
94 | "language_info": {
95 | "codemirror_mode": {
96 | "name": "ipython",
97 | "version": 3
98 | },
99 | "file_extension": ".py",
100 | "mimetype": "text/x-python",
101 | "name": "python",
102 | "nbconvert_exporter": "python",
103 | "pygments_lexer": "ipython3",
104 | "version": "3.7.9"
105 | }
106 | },
107 | "nbformat": 4,
108 | "nbformat_minor": 4
109 | }
110 |
--------------------------------------------------------------------------------
/diarizen/models/pruning/model_distill_prune.py:
--------------------------------------------------------------------------------
1 | # Licensed under the MIT license.
2 | # Copyright 2025 Brno University of Technology (author: Jiangyu Han, ihan@fit.vut.cz)
3 | # This work is inspired by: https://github.com/pyf98/DPHuBERT
4 |
5 | import torch
6 | import torch.nn as nn
7 |
8 | from diarizen.models.module.wav2vec2.model import wav2vec2_model
9 |
10 | class Model(nn.Module):
11 | def __init__(
12 | self,
13 | teacher_ckpt: str,
14 | student_ckpt: str,
15 | pruning_units: str = "conv,head,interm",
16 | distill_layers: str = "0,4,8,12",
17 | ):
18 | super().__init__()
19 |
20 | self.distill_layers = [int(l) for l in distill_layers.split(",")]
21 |
22 | self.teacher_model = self.build_teacher(teacher_ckpt)
23 | self.student_model, self.student_config = self.build_student(student_ckpt, pruning_units)
24 |
25 | def build_teacher(self, teacher_ckpt):
26 | teacher_ckpt = torch.load(teacher_ckpt, map_location="cpu")
27 | teacher_model = wav2vec2_model(**teacher_ckpt["config"])
28 | teacher_result = teacher_model.load_state_dict(teacher_ckpt["state_dict"], strict=False)
29 | print(f"Load pretrained ckpt to teacher: missing {teacher_result.missing_keys}, unexpected {teacher_result.unexpected_keys}")
30 |
31 | # freeze teacher
32 | for p in teacher_model.parameters():
33 | p.requires_grad = False
34 | print("Freeze parameters of the teacher model by setting requires_grad=False")
35 | teacher_model.eval()
36 | return teacher_model
37 |
38 | def build_student(self, student_ckpt, pruning_units):
39 | student_ckpt = torch.load(student_ckpt, map_location="cpu")
40 | pruning_units = pruning_units.split(",")
41 | print(f"Pruning units: {pruning_units}")
42 | student_config = student_ckpt['config']
43 | student_config.update(
44 | dict(
45 | extractor_prune_conv_channels = "conv" in pruning_units,
46 | encoder_prune_attention_heads = "head" in pruning_units,
47 | encoder_prune_attention_layer = "attlayer" in pruning_units,
48 | encoder_prune_feed_forward_intermediate = "interm" in pruning_units,
49 | encoder_prune_feed_forward_layer = "ffnlayer" in pruning_units,
50 | )
51 | )
52 | student_model = wav2vec2_model(**student_config)
53 | student_result = student_model.load_state_dict(student_ckpt["state_dict"], strict=False)
54 | print(f"Load pretrained ckpt to student: missing {student_result.missing_keys}, unexpected {student_result.unexpected_keys}")
55 | return student_model, student_config
56 |
57 | def forward(self, waveforms: torch.Tensor) -> torch.Tensor:
58 | self.teacher_model.eval()
59 | with torch.no_grad():
60 | teacher_hiddens, _ = self.teacher_model.extract_features(waveforms)
61 | teacher_hiddens = torch.stack(
62 | [teacher_hiddens[idx] for idx in self.distill_layers], dim=1
63 | ) # (batch, layer, time, feature)
64 |
65 | student_hiddens, _ = self.student_model.extract_features(waveforms)
66 | student_hiddens = torch.stack(
67 | [student_hiddens[idx] for idx in self.distill_layers], dim=1
68 | ) # (batch, layer, time, feature)
69 |
70 | return student_hiddens, teacher_hiddens
--------------------------------------------------------------------------------
/pyannote-audio/setup.cfg:
--------------------------------------------------------------------------------
1 | # This file is used to configure your project.
2 | # Read more about the various options under:
3 | # http://setuptools.readthedocs.io/en/latest/setuptools.html#configuring-setup-using-setup-cfg-files
4 |
5 | [metadata]
6 | name = pyannote-audio
7 | description = Neural speaker diarization
8 | author = Herve Bredin
9 | author-email = herve.bredin@irit.fr
10 | license = mit
11 | long-description = file: README.md
12 | long-description-content-type = text/markdown; charset=UTF-8; variant=GFM
13 | # Change if running only on Windows, Mac or Linux (comma-separated)
14 | platforms = Linux, Mac
15 | # Add here all kinds of additional classifiers as defined under
16 | # https://pypi.python.org/pypi?%3Aaction=list_classifiers
17 | classifiers =
18 | Development Status :: 4 - Beta
19 | Programming Language :: Python
20 |
21 | [options]
22 | zip_safe = False
23 | packages = find:
24 | include_package_data = True
25 | # DON'T CHANGE THE FOLLOWING LINE! IT WILL BE UPDATED BY PYSCAFFOLD!
26 | setup_requires = pyscaffold>=3.2a0,<3.3a0
27 | # Add here dependencies of your project (semicolon/line-separated), e.g.
28 | # install_requires = numpy; scipy
29 | # Require a specific Python version, e.g. Python 2.7 or >= 3.4
30 | python_requires = >=3.7
31 |
32 | [options.packages.find]
33 | where = .
34 | exclude =
35 | tests
36 |
37 | [options.extras_require]
38 | # Add here additional requirements for extra features, to install with:
39 | # `pip install fastaudio[PDF]` like:
40 | # PDF = ReportLab; RXP
41 | # Add here test requirements (semicolon/line-separated)
42 | testing =
43 | pytest>=6.0
44 | pytest-cov>=2.10
45 | jupyter
46 | papermill
47 | dev =
48 | pre_commit>=2.7
49 | recommonmark>=0.6
50 | black>=22.3.0
51 | cli =
52 | hydra-core >=1.1,<1.2
53 | typer >= 0.4.0,<0.5.0
54 |
55 | [options.entry_points]
56 |
57 | console_scripts =
58 | pyannote-audio-train=pyannote.audio.cli.train:train
59 | pyannote-audio-eval=pyannote.audio.cli.evaluate:evaluate
60 |
61 |
62 | [test]
63 | # py.test options when running `python setup.py test`
64 | # addopts = --verbose
65 | extras = True
66 |
67 | [tool:pytest]
68 | # Options for py.test:
69 | # Specify command line options as you would do when invoking py.test directly.
70 | # e.g. --cov-report html (or xml) for html/xml output or --junitxml junit.xml
71 | # in order to write a coverage file that can be read by Jenkins.
72 | addopts =
73 | --cov pyannote --cov-report term-missing
74 | --verbose
75 | norecursedirs =
76 | dist
77 | build
78 | .tox
79 | testpaths = tests
80 |
81 | [aliases]
82 | dists = bdist_wheel
83 |
84 | [bdist_wheel]
85 | # Use this option if your package is pure-python
86 | universal = 1
87 |
88 | [build_sphinx]
89 | source_dir = doc
90 | build_dir = build/sphinx
91 |
92 | [devpi:upload]
93 | # Options for the devpi: PyPI server and packaging tool
94 | # VCS export must be deactivated since we are using setuptools-scm
95 | no-vcs = 1
96 | formats = bdist_wheel
97 |
98 | [flake8]
99 | # Some sane defaults for the code style checker flake8
100 | exclude =
101 | .tox
102 | build
103 | dist
104 | .eggs
105 | docs/conf.py
106 |
107 | [pyscaffold]
108 | # PyScaffold's parameters when the project was created.
109 | # This will be used when updating. Do not change!
110 | version = 3.2.3
111 | package = pyannote-audio
112 | extensions =
113 | markdown
114 | no_skeleton
115 | pre_commit
116 | dsproject
117 |
--------------------------------------------------------------------------------
/pyannote-audio/tests/utils/test_powerset.py:
--------------------------------------------------------------------------------
1 | # MIT License
2 | #
3 | # Copyright (c) 2023- CNRS
4 | #
5 | # Permission is hereby granted, free of charge, to any person obtaining a copy
6 | # of this software and associated documentation files (the "Software"), to deal
7 | # in the Software without restriction, including without limitation the rights
8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | # copies of the Software, and to permit persons to whom the Software is
10 | # furnished to do so, subject to the following conditions:
11 | #
12 | # The above copyright notice and this permission notice shall be included in all
13 | # copies or substantial portions of the Software.
14 | #
15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | # SOFTWARE.
22 |
23 |
24 | import torch
25 |
26 | from pyannote.audio.utils.powerset import Powerset
27 |
28 |
29 | def test_roundtrip():
30 | for num_classes in range(2, 5):
31 | for max_set_size in range(1, num_classes + 1):
32 | powerset = Powerset(num_classes, max_set_size)
33 |
34 | # simulate a sequence where each frame is assigned to a different powerset class
35 | one_sequence = [
36 | [0] * powerset.num_powerset_classes
37 | for _ in range(powerset.num_powerset_classes)
38 | ]
39 | for i in range(powerset.num_powerset_classes):
40 | one_sequence[i][i] = 1.0
41 |
42 | # make a batch out of this sequence and the same sequence in reverse order
43 | batch_powerset = torch.tensor([one_sequence, one_sequence[::-1]])
44 |
45 | # convert from powerset to multi-label
46 | batch_multilabel = powerset.to_multilabel(batch_powerset)
47 |
48 | # convert batch back to powerset
49 | reconstruction = powerset.to_powerset(batch_multilabel)
50 |
51 | assert torch.equal(batch_powerset, reconstruction)
52 |
53 |
54 | def test_permutate_powerset():
55 | for num_classes in range(1, 6):
56 | for max_set_size in range(1, num_classes + 1):
57 | powerset = Powerset(num_classes, max_set_size)
58 |
59 | # create (num_powerset_class, num_powerset_class)-shaped tensor, where each frame is assigned to a different powerset class
60 | # and convert it to its multi-label equivalent
61 | t1 = torch.nn.functional.one_hot(
62 | torch.arange(powerset.num_powerset_classes),
63 | powerset.num_powerset_classes,
64 | )
65 | t1_ml = powerset.to_multilabel(t1)
66 |
67 | # then permutate the powerset class in powerset space AND the multilabel equivalent in its native space
68 | # and check it has the same result.
69 | # perm = torch.randperm(num_classes)
70 | perm = tuple(torch.randperm(num_classes).tolist())
71 | t1_ml_perm = t1_ml[:, perm]
72 | perm_ps = powerset.permutation_mapping[perm]
73 | t1_ps_perm = t1[..., perm_ps]
74 | t1_ps_perm_ml = powerset.to_multilabel(t1_ps_perm)
75 |
76 | assert t1_ml_perm.equal(t1_ps_perm_ml)
77 |
--------------------------------------------------------------------------------
/recipes/diar_ssl/data/AMI_AliMeeting_AISHELL4/dev/wav.scp:
--------------------------------------------------------------------------------
1 | ES2011a /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/ES2011a.wav
2 | ES2011b /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/ES2011b.wav
3 | ES2011c /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/ES2011c.wav
4 | ES2011d /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/ES2011d.wav
5 | IB4001 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/IB4001.wav
6 | IB4002 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/IB4002.wav
7 | IB4003 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/IB4003.wav
8 | IB4004 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/IB4004.wav
9 | IB4010 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/IB4010.wav
10 | IB4011 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/IB4011.wav
11 | IS1008a /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/IS1008a.wav
12 | IS1008b /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/IS1008b.wav
13 | IS1008c /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/IS1008c.wav
14 | IS1008d /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/IS1008d.wav
15 | TS3004a /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/TS3004a.wav
16 | TS3004b /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/TS3004b.wav
17 | TS3004c /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/TS3004c.wav
18 | TS3004d /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/TS3004d.wav
19 | R8001_M8004_MS801 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/R8001_M8004_MS801.wav
20 | R8003_M8001_MS801 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/R8003_M8001_MS801.wav
21 | R8007_M8010_MS803 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/R8007_M8010_MS803.wav
22 | R8007_M8011_MS806 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/R8007_M8011_MS806.wav
23 | R8008_M8013_MS807 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/R8008_M8013_MS807.wav
24 | R8009_M8018_MS809 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/R8009_M8018_MS809.wav
25 | R8009_M8019_MS810 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/R8009_M8019_MS810.wav
26 | R8009_M8020_MS810 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/R8009_M8020_MS810.wav
27 | 20200706_L_R001S01C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200706_L_R001S01C01.wav
28 | 20200708_L_R002S07C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200708_L_R002S07C01.wav
29 | 20200709_L_R002S08C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200709_L_R002S08C01.wav
30 | 20200616_M_R001S01C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200616_M_R001S01C01.wav
31 | 20200715_M_R002S06C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200715_M_R002S06C01.wav
32 | 20200620_M_R002S08C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200620_M_R002S08C01.wav
33 | 20200620_M_R002S07C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200620_M_R002S07C01.wav
34 | 20200710_M_R002S03C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200710_M_R002S03C01.wav
35 | 20200710_M_R002S07C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200710_M_R002S07C01.wav
36 | 20200712_M_R002S05C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200712_M_R002S05C01.wav
37 | 20200704_M_R002S06C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200704_M_R002S06C01.wav
38 | 20200622_M_R002S02C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200622_M_R002S02C01.wav
39 | 20200715_M_R002S03C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200715_M_R002S03C01.wav
40 | 20200623_S_R001S06C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200623_S_R001S06C01.wav
41 | 20200805_S_R001S01C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200805_S_R001S01C01.wav
42 | 20200701_S_R001S03C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200701_S_R001S03C01.wav
43 | 20200805_S_R001S03C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200805_S_R001S03C01.wav
44 | 20200702_S_R001S02C01 /YOUR_PATH/AMI_AliMeeting_AISHELL4/wavs/dev/20200702_S_R001S02C01.wav
45 |
--------------------------------------------------------------------------------
/recipes/diar_ssl/data/NSF/dev_sc/all.uem:
--------------------------------------------------------------------------------
1 | S30500101 1 0.000 380.935
2 | S30500201 1 0.000 380.935
3 | S30500211 1 0.000 380.935
4 | S30502101 1 0.000 536.593
5 | S30502201 1 0.000 536.593
6 | S30502211 1 0.000 536.593
7 | S30503101 1 0.000 396.243
8 | S30503201 1 0.000 396.243
9 | S30503211 1 0.000 396.243
10 | S30504101 1 0.000 410.198
11 | S30504201 1 0.000 410.198
12 | S30504211 1 0.000 410.198
13 | S30505101 1 0.000 341.835
14 | S30505201 1 0.000 341.835
15 | S30505211 1 0.000 341.835
16 | S30508101 1 0.000 357.795
17 | S30508201 1 0.000 357.795
18 | S30508211 1 0.000 357.795
19 | S30509101 1 0.000 371.505
20 | S30509201 1 0.000 371.505
21 | S30509211 1 0.000 371.505
22 | S30519101 1 0.000 291.910
23 | S30519201 1 0.000 291.910
24 | S30519211 1 0.000 291.910
25 | S30520101 1 0.000 392.993
26 | S30520201 1 0.000 392.993
27 | S30520211 1 0.000 392.993
28 | S30522101 1 0.000 358.823
29 | S30522201 1 0.000 358.823
30 | S30522211 1 0.000 358.823
31 | S30552101 1 0.000 318.121
32 | S30552201 1 0.000 318.121
33 | S30552211 1 0.000 318.121
34 | S30552301 1 0.000 318.121
35 | S30560101 1 0.000 298.760
36 | S30560201 1 0.000 298.760
37 | S30560211 1 0.000 298.760
38 | S30560301 1 0.000 298.760
39 | S30563101 1 0.000 376.491
40 | S30563201 1 0.000 376.491
41 | S30563211 1 0.000 376.491
42 | S30563301 1 0.000 376.491
43 | S30569101 1 0.000 320.840
44 | S30569201 1 0.000 320.840
45 | S30569211 1 0.000 320.840
46 | S30569301 1 0.000 320.840
47 | S30570101 1 0.000 358.233
48 | S30570201 1 0.000 358.233
49 | S30570211 1 0.000 358.233
50 | S30570301 1 0.000 358.233
51 | S30601101 1 0.000 362.679
52 | S30601201 1 0.000 362.679
53 | S30601211 1 0.000 362.679
54 | S30601301 1 0.000 362.679
55 | S30603101 1 0.000 414.218
56 | S30603201 1 0.000 414.218
57 | S30603211 1 0.000 414.218
58 | S30603301 1 0.000 414.218
59 | S30605101 1 0.000 348.659
60 | S30605201 1 0.000 348.659
61 | S30605211 1 0.000 348.659
62 | S30605301 1 0.000 348.659
63 | S30609101 1 0.000 367.714
64 | S30609201 1 0.000 367.714
65 | S30609211 1 0.000 367.714
66 | S30610101 1 0.000 492.053
67 | S30610201 1 0.000 492.053
68 | S30610211 1 0.000 492.053
69 | S30633101 1 0.000 373.159
70 | S30633201 1 0.000 373.159
71 | S30633211 1 0.000 373.159
72 | S30633301 1 0.000 373.159
73 | S30636101 1 0.000 404.768
74 | S30636201 1 0.000 404.768
75 | S30636211 1 0.000 404.768
76 | S30636301 1 0.000 404.768
77 | S30639101 1 0.000 362.749
78 | S30639201 1 0.000 362.749
79 | S30639211 1 0.000 362.749
80 | S30639301 1 0.000 362.749
81 | S30716101 1 0.000 251.385
82 | S30716201 1 0.000 251.385
83 | S30716211 1 0.000 251.385
84 | S30716301 1 0.000 251.385
85 | S30718101 1 0.000 356.323
86 | S30718201 1 0.000 356.323
87 | S30718211 1 0.000 356.323
88 | S30718301 1 0.000 356.323
89 | S30719101 1 0.000 406.550
90 | S30719201 1 0.000 406.550
91 | S30719211 1 0.000 406.550
92 | S30719301 1 0.000 406.550
93 | S30721201 1 0.000 367.565
94 | S30721211 1 0.000 367.565
95 | S30721301 1 0.000 367.565
96 | S30724201 1 0.000 378.140
97 | S30724211 1 0.000 378.140
98 | S30724301 1 0.000 378.140
99 | S30725201 1 0.000 391.870
100 | S30725211 1 0.000 391.870
101 | S30725301 1 0.000 391.870
102 | S30752101 1 0.000 385.781
103 | S30752201 1 0.000 385.781
104 | S30752211 1 0.000 385.781
105 | S30752301 1 0.000 385.781
106 | S30754101 1 0.000 444.688
107 | S30754201 1 0.000 444.688
108 | S30754211 1 0.000 444.688
109 | S30754301 1 0.000 444.688
110 | S30773101 1 0.000 420.119
111 | S30773201 1 0.000 420.119
112 | S30773211 1 0.000 420.119
113 | S30773301 1 0.000 420.119
114 | S30774101 1 0.000 372.440
115 | S30774201 1 0.000 372.440
116 | S30774211 1 0.000 372.440
117 | S30774301 1 0.000 372.440
118 |
--------------------------------------------------------------------------------
/pyannote-audio/.github/ISSUE_TEMPLATE/bug_report.yml:
--------------------------------------------------------------------------------
1 | name: Bug report
2 | description: Report a bug in pyannote.audio
3 | body:
4 |
5 | - type: markdown
6 | attributes:
7 | value: |
8 | When reporting bugs, please follow the guidelines in this template. This helps identify the problem precisely and thus enables contributors to fix it faster.
9 | - Write a descriptive issue title above.
10 | - The golden rule is to **always open *one* issue for *one* bug**. If you notice several bugs and want to report them, make sure to create one new issue for each of them.
11 | - Search [open](https://github.com/pyannote/pyannote-audio/issues) and [closed](https://github.com/pyannote/pyannote-audio/issues?q=is%3Aissue+is%3Aclosed) issues to ensure it has not already been reported. If you don't find a relevant match or if you're unsure, don't hesitate to **open a new issue**. The bugsquad will handle it from there if it's a duplicate.
12 | - Please always check if your issue is reproducible in the latest version – it may already have been fixed!
13 | - If you use a custom build, please test if your issue is reproducible in official releases too.
14 |
15 | - type: textarea
16 | attributes:
17 | label: Tested versions
18 | description: |
19 | To properly fix a bug, we need to identify if the bug was recently introduced in the engine, or if it was always present.
20 | - Please specify the pyannote.audio version you found the issue in, including the **Git commit hash** if using a development build.
21 | - If you can, **please test earlier pyannote.audio versions** and, if applicable, newer versions (development branch). Mention whether the bug is reproducible or not in the versions you tested.
22 | - The aim is for us to identify whether a bug is a **regression**, i.e. an issue that didn't exist in a previous version, but was introduced later on, breaking existing functionality. For example, if a bug is reproducible in 3.2 but not in 3.0, we would like you to test intermediate 3.1 to find which version is the first one where the issue can be reproduced.
23 | placeholder: |
24 | - Reproducible in: 3.1, 3.2, and later
25 | - Not reproducible in: 3.0
26 | validations:
27 | required: true
28 |
29 | - type: input
30 | attributes:
31 | label: System information
32 | description: |
33 | - Specify the OS version, and when relevant hardware information.
34 | - For issues that are likely OS-specific and/or GPU-related, please specify the GPU model and architecture.
35 | - **Bug reports not including the required information may be closed at the maintainers' discretion.** If in doubt, always include all the requested information; it's better to include too much information than not enough information.
36 | placeholder: macOS 13.6 - pyannote.audio 3.1.1 - M1 Pro
37 | validations:
38 | required: true
39 |
40 | - type: textarea
41 | attributes:
42 | label: Issue description
43 | description: |
44 | Describe your issue briefly. What doesn't work, and how do you expect it to work instead?
45 | You can include audio, images or videos with drag and drop, and format code blocks or logs with ``` tags.
46 | validations:
47 | required: true
48 |
49 | - type: input
50 | attributes:
51 | label: Minimal reproduction example (MRE)
52 | description: |
53 | Having reproducible issues is a prerequisite for contributors to be able to solve them.
54 | Include a link to minimal reproduction example using [this Google Colab notebook](https://colab.research.google.com/github/pyannote/pyannote-audio/blob/develop/tutorials/MRE_template.ipynb) as a starting point.
55 | validations:
56 | required: true
57 |
--------------------------------------------------------------------------------
/pyannote-audio/FAQ.md:
--------------------------------------------------------------------------------
1 |
2 | # Frequently Asked Questions
3 | - [Can I apply pretrained pipelines on audio already loaded in memory?](#can-i-apply-pretrained-pipelines-on-audio-already-loaded-in-memory)
4 | - [Can I use gated models (and pipelines) offline?](#can-i-use-gated-models-(and-pipelines)-offline)
5 | - [Does pyannote support streaming speaker diarization?](#does-pyannote-support-streaming-speaker-diarization)
6 | - [How can I improve performance?](#how-can-i-improve-performance)
7 | - [How does one spell and pronounce pyannote.audio?](#how-does-one-spell-and-pronounce-pyannoteaudio)
8 |
9 |
10 | ## Can I apply pretrained pipelines on audio already loaded in memory?
11 |
12 | Yes: read [this tutorial](tutorials/applying_a_pipeline.ipynb) until the end.
13 |
14 |
15 | ## Can I use gated models (and pipelines) offline?
16 |
17 | **Short answer**: yes, see [this tutorial](tutorials/applying_a_model.ipynb) for models and [that one](tutorials/applying_a_pipeline.ipynb) for pipelines.
18 |
19 | **Long answer**: gating models and pipelines allows [me](https://herve.niderb.fr) to know a bit more about `pyannote.audio` user base and eventually help me write grant proposals to make `pyannote.audio` even better. So, please fill gating forms as precisely as possible.
20 |
21 | For instance, before gating `pyannote/speaker-diarization`, I had no idea that so many people were relying on it in production. Hint: sponsors are more than welcome! Maintaining open source libraries is time consuming.
22 |
23 | That being said, this whole authentication process does not prevent you from using official `pyannote.audio` models offline (i.e. without going through the authentication process in every `docker run ...` or whatever you are using in production): see [this tutorial](tutorials/applying_a_model.ipynb) for models and [that one](tutorials/applying_a_pipeline.ipynb) for pipelines.
24 |
25 |
26 | ## Does pyannote support streaming speaker diarization?
27 |
28 | **Short answer:** not out of the box, no.
29 |
30 | **Long answer:** [I](https://herve.niderb.fr) am looking for sponsors to add this feature. In the meantime, [`diart`](https://github.com/juanmc2005/StreamingSpeakerDiarization) is the closest you can get from a streaming `pyannote.audio`. You might also be interested in [this blog post](https://herve.niderb.fr/fastpages/2021/08/05/Streaming-voice-activity-detection-with-pyannote.html) about streaming voice activity detection based on `pyannote.audio`.
31 |
32 |
33 | ## How can I improve performance?
34 |
35 | **Long answer:**
36 |
37 | 1. Manually annotate dozens of conversations as precisely as possible.
38 | 2. Separate them into train (80%), development (10%) and test (10%) subsets.
39 | 3. Setup the data for use with [`pyannote.database`](https://github.com/pyannote/pyannote-database#speaker-diarization).
40 | 4. Follow [this recipe](https://github.com/pyannote/pyannote-audio/blob/develop/tutorials/adapting_pretrained_pipeline.ipynb).
41 | 5. Enjoy.
42 |
43 | **Also:** [I am available](https://herve.niderb.fr) for contracting to help you with that.
44 |
45 |
46 | ## How does one spell and pronounce pyannote.audio?
47 |
48 | 📝 Written in lower case: `pyannote.audio` (or `pyannote` if you are lazy). Not `PyAnnote` nor `PyAnnotate` (sic).
49 | 📢 Pronounced like the french verb `pianoter`. `pi` like in `pi`ano, not `py` like in `py`thon.
50 | 🎹 `pianoter` means to play the piano (hence the logo 🤯).
51 |
52 |
53 |
54 | Generated by [FAQtory](https://github.com/willmcgugan/faqtory)
55 |
--------------------------------------------------------------------------------
/pyannote-audio/pyannote/audio/utils/reproducibility.py:
--------------------------------------------------------------------------------
1 | # MIT License
2 | #
3 | # Copyright (c) 2023- CNRS
4 | #
5 | # Permission is hereby granted, free of charge, to any person obtaining a copy
6 | # of this software and associated documentation files (the "Software"), to deal
7 | # in the Software without restriction, including without limitation the rights
8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | # copies of the Software, and to permit persons to whom the Software is
10 | # furnished to do so, subject to the following conditions:
11 | #
12 | # The above copyright notice and this permission notice shall be included in all
13 | # copies or substantial portions of the Software.
14 | #
15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | # SOFTWARE.
22 |
23 | # Context: https://github.com/pyannote/pyannote-audio/issues/1370
24 |
25 | import warnings
26 |
27 | import torch
28 |
29 |
30 | class ReproducibilityError(Exception):
31 | ...
32 |
33 |
34 | class ReproducibilityWarning(UserWarning):
35 | ...
36 |
37 |
38 | def raise_reproducibility(device: torch.device):
39 | if (device.type == "cuda") and (
40 | torch.backends.cuda.matmul.allow_tf32 or torch.backends.cudnn.allow_tf32
41 | ):
42 | raise ReproducibilityError(
43 | "Please disable TensorFloat-32 (TF32) by calling\n"
44 | " >>> import torch\n"
45 | " >>> torch.backends.cuda.matmul.allow_tf32 = False\n"
46 | " >>> torch.backends.cudnn.allow_tf32 = False\n"
47 | "or you might face reproducibility issues and obtain lower accuracy.\n"
48 | "See https://github.com/pyannote/pyannote-audio/issues/1370 for more details."
49 | )
50 |
51 |
52 | def warn_reproducibility(device: torch.device):
53 | if (device.type == "cuda") and (
54 | torch.backends.cuda.matmul.allow_tf32 or torch.backends.cudnn.allow_tf32
55 | ):
56 | warnings.warn(
57 | ReproducibilityWarning(
58 | "Please disable TensorFloat-32 (TF32) by calling\n"
59 | " >>> import torch\n"
60 | " >>> torch.backends.cuda.matmul.allow_tf32 = False\n"
61 | " >>> torch.backends.cudnn.allow_tf32 = False\n"
62 | "or you might face reproducibility issues and obtain lower accuracy.\n"
63 | "See https://github.com/pyannote/pyannote-audio/issues/1370 for more details."
64 | )
65 | )
66 |
67 |
68 | def fix_reproducibility(device: torch.device):
69 | if (device.type == "cuda") and (
70 | torch.backends.cuda.matmul.allow_tf32 or torch.backends.cudnn.allow_tf32
71 | ):
72 | torch.backends.cuda.matmul.allow_tf32 = False
73 | torch.backends.cudnn.allow_tf32 = False
74 | warnings.warn(
75 | ReproducibilityWarning(
76 | "TensorFloat-32 (TF32) has been disabled as it might lead to reproducibility issues and lower accuracy.\n"
77 | "It can be re-enabled by calling\n"
78 | " >>> import torch\n"
79 | " >>> torch.backends.cuda.matmul.allow_tf32 = True\n"
80 | " >>> torch.backends.cudnn.allow_tf32 = True\n"
81 | "See https://github.com/pyannote/pyannote-audio/issues/1370 for more details.\n"
82 | )
83 | )
84 |
--------------------------------------------------------------------------------
/recipes/diar_ssl/run_stage.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Licensed under the MIT license.
4 | # Copyright 2024 Brno University of Technology (author: Jiangyu Han, ihan@fit.vut.cz)
5 |
6 | set -eu
7 | ulimit -n 2048
8 |
9 | # general setup
10 | stage=1
11 | recipe_root=/YOUR_PATH/DiariZen/recipes/diar_ssl
12 | exp_root=$recipe_root/exp
13 | conf_dir=$recipe_root/conf
14 |
15 | # training setup
16 | use_dual_opt=true # true for wavlm_updated_conformer.toml; false for the others
17 | train_conf=$conf_dir/wavlm_updated_conformer.toml
18 | # train_conf=$conf_dir/wavlm_frozen_conformer.toml
19 | # train_conf=$conf_dir/fbank_conformer.toml
20 | # train_conf=$conf_dir/pyannote_baseline.toml
21 |
22 | conf_name=`ls $train_conf | awk -F '/' '{print $NF}' | awk -F '.' '{print $1}'`
23 |
24 | # inference setup
25 | dtype=test
26 | data_dir=$recipe_root/data/AMI_AliMeeting_AISHELL4
27 | seg_duration=8
28 |
29 | # clustering setup
30 | clustering_method=AgglomerativeClustering
31 | ahc_threshold=0.70
32 | min_cluster_size=30
33 | infer_affix=_constrained_AHC_thres_${ahc_threshold}_mcs_${min_cluster_size}
34 |
35 | avg_ckpt_num=5
36 | val_metric=Loss # Loss or DER
37 | val_mode=best # [prev, best, center]
38 |
39 | # scoring setup
40 | collar=0
41 | REF_DIR=$data_dir
42 | dscore_dir=/YOUR_PATH/DiariZen/dscore
43 |
44 | # =======================================
45 | # =======================================
46 | if [ $stage -le 1 ]; then
47 | if (! $use_dual_opt); then
48 | echo "stage1: use single-opt for model training..."
49 | conda activate diarizen && CUDA_VISIBLE_DEVICES="0,1" accelerate launch \
50 | --num_processes 2 --main_process_port 1134 \
51 | run_single_opt.py -C $train_conf -M validate
52 | else
53 | echo "stage1: use dual-opt for model training..."
54 | conda activate diarizen && CUDA_VISIBLE_DEVICES="0,1,2,3" accelerate launch \
55 | --num_processes 4 --main_process_port 1134 \
56 | run_dual_opt.py -C $train_conf -M train
57 | fi
58 | fi
59 |
60 | diarization_dir=$exp_root/$conf_name # can be replaced by our pre-trained models, e.g. diarization_dir=/YOUR_PATH/checkpoints/wavlm_updated_conformer
61 | config_dir=`ls $diarization_dir/*.toml | sort -r | head -n 1`
62 | embedding_model=/YOUR_PATH/pretrained/pyannote3/wespeaker-voxceleb-resnet34-LM/pytorch_model.bin # it's necessary to have "pyannote" in your directory path
63 |
64 | if [ $stage -le 2 ]; then
65 | echo "stage2: model inference..."
66 | export CUDA_VISIBLE_DEVICES=0
67 |
68 | train_log=`du -h $diarization_dir/*.log | sort -rh | head -n 1 | awk '{print $NF}'`
69 | cat $train_log | grep 'Loss/DER' | awk -F ']:' '{print $NF}' > $diarization_dir/val_metric_summary.lst
70 |
71 | for dset in AMI AliMeeting AISHELL4; do
72 | conda activate diarizen && python infer_avg.py -C $config_dir \
73 | -i ${data_dir}/${dtype}/${dset}/wav.scp \
74 | -o ${diarization_dir}/infer$infer_affix/metric_${val_metric}_${val_mode}/avg_ckpt${avg_ckpt_num}/${dtype}/${dset} \
75 | --embedding_model $embedding_model \
76 | --avg_ckpt_num $avg_ckpt_num \
77 | --val_metric $val_metric \
78 | --val_mode $val_mode \
79 | --val_metric_summary $diarization_dir/val_metric_summary.lst \
80 | --seg_duration $seg_duration \
81 | --clustering_method $clustering_method \
82 | --ahc_threshold $ahc_threshold \
83 | --min_cluster_size $min_cluster_size
84 |
85 | echo "stage3: scoring..."
86 | SYS_DIR=${diarization_dir}/infer$infer_affix/metric_${val_metric}_${val_mode}/avg_ckpt${avg_ckpt_num}
87 | OUT_DIR=${SYS_DIR}/${dtype}/${dset}
88 | conda activate diarizen && python ${dscore_dir}/score.py \
89 | -r ${REF_DIR}/${dtype}/${dset}/rttm \
90 | -s $OUT_DIR/*.rttm --collar ${collar} \
91 | > $OUT_DIR/result_collar${collar}
92 | done
93 | fi
94 |
--------------------------------------------------------------------------------
/pyannote-audio/pyannote/audio/cli/evaluate.py:
--------------------------------------------------------------------------------
1 | # MIT License
2 | #
3 | # Copyright (c) 2022- CNRS
4 | #
5 | # Permission is hereby granted, free of charge, to any person obtaining a copy
6 | # of this software and associated documentation files (the "Software"), to deal
7 | # in the Software without restriction, including without limitation the rights
8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | # copies of the Software, and to permit persons to whom the Software is
10 | # furnished to do so, subject to the following conditions:
11 | #
12 | # The above copyright notice and this permission notice shall be included in all
13 | # copies or substantial portions of the Software.
14 | #
15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | # SOFTWARE.
22 |
23 |
24 | from typing import Optional
25 |
26 | import hydra
27 | from omegaconf import DictConfig
28 | from pyannote.database import FileFinder, ProtocolFile, registry
29 | from rich.progress import Progress
30 |
31 | from pyannote.audio import Inference, Model
32 | from pyannote.audio.pipelines.utils import get_devices
33 | from pyannote.audio.utils.metric import DiscreteDiarizationErrorRate
34 | from pyannote.audio.utils.signal import binarize
35 |
36 |
37 | @hydra.main(config_path="evaluate_config", config_name="config")
38 | def evaluate(cfg: DictConfig) -> Optional[float]:
39 |
40 | # load pretrained model
41 | (device,) = get_devices(needs=1)
42 | model = Model.from_pretrained(cfg.model, device=device)
43 |
44 | # load databases into registry if it was specified
45 | if "registry" in cfg:
46 | for database_yml in cfg.registry.split(","):
47 | registry.load_database(database_yml)
48 |
49 | # load evaluation files
50 | protocol = registry.get_protocol(
51 | cfg.protocol, preprocessors={"audio": FileFinder()}
52 | )
53 |
54 | files = list(getattr(protocol, cfg.subset)())
55 |
56 | # load evaluation metric
57 | metric = DiscreteDiarizationErrorRate()
58 |
59 | with Progress() as progress:
60 |
61 | main_task = progress.add_task(protocol.name, total=len(files))
62 | file_task = progress.add_task("Processing", total=1.0)
63 |
64 | def progress_hook(completed: Optional[int] = None, total: Optional[int] = None):
65 | progress.update(file_task, completed=completed / total)
66 |
67 | inference = Inference(model, device=device)
68 | warm_up = cfg.warm_up / inference.duration
69 |
70 | def hypothesis(file: ProtocolFile):
71 | return Inference.trim(
72 | binarize(inference(file, hook=progress_hook)),
73 | warm_up=(warm_up, warm_up),
74 | )
75 |
76 | for file in files:
77 | progress.update(file_task, description=file["uri"])
78 | reference = file["annotation"]
79 | uem = file["annotated"]
80 | _ = metric(reference, hypothesis(file), uem=uem)
81 | progress.advance(main_task)
82 |
83 | report = metric.report(display=False)
84 |
85 | with open("report.txt", "w") as f:
86 |
87 | f.write(f"# Model: {cfg.model}\n")
88 | f.write(f"# Protocol: {protocol.name}\n")
89 | f.write(f"# Subset: {cfg.subset}\n")
90 | f.write("\n")
91 | report = report.to_string(
92 | index=True,
93 | sparsify=False,
94 | justify="right",
95 | float_format=lambda f: "{0:.2f}".format(f),
96 | )
97 | f.write(f"{report}")
98 |
99 |
100 | if __name__ == "__main__":
101 | evaluate()
102 |
--------------------------------------------------------------------------------
/diarizen/ckpt_utils.py:
--------------------------------------------------------------------------------
1 | # Licensed under the MIT license.
2 | # Copyright 2022 Brno University of Technology (author: Federico Landini, landini@fit.vut.cz)
3 | # Copyright 2024 Brno University of Technology (author: Jiangyu Han, ihan@fit.vut.cz)
4 |
5 | import os
6 | from pathlib import Path
7 |
8 | import torch
9 | import torch.nn as nn
10 |
11 | import copy
12 |
13 | from typing import List, Dict
14 |
15 |
16 | def average_checkpoints(
17 | model: nn.Module,
18 | checkpoint_list: str,
19 | ) -> nn.Module:
20 | states_dict_list = []
21 | for ckpt_data in checkpoint_list:
22 | ckpt_path = ckpt_data['bin_path']
23 | copy_model = copy.deepcopy(model)
24 | checkpoint = torch.load(ckpt_path, map_location=torch.device('cpu'))
25 | copy_model.load_state_dict(checkpoint)
26 | states_dict_list.append(copy_model.state_dict())
27 | avg_state_dict = average_states(states_dict_list, torch.device('cpu'))
28 | avg_model = copy.deepcopy(model)
29 | avg_model.load_state_dict(avg_state_dict)
30 | return avg_model
31 |
32 | def average_states(
33 | states_list: List[Dict[str, torch.Tensor]],
34 | device: torch.device,
35 | ) -> List[Dict[str, torch.Tensor]]:
36 | qty = len(states_list)
37 | avg_state = states_list[0]
38 | for i in range(1, qty):
39 | for key in avg_state:
40 | avg_state[key] += states_list[i][key].to(device)
41 | for key in avg_state:
42 | avg_state[key] = avg_state[key] / qty
43 | return avg_state
44 |
45 | def load_metric_summary(metric_file, ckpt_path):
46 | with open(metric_file, "r") as f:
47 | lines = f.readlines()
48 | out_lst = []
49 | for line in lines:
50 | assert "Validation Loss/DER" in line
51 | epoch = line.split()[4].split(':')[0]
52 | Loss, DER = line.split()[-3], line.split()[-1]
53 | bin_path = f"epoch_{str(epoch).zfill(4)}/pytorch_model.bin"
54 | out_lst.append({
55 | 'epoch': int(epoch),
56 | 'bin_path': ckpt_path / bin_path,
57 | 'Loss': float(Loss),
58 | 'DER': float(DER)
59 | })
60 | return out_lst
61 |
62 | def average_ckpt(ckpt_dir, model, wavlm_only=False, val_metric='Loss', avg_ckpt_num=5, val_mode="best"):
63 | if os.path.isfile(ckpt_dir):
64 | ckpt_loaded = torch.load(ckpt_dir, map_location=torch.device('cpu'))
65 | if not wavlm_only:
66 | print(f"No model averaging | Fine-tune model from: {ckpt_dir}")
67 | model.load_state_dict(ckpt_loaded, strict=True)
68 | else:
69 | print(f"Only initialize wavlm model from: {ckpt_dir}")
70 | wavlm_prefix = "wavlm_model."
71 | wavlm_state_dict = {
72 | k[len(wavlm_prefix):]: v
73 | for k, v in ckpt_loaded.items()
74 | if k.startswith(wavlm_prefix)
75 | }
76 | model.wavlm_model.load_state_dict(wavlm_state_dict, strict=False)
77 | return model
78 |
79 | if 'checkpoints/epoch_' in ckpt_dir:
80 | print(f"No model averaging | Fine-tune model from certain epoch: {ckpt_dir.split('/')[-1]}")
81 | ckpt_loaded = torch.load(os.path.join(ckpt_dir, 'pytorch_model.bin'), map_location=torch.device('cpu'))
82 | model.load_state_dict(ckpt_loaded)
83 | return model
84 |
85 | assert val_metric == "Loss" and val_mode == "best"
86 | print(f'averaging best {avg_ckpt_num} checkpoints to the converged moment...')
87 |
88 | ckpt_dir = Path(ckpt_dir).expanduser().absolute()
89 | ckpt_path = ckpt_dir / 'checkpoints'
90 | val_metric_path = ckpt_dir / 'val_metric_summary.lst'
91 |
92 | val_metric_lst = load_metric_summary(val_metric_path, ckpt_path)
93 | val_metric_lst_sorted = sorted(val_metric_lst, key=lambda i: i[val_metric])
94 | best_val_metric_idx = val_metric_lst.index(val_metric_lst_sorted[0])
95 | val_metric_lst_out = val_metric_lst[
96 | best_val_metric_idx - avg_ckpt_num + 1 :
97 | best_val_metric_idx + 1
98 | ]
99 |
100 | return average_checkpoints(model, val_metric_lst_out)
101 |
--------------------------------------------------------------------------------