├── models ├── __init__.py ├── model.py ├── predictor.py └── attacker.py ├── img └── procedure.png ├── data ├── 237-134500-0001.flac └── benchmark.txt ├── requirements.txt ├── conf ├── strategy │ └── predictor.yaml └── config.yaml ├── main.py ├── LICENSE └── README.md /models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /img/procedure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mariegold/NP-Attack/HEAD/img/procedure.png -------------------------------------------------------------------------------- /data/237-134500-0001.flac: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mariegold/NP-Attack/HEAD/data/237-134500-0001.flac -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch==1.10.1 2 | torchaudio==0.10.1 3 | librosa 4 | matplotlib==3.4.3 5 | numpy==1.21.2 6 | omegaconf==2.1.1 7 | pandas==1.3.2 8 | SoundFile==0.10.3.post1 9 | speechbrain==0.5.9 10 | hydra-core 11 | jiwer 12 | tqdm -------------------------------------------------------------------------------- /conf/strategy/predictor.yaml: -------------------------------------------------------------------------------- 1 | name: predictor 2 | n_points: 64 3 | norm: inf 4 | 5 | predictor: 6 | norm: ${strategy.norm} 7 | n_layers: 4 8 | batch_size: 32 9 | epochs: 300 10 | search_step: 200 11 | sample_size: 8 12 | lr: 0.0001 -------------------------------------------------------------------------------- /conf/config.yaml: -------------------------------------------------------------------------------- 1 | defaults: 2 | - strategy: predictor 3 | - _self_ 4 | 5 | dir: ${hydra:runtime.cwd}/data # /Users/MB/Desktop/black-box-asr/data 6 | attacker: ${strategy.name} 7 | 8 | sr: 16000 9 | budget: 5000 10 | eps_perb: 0.00 # 0.01 11 | min_wer: 1e-9 12 | seed: 1234 13 | upper_lim: 2.0 14 | norm: inf 15 | 16 | asr: 17 | name: asr-transformer-transformerlm-librispeech 18 | source: "speechbrain/${asr.name}" 19 | savedir: "${dir}/pretrained_models/${asr.name}" 20 | 21 | wave_file: "${dir}/237-134500-0001.flac" 22 | out: False 23 | out_file: res.out 24 | 25 | hydra: 26 | verbose: false 27 | run: 28 | dir: logs/ -------------------------------------------------------------------------------- /models/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from speechbrain.pretrained import EncoderDecoderASR 3 | 4 | 5 | class ASR: 6 | def __init__(self, hp, device) -> None: 7 | self.hp = hp 8 | self.model = EncoderDecoderASR.from_hparams( 9 | source=hp.source, 10 | savedir=hp.savedir, 11 | run_opts={"device": str(device)}, 12 | freeze_params=True 13 | ) 14 | self.b_len = torch.tensor([1.]).to(device) 15 | 16 | def transcribe(self, wave): 17 | wave = torch.tensor(wave).to(self.model.device).unsqueeze(0) 18 | pred = self.model.transcribe_batch(wave, self.b_len)[0][0] 19 | return pred 20 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import hydra 2 | import soundfile as sf 3 | import torch 4 | from omegaconf import DictConfig, OmegaConf 5 | 6 | from models.attacker import NPAttacker 7 | 8 | 9 | @hydra.main(config_path="conf", config_name="config") 10 | def main(args: DictConfig): 11 | print(OmegaConf.to_yaml(args)) 12 | torch.manual_seed(args.seed) 13 | 14 | model = NPAttacker(args) 15 | wave = model.attack(args.wave_file) 16 | 17 | if wave is not None: 18 | # retest the output 19 | out = args.strategy.name + str(args.seed) + '.wav' 20 | sf.write(out, wave, args.sr) 21 | print(model.asr.model.transcribe_file(out)) 22 | 23 | if args.out: 24 | model.eval_attack(wave) 25 | 26 | if __name__ == '__main__': 27 | main() 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Marie Biolková 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /data/benchmark.txt: -------------------------------------------------------------------------------- 1 | 6930-81414-0017 2 | 5142-36586-0004 3 | 1188-133604-0013 4 | 5142-33396-0061 5 | 8463-294828-0030 6 | 2830-3980-0059 7 | 2830-3980-0014 8 | 3575-170457-0031 9 | 1580-141083-0028 10 | 121-121726-0008 11 | 1995-1836-0002 12 | 5683-32865-0015 13 | 5142-33396-0047 14 | 5142-33396-0026 15 | 672-122797-0029 16 | 4446-2271-0015 17 | 7127-75947-0011 18 | 121-127105-0018 19 | 121-121726-0012 20 | 672-122797-0039 21 | 2830-3980-0034 22 | 4446-2275-0023 23 | 1995-1826-0003 24 | 1580-141084-0017 25 | 6829-68771-0028 26 | 1580-141084-0012 27 | 4970-29093-0017 28 | 61-70968-0048 29 | 8555-292519-0013 30 | 237-134500-0035 31 | 4992-41797-0003 32 | 5142-36586-0001 33 | 260-123286-0024 34 | 121-127105-0008 35 | 6829-68769-0041 36 | 6829-68771-0003 37 | 6930-81414-0021 38 | 7127-75946-0021 39 | 2094-142345-0031 40 | 1995-1837-0028 41 | 7729-102255-0034 42 | 8463-294828-0024 43 | 4970-29093-0008 44 | 3729-6852-0025 45 | 1580-141083-0014 46 | 5683-32866-0026 47 | 5142-33396-0014 48 | 6829-68769-0043 49 | 1580-141083-0017 50 | 2094-142345-0047 51 | 5105-28240-0002 52 | 260-123440-0017 53 | 8455-210777-0043 54 | 2094-142345-0007 55 | 1995-1826-0004 56 | 5639-40744-0011 57 | 4970-29095-0005 58 | 260-123288-0006 59 | 2830-3979-0012 60 | 5105-28240-0014 61 | 8463-287645-0008 62 | 1995-1837-0013 63 | 237-134500-0017 64 | 4507-16021-0015 65 | 908-31957-0018 66 | 1995-1837-0002 67 | 121-127105-0017 68 | 6829-68769-0016 69 | 3729-6852-0021 70 | 1188-133604-0042 71 | 4446-2275-0032 72 | 61-70968-0058 73 | 5683-32879-0008 74 | 4970-29095-0011 75 | 1995-1837-0008 76 | 7021-79730-0000 77 | 1320-122612-0014 78 | 2094-142345-0044 79 | 5142-36377-0019 80 | 1580-141083-0052 81 | 6829-68769-0013 82 | 61-70970-0017 83 | 3729-6852-0031 84 | 8455-210777-0054 85 | 237-126133-0018 86 | 6930-75918-0007 87 | 121-127105-0015 88 | 2830-3980-0002 89 | 672-122797-0054 90 | 5142-33396-0029 91 | 6930-81414-0025 92 | 61-70968-0012 93 | 8555-284449-0006 94 | 4446-2273-0027 95 | 6930-81414-0003 96 | 2094-142345-0025 97 | 1221-135766-0015 98 | 5105-28240-0020 99 | 1995-1837-0005 100 | 3575-170457-0049 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NP-Attack: Neural Predictor for Black-Box Adversarial Attacks on Speech Recognition 2 | 3 | [![Python 3.8](https://img.shields.io/badge/python-3.8-blue.svg)](https://www.python.org/downloads/release/python-380/) 4 | [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) 5 | [![arXiv](https://img.shields.io/badge/arXiv-2203.09849-b31b1b.svg)](https://arxiv.org/abs/2203.09849) 6 | 7 | ## Abstract 8 | 9 | Recent works have revealed the vulnerability of automatic speech recognition (ASR) models to adversarial examples (AEs), *i.e.*, small perturbations that cause an error in the transcription of the audio signal. Studying audio adversarial attacks is the first step towards robust ASR. Despite the significant progress made in attacking audio examples, the black-box attack remains challenging because only the hard-label information of transcriptions is provided. Due to this limited information, existing black-box methods often require an excessive number of queries to attack a single audio example. In this paper, we introduce NP-Attack, a neural predictor-based method, which progressively evolves the search towards the small adversarial perturbation. Given a perturbation direction, our neural predictor directly estimates the smallest perturbation that causes a mistranscription. In doing so, NP-Attack accurately learns promising perturbation directions of the AE via gradient-based optimization. Experimental results show that NP-Attack achieves competitive results with other state-of-the-art black-box adversarial attacks while requiring a significantly smaller number of queries. 10 | 11 | procedure 12 | 13 | ## Requirements 14 | 15 | ```python 16 | pip install -r requirements.txt 17 | ``` 18 | 19 | ## Usage 20 | 21 | To run the NP-attack, either modify `conf/config.yaml` directly, or via command line: 22 | 23 | ```python 24 | python main.py [wave_file=] [budget=] [eps_perb=] 25 | ``` 26 | 27 | For more information, refer to [Hydra documentation](https://hydra.cc/docs/intro/). 28 | 29 | ## Data 30 | 31 | The LibriSpeech test-clean dataset can be downloaded: 32 | 33 | ``` 34 | wget https://www.openslr.org/resources/12/test-clean.tar.gz 35 | tar -xzvf test-clean.tar.gz 36 | ``` 37 | 38 | The list of samples used for benchmarking can be found in `data/benchmark.txt`. 39 | 40 | ## Reference 41 | 42 | ``` 43 | @article{biolkova2022npattack, 44 | title={Neural Predictor for Black-Box Adversarial Attacks on Speech Recognition}, 45 | author={Biolková Marie and Nguyen Bac}, 46 | journal={arXiv preprint arXiv:2203.09849}, 47 | year={2022} 48 | } 49 | ``` 50 | -------------------------------------------------------------------------------- /models/predictor.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn.functional as F 4 | from torch import nn 5 | from torch.nn.utils import weight_norm 6 | from torch.utils.data import DataLoader, TensorDataset 7 | from tqdm import tqdm 8 | 9 | 10 | def wn_conv1d(*args, **kwargs): 11 | return weight_norm(nn.Conv1d(*args, **kwargs)) 12 | 13 | 14 | def weight_reset(m): 15 | if isinstance(m, nn.Linear): 16 | m.reset_parameters() 17 | 18 | 19 | class Audio2Spec(nn.Module): 20 | r"""Waveform to spectrogram.""" 21 | 22 | def __init__( 23 | self, 24 | n_fft=1024, 25 | hop_length=256, 26 | win_length=1024, 27 | ): 28 | super().__init__() 29 | window = torch.hann_window(win_length).float() 30 | self.register_buffer("window", window) 31 | self.n_fft = n_fft 32 | self.hop_length = hop_length 33 | self.win_length = win_length 34 | 35 | def forward(self, audio): 36 | p = (self.n_fft - self.hop_length) // 2 37 | audio = F.pad(audio, (p, p), "reflect").squeeze(1) 38 | re, im = torch.stft( 39 | audio, 40 | n_fft=self.n_fft, 41 | hop_length=self.hop_length, 42 | win_length=self.win_length, 43 | window=self.window, 44 | return_complex=False, 45 | ).unbind(-1) 46 | return re ** 2 + im ** 2 47 | # return torch.sqrt(torch.clamp(re ** 2 + im ** 2, min=1e-9)) 48 | 49 | 50 | class MLP(nn.Module): 51 | def __init__(self, hp, device): 52 | super().__init__() 53 | self.hp = hp 54 | self.device = device 55 | 56 | modules = [] 57 | for p, h in zip(hp.hiddens, hp.hiddens[1:]): 58 | modules.append(nn.Linear(p, h)) 59 | modules.append(nn.ReLU()) 60 | modules.append(nn.Linear(hp.hiddens[-1], 1)) 61 | 62 | self.net = nn.Sequential(*modules) 63 | 64 | def forward(self, x): 65 | x = self.net(x) 66 | return x 67 | 68 | 69 | class CNN(nn.Module): 70 | def __init__(self, hp, device): 71 | super().__init__() 72 | self.hp = hp 73 | self.device = device 74 | 75 | self.norm = np.inf if hp.norm == 'inf' else hp.norm 76 | 77 | dim = 32 78 | self.spec = Audio2Spec() 79 | modules = [wn_conv1d(513, dim, 3, padding=1)] 80 | for _ in range(hp.n_layers): 81 | modules.append(nn.LeakyReLU(0.2)) 82 | modules.append(wn_conv1d(dim, min(512, dim * 2), 3, padding=1)) 83 | modules.append(nn.AvgPool1d(2)) 84 | dim = min(512, dim * 2) 85 | 86 | self.net = nn.Sequential(*modules) 87 | self.proj = nn.Linear(512, 1) 88 | 89 | def forward(self, x): 90 | x = x / torch.norm(x, p=self.norm, keepdim=True, dim=1) 91 | x = x.reshape(x.size(0), 1, -1) 92 | x = self.net(self.spec(x)) 93 | x = torch.mean(x, dim=-1) 94 | x = self.proj(x) 95 | return x 96 | 97 | 98 | class Predictor(nn.Module): 99 | def __init__(self, hp, device): 100 | super().__init__() 101 | self.hp = hp 102 | self.device = device 103 | self.net = CNN(hp, device) 104 | 105 | def forward(self, x): 106 | return self.net(x) 107 | 108 | def fit(self, x_train, y_train): 109 | hp = self.hp 110 | device = self.device 111 | self.inp_dim = x_train.shape[-1] 112 | 113 | self.net.apply(weight_reset) 114 | self.to(device) 115 | self.train() 116 | 117 | samples = TensorDataset(torch.tensor(x_train).float(), torch.tensor(y_train).float()) 118 | dataloader = DataLoader(samples, batch_size=hp.batch_size, 119 | drop_last=False, shuffle=True) 120 | criteria = nn.MSELoss() 121 | optimizer = torch.optim.Adam(self.parameters(), lr=hp.lr) 122 | scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=0.99) 123 | 124 | for epoch in (pbar := tqdm(range(hp.epochs))): 125 | mae, size = 0, 0 126 | for x, y in dataloader: 127 | loss = criteria(self(x.to(device)), torch.log(y.to(device))) 128 | 129 | optimizer.zero_grad() 130 | loss.backward() 131 | optimizer.step() 132 | 133 | mae += loss.item()*x.shape[0] 134 | size += x.shape[0] 135 | 136 | scheduler.step() 137 | pbar.set_description(f"Loss={mae/size:.4f}") 138 | 139 | return mae / size 140 | 141 | def optim_inputs(self): 142 | self.to(self.device) 143 | self.eval() 144 | 145 | x = torch.FloatTensor(self.hp.sample_size, self.inp_dim).uniform_(-1.0, 1.0).to(self.device) 146 | x.requires_grad_(True) 147 | optimizer = torch.optim.Adam([x], lr=0.01) 148 | 149 | for _ in (pbar := tqdm(range(self.hp.search_step))): 150 | optimizer.zero_grad() 151 | 152 | y_pred = self(x) 153 | loss = torch.mean(y_pred) 154 | loss.backward() 155 | 156 | optimizer.step() 157 | 158 | pbar.set_description(f'avg-min={torch.mean(torch.exp(y_pred)).item():.4f}') 159 | 160 | return x.detach().cpu().numpy(), y_pred.detach().cpu().numpy() 161 | -------------------------------------------------------------------------------- /models/attacker.py: -------------------------------------------------------------------------------- 1 | import librosa as lr 2 | import numpy as np 3 | import torch 4 | from jiwer import wer 5 | from pathlib import Path 6 | 7 | from .model import ASR 8 | from .predictor import Predictor 9 | 10 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 11 | 12 | class NPAttacker: 13 | def __init__(self, hp): 14 | self.hp = hp 15 | self.asr = ASR(hp.asr, device) 16 | self.rng = np.random.RandomState(hp.seed) 17 | self.norm = np.inf if hp.norm == 'inf' else None 18 | self.sample_id = Path(hp.wave_file).stem 19 | self.pre = Predictor(hp.strategy.predictor, device) 20 | 21 | def attack(self, wave_file): 22 | """ Finds a minimal adversarial perturbation to an audio sample. 23 | 24 | Args: 25 | wav_file (str): Path to the original audio file. 26 | 27 | Returns: 28 | np.array: Adversarial example. 29 | """ 30 | wave = lr.load(wave_file, sr=self.hp.sr)[0].astype(np.float32) 31 | 32 | # set the ground-truth 33 | self.wave = wave 34 | self.label = self.asr.transcribe(wave) 35 | self.dim_attack = len(wave) 36 | self.num_queries = 0 37 | hp = self.hp 38 | success = False 39 | 40 | # warmup for the predictor 41 | x_train = self.rng.uniform(-1.0, 1.0, size=(hp.strategy.n_points, self.dim_attack)) 42 | y_train = np.full((hp.strategy.n_points, 1), np.inf, dtype=np.float32) 43 | 44 | curr_queries = self.num_queries 45 | 46 | for i in range(hp.strategy.n_points): 47 | d = self.b_dist(x_train[i,:]) 48 | if self.num_queries < hp.budget: 49 | curr_queries = self.num_queries 50 | y_train[i] = d 51 | print(f"queries={self.num_queries}" 52 | f"\tperb={y_train.min():.4f}") 53 | else: 54 | break 55 | if d <= hp.eps_perb: 56 | break 57 | 58 | # search the best theta 59 | while self.num_queries < hp.budget and y_train.min() > hp.eps_perb: 60 | self.pre.fit(x_train, y_train) 61 | x_new, y_pred = self.pre.optim_inputs() 62 | 63 | y_new = np.full((hp.strategy.predictor.sample_size, 1), np.inf, dtype=np.float32) 64 | for i in range(hp.strategy.predictor.sample_size): 65 | y_new[i] = self.b_dist(x_new[i,:]) 66 | if y_new[i] <= hp.eps_perb: 67 | success = True # avoid adding random points 68 | break 69 | 70 | test = np.mean((np.log(y_new) - y_pred)**2) 71 | 72 | x_rand = self.rng.uniform(-1.0, 1.0, size=(2, x_train.shape[-1])) 73 | y_rand = np.full((2, 1), np.inf, dtype=np.float32) 74 | if not success: # add a few random candidates 75 | for i in range(2): 76 | y_rand[i] = self.b_dist(x_rand[i,:]) 77 | if y_rand[i] <= hp.eps_perb: 78 | break 79 | 80 | if self.num_queries <= hp.budget: 81 | curr_queries = self.num_queries 82 | 83 | x_train = np.concatenate((x_train, x_new, x_rand)) 84 | y_train = np.concatenate((y_train, y_new, y_rand)) 85 | 86 | print(f"queries={curr_queries}" 87 | f"\tperb={y_train.min():.4f}", 88 | f"\tL-test={test:.4f}") 89 | else: 90 | break 91 | 92 | index = np.argmin(y_train.flatten()) 93 | theta = self.get_theta(x_train[index]) 94 | self.num_queries = curr_queries 95 | 96 | return self.get_wave(theta, y_train[index][0]) 97 | 98 | def get_theta(self, x): 99 | """ Normalizes the perturbation. 100 | 101 | Args: 102 | x (np.array): Perturbation direction. 103 | 104 | Returns: 105 | np.array: Normalized direction. 106 | """ 107 | return x / np.linalg.norm(x, ord=self.norm) 108 | 109 | def get_wave(self, theta, mag): 110 | """ Applies a perturbation to the original sample. 111 | 112 | Args: 113 | theta (np.array): Perturbation direction, normalized. 114 | mag (float): Perturbation magnitude. 115 | 116 | Returns: 117 | np.array: Perturbed sample clipped to range. 118 | """ 119 | return np.clip(self.wave + theta*mag, -1.0, 1.0).astype(np.float32) 120 | 121 | def query(self, theta, mag): 122 | """ Queries the model with a perturbed sample. 123 | 124 | Args: 125 | theta (np.array): Perturbation direction, normalized. 126 | mag (float): Perturbation magnitude. 127 | 128 | Returns: 129 | bool: Attack success. 130 | """ 131 | self.num_queries += 1 132 | wave = self.get_wave(theta, mag) 133 | pred = self.asr.transcribe(wave.astype(np.float32)) 134 | 135 | return wer(self.label, pred) > self.hp.min_wer 136 | 137 | def b_dist(self, x, tol=1e-4, incr=0.1): 138 | """Returns the minimum distance to the decision boundary. 139 | 140 | Args: 141 | x (np.ndarray): Search direction, normalized. 142 | tol (float): Precision tolerance for binary search. 143 | Defaults to 1e-4. 144 | incr (float): Step size for finding the binary search bound. 145 | Defaults to 0.1. 146 | 147 | Returns: 148 | float: Minimum distance. 149 | """ 150 | upper=self.hp.upper_lim 151 | theta = self.get_theta(x) 152 | hi = incr 153 | while not self.query(theta, hi) and hi < upper: 154 | hi += incr 155 | 156 | if hi >= upper: 157 | return 1e9 158 | 159 | lo = hi - incr 160 | while hi - lo > tol and hi < upper: 161 | mid = (lo + hi) / 2 162 | if self.query(theta, mid): 163 | hi = mid 164 | else: 165 | lo = mid 166 | 167 | return hi 168 | 169 | def eval_attack(self, ae): 170 | """ Exports the results of the attack. 171 | 172 | Args: 173 | ae (np.array): Adversarial example. 174 | """ 175 | try: 176 | diff = ae - self.wave 177 | except NameError: 178 | "Run the attack first!" 179 | 180 | linf = np.linalg.norm(diff, ord=np.inf) 181 | orig_linf = np.linalg.norm(self.wave, ord=np.inf) 182 | snr_linf = 20*np.log10(orig_linf/linf) 183 | 184 | out_list = [self.sample_id, str(self.num_queries), 185 | str(linf), str(snr_linf)] 186 | with open(self.hp.out_file, 'a') as f: 187 | f.write(",".join(out_list)) 188 | f.write('\n') 189 | 190 | --------------------------------------------------------------------------------