├── .gitignore ├── LICENSE ├── PATE.ipynb ├── README.md └── img └── pate.jpeg /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # data 107 | data/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Diego Fernando Muñoz C 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /PATE.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Private Aggregation of Teacher Ensembles (PATE)\n", 8 | "\n", 9 | "\n", 10 | "\n", 11 | "![PATE chart](img/pate.jpeg)" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "## Import libraries" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 1, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "import torch\n", 28 | "\n", 29 | "import numpy as np\n", 30 | "from torchvision import datasets\n", 31 | "import torchvision.transforms as transforms\n", 32 | "from torch.utils.data import Subset" 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "## Load the [Data](http://pytorch.org/docs/stable/torchvision/datasets.html)\n", 40 | "\n", 41 | "Downloading may take a few moments, and you should see your progress as the data is loading. You may also choose to change the `batch_size` if you want to load more data at a time." 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 2, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "# number of subprocesses to use for data loading\n", 51 | "num_workers = 0\n", 52 | "# how many samples per batch to load\n", 53 | "batch_size = 32\n", 54 | "\n", 55 | "# convert data to torch.FloatTensor\n", 56 | "transform = transforms.Compose([transforms.ToTensor(),\n", 57 | " transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n", 58 | "\n", 59 | "# choose the training and test datasets\n", 60 | "train_data = datasets.MNIST(root='data', train=True,\n", 61 | " download=True, transform=transform)\n", 62 | "test_data = datasets.MNIST(root='data', train=False,\n", 63 | " download=True, transform=transform)\n" 64 | ] 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "metadata": {}, 69 | "source": [ 70 | "Function for returning dataloaders for a specified number of teachers." 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 3, 76 | "metadata": {}, 77 | "outputs": [], 78 | "source": [ 79 | "# number of teachers to essemble\n", 80 | "num_teachers = 100\n", 81 | "\n", 82 | "def get_data_loaders(train_data, num_teachers = 10):\n", 83 | " teacher_loaders = []\n", 84 | " data_size = len(train_data) // num_teachers\n", 85 | "\n", 86 | " for i in range(num_teachers):\n", 87 | " indices = list(range(i*data_size, (i+1) *data_size))\n", 88 | " subset_data = Subset(train_data, indices)\n", 89 | " loader = torch.utils.data.DataLoader(subset_data, batch_size=batch_size, num_workers=num_workers)\n", 90 | " teacher_loaders.append(loader)\n", 91 | "\n", 92 | " return teacher_loaders\n", 93 | "\n", 94 | "teacher_loaders = get_data_loaders(train_data, num_teachers)" 95 | ] 96 | }, 97 | { 98 | "cell_type": "markdown", 99 | "metadata": {}, 100 | "source": [ 101 | "Define a train student set of 9000 examples and 1000 test examples" 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "execution_count": 4, 107 | "metadata": {}, 108 | "outputs": [], 109 | "source": [ 110 | "student_train_data = Subset(test_data, list(range(9000)))\n", 111 | "student_test_data = Subset(test_data, list(range(9000, 10000)))\n", 112 | "\n", 113 | "student_train_loader = torch.utils.data.DataLoader(student_train_data, batch_size=batch_size, \n", 114 | " num_workers=num_workers)\n", 115 | "student_test_loader = torch.utils.data.DataLoader(student_test_data, batch_size=batch_size, \n", 116 | " num_workers=num_workers)" 117 | ] 118 | }, 119 | { 120 | "cell_type": "markdown", 121 | "metadata": {}, 122 | "source": [ 123 | "## Defining models\n", 124 | "\n", 125 | "I'm going to define a single model for all the teachers, the analysis does not depends on the model" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": 5, 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "import torch.nn as nn\n", 135 | "import torch.nn.functional as F\n", 136 | "import torch.optim as optim\n", 137 | "\n", 138 | "class Net(nn.Module):\n", 139 | " def __init__(self):\n", 140 | " super(Net, self).__init__()\n", 141 | " self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n", 142 | " self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n", 143 | " self.conv2_drop = nn.Dropout2d()\n", 144 | " self.fc1 = nn.Linear(320, 50)\n", 145 | " self.fc2 = nn.Linear(50, 10)\n", 146 | "\n", 147 | " def forward(self, x):\n", 148 | " x = F.relu(F.max_pool2d(self.conv1(x), 2))\n", 149 | " x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n", 150 | " x = x.view(-1, 320)\n", 151 | " x = F.relu(self.fc1(x))\n", 152 | " x = F.dropout(x, training=self.training)\n", 153 | " x = self.fc2(x)\n", 154 | " return F.log_softmax(x)\n", 155 | " " 156 | ] 157 | }, 158 | { 159 | "cell_type": "code", 160 | "execution_count": 79, 161 | "metadata": {}, 162 | "outputs": [], 163 | "source": [ 164 | "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", 165 | "\n", 166 | "def train(model, trainloader, criterion, optimizer, epochs=10, print_every=120):\n", 167 | " model.to(device)\n", 168 | " steps = 0\n", 169 | " running_loss = 0\n", 170 | " for e in range(epochs):\n", 171 | " # Model in training mode, dropout is on\n", 172 | " model.train()\n", 173 | " for images, labels in trainloader:\n", 174 | " images, labels = images.to(device), labels.to(device)\n", 175 | " steps += 1\n", 176 | " \n", 177 | " optimizer.zero_grad()\n", 178 | " \n", 179 | " output = model.forward(images)\n", 180 | " loss = criterion(output, labels)\n", 181 | " loss.backward()\n", 182 | " optimizer.step()\n", 183 | " \n", 184 | " running_loss += loss.item()\n" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": 80, 190 | "metadata": {}, 191 | "outputs": [], 192 | "source": [ 193 | "def predict(model, dataloader):\n", 194 | " outputs = torch.zeros(0, dtype=torch.long).to(device)\n", 195 | " model.to(device)\n", 196 | " model.eval()\n", 197 | " for images, labels in dataloader:\n", 198 | " images, labels = images.to(device), labels.to(device)\n", 199 | " output = model.forward(images)\n", 200 | " ps = torch.argmax(torch.exp(output), dim=1)\n", 201 | " outputs = torch.cat((outputs, ps))\n", 202 | " \n", 203 | " return outputs \n" 204 | ] 205 | }, 206 | { 207 | "cell_type": "markdown", 208 | "metadata": {}, 209 | "source": [ 210 | "## Training all the teacher models\n", 211 | "\n", 212 | "Here we define and train the teachers" 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": 81, 218 | "metadata": {}, 219 | "outputs": [ 220 | { 221 | "name": "stdout", 222 | "output_type": "stream", 223 | "text": [ 224 | "Training teacher 1\n" 225 | ] 226 | }, 227 | { 228 | "name": "stderr", 229 | "output_type": "stream", 230 | "text": [ 231 | "/home/diego/anaconda3/envs/tf_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:21: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n" 232 | ] 233 | }, 234 | { 235 | "name": "stdout", 236 | "output_type": "stream", 237 | "text": [ 238 | "Training teacher 2\n", 239 | "Training teacher 3\n", 240 | "Training teacher 4\n", 241 | "Training teacher 5\n", 242 | "Training teacher 6\n", 243 | "Training teacher 7\n", 244 | "Training teacher 8\n", 245 | "Training teacher 9\n", 246 | "Training teacher 10\n", 247 | "Training teacher 11\n", 248 | "Training teacher 12\n", 249 | "Training teacher 13\n", 250 | "Training teacher 14\n", 251 | "Training teacher 15\n", 252 | "Training teacher 16\n", 253 | "Training teacher 17\n", 254 | "Training teacher 18\n", 255 | "Training teacher 19\n", 256 | "Training teacher 20\n", 257 | "Training teacher 21\n", 258 | "Training teacher 22\n", 259 | "Training teacher 23\n", 260 | "Training teacher 24\n", 261 | "Training teacher 25\n", 262 | "Training teacher 26\n", 263 | "Training teacher 27\n", 264 | "Training teacher 28\n", 265 | "Training teacher 29\n", 266 | "Training teacher 30\n", 267 | "Training teacher 31\n", 268 | "Training teacher 32\n", 269 | "Training teacher 33\n", 270 | "Training teacher 34\n", 271 | "Training teacher 35\n", 272 | "Training teacher 36\n", 273 | "Training teacher 37\n", 274 | "Training teacher 38\n", 275 | "Training teacher 39\n", 276 | "Training teacher 40\n", 277 | "Training teacher 41\n", 278 | "Training teacher 42\n", 279 | "Training teacher 43\n", 280 | "Training teacher 44\n", 281 | "Training teacher 45\n", 282 | "Training teacher 46\n", 283 | "Training teacher 47\n", 284 | "Training teacher 48\n", 285 | "Training teacher 49\n", 286 | "Training teacher 50\n", 287 | "Training teacher 51\n", 288 | "Training teacher 52\n", 289 | "Training teacher 53\n", 290 | "Training teacher 54\n", 291 | "Training teacher 55\n", 292 | "Training teacher 56\n", 293 | "Training teacher 57\n", 294 | "Training teacher 58\n", 295 | "Training teacher 59\n", 296 | "Training teacher 60\n", 297 | "Training teacher 61\n", 298 | "Training teacher 62\n", 299 | "Training teacher 63\n", 300 | "Training teacher 64\n", 301 | "Training teacher 65\n", 302 | "Training teacher 66\n", 303 | "Training teacher 67\n", 304 | "Training teacher 68\n", 305 | "Training teacher 69\n", 306 | "Training teacher 70\n", 307 | "Training teacher 71\n", 308 | "Training teacher 72\n", 309 | "Training teacher 73\n", 310 | "Training teacher 74\n", 311 | "Training teacher 75\n", 312 | "Training teacher 76\n", 313 | "Training teacher 77\n", 314 | "Training teacher 78\n", 315 | "Training teacher 79\n", 316 | "Training teacher 80\n", 317 | "Training teacher 81\n", 318 | "Training teacher 82\n", 319 | "Training teacher 83\n", 320 | "Training teacher 84\n", 321 | "Training teacher 85\n", 322 | "Training teacher 86\n", 323 | "Training teacher 87\n", 324 | "Training teacher 88\n", 325 | "Training teacher 89\n", 326 | "Training teacher 90\n", 327 | "Training teacher 91\n", 328 | "Training teacher 92\n", 329 | "Training teacher 93\n", 330 | "Training teacher 94\n", 331 | "Training teacher 95\n", 332 | "Training teacher 96\n", 333 | "Training teacher 97\n", 334 | "Training teacher 98\n", 335 | "Training teacher 99\n", 336 | "Training teacher 100\n" 337 | ] 338 | } 339 | ], 340 | "source": [ 341 | "# Instantiate and train the models for each teacher\n", 342 | "def train_models(num_teachers):\n", 343 | " models = []\n", 344 | " for t in range(num_teachers):\n", 345 | " print(\"Training teacher {}\".format(t+1))\n", 346 | " model = Net()\n", 347 | " criterion = nn.NLLLoss()\n", 348 | " optimizer = optim.Adam(model.parameters(), lr=0.003)\n", 349 | " train(model, teacher_loaders[t], criterion, optimizer)\n", 350 | " models.append(model)\n", 351 | " return models\n", 352 | "\n", 353 | "models = train_models(num_teachers) " 354 | ] 355 | }, 356 | { 357 | "cell_type": "markdown", 358 | "metadata": {}, 359 | "source": [ 360 | "## Aggregated teacher\n", 361 | "\n", 362 | "This function predict the labels from all the dataset in each of the teachers, then return all the predictions and the maximum votation after adding laplacian noise" 363 | ] 364 | }, 365 | { 366 | "cell_type": "code", 367 | "execution_count": 82, 368 | "metadata": {}, 369 | "outputs": [], 370 | "source": [ 371 | "import numpy as np" 372 | ] 373 | }, 374 | { 375 | "cell_type": "code", 376 | "execution_count": 83, 377 | "metadata": {}, 378 | "outputs": [], 379 | "source": [ 380 | "# define epsilon\n", 381 | "epsilon = 0.2" 382 | ] 383 | }, 384 | { 385 | "cell_type": "markdown", 386 | "metadata": {}, 387 | "source": [ 388 | "# Aggregated teacher\n", 389 | "\n", 390 | "This function makes the predictions in all the teachers, count the votes and add noise, then returns the votation and the argmax results." 391 | ] 392 | }, 393 | { 394 | "cell_type": "code", 395 | "execution_count": 84, 396 | "metadata": {}, 397 | "outputs": [], 398 | "source": [ 399 | "def aggregated_teacher(models, data_loader, epsilon):\n", 400 | " preds = torch.torch.zeros((len(models), 9000), dtype=torch.long)\n", 401 | " for i, model in enumerate(models):\n", 402 | " results = predict(model, data_loader)\n", 403 | " preds[i] = results\n", 404 | " \n", 405 | " labels = np.array([]).astype(int)\n", 406 | " for image_preds in np.transpose(preds):\n", 407 | " label_counts = np.bincount(image_preds, minlength=10)\n", 408 | " beta = 1 / epsilon\n", 409 | "\n", 410 | " for i in range(len(label_counts)):\n", 411 | " label_counts[i] += np.random.laplace(0, beta, 1)\n", 412 | "\n", 413 | " new_label = np.argmax(label_counts)\n", 414 | " labels = np.append(labels, new_label)\n", 415 | " \n", 416 | " return preds.numpy(), labels" 417 | ] 418 | }, 419 | { 420 | "cell_type": "code", 421 | "execution_count": 85, 422 | "metadata": { 423 | "scrolled": true 424 | }, 425 | "outputs": [ 426 | { 427 | "name": "stderr", 428 | "output_type": "stream", 429 | "text": [ 430 | "/home/diego/anaconda3/envs/tf_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:21: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n" 431 | ] 432 | } 433 | ], 434 | "source": [ 435 | "teacher_models = models\n", 436 | "preds, student_labels = aggregated_teacher(teacher_models, student_train_loader, epsilon)" 437 | ] 438 | }, 439 | { 440 | "cell_type": "markdown", 441 | "metadata": {}, 442 | "source": [ 443 | "# PATE Analysis\n", 444 | "\n", 445 | "Perform PATE analysis and show the results" 446 | ] 447 | }, 448 | { 449 | "cell_type": "code", 450 | "execution_count": 87, 451 | "metadata": {}, 452 | "outputs": [ 453 | { 454 | "name": "stdout", 455 | "output_type": "stream", 456 | "text": [ 457 | "Data Independent Epsilon: 1451.5129254649705\n", 458 | "Data Dependent Epsilon: 15.661427783915407\n" 459 | ] 460 | } 461 | ], 462 | "source": [ 463 | "from syft.frameworks.torch.differential_privacy import pate\n", 464 | "\n", 465 | "data_dep_eps, data_ind_eps = pate.perform_analysis(teacher_preds=preds, indices=student_labels, noise_eps=epsilon, delta=1e-5)\n", 466 | "print(\"Data Independent Epsilon:\", data_ind_eps)\n", 467 | "print(\"Data Dependent Epsilon:\", data_dep_eps)" 468 | ] 469 | }, 470 | { 471 | "cell_type": "markdown", 472 | "metadata": {}, 473 | "source": [ 474 | "# Training the student\n", 475 | "\n", 476 | "Now we will train the student with the aggregated teacher labels" 477 | ] 478 | }, 479 | { 480 | "cell_type": "code", 481 | "execution_count": 88, 482 | "metadata": {}, 483 | "outputs": [], 484 | "source": [ 485 | "def student_loader(student_train_loader, labels):\n", 486 | " for i, (data, _) in enumerate(iter(student_train_loader)):\n", 487 | " yield data, torch.from_numpy(labels[i*len(data):(i+1)*len(data)])" 488 | ] 489 | }, 490 | { 491 | "cell_type": "code", 492 | "execution_count": null, 493 | "metadata": {}, 494 | "outputs": [ 495 | { 496 | "name": "stderr", 497 | "output_type": "stream", 498 | "text": [ 499 | "/home/diego/anaconda3/envs/tf_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:21: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n" 500 | ] 501 | }, 502 | { 503 | "name": "stdout", 504 | "output_type": "stream", 505 | "text": [ 506 | "Epoch: 1/10.. Training Loss: 0.392.. Test Loss: 1.955.. Test Accuracy: 0.487\n", 507 | "Epoch: 1/10.. Training Loss: 0.287.. Test Loss: 1.063.. Test Accuracy: 0.719\n", 508 | "Epoch: 1/10.. Training Loss: 0.196.. Test Loss: 0.700.. Test Accuracy: 0.784\n", 509 | "Epoch: 1/10.. Training Loss: 0.127.. Test Loss: 0.570.. Test Accuracy: 0.836\n", 510 | "Epoch: 1/10.. Training Loss: 0.110.. Test Loss: 0.376.. Test Accuracy: 0.890\n", 511 | "Epoch: 2/10.. Training Loss: 0.121.. Test Loss: 0.380.. Test Accuracy: 0.902\n", 512 | "Epoch: 2/10.. Training Loss: 0.097.. Test Loss: 0.316.. Test Accuracy: 0.904\n", 513 | "Epoch: 2/10.. Training Loss: 0.087.. Test Loss: 0.317.. Test Accuracy: 0.899\n", 514 | "Epoch: 2/10.. Training Loss: 0.081.. Test Loss: 0.292.. Test Accuracy: 0.908\n" 515 | ] 516 | } 517 | ], 518 | "source": [ 519 | "student_model = Net()\n", 520 | "criterion = nn.NLLLoss()\n", 521 | "optimizer = optim.Adam(student_model.parameters(), lr=0.001)\n", 522 | "epochs = 10\n", 523 | "student_model.to(device)\n", 524 | "steps = 0\n", 525 | "running_loss = 0\n", 526 | "for e in range(epochs):\n", 527 | " # Model in training mode, dropout is on\n", 528 | " student_model.train()\n", 529 | " train_loader = student_loader(student_train_loader, student_labels)\n", 530 | " for images, labels in train_loader:\n", 531 | " images, labels = images.to(device), labels.to(device)\n", 532 | " steps += 1\n", 533 | "\n", 534 | " optimizer.zero_grad()\n", 535 | " output = student_model.forward(images)\n", 536 | " loss = criterion(output, labels)\n", 537 | " loss.backward()\n", 538 | " optimizer.step()\n", 539 | "\n", 540 | " running_loss += loss.item()\n", 541 | " if steps % 50 == 0:\n", 542 | " test_loss = 0\n", 543 | " accuracy = 0\n", 544 | " student_model.eval()\n", 545 | " with torch.no_grad():\n", 546 | " for images, labels in student_test_loader:\n", 547 | " images, labels = images.to(device), labels.to(device)\n", 548 | " log_ps = student_model(images)\n", 549 | " test_loss += criterion(log_ps, labels).item()\n", 550 | " \n", 551 | " # Accuracy\n", 552 | " ps = torch.exp(log_ps)\n", 553 | " top_p, top_class = ps.topk(1, dim=1)\n", 554 | " equals = top_class == labels.view(*top_class.shape)\n", 555 | " accuracy += torch.mean(equals.type(torch.FloatTensor))\n", 556 | " student_model.train()\n", 557 | " print(\"Epoch: {}/{}.. \".format(e+1, epochs),\n", 558 | " \"Training Loss: {:.3f}.. \".format(running_loss/len(student_train_loader)),\n", 559 | " \"Test Loss: {:.3f}.. \".format(test_loss/len(student_test_loader)),\n", 560 | " \"Test Accuracy: {:.3f}\".format(accuracy/len(student_test_loader)))\n", 561 | " running_loss = 0" 562 | ] 563 | }, 564 | { 565 | "cell_type": "code", 566 | "execution_count": 73, 567 | "metadata": {}, 568 | "outputs": [ 569 | { 570 | "name": "stdout", 571 | "output_type": "stream", 572 | "text": [ 573 | "Test Loss: 15.074.. Test Accuracy: 27.719\n" 574 | ] 575 | }, 576 | { 577 | "name": "stderr", 578 | "output_type": "stream", 579 | "text": [ 580 | "/home/diego/anaconda3/envs/tf_gpu/lib/python3.6/site-packages/ipykernel_launcher.py:21: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n" 581 | ] 582 | } 583 | ], 584 | "source": [ 585 | "t1_model = models[99]\n", 586 | "t1_model.eval()\n", 587 | "with torch.no_grad():\n", 588 | " test_loss = 0\n", 589 | " accuracy = 0\n", 590 | " for images, labels in student_test_loader:\n", 591 | " images, labels = images.to(device), labels.to(device)\n", 592 | " log_ps = t1_model(images)\n", 593 | " test_loss += criterion(log_ps, labels).item()\n", 594 | "\n", 595 | " # Accuracy\n", 596 | " ps = torch.exp(log_ps)\n", 597 | " top_p, top_class = ps.topk(1, dim=1)\n", 598 | " equals = top_class == labels.view(*top_class.shape)\n", 599 | " accuracy += torch.mean(equals.type(torch.FloatTensor))\n", 600 | " t1_model.train()\n", 601 | " print(\"Test Loss: {:.3f}.. \".format(test_loss),\n", 602 | " \"Test Accuracy: {:.3f}\".format(accuracy))" 603 | ] 604 | }, 605 | { 606 | "cell_type": "code", 607 | "execution_count": null, 608 | "metadata": {}, 609 | "outputs": [], 610 | "source": [] 611 | }, 612 | { 613 | "cell_type": "code", 614 | "execution_count": null, 615 | "metadata": {}, 616 | "outputs": [], 617 | "source": [] 618 | } 619 | ], 620 | "metadata": { 621 | "kernelspec": { 622 | "display_name": "Python (tf_gpu)", 623 | "language": "python", 624 | "name": "tf_gpu" 625 | }, 626 | "language_info": { 627 | "codemirror_mode": { 628 | "name": "ipython", 629 | "version": 3 630 | }, 631 | "file_extension": ".py", 632 | "mimetype": "text/x-python", 633 | "name": "python", 634 | "nbconvert_exporter": "python", 635 | "pygments_lexer": "ipython3", 636 | "version": "3.6.8" 637 | } 638 | }, 639 | "nbformat": 4, 640 | "nbformat_minor": 2 641 | } 642 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pate_torch 2 | Implementation of PATE technique described in https://arxiv.org/pdf/1610.05755.pdf 3 | -------------------------------------------------------------------------------- /img/pate.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimun/pate_torch/1bd07360eaccb2734cbba4f275c7de91b7a6d5a6/img/pate.jpeg --------------------------------------------------------------------------------