├── .gitignore ├── 01_Intro_to_Time_Series_Classification.ipynb ├── 02_UCR_TCS.ipynb ├── 03_New_Time_Series_Data_Augmentation.ipynb ├── 04_TS_Multivariate_Scaling_LSST.ipynb ├── 05_ROCKET_a_new_SOTA_classifier.ipynb ├── 06_TS_transforms-RandAugment-UPDATED.ipynb ├── README.md ├── fastai_timeseries ├── __init__.py └── exp │ ├── __init__.py │ ├── nb_ColorfulDim.py │ ├── nb_ImageDataAugmentation.py │ ├── nb_Initialization.py │ ├── nb_NewDataAugmentation.py │ ├── nb_Optimizers.py │ ├── nb_TSBasicData.py │ ├── nb_TSCallbacks.py │ ├── nb_TSCharts.py │ ├── nb_TSDataAugmentation.py │ ├── nb_TSDatasets.py │ ├── nb_TSImageData.py │ ├── nb_TSTrain.py │ ├── nb_TSUtilities.py │ └── rocket_functions.py ├── images ├── LSST_scalling.jpg ├── LSST_scalling2.jpg ├── MixMatch_comparison_performance.jpg ├── MixMatch_time_comparison.jpg ├── ROCKET_InceptionTime_multi.png ├── Rocket.svg ├── TSAugmentation.png ├── Time_ROCKET_comp.png ├── accuracy_blf.jpg ├── beef_mixup.jpg ├── sel_samples_blf.jpg ├── time_blf.jpg ├── tweet_blf.jpg └── valid_loss_blf.jpg ├── old ├── 06_TS_transforms.ipynb └── nb_TSDataAugmentation.py └── torchtimeseries └── models ├── FCN.py ├── InceptionTime.py ├── ROCKET.py ├── ResCNN.py ├── ResNet.py ├── __init__.py └── layers.py /.gitignore: -------------------------------------------------------------------------------- 1 | #*.jpg 2 | *.npy 3 | *.arff 4 | #*.png 5 | *.pyc 6 | *.pth 7 | *.txt 8 | *.csv 9 | 10 | 11 | # Packages # 12 | ########### 13 | *.7z 14 | *.dmg 15 | *.gz 16 | *.iso 17 | *.jar 18 | *.rar 19 | *.tar 20 | *.zip 21 | 22 | # Folders # 23 | ########## 24 | coursev3_2/ 25 | data/models/ 26 | data/UCR/ 27 | data/trading/images 28 | sequentia/models/ 29 | tests/ 30 | #old/ 31 | 32 | .ipynb_checkpoints 33 | 34 | -------------------------------------------------------------------------------- /02_UCR_TCS.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "created by Ignacio Oguiza - email: oguiza@gmail.com" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": { 13 | "ExecuteTime": { 14 | "end_time": "2019-10-01T16:33:54.512402Z", 15 | "start_time": "2019-10-01T16:33:54.507048Z" 16 | } 17 | }, 18 | "source": [ 19 | "If you are using Google Colab uncomment the following code:" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 5, 25 | "metadata": { 26 | "ExecuteTime": { 27 | "end_time": "2019-10-01T16:38:42.227769Z", 28 | "start_time": "2019-10-01T16:38:42.224773Z" 29 | } 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "#!git clone https://github.com/timeseriesAI/timeseriesAI.git\n", 34 | "#%cd timeseriesAI\n", 35 | "#!pip install tslearn\n", 36 | "#!pip install pyts" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 6, 42 | "metadata": { 43 | "ExecuteTime": { 44 | "end_time": "2019-10-01T16:38:44.251012Z", 45 | "start_time": "2019-10-01T16:38:42.230947Z" 46 | } 47 | }, 48 | "outputs": [ 49 | { 50 | "name": "stdout", 51 | "output_type": "stream", 52 | "text": [ 53 | "/home/oguizadl/timeseriesAI\n", 54 | "fastai : 1.0.58\n", 55 | "torch : 1.3.0\n", 56 | "device : cuda\n" 57 | ] 58 | } 59 | ], 60 | "source": [ 61 | "import fastai, os\n", 62 | "from fastai_timeseries import *\n", 63 | "from torchtimeseries.models import *\n", 64 | "path = Path(os.getcwd())\n", 65 | "print(path)\n", 66 | "print('fastai :', fastai.__version__)\n", 67 | "print('torch :', torch.__version__)\n", 68 | "print('device :', device)" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 7, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "def run_UCR_test(iters, epochs, datasets, arch, \n", 78 | " bs=64, max_lr=3e-3, pct_start=.7, warmup=False, wd=1e-2, metrics=[accuracy], \n", 79 | " scale_type ='standardize', scale_by_channel=True, scale_by_sample=False, scale_range=(-1, 1), \n", 80 | " opt_func=functools.partial(torch.optim.Adam, betas=(0.9, 0.99)), \n", 81 | " loss_func=None, **arch_kwargs):\n", 82 | " ds_, acc_, acces_, accmax_, iter_, time_, epochs_, loss_, val_loss_ = [], [], [], [], [], [], [], [], []\n", 83 | " datasets = listify(datasets)\n", 84 | " for ds in datasets: \n", 85 | " db = create_UCR_databunch(ds, scale_by_channel=scale_by_channel, \n", 86 | " scale_by_sample=scale_by_sample, scale_range=scale_range,)\n", 87 | " for i in range(iters):\n", 88 | " print('\\n', ds, i)\n", 89 | " ds_.append(ds)\n", 90 | " iter_.append(i)\n", 91 | " epochs_.append(epochs)\n", 92 | " model = arch(db.features, db.c, **arch_kwargs).to(defaults.device)\n", 93 | " learn = Learner(db, model, opt_func=opt_func, loss_func=loss_func)\n", 94 | " learn.metrics = metrics\n", 95 | " start_time = time.time()\n", 96 | " learn.fit_one_cycle(epochs, max_lr=max_lr, pct_start=pct_start, moms=(.95, .85) if warmup else (.95, .95),\n", 97 | " div_factor=25.0 if warmup else 1., wd=wd)\n", 98 | " duration = time.time() - start_time\n", 99 | " time_.append('{:.0f}'.format(duration))\n", 100 | " early_stop = math.ceil(np.argmin(learn.recorder.losses) / len(learn.data.train_dl))\n", 101 | " acc_.append(learn.recorder.metrics[-1][0].item())\n", 102 | " acces_.append(learn.recorder.metrics[early_stop - 1][0].item())\n", 103 | " accmax_.append(np.max(learn.recorder.metrics))\n", 104 | " loss_.append(learn.recorder.losses[-1].item())\n", 105 | " val_loss_.append(learn.recorder.val_losses[-1].item())\n", 106 | " if len(datasets) * iters >1: clear_output()\n", 107 | " df = (pd.DataFrame(np.stack((ds_, iter_, epochs_, loss_, val_loss_ ,acc_, acces_, accmax_, time_)).T,\n", 108 | " columns=['dataset', 'iter', 'epochs', 'loss', 'val_loss', \n", 109 | " 'accuracy', 'accuracy_ts', \n", 110 | " 'max_accuracy', 'time (s)'])\n", 111 | " )\n", 112 | " df = df.astype({'loss': float, 'val_loss': float, 'accuracy': float, \n", 113 | " 'accuracy_ts': float, 'max_accuracy': float})\n", 114 | " display(df)\n", 115 | " return learn, df" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 8, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "# This is an unofficial PyTorch implementation by Ignacio Oguiza - oguiza@gmail.com based on:\n", 125 | "\n", 126 | "# Fawaz, H. I., Lucas, B., Forestier, G., Pelletier, C., Schmidt, D. F., Weber, J., ... & Petitjean, F. (2019). InceptionTime: Finding AlexNet for Time Series Classification. arXiv preprint arXiv:1909.04939.\n", 127 | "# Official InceptionTime tensorflow implementation: https://github.com/hfawaz/InceptionTime\n", 128 | "\n", 129 | "import torch\n", 130 | "import torch.nn as nn\n", 131 | "\n", 132 | "def noop(x):\n", 133 | " return x\n", 134 | "\n", 135 | "def shortcut(c_in, c_out):\n", 136 | " return nn.Sequential(*[nn.Conv1d(c_in, c_out, kernel_size=1), \n", 137 | " nn.BatchNorm1d(c_out)])\n", 138 | " \n", 139 | "class Inception(nn.Module):\n", 140 | " def __init__(self, c_in, bottleneck=32, ks=40, nb_filters=32):\n", 141 | "\n", 142 | " super().__init__()\n", 143 | " self.bottleneck = nn.Conv1d(c_in, bottleneck, 1) if bottleneck and c_in > 1 else noop\n", 144 | " mts_feat = bottleneck or c_in\n", 145 | " conv_layers = []\n", 146 | " kss = [ks // (2**i) for i in range(3)]\n", 147 | " # ensure odd kss until nn.Conv1d with padding='same' is available in pytorch 1.3\n", 148 | " kss = [ksi if ksi % 2 != 0 else ksi - 1 for ksi in kss] \n", 149 | " for i in range(len(kss)):\n", 150 | " conv_layers.append(\n", 151 | " nn.Conv1d(mts_feat, nb_filters, kernel_size=kss[i], padding=kss[i] // 2))\n", 152 | " self.conv_layers = nn.ModuleList(conv_layers)\n", 153 | " self.maxpool = nn.MaxPool1d(3, stride=1, padding=1)\n", 154 | " self.conv = nn.Conv1d(c_in, nb_filters, kernel_size=1)\n", 155 | " self.bn = nn.BatchNorm1d(nb_filters * 4)\n", 156 | " self.act = nn.ReLU()\n", 157 | "\n", 158 | " def forward(self, x):\n", 159 | " input_tensor = x\n", 160 | " x = self.bottleneck(input_tensor)\n", 161 | " for i in range(3):\n", 162 | " out_ = self.conv_layers[i](x)\n", 163 | " if i == 0: out = out_\n", 164 | " else: out = torch.cat((out, out_), 1)\n", 165 | " mp = self.conv(self.maxpool(input_tensor))\n", 166 | " inc_out = torch.cat((out, mp), 1)\n", 167 | " return self.act(self.bn(inc_out))\n", 168 | "\n", 169 | "\n", 170 | "class InceptionBlock(nn.Module):\n", 171 | " def __init__(self,c_in,bottleneck=32,ks=40,nb_filters=32,residual=True,depth=6):\n", 172 | "\n", 173 | " super().__init__()\n", 174 | "\n", 175 | " self.residual = residual\n", 176 | " self.depth = depth\n", 177 | "\n", 178 | " #inception & residual layers\n", 179 | " inc_mods = []\n", 180 | " res_layers = []\n", 181 | " res = 0\n", 182 | " for d in range(depth):\n", 183 | " inc_mods.append(\n", 184 | " Inception(c_in if d == 0 else nb_filters * 4, bottleneck=bottleneck if d > 0 else 0,ks=ks,\n", 185 | " nb_filters=nb_filters))\n", 186 | " if self.residual and d % 3 == 2:\n", 187 | " res_layers.append(shortcut(c_in if res == 0 else nb_filters * 4, nb_filters * 4))\n", 188 | " res += 1\n", 189 | " else: res_layer = res_layers.append(None)\n", 190 | " self.inc_mods = nn.ModuleList(inc_mods)\n", 191 | " self.res_layers = nn.ModuleList(res_layers)\n", 192 | " self.act = nn.ReLU()\n", 193 | " \n", 194 | " def forward(self, x):\n", 195 | " res = x\n", 196 | " for d, l in enumerate(range(self.depth)):\n", 197 | " x = self.inc_mods[d](x)\n", 198 | " if self.residual and d % 3 == 2:\n", 199 | " res = self.res_layers[d](res)\n", 200 | " x += res\n", 201 | " res = x\n", 202 | " x = self.act(x)\n", 203 | " return x\n", 204 | " \n", 205 | "class InceptionTime(nn.Module):\n", 206 | " def __init__(self,c_in,c_out,bottleneck=32,ks=40,nb_filters=32,residual=True,depth=6):\n", 207 | " super().__init__()\n", 208 | " self.block = InceptionBlock(c_in,bottleneck=bottleneck,ks=ks,nb_filters=nb_filters,\n", 209 | " residual=residual,depth=depth)\n", 210 | " self.gap = nn.AdaptiveAvgPool1d(1)\n", 211 | " self.fc = nn.Linear(nb_filters * 4, c_out)\n", 212 | "\n", 213 | " def forward(self, x):\n", 214 | " x = self.block(x)\n", 215 | " x = self.gap(x).squeeze(-1)\n", 216 | " x = self.fc(x)\n", 217 | " return x" 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "execution_count": 9, 223 | "metadata": { 224 | "ExecuteTime": { 225 | "end_time": "2019-10-01T16:42:10.837080Z", 226 | "start_time": "2019-10-01T16:42:10.830312Z" 227 | } 228 | }, 229 | "outputs": [], 230 | "source": [ 231 | "# Data\n", 232 | "bottom10 = [ 'Wine', 'BeetleFly', #'CinCECGtorso', not available for download \n", 233 | " 'InlineSkate', 'MiddlePhalanxTW', 'OliveOil', 'SmallKitchenAppliances', 'WordSynonyms', \n", 234 | " 'MiddlePhalanxOutlineAgeGroup', 'MoteStrain', 'Phoneme']\n", 235 | "top3 = ['Herring', 'ScreenType', 'ChlorineConcentration']\n", 236 | "datasets = bottom10 + top3\n", 237 | "bs = 64\n", 238 | "scale_type = 'standardize'\n", 239 | "scale_by_channel = True\n", 240 | "scale_by_sample = False \n", 241 | "scale_range = (-1, 1) \n", 242 | "\n", 243 | "# Arch\n", 244 | "arch = InceptionTime\n", 245 | "arch_kwargs = dict()\n", 246 | "\n", 247 | "# Training\n", 248 | "iters = 1\n", 249 | "epochs = 500\n", 250 | "max_lr = 3e-3\n", 251 | "warmup = False\n", 252 | "pct_start = .7\n", 253 | "metrics = [accuracy]\n", 254 | "wd = 1e-2\n", 255 | "opt_func = Ranger\n", 256 | "loss_func = LabelSmoothingCrossEntropy()" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": 6, 262 | "metadata": { 263 | "ExecuteTime": { 264 | "end_time": "2019-10-01T16:51:01.204577Z", 265 | "start_time": "2019-10-01T16:42:13.516196Z" 266 | } 267 | }, 268 | "outputs": [ 269 | { 270 | "data": { 271 | "text/html": [ 272 | "
\n", 273 | "\n", 286 | "\n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | " \n", 296 | " \n", 297 | " \n", 298 | " \n", 299 | " \n", 300 | " \n", 301 | " \n", 302 | " \n", 303 | " \n", 304 | " \n", 305 | " \n", 306 | " \n", 307 | " \n", 308 | " \n", 309 | " \n", 310 | " \n", 311 | " \n", 312 | " \n", 313 | " \n", 314 | " \n", 315 | " \n", 316 | " \n", 317 | " \n", 318 | " \n", 319 | " \n", 320 | " \n", 321 | " \n", 322 | " \n", 323 | " \n", 324 | " \n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | " \n", 352 | " \n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | " \n", 375 | " \n", 376 | " \n", 377 | " \n", 378 | " \n", 379 | " \n", 380 | " \n", 381 | " \n", 382 | " \n", 383 | " \n", 384 | " \n", 385 | " \n", 386 | " \n", 387 | " \n", 388 | " \n", 389 | " \n", 390 | " \n", 391 | " \n", 392 | " \n", 393 | " \n", 394 | " \n", 395 | " \n", 396 | " \n", 397 | " \n", 398 | " \n", 399 | " \n", 400 | " \n", 401 | " \n", 402 | " \n", 403 | " \n", 404 | " \n", 405 | " \n", 406 | " \n", 407 | " \n", 408 | " \n", 409 | " \n", 410 | " \n", 411 | " \n", 412 | " \n", 413 | " \n", 414 | " \n", 415 | " \n", 416 | " \n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | "
datasetiterepochslossval_lossaccuracyaccuracy_tsmax_accuracytime (s)
0Wine010.6924240.6937930.5000000.5000000.5000002
1BeetleFly010.7370830.6976680.5000000.5000000.5000002
2InlineSkate011.9610441.9514940.1527270.1527270.15272742
3MiddlePhalanxTW011.5896551.7491700.2727270.2727270.2727275
4OliveOil011.6674271.4289590.1666670.1666670.1666673
5SmallKitchenAppliances011.1356671.1098950.3333330.3333330.33333326
6WordSynonyms013.3821293.2685700.0219440.0219440.02194412
7MiddlePhalanxOutlineAgeGroup011.0002101.0805970.5714290.5714290.5714295
8MoteStrain010.7095740.6902080.5391370.5391370.5391374
9Phoneme013.7373373.6576710.0047470.0047470.00474772
10Herring010.8625920.7298970.4062500.4062500.4062506
11ScreenType011.1171271.1008620.3333330.3333330.33333327
12ChlorineConcentration011.0326401.0656730.5325520.5325520.53255225
\n", 460 | "
" 461 | ], 462 | "text/plain": [ 463 | " dataset iter epochs loss val_loss accuracy \\\n", 464 | "0 Wine 0 1 0.692424 0.693793 0.500000 \n", 465 | "1 BeetleFly 0 1 0.737083 0.697668 0.500000 \n", 466 | "2 InlineSkate 0 1 1.961044 1.951494 0.152727 \n", 467 | "3 MiddlePhalanxTW 0 1 1.589655 1.749170 0.272727 \n", 468 | "4 OliveOil 0 1 1.667427 1.428959 0.166667 \n", 469 | "5 SmallKitchenAppliances 0 1 1.135667 1.109895 0.333333 \n", 470 | "6 WordSynonyms 0 1 3.382129 3.268570 0.021944 \n", 471 | "7 MiddlePhalanxOutlineAgeGroup 0 1 1.000210 1.080597 0.571429 \n", 472 | "8 MoteStrain 0 1 0.709574 0.690208 0.539137 \n", 473 | "9 Phoneme 0 1 3.737337 3.657671 0.004747 \n", 474 | "10 Herring 0 1 0.862592 0.729897 0.406250 \n", 475 | "11 ScreenType 0 1 1.117127 1.100862 0.333333 \n", 476 | "12 ChlorineConcentration 0 1 1.032640 1.065673 0.532552 \n", 477 | "\n", 478 | " accuracy_ts max_accuracy time (s) \n", 479 | "0 0.500000 0.500000 2 \n", 480 | "1 0.500000 0.500000 2 \n", 481 | "2 0.152727 0.152727 42 \n", 482 | "3 0.272727 0.272727 5 \n", 483 | "4 0.166667 0.166667 3 \n", 484 | "5 0.333333 0.333333 26 \n", 485 | "6 0.021944 0.021944 12 \n", 486 | "7 0.571429 0.571429 5 \n", 487 | "8 0.539137 0.539137 4 \n", 488 | "9 0.004747 0.004747 72 \n", 489 | "10 0.406250 0.406250 6 \n", 490 | "11 0.333333 0.333333 27 \n", 491 | "12 0.532552 0.532552 25 " 492 | ] 493 | }, 494 | "metadata": {}, 495 | "output_type": "display_data" 496 | } 497 | ], 498 | "source": [ 499 | "output = run_UCR_test(iters,\n", 500 | " epochs,\n", 501 | " datasets,\n", 502 | " arch,\n", 503 | " bs=bs,\n", 504 | " max_lr=max_lr,\n", 505 | " pct_start=pct_start,\n", 506 | " warmup=warmup,\n", 507 | " wd=wd,\n", 508 | " metrics=metrics,\n", 509 | " scale_type=scale_type,\n", 510 | " scale_by_channel=scale_by_channel,\n", 511 | " scale_by_sample=scale_by_sample,\n", 512 | " scale_range=scale_range,\n", 513 | " opt_func=opt_func,\n", 514 | " loss_func=loss_func,\n", 515 | " **arch_kwargs)" 516 | ] 517 | }, 518 | { 519 | "cell_type": "code", 520 | "execution_count": null, 521 | "metadata": {}, 522 | "outputs": [], 523 | "source": [] 524 | } 525 | ], 526 | "metadata": { 527 | "kernelspec": { 528 | "display_name": "fastai-v1", 529 | "language": "python", 530 | "name": "fastai-v1" 531 | }, 532 | "language_info": { 533 | "codemirror_mode": { 534 | "name": "ipython", 535 | "version": 3 536 | }, 537 | "file_extension": ".py", 538 | "mimetype": "text/x-python", 539 | "name": "python", 540 | "nbconvert_exporter": "python", 541 | "pygments_lexer": "ipython3", 542 | "version": "3.7.5" 543 | } 544 | }, 545 | "nbformat": 4, 546 | "nbformat_minor": 2 547 | } 548 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## timeseriesAI is based on fastai v1. I'd recommend you use [timeseriesAI/tsai](https://github.com/timeseriesAI/tsai) which is based on fastai v2. 2 | 3 | 4 | timeseriesAI is a library built on top of fastai/ Pytorch to help you apply Deep Learning to your time series/ sequential datasets, in particular Time Series Classification (TSC) and Time Series Regression (TSR) problems. 5 | 6 | 7 | The library contains 3 major components: 8 | 9 | 1. **Notebooks** 📒: they are very practical, and show you how certain techniques can be easily applied. 10 | 11 | 2. **fastai_timeseries** 🏃🏽‍♀️: it's an extension of fastai's library that focuses on time series/ sequential problems. 12 | 13 | 3. **torchtimeseries.models** 👫: it's a collection of some state-of-the-art time series/ sequential models. 14 | 15 | 16 | The 3 components of this library will keep growing in the future as new techniques are added and/or new state-of-the-art models appear. In those cases, I will keep adding notebooks to demonstrate how you can apply them in a practical way. 17 | 18 | 19 | ## Notebooks 20 | 21 | #### 1. Introduction to Time Series Classification (TSC) 🤝: 22 | - This is an intro that nb that shows you how you can achieve high performance in 4 simple steps. 23 | 24 | #### 2. UCR_TCS 🧪: 25 | - The UCR datasets are broadly used in TSC problems as s bechmark to measure performance. This notebook will allow you to test any of the available datasets, with the model of your choice and any training scheme. You can easily tweak any of them to try to beat a SOTA. 26 | 27 | #### 3. New TS data augmentations 🔎: 28 | - You will see how you can apply successful data augmentation techniques (like mixup, cutout, and cutmix) to time series problems. 29 | 30 | #### 4. The importance of scaling ⚖️: 31 | - In this notebook you'll learn more about the options to scale your data and the impact it may have on performance, which is huge! 32 | 33 | #### 5. Multivariate ROCKET on GPU 🚀: 34 | - On October 29, 2019, there was a major milestone in the area of Time Series Classification. 35 | A new method, called ROCKET (RandOm Convolutional KErnel Transform) was released (Dempster A, Petitjean F, Webb GI (2019) [ROCKET: Exceptionally fast and accurate time series classification using random convolutional kernels](https://arxiv.org/pdf/1910.13051)) together with the code they used. 36 | This new method not only beat the previous recognized state of the art (HIVE-COTE) on a TSC benchmark, but it does it in record time, many orders of magnitude faster than any other method. 37 | I’ve been using it for a couple of days and the results are IMPRESSIVE!! 38 | The release code however has 2 limitations: 39 | - it can only handle univariate time series 40 | - it doesn’t support GPU 41 | 42 | - I have developed ROCKET with GPU support in Pytorch that you can now use it with univariate of multivariate time series. In this notebook you will see how you can use ROCKET in your time series problems. 43 | 44 | #### 6. TS data augmentations: single-item transforms 👯 - UPDATED with new tfms, GPU tfms and RandAugment 45 | - In this notebook, you'll find some TS transforms that can be used to augment your data. Most of the transforms are adapted from/ inspired by research papers on time series augmentation. Transforms include: TSjittering, TSmagscale, TSmagwarp, TStimenoise, TStimewarp, TSlookback, TStimestepsout, TSchannelout, TScutout, TScrop, TSwindowslice and TSzoom. 46 | 47 | - UPDATED: I have now updated all tfms so that we can use them as regular tfms, or pass them to a DataLoader and use them as a batch transform (which is much faster). I have also included RandAugment (a new technique developed by Google to eliminate the data augmentation search phase). This applies a random data augmentation to each individual batch. 48 | 49 | -------------------------------------------------------------------------------- /fastai_timeseries/__init__.py: -------------------------------------------------------------------------------- 1 | from .exp import * -------------------------------------------------------------------------------- /fastai_timeseries/exp/__init__.py: -------------------------------------------------------------------------------- 1 | from .nb_TSDatasets import * 2 | from .nb_TSBasicData import * 3 | from .nb_TSUtilities import * 4 | from .nb_TSDataAugmentation import * 5 | from .nb_TSImageData import * 6 | from .nb_TSCallbacks import * 7 | from .nb_TSCharts import * 8 | from .nb_TSTrain import * 9 | from .nb_ColorfulDim import * 10 | from .nb_Initialization import * 11 | from .nb_Optimizers import * 12 | from .rocket_functions import * 13 | #from .nb_ImageDataAugmentation import * 14 | #from .nb_NewDataAugmentation import * -------------------------------------------------------------------------------- /fastai_timeseries/exp/nb_ColorfulDim.py: -------------------------------------------------------------------------------- 1 | #AUTOGENERATED! DO NOT EDIT! file to edit: ./ColorfulDim.ipynb (unless otherwise specified) 2 | 3 | from fastai.torch_core import * 4 | from fastai.vision import Learner, Tensors, Rank0Tensor, flatten_model, listify 5 | from torch import nn 6 | import torch 7 | from fastai.metrics import error_rate 8 | from fastai.callbacks import HookCallback 9 | import numpy as np 10 | import math 11 | import matplotlib.pyplot as plt 12 | 13 | def splitAtFirstParenthesis(s,showDetails,shapeData): 14 | pos=len(s.split('(')[0]) 15 | ret = s[:pos] 16 | if (showDetails): 17 | ret += shapeData + '\n' + s[pos:] 18 | return ret 19 | 20 | class ActivationsHistogram(HookCallback): 21 | "Callback that record histogram of activations." 22 | "NOTE: learner property name will be 'activations_histogram'" 23 | 24 | def __init__(self, learn:Learner, do_remove:bool=True, 25 | hMin=-1, 26 | hMax=1, 27 | nBins=100, 28 | useClasses=False, # if true compute histogram of classes in the last layer 29 | liveChart = True, # show live chart of last layer 30 | modulesId=-1, # array of modules to keep 31 | ): 32 | self.hMin = hMin or (-hMax) 33 | self.hMax = hMax 34 | self.nBins = nBins 35 | self.zero_bin = math.floor(- self.nBins * self.hMin / (self.hMax - self.hMin)) 36 | self.liveChart = liveChart 37 | self.allModules = [m for m in flatten_model(learn.model)] 38 | self.useClasses = useClasses 39 | modules = self.allModules 40 | if modulesId: 41 | modules = [self.allModules[i] for i in listify(modulesId)] 42 | self.allModules = modules if modules else self.allModules 43 | self.c = learn.data.c # Number of Calsses 44 | super().__init__(learn,modules,do_remove) 45 | 46 | def mkHist(self, x, useClasses): 47 | ret = x.clone().detach().cpu() # Need this for Pytorch 1.0 48 | #ret = x.clone().detach() #Pytorch 1.1 # WARNING: so bad performance on GPU! (x10) 49 | if useClasses: 50 | ret = torch.stack([ret[:,i].histc(self.nBins, self.hMin, self.hMax) for i in range(ret.shape[1])],dim=1) #histogram by class... 51 | else: 52 | ret = ret.histc(self.nBins, self.hMin, self.hMax).unsqueeze(1) # histogram by activation 53 | return ret 54 | 55 | def on_train_begin(self, **kwargs): 56 | "Initialize stats." 57 | super().on_train_begin(**kwargs) 58 | self.stats_hist = None # empty at start 59 | self.stats_valid_hist = None # empty at start 60 | self.stats_epoch = [] 61 | self.cur_epoch = -1 62 | self.cur_train_batch = -1 63 | self.stats_valid_epoch = [] 64 | self.shape_out={} 65 | 66 | def on_epoch_begin(self, **kwargs): 67 | self.cur_epoch += 1 68 | 69 | def on_batch_begin(self, train, **kwargs): 70 | if train: 71 | self.cur_train_batch += 1 72 | 73 | def hook(self, m:nn.Module, i:Tensors, o:Tensors)->Rank0Tensor: 74 | if (isinstance(o,torch.Tensor)) and (m not in self.shape_out): 75 | outShape = o.shape; 76 | self.shape_out[m]=outShape; 77 | return self.mkHist(o,self.useClasses) 78 | 79 | def on_batch_end(self, train, **kwargs): 80 | "Take the stored results and puts it in `self.stats_hist`" 81 | hasValues = True if ((len(self.hooks.stored)>0) and (not (self.hooks.stored[0] is None))) else False 82 | stacked = torch.stack(self.hooks.stored).unsqueeze(1) if hasValues else None 83 | if train and hasValues: 84 | if self.stats_hist is None: self.stats_hist = stacked #start 85 | else: self.stats_hist = torch.cat([self.stats_hist,stacked],dim=1) #cat 86 | if (not train) and hasValues: 87 | if self.stats_valid_hist is None: self.stats_valid_hist = stacked #start 88 | else: self.stats_valid_hist = torch.cat([self.stats_valid_hist,stacked],dim=1) #cat 89 | 90 | def on_epoch_end(self, **kwargs): 91 | self.stats_epoch.append(self.cur_train_batch) 92 | start = 0 if 1==len(self.stats_epoch) else self.stats_epoch[-2] 93 | end = self.stats_epoch[-1] 94 | startValid = 0 if 0==len(self.stats_valid_epoch) else self.stats_valid_epoch[-1] 95 | if not(self.stats_hist is None): 96 | hScale = 1 97 | domain = self.stats_hist[-1][start:end] 98 | yy = np.arange(self.hMin,self.hMax,(self.hMax-self.hMin)/self.nBins) 99 | xx,_ = self.computeXY(domain,hScale,.25,start) # average on last quarter of epoch 100 | xx = xx.sum(-1) # useClasses support 101 | toDisplay = [(xx,yy)] 102 | 103 | if not (self.stats_valid_hist is None): 104 | domainValid = self.stats_valid_hist[-1][startValid:] #till end 105 | xxv,_ = self.computeXY(domainValid,hScale,1,start) # validation average all available data 106 | xxv = xxv.sum(-1) # useClasses support 107 | toDisplay += [(xxv,yy)] 108 | self.stats_valid_epoch += [self.stats_valid_hist.shape[1]] 109 | if self.liveChart: 110 | rec = rec = self.learn.recorder 111 | rec.pbar.update_graph(toDisplay) 112 | 113 | def on_train_end(self, **kwargs): 114 | "Polish the final result." 115 | super().on_train_end(**kwargs) 116 | self.activations_histogram.plotActsHist() 117 | 118 | @staticmethod 119 | def get_color_value_from_map(idx:int, cmap='Reds', scale=1): 120 | return plt.get_cmap(cmap)(idx*scale) 121 | 122 | @staticmethod 123 | def getHistImg(act,useClasses): 124 | dd = act.squeeze(2) if not useClasses else act.sum(dim=2) # Reassemble... 125 | dd = dd.log() # Scale for visualizaion 126 | dd = dd.t() # rotate 127 | return dd 128 | 129 | @staticmethod 130 | def getMin(act,useClasses,zero_bin): 131 | dd = act.squeeze(2) if not useClasses else act.sum(dim=2) # Reassemble... 132 | dd = dd.t().float() # rotate 133 | return dd[zero_bin - 1:zero_bin + 2].sum(0)/dd.sum(0) 134 | 135 | 136 | @staticmethod 137 | def computeXY(l,hscale,perc,hshift=0): 138 | start = int(l.shape[0]*(1-perc)) 139 | end = l.shape[0] 140 | m = l[start:end].mean(dim=0) # all data mean 141 | xx = hshift + m*hscale 142 | yy = +np.array(range(l.shape[1])) 143 | return xx,yy 144 | 145 | @staticmethod 146 | def plotPerc(ax,l,hscale,perc,hshift=0,colorById=False,linewidth=1,addLabel=False): 147 | xx,yy = ActivationsHistogram.computeXY(l,hscale,perc,hshift) 148 | if colorById: 149 | classes = xx.shape[1] 150 | for i in range(classes): 151 | xx_cur = xx[:,i] 152 | color = ActivationsHistogram.get_color_value_from_map(i/classes, cmap='rainbow') 153 | label = i if addLabel else None 154 | ax.plot(xx_cur,yy,linewidth=linewidth, color=color, label=label); 155 | else: 156 | color = [1-perc,1-perc,1-perc] 157 | ax.plot(xx,yy,linewidth=linewidth, color=color); 158 | 159 | def plotActsHist(self, cols=10, toDisplay=None, hScale = .05, 160 | showEpochs=False, showLayerInfo=True, aspectAuto=True, showImage=True): 161 | histsTensor = self.activations_histogram.stats_hist 162 | hists = [histsTensor[i] for i in range(histsTensor.shape[0])] 163 | if toDisplay: hists = [hists[i] for i in listify(toDisplay)] # optionally focus 164 | 165 | n=len(hists) 166 | cols = cols or 3 167 | cols = min(cols,n) 168 | rows = int(math.ceil(n/cols)) 169 | fig = plt.figure(figsize=(20, rows * 4.5)) 170 | grid = plt.GridSpec(rows, cols, figure=fig ,left=None, bottom=None, 171 | right=None, top=None, wspace=.25, hspace=.25) 172 | 173 | for i,l in enumerate(hists): 174 | img=self.getHistImg(l,self.useClasses) 175 | dead = self.getMin(l,self.useClasses,self.zero_bin) 176 | cr = math.floor(i/cols) 177 | cc = i%cols 178 | main_ax = fig.add_subplot(grid[cr,cc]) 179 | if showImage: main_ax.imshow(img); 180 | layerId = listify(toDisplay)[i] if toDisplay else i 181 | m = self.allModules[layerId] 182 | outShapeText = f' (out: {list(self.shape_out[m])})' if (m in self.shape_out) else '' 183 | title = f'L:{layerId}' + '\n' + splitAtFirstParenthesis(str(m),False,outShapeText) 184 | main_ax.set_title(title, fontsize=8, weight='bold') 185 | imgH=img.shape[0] 186 | main_ax.set_yticks([]) 187 | main_ax.set_ylabel(str(self.hMin) + " : " + str(self.hMax)) 188 | if aspectAuto: main_ax.set_aspect('auto') 189 | imgW=img.shape[1] 190 | imgH=img.shape[0] 191 | ratioH=-self.hMin/(self.hMax-self.hMin) 192 | zeroPosH = imgH*ratioH 193 | main_ax.plot(dead * l.shape[1],'r', linewidth=2) # X Axis 194 | main_ax.plot([0,imgW],[zeroPosH,zeroPosH],'black', linewidth=.5) # X Axis 195 | if (showEpochs): 196 | start = 0 197 | nEpochs = len(self.activations_histogram.stats_epoch) 198 | for i,hh in enumerate(self.activations_histogram.stats_epoch): 199 | if(i<(nEpochs-1)): main_ax.plot([hh,hh],[0,imgH],color=[0,0,1]) 200 | end = hh # rolling 201 | domain = l[start:end] 202 | domain_mean = domain.mean(-1) # mean on classes 203 | if self.useClasses: 204 | self.plotPerc(main_ax,domain,hScale,1,start,colorById=True,addLabel=(0==i)) #plot all 205 | main_ax.legend(loc='upper left') 206 | else: 207 | self.plotPerc(main_ax,domain_mean,hScale,.5,start) 208 | self.plotPerc(main_ax,domain_mean,hScale,1,start,linewidth=1.5) 209 | start = hh 210 | main_ax.set_xlim([0,imgW]) 211 | main_ax.set_ylim([0,imgH]) 212 | plt.show() 213 | if showLayerInfo: 214 | for i, l in enumerate(self.allModules): print('{:2} {}'.format(i, l)) 215 | 216 | 217 | 218 | def telemetry(learn:Learner, ah:Callable=None, modulesId:Union[bool, list]=None, filt:Callable=None, hMin:int=-10, hMax:int=10, nBins:int=200) -> Learner: 219 | if modulesId == None and filt != None: modulesId = get_layers_idx(learn, filt) 220 | if ah is None: ah = partial(ActivationsHistogram, modulesId=modulesId, hMin=hMin, hMax=hMax, nBins=nBins, liveChart=False) 221 | learn.callback_fns.append(ah) 222 | return learn 223 | 224 | Learner.telemetry = telemetry 225 | 226 | 227 | def noop(x): return x 228 | 229 | 230 | def get_layers(model, cond=noop): 231 | if isinstance(model, Learner): model=model.model 232 | def _get_layers(model, cond=noop): 233 | return [m for m in flatten_model(model) if cond(m)] 234 | if hasattr(model, 'body') or hasattr(model, 'head'): 235 | all_layers = [] 236 | if hasattr(model, 'body'): 237 | for i in range(len(model.body)): 238 | all_layers.extend(_get_layers(model.body[i], cond=cond)) 239 | if hasattr(model, 'head'): 240 | for i in range(len(model.head)): 241 | all_layers.extend(_get_layers(model.head[i], cond=cond)) 242 | return all_layers 243 | else: return _get_layers(model, cond=cond) 244 | 245 | 246 | def get_layers_idx(learn, cond=noop): 247 | mods = get_layers(learn) 248 | idx = [] 249 | for i,m in enumerate(mods): 250 | if cond(m): idx.append(i) 251 | return idx -------------------------------------------------------------------------------- /fastai_timeseries/exp/nb_ImageDataAugmentation.py: -------------------------------------------------------------------------------- 1 | #AUTOGENERATED! DO NOT EDIT! file to edit: ./ImageDataAugmentation.ipynb (unless otherwise specified) 2 | 3 | from fastai.torch_core import * 4 | from fastai.callback import * 5 | from fastai.callbacks.mixup import MixUpLoss 6 | from fastai.basic_train import Learner, LearnerCallback 7 | 8 | class CutMixCallback(LearnerCallback): 9 | "Callback that creates the cutmixed input and target." 10 | def __init__(self, learn:Learner, α:float=1., stack_y:bool=True, true_λ:bool=True): 11 | super().__init__(learn) 12 | self.α,self.stack_y,self.true_λ = α,stack_y,true_λ 13 | 14 | def on_train_begin(self, **kwargs): 15 | if self.stack_y: self.learn.loss_func = MixUpLoss(self.learn.loss_func) 16 | 17 | def on_batch_begin(self, last_input, last_target, train, **kwargs): 18 | "Applies cutmix to `last_input` and `last_target` if `train`." 19 | if not train: return 20 | λ = np.random.beta(self.α, self.α) 21 | λ = max(λ, 1- λ) 22 | shuffle = torch.randperm(last_target.size(0)).to(last_input.device) 23 | x1, y1 = last_input[shuffle], last_target[shuffle] 24 | #Get new input 25 | last_input_size = last_input.shape 26 | bbx1, bby1, bbx2, bby2 = rand_bbox(last_input.size(), λ) 27 | new_input = last_input.clone() 28 | new_input[:, ..., bby1:bby2, bbx1:bbx2] = last_input[shuffle, ..., bby1:bby2, bbx1:bbx2] 29 | λ = last_input.new([λ]) 30 | if self.true_λ: 31 | λ = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (last_input_size[-1] * last_input_size[-2])) 32 | λ = last_input.new([λ]) 33 | if self.stack_y: 34 | new_target = torch.cat([last_target.unsqueeze(1).float(), y1.unsqueeze(1).float(), 35 | λ.repeat(last_input_size[0]).unsqueeze(1).float()], 1) 36 | else: 37 | if len(last_target.shape) == 2: 38 | λ = λ.unsqueeze(1).float() 39 | new_target = last_target.float() * λ + y1.float() * (1-λ) 40 | return {'last_input': new_input, 'last_target': new_target} 41 | 42 | def on_train_end(self, **kwargs): 43 | if self.stack_y: self.learn.loss_func = self.learn.loss_func.get_old() 44 | 45 | 46 | def rand_bbox(last_input_size, λ): 47 | '''lambd is always between .5 and 1''' 48 | 49 | W = last_input_size[-1] 50 | H = last_input_size[-2] 51 | cut_rat = np.sqrt(1. - λ) # 0. - .707 52 | cut_w = np.int(W * cut_rat) 53 | cut_h = np.int(H * cut_rat) 54 | 55 | # uniform 56 | cx = np.random.randint(W) 57 | cy = np.random.randint(H) 58 | 59 | bbx1 = np.clip(cx - cut_w // 2, 0, W) 60 | bby1 = np.clip(cy - cut_h // 2, 0, H) 61 | bbx2 = np.clip(cx + cut_w // 2, 0, W) 62 | bby2 = np.clip(cy + cut_h // 2, 0, H) 63 | 64 | return bbx1, bby1, bbx2, bby2 65 | 66 | 67 | def cutmix(learn:Learner, α:float=1., stack_x:bool=False, stack_y:bool=True, true_λ:bool=True) -> Learner: 68 | "Add mixup https://arxiv.org/pdf/1905.04899.pdf to `learn`." 69 | learn.callback_fns.append(partial(CutMixCallback, α=α, stack_y=stack_y, true_λ=true_λ)) 70 | return learn 71 | 72 | Learner.cutmix = cutmix 73 | 74 | 75 | from fastai.torch_core import * 76 | from fastai.callback import * 77 | from fastai.basic_train import Learner, LearnerCallback 78 | 79 | class RicapLoss(nn.Module): 80 | "Adapt the loss function `crit` to go with ricap data augmentations." 81 | 82 | def __init__(self, crit, reduction='mean'): 83 | super().__init__() 84 | if hasattr(crit, 'reduction'): 85 | self.crit = crit 86 | self.old_red = crit.reduction 87 | setattr(self.crit, 'reduction', 'none') 88 | else: 89 | self.crit = partial(crit, reduction='none') 90 | self.old_crit = crit 91 | self.reduction = reduction 92 | 93 | def forward(self, output, target): 94 | if target.ndim == 2: 95 | c_ = target[:, 1:5] 96 | W_ = target[:, 5:] 97 | loss = [W_[:, k] * self.crit(output, c_[:, k].long()) for k in range(4)] 98 | d = torch.mean(torch.stack(loss)) 99 | else: d = self.crit(output, target) 100 | if self.reduction == 'mean': return d.mean() 101 | elif self.reduction == 'sum': return d.sum() 102 | return d 103 | 104 | def get_old(self): 105 | if hasattr(self, 'old_crit'): return self.old_crit 106 | elif hasattr(self, 'old_red'): 107 | setattr(self.crit, 'reduction', self.old_red) 108 | return self.crit 109 | 110 | class RicapCallback(LearnerCallback): 111 | "Callback that creates the ricap input and target." 112 | def __init__(self, learn:Learner, stack_y:bool=True): 113 | super().__init__(learn) 114 | self.stack_y = stack_y 115 | 116 | def on_train_begin(self, **kwargs): 117 | if self.stack_y: self.learn.loss_func = RicapLoss(self.learn.loss_func) 118 | 119 | def on_batch_begin(self, last_input, last_target, train, **kwargs): 120 | "Applies ricap to `last_input` and `last_target` if `train`." 121 | if not train: return 122 | I_x, I_y = last_input.size()[2:] 123 | w = int(np.round(I_x * np.random.beta(1, 1))) 124 | h = int(np.round(I_y * np.random.beta(1, 1))) 125 | w_ = [w, I_x - w, w, I_x - w] 126 | h_ = [h, h, I_y - h, I_y - h] 127 | cropped_images = {} 128 | bs = last_input.size(0) 129 | c_ = torch.zeros((bs, 4)).float().to(last_input.device) 130 | W_ = torch.zeros(4).float().to(last_input.device) 131 | for k in range(4): 132 | idx = torch.randperm(bs) 133 | x_k = np.random.randint(0, I_x - w_[k] + 1) 134 | y_k = np.random.randint(0, I_y - h_[k] + 1) 135 | cropped_images[k] = last_input[idx][:, :, x_k:x_k + w_[k], y_k:y_k + h_[k]] 136 | c_[:, k] = last_target[idx].float() 137 | W_[k] = w_[k] * h_[k] / (I_x * I_y) 138 | patched_images = torch.cat( 139 | (torch.cat((cropped_images[0], cropped_images[1]), 2), 140 | torch.cat((cropped_images[2], cropped_images[3]), 2)), 3) #.cuda() 141 | if self.stack_y: 142 | new_target = torch.cat((last_target[:,None].float(), c_, 143 | W_[None].repeat(last_target.size(0), 1)), dim=1) 144 | else: 145 | new_target = c_ * W_ 146 | return {'last_input': patched_images, 'last_target': new_target} 147 | 148 | def on_train_end(self, **kwargs): 149 | if self.stack_y: self.learn.loss_func = self.learn.loss_func.get_old() 150 | 151 | 152 | def ricap(learn:Learner, stack_y:bool=True) -> Learner: 153 | "Add ricap https://arxiv.org/pdf/1811.09030.pdf to `learn`." 154 | learn.callback_fns.append(partial(RicapCallback, stack_y=stack_y)) 155 | return learn 156 | 157 | Learner.ricap = ricap -------------------------------------------------------------------------------- /fastai_timeseries/exp/nb_Initialization.py: -------------------------------------------------------------------------------- 1 | #AUTOGENERATED! DO NOT EDIT! file to edit: ./Initialization.ipynb (unless otherwise specified) 2 | 3 | # To develop .zmuv() I've taken ideas from: 4 | # - https://github.com/ducha-aiki/LSUV-pytorch/blob/master/LSUV.py 5 | # and from the fastai community, in particular from Jeremy, @kevinB and @SteveR 6 | 7 | from fastai.torch_core import * 8 | from fastai.basic_train import Learner 9 | 10 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 11 | 12 | 13 | class ListContainer(): 14 | def __init__(self, items): 15 | self.items = listify(items) 16 | def __getitem__(self, idx): 17 | try: 18 | print(idx) 19 | return self.items[idx] 20 | except TypeError: 21 | if isinstance(idx[0], bool): 22 | assert len(idx)==len(self) 23 | return [o for m,o in zip(idx,self.items) if m] 24 | return [self.items[i] for i in idx] 25 | def __len__(self): return len(self.items) 26 | def __iter__(self): return iter(self.items) 27 | def __setitem__(self, i, o): self.items[i] = o 28 | def __delitem__(self, i): del(self.items[i]) 29 | def __repr__(self): 30 | res = f'{self.__class__.__name__} ({len(self)} items)\n{self.items[:10]}' 31 | if len(self)>10: res = res[:-1]+ '...]' 32 | return res 33 | 34 | class Hook(): 35 | def __init__(self, m, f): 36 | self.hook = m.register_forward_hook(partial(f, self)) 37 | def remove(self): self.hook.remove() 38 | def __del__(self): self.remove() 39 | 40 | class Hooks(ListContainer): 41 | def __init__(self, ms, f): super().__init__([Hook(m, f) for m in ms]) 42 | def __enter__(self, *args): return self 43 | def __exit__ (self, *args): self.remove() 44 | def __del__(self): self.remove() 45 | def __delitem__(self, i): 46 | self[i].remove() 47 | super().__delitem__(i) 48 | def remove(self): 49 | for h in self: h.remove() 50 | 51 | 52 | def noop(x): return x 53 | 54 | def is_layer(*args): 55 | def _is_layer(l, cond=args): 56 | return isinstance(l, cond) 57 | return partial(_is_layer, cond=args) 58 | 59 | def is_lin_layer(l): 60 | return isinstance(l, nn.Linear) 61 | 62 | def is_conv_lin_layer(l): 63 | lin_layers = (nn.Conv1d, nn.Conv2d, nn.Conv3d, nn.Linear) 64 | return isinstance(l, lin_layers) 65 | 66 | def is_affine_layer(l): 67 | return has_bias(l) or has_weight(l) 68 | 69 | def is_conv_layer(l): 70 | conv_layers = (nn.Conv1d, nn.Conv2d, nn.Conv3d) 71 | return isinstance(l, conv_layers) 72 | 73 | def has_bias(l): 74 | return (hasattr(l, 'bias') and l.bias is not None) 75 | 76 | def has_weight(l): 77 | return (hasattr(l, 'weight')) 78 | 79 | def has_weight_or_bias(l): 80 | return any((has_weight(l), has_bias(l))) 81 | 82 | def find_modules(m, cond=noop): 83 | if isinstance(m, Learner): m=m.model 84 | if cond(m): return [m] 85 | return sum([find_modules(o,cond) for o in m.children()], []) 86 | 87 | def get_layers(model, cond=noop): 88 | if isinstance(model, Learner): model=model.model 89 | return [m for m in flatten_model(model) if any([c(m) for c in listify(cond)])] 90 | 91 | Learner.layers = get_layers 92 | 93 | 94 | 95 | def append_stat(hook, mod, inp, out): 96 | i = inp[0].data 97 | try: o = out.data 98 | except: o = out[0].data # for RNNs 99 | hook.inp_mean, hook.inp_std = i.mean().item(), i.std().item() 100 | hook.out_mean, hook.out_std = o.mean().item(), o.std().item() 101 | 102 | 103 | def zmuv_layer(model: Callable, m: Callable, xb: Tensor, max_attempts: int = 5, 104 | exp_mean: float = 0., exp_std: float = 1., tol: float = 1e-5, 105 | ε: float = 1e-8): 106 | h = Hook(m, append_stat) 107 | attempts = 0 108 | while model(xb) is not None: 109 | if attempts == 0: pre_mean, pre_std = h.out_mean, h.out_std 110 | if has_weight(m) and abs(h.out_std - exp_std) > tol: m.weight.data *= exp_std / (h.out_std + ε) 111 | elif has_bias(m) and abs(h.out_mean - exp_mean) > tol: m.bias.data -= h.out_mean - exp_mean 112 | else: break 113 | attempts += 1 114 | if attempts >= max_attempts: break 115 | h.remove() 116 | return pre_mean, pre_std, h.out_mean, h.out_std 117 | 118 | 119 | def zmuv(learn: Learner, tol: float = 1e-5, exp_mean: float = 0., 120 | exp_std: float = 1., orthonorm: bool = True, 121 | cond: callable = has_weight_or_bias, verbose: bool = False) -> Learner: 122 | print('ZMUV initialization...') 123 | xb, yb = next(iter(learn.data.train_dl)) 124 | if orthonorm: learn.model.apply(orthogonal_weights_init) 125 | mods = get_layers(learn.model, cond=cond) 126 | mean_act, std_act = [], [] 127 | from fastprogress import progress_bar 128 | from time import sleep 129 | pb = progress_bar(mods) 130 | for m in pb: 131 | sleep(0.01) 132 | if has_weight(m) or has_bias(m): 133 | pre_mean, pre_std, mean, std = zmuv_layer(learn.model, m, xb, tol=tol, 134 | exp_mean=exp_mean,exp_std=exp_std) 135 | if mean==0 and std==0: continue 136 | mean_act.append(mean) 137 | std_act.append(std) 138 | if verbose >= 2: 139 | print(m) 140 | print(' pre-zmuv activations : mean = {:9.5f} std = {:9.5f}'. 141 | format(pre_mean, pre_std)) 142 | print(' post-zmuv activations : mean = {:9.5f} std = {:9.5f}'. 143 | format(mean, std)) 144 | print('\noverall post-zmuv activations: mean = {:9.5f} std = {:9.5f}' 145 | .format(np.mean(mean_act), np.mean(std_act))) 146 | print('...ZMUV initialization complete\n') 147 | return learn 148 | 149 | Learner.zmuv = zmuv 150 | 151 | def svd_orthonormal(w): 152 | shape = w.shape 153 | if len(shape) < 2: flat_shape = (shape[0], 1) 154 | else: flat_shape = (shape[0], np.prod(shape[1:])) 155 | a = np.random.normal(0.0, 1.0, flat_shape) #w; 156 | u, _, v = np.linalg.svd(a, full_matrices=False) 157 | q = u if u.shape == flat_shape else v 158 | q = q.reshape(shape) 159 | return q.astype(np.float32) 160 | 161 | 162 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 163 | def orthogonal_weights_init(m): 164 | lin_layers = (nn.Conv1d, nn.Conv2d, nn.Conv3d, nn.Linear) 165 | if isinstance(m, lin_layers): 166 | if hasattr(m, 'weight_v'): 167 | w_ortho = svd_orthonormal(m.weight_v.data.cpu().numpy()) 168 | m.weight_v.data = torch.from_numpy(w_ortho).to(device) 169 | try:nn.init.constant_(m.bias, 0) 170 | except:pass 171 | else: 172 | w_ortho = svd_orthonormal(m.weight.data.cpu().numpy()) 173 | m.weight.data = torch.from_numpy(w_ortho).to(device) 174 | try:nn.init.constant_(m.bias, 0) 175 | except:pass 176 | return 177 | 178 | 179 | def ortho_w_init(learn: Learner) -> Learner: 180 | learn.model.apply(orthogonal_weights_init) 181 | return learn 182 | 183 | Learner.ortho = ortho_w_init 184 | 185 | 186 | 187 | def kaiming(learn: Learner, normal: bool = True, a: float = 0, 188 | verbose:bool=False) -> Learner: 189 | f = nn.init.kaiming_normal_ if normal else nn.init.kaiming_uniform_ 190 | for i, m in enumerate(get_layers(learn, cond=noop)): 191 | if hasattr(m, 'weight') and m.weight.ndim >= 2: 192 | if verbose: print(i,'w',m) 193 | f(m.weight, a=a) 194 | if has_bias(m): 195 | if verbose: print(i,'b',m) 196 | m.bias.data.zero_() 197 | return learn 198 | 199 | Learner.kaiming = kaiming 200 | 201 | 202 | def activations(learn, thr=.1, cond=noop) -> Learner: 203 | mods = get_layers(learn, cond=cond) 204 | xb, yb = next(iter(learn.data.train_dl)) 205 | with Hooks(mods, layer_stats) as hooks: 206 | learn.model(xb) 207 | ms, inp_act_mean, out_act_mean, inp_act_std, out_act_std, inp_dead, \ 208 | out_dead, inp_size, out_size = [], [], [], [], [], [], [], [], [] 209 | i = 0 210 | for m, h in zip(mods, hooks): 211 | if not hasattr(h, 'out_mean') or\ 212 | (h.out_mean == h.inp_mean and h.out_std == h.inp_std): continue 213 | ms.append(' {:3d}-{} {}'.format(i,type(m).__name__,h.out_size[1:])) 214 | inp_act_mean.append(h.inp_mean) 215 | out_act_mean.append(h.out_mean) 216 | inp_act_std.append(h.inp_std) 217 | out_act_std.append(h.out_std) 218 | inp_dead.append(h.inp_dead) 219 | out_dead.append(h.out_dead) 220 | inp_size.append(h.inp_size) 221 | out_size.append(h.out_size) 222 | i += 1 223 | 224 | width = 0.4 225 | 226 | # MEAN 227 | x = inp_act_mean 228 | z = out_act_mean 229 | y = np.arange(len(x)) 230 | figsize = (10, len(y) * .3) 231 | 232 | plt.figure(figsize=figsize) 233 | plt.barh(y - width / 2, x, width, color='lavender', align='center', label='input') 234 | plt.barh(y + width / 2,z,width,color='purple',align='center',label='output') 235 | plt.vlines(0, y[0] - 1, y[-1] + 1, linewidth=3, color='g', label='expected') 236 | plt.vlines(np.mean(z),y[0] - 1,y[-1] + 1,linewidth=3,color='r',label='actual') 237 | plt.yticks(y, ms) 238 | plt.xlim(min(-1., min(x) - .1, min(z) - .1), max(1, max(x) + .1, max(z) + .1)) 239 | plt.ylim(len(y), -1) 240 | plt.legend(loc='best') 241 | plt.title('Initial activations (mean)') 242 | plt.grid(axis='x', color='silver', alpha=.3) 243 | plt.show() 244 | 245 | #STD 246 | x = inp_act_std 247 | z = out_act_std 248 | y = np.arange(len(x)) 249 | plt.figure(figsize=figsize) 250 | plt.barh( 251 | y - width / 2, x, width, color='lavender', align='center', label='input') 252 | plt.barh(y + width / 2,z,width,color='purple',align='center',label='output') 253 | plt.vlines(1, y[0] - 1, y[-1] + 1, linewidth=3, color='g', label='expected') 254 | plt.vlines(np.mean(z),y[0] - 1,y[-1] + 1,linewidth=3,color='r',label='actual') 255 | plt.yticks(y, ms) 256 | plt.xlim(0, max(1.2, max(1, max(x) + .1, max(z) + .1))) 257 | plt.ylim(len(y), -1) 258 | plt.legend(loc='best') 259 | plt.title('Initial activations (std)') 260 | plt.grid(axis='x', color='silver', alpha=.3) 261 | plt.show() 262 | 263 | #DEAD 264 | x = inp_dead 265 | z = out_dead 266 | max_exp = .1 267 | plt.figure(figsize=figsize) 268 | plt.barh(y - width / 2, x, width, color='lavender', align='center', label='input') 269 | plt.barh( y + width / 2,z,width, color='purple', align='center',label='output') 270 | plt.vlines(max_exp, y[0] - 1, y[-1] + 1, linewidth=3, color='g', label='expected') 271 | plt.vlines(np.mean(z),y[0] - 1,y[-1] + 1,linewidth=3, color='r',label='actual') 272 | plt.yticks(y, ms) 273 | plt.xlim(0, 1) 274 | plt.ylim(len(y), -1) 275 | plt.legend(loc='best') 276 | plt.title('Initial dead activations <={}'.format(str(thr))) 277 | plt.grid(axis='x', color='silver', alpha=.3) 278 | plt.show() 279 | 280 | return learn 281 | 282 | Learner.activations = activations 283 | Learner.act = activations 284 | 285 | def layer_stats(hook, mod, inp, out, thr=.1): 286 | i = inp[0].data 287 | o = out.data 288 | hook.inp_mean, hook.inp_std = i.mean().item(), i.std().item() 289 | hook.out_mean, hook.out_std = o.mean().item(), o.std().item() 290 | hook.inp_size = list(i.shape) 291 | hook.out_size = list(o.shape) 292 | hook.inp_dead = len(i[torch.abs(i)<=thr].flatten()) / len(i.flatten()) 293 | hook.out_dead = len(o[torch.abs(o)<=thr].flatten()) / len(o.flatten()) -------------------------------------------------------------------------------- /fastai_timeseries/exp/nb_TSBasicData.py: -------------------------------------------------------------------------------- 1 | #AUTOGENERATED! DO NOT EDIT! file to edit: ./TSBasicData.ipynb (unless otherwise specified) 2 | import fastai 3 | from fastai.torch_core import * 4 | from fastai.basic_data import * 5 | from fastai.data_block import * 6 | from fastai.core import * 7 | 8 | 9 | try: 10 | from exp.nb_TSUtilities import * 11 | from exp.nb_TSDatasets import * 12 | except ImportError: 13 | from .nb_TSUtilities import * 14 | from .nb_TSDatasets import * 15 | 16 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 17 | 18 | 19 | class TSItem(ItemBase): 20 | "`ItemBase` suitable for time series" 21 | 22 | def __init__(self, item, *args, **kwargs): 23 | super().__init__(item, *args, **kwargs) 24 | self.data = item 25 | self.obj = item 26 | self.channels = item.shape[-2] 27 | self.seq_len = item.shape[-1] 28 | 29 | 30 | def __str__(self): 31 | return 'TimeSeries(ch={:.0f}, seq_len={:.0f})'.format( 32 | self.channels, self.seq_len) 33 | 34 | def clone(self): 35 | return self.__class__(self.data.clone()) 36 | 37 | def apply_tfms(self, tfms=None, **kwargs): 38 | if tfms is None: return self 39 | x = self.clone() 40 | for tfm in tfms: x.data = tfm(x.data) 41 | return x 42 | 43 | def reconstruct(self, item): 44 | return TSItem(item) 45 | 46 | def show(self, ax=None, title=None, **kwargs): 47 | x = self.clone() 48 | if ax is None: 49 | plt.plot(x.data.transpose_(0, 1)) 50 | plt.title(title) 51 | plt.show() 52 | else: 53 | ax.plot(x.data.transpose_(0, 1)) 54 | ax.title.set_text(title) 55 | ax.tick_params( 56 | axis='both', 57 | which='both', 58 | bottom='off', 59 | top='off', 60 | labelbottom='off', 61 | right='off', 62 | left='off', 63 | labelleft='off') 64 | return ax 65 | 66 | class TimeSeriesItem(TSItem): pass 67 | 68 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 69 | 70 | class TSDataBunch(DataBunch): 71 | 72 | def scale(self, scale_type='standardize', scale_by_channel=False, scale_by_sample=False, 73 | scale_range=(-1, 1)) -> None: 74 | self.scale_type = scale_type 75 | self.scale_by_channel = scale_by_channel 76 | self.scale_by_sample = scale_by_sample 77 | self.scale_range = scale_range 78 | if scale_type is None: 79 | self.stats = None 80 | return self 81 | assert scale_type in ['normalize', 'standardize', 'robustscale'], print('Select a correct type', scale_type) 82 | 83 | train = self.train_ds.x.items.astype(float) 84 | valid = self.valid_ds.x.items.astype(float) 85 | if self.test_ds is not None: test = self.test_ds.x.items.astype(float) 86 | 87 | if scale_by_channel and scale_by_sample: axis = -1 # mean 88 | elif scale_by_channel: axis = (0, 2) # mean for the entire dataset by channel 89 | elif scale_by_sample: axis = (1, 2) # mean for each sample 90 | else: axis = None 91 | 92 | 93 | if scale_by_sample: 94 | self.stats = None 95 | if scale_type == 'normalize': 96 | train_min = np.nanmin(train, axis=axis, keepdims=True) 97 | train_max = np.nanmax(train, axis=axis, keepdims=True) 98 | self.train_ds.x.items = (((train - train_min)) / (train_max - train_min)) *\ 99 | (self.scale_range[1] - self.scale_range[0]) + self.scale_range[0] 100 | valid_min = np.nanmin(valid, axis=axis, keepdims=True) 101 | valid_max = np.nanmax(valid, axis=axis, keepdims=True) 102 | self.valid_ds.x.items = (((valid - valid_min)) / (valid_max - valid_min)) *\ 103 | (self.scale_range[1] - self.scale_range[0]) + self.scale_range[0] 104 | if self.test_ds is not None: 105 | test_min = np.nanmin(test, axis=axis, keepdims=True) 106 | test_max = np.nanmax(test, axis=axis, keepdims=True) 107 | self.test_ds.x.items = (((test - test_min)) / (test_max - test_min)) *\ 108 | (self.scale_range[1] - self.scale_range[0]) + self.scale_range[0] 109 | return self 110 | 111 | elif scale_type == 'standardize': 112 | train_mean = np.nanmean(train, axis=axis, keepdims=True) 113 | train_std = np.nanstd(train, axis=axis, keepdims=True) + 1e-8 114 | self.train_ds.x.items = (train - train_mean) / train_std 115 | valid_mean = np.nanmean(valid, axis=axis, keepdims=True) 116 | valid_std = np.nanstd(valid, axis=axis, keepdims=True) + 1e-8 117 | self.valid_ds.x.items = (valid - valid_mean) / valid_std 118 | if self.test_ds is not None: 119 | test_mean = np.nanmean(test, axis=axis, keepdims=True) 120 | test_std = np.nanstd(test, axis=axis, keepdims=True) + 1e-8 121 | self.test_ds.x.items = (test - test_mean) / test_std 122 | return self 123 | 124 | elif scale_type == 'robustscale': 125 | train_median = np.nanmedian(train, axis=axis, keepdims=True) 126 | train_perc_25 = np.nanpercentile(train, 25, axis=axis, keepdims=True) 127 | train_perc_75 = np.nanpercentile(train, 75, axis=axis, keepdims=True) 128 | train_scale = train_perc_75 - train_perc_25 129 | self.train_ds.x.items = (train - train_median) / train_scale 130 | 131 | valid_median = np.nanmedian(valid, axis=axis, keepdims=True) 132 | valid_perc_25 = np.nanpercentile(valid, 25, axis=axis, keepdims=True) 133 | valid_perc_75 = np.nanpercentile(valid, 75, axis=axis, keepdims=True) 134 | valid_scale = valid_perc_75 - valid_perc_25 135 | self.valid_ds.x.items = (valid - valid_median) / valid_scale 136 | 137 | if self.test_ds is not None: 138 | test_median = np.nanmedian(test, axis=axis, keepdims=True) 139 | test_perc_25 = np.nanpercentile(test, 25, axis=axis, keepdims=True) 140 | test_perc_75 = np.nanpercentile(test, 75, axis=axis, keepdims=True) 141 | test_scale = test_perc_75 - test_perc_25 142 | self.test_ds.x.items = (test - test_median) / test_scale 143 | return self 144 | 145 | else: 146 | if scale_type == 'normalize': 147 | train_min = np.nanmin(train, axis=axis, keepdims=True) 148 | train_max = np.nanmax(train, axis=axis, keepdims=True) 149 | self.stats = train_min, train_max 150 | self.train_ds.x.items = (((self.train_ds.x.items - train_min)) / (train_max - train_min)) *\ 151 | (self.scale_range[1] - self.scale_range[0]) + self.scale_range[0] 152 | self.valid_ds.x.items = (((self.valid_ds.x.items - train_min)) / (train_max - train_min)) *\ 153 | (self.scale_range[1] - self.scale_range[0]) + self.scale_range[0] 154 | if self.test_ds is not None: 155 | self.test_ds.x.items = (((self.test_ds.x.items - train_min)) / (train_max - train_min)) *\ 156 | (self.scale_range[1] - self.scale_range[0]) + self.scale_range[0] 157 | return self 158 | elif scale_type == 'standardize': 159 | train_mean = np.nanmean(train, axis=axis, keepdims=True) 160 | train_std = np.nanstd(train, axis=axis, keepdims=True) + 1e-8 161 | self.stats = train_mean, train_std 162 | self.train_ds.x.items = (self.train_ds.x.items - train_mean) / train_std 163 | self.valid_ds.x.items = (self.valid_ds.x.items - train_mean) / train_std 164 | if self.test_ds is not None: 165 | self.test_ds.x.items = (self.test_ds.x.items - train_mean) / train_std 166 | return self 167 | elif scale_type == 'robustscale': 168 | train_median = np.nanmedian(train, axis=axis, keepdims=True) 169 | train_perc_25 = np.nanpercentile(train, 25, axis=axis, keepdims=True) 170 | train_perc_75 = np.nanpercentile(train, 75, axis=axis, keepdims=True) 171 | train_scale = train_perc_75 - train_perc_25 172 | self.stats = train_median, train_scale 173 | self.train_ds.x.items = (train - train_median) / train_scale 174 | self.valid_ds.x.items = (valid - train_median) / train_scale 175 | if self.test_ds is not None: self.test_ds.x.items = (test - train_median) / train_scale 176 | return self 177 | 178 | @property 179 | def cw(self)->None: return self._get_cw(self.train_dl) 180 | 181 | @property 182 | def dbtype(self)->str: return '1D' 183 | 184 | def _get_cw(self, train_dl): 185 | target = torch.Tensor(train_dl.dataset.y.items).to(dtype=torch.int64) 186 | # Compute samples weight (each sample should get its own weight) 187 | class_sample_count = torch.tensor( 188 | [(target == t).sum() for t in torch.unique(target, sorted=True)]) 189 | weights = 1. / class_sample_count.float() 190 | return (weights / weights.sum()).to(device) 191 | 192 | 193 | def show_counts(databunch): 194 | labels, counts = np.unique(databunch.train_ds.y.items, return_counts=True) 195 | plt.bar(labels, counts) 196 | plt.title('labels') 197 | plt.xticks(labels) 198 | plt.show() 199 | 200 | DataBunch.show_counts = show_counts 201 | 202 | 203 | 204 | class TSPreProcessor(PreProcessor): 205 | 206 | def __init__(self, ds: ItemList): self.ds = ds 207 | 208 | def process(self, ds: ItemList): 209 | ds.features, ds.seq_len = self.ds.get(0).data.size(-2), self.ds.get(0).data.size(-1) 210 | ds.f = ds.features 211 | ds.s = ds.seq_len 212 | 213 | 214 | class TimeSeriesList(ItemList): 215 | "`ItemList` suitable for time series" 216 | _bunch = TSDataBunch 217 | _processor = TSPreProcessor 218 | _label_cls = None 219 | _square_show = True 220 | 221 | def __init__(self, items, *args, mask=None, tfms=None, **kwargs): 222 | items = To3dTensor(items) 223 | super().__init__(items, *args, **kwargs) 224 | self.tfms,self.mask = tfms,mask 225 | self.copy_new.append('tfms') 226 | 227 | def get(self, i): 228 | item = super().get(i) 229 | if self.mask is None: return TSItem(To2dTensor(item)) 230 | else: return[TSItem(To2dTensor(item[m])) for m in self.mask] 231 | 232 | 233 | def show_xys(self, xs, ys, figsize=(10, 10), **kwargs): 234 | "Show the `xs` and `ys` on a figure of `figsize`. `kwargs` are passed to the show method." 235 | rows = int(math.sqrt(len(xs))) 236 | fig, axs = plt.subplots(rows, rows, figsize=figsize) 237 | for x, y, ax in zip(xs, ys, axs.flatten()): 238 | x.show(ax=ax, title=str(y), **kwargs) 239 | plt.tight_layout() 240 | plt.show() 241 | 242 | def show_xyzs(self, xs, ys, zs, figsize=(10, 10), **kwargs): 243 | if self._square_show_res: 244 | rows = int(np.ceil(math.sqrt(len(xs)))) 245 | fig, axs = plt.subplots( 246 | rows, 247 | rows, 248 | figsize=figsize, 249 | title='Ground truth\nPredictions', 250 | weight='bold', 251 | size=12) 252 | for x, y, z, ax in zip(xs, ys, zs, axs.flatten()): 253 | x.show(ax=ax, title=f'{str(y)}\n{str(z)}', **kwargs) 254 | else: 255 | fig, axs = plt.subplots( 256 | len(xs), 257 | 2, 258 | figsize=figsize, 259 | title='Ground truth/Predictions', 260 | weight='bold', 261 | size=12) 262 | for i, (x, y, z) in enumerate(zip(xs, ys, zs)): 263 | x.show(ax=axs[i, 0], title=str(y), **kwargs) 264 | x.show(ax=axs[i, 1], title=str(z), **kwargs) 265 | plt.tight_layout() 266 | plt.show() 267 | 268 | @classmethod 269 | def from_array(cls, ts, **kwargs): 270 | return cls(ts) 271 | 272 | @classmethod 273 | def from_df(cls, df, path='.', cols=None, feat=None, processor=None, **kwargs) -> 'ItemList': 274 | "Create an `ItemList` in `path` from the inputs in the `cols` of `df`." 275 | if cols is None: 276 | inputs = df 277 | else: 278 | cols = listify(cols) 279 | if feat is not None and feat not in cols: cols = cols + listify(feat) 280 | col_idxs = df_names_to_idx(list(cols), df) 281 | inputs = df.iloc[:, col_idxs] 282 | assert inputs.isna().sum().sum( 283 | ) == 0, f"You have NaN values in column(s) {cols} of your dataframe, please fix it." 284 | inputs = df2array(inputs, feat) 285 | res = cls( 286 | items=inputs, 287 | path=path, 288 | inner_df=df, 289 | processor=processor, 290 | **kwargs) 291 | return res 292 | 293 | class TSList(TimeSeriesList): pass 294 | 295 | 296 | 297 | class MixedTimeSeriesList(ItemList): 298 | "`ItemList` suitable for time series" 299 | _bunch = TSDataBunch 300 | _processor = TSPreProcessor 301 | _label_cls = None 302 | _square_show = True 303 | 304 | def __init__(self, items, *args, mask=None, tfms=None, **kwargs): 305 | items = To3dTensor(items) 306 | super().__init__(items, *args, **kwargs) 307 | self.tfms,self.mask = tfms,mask 308 | self.copy_new.append('tfms') 309 | 310 | def get(self, i): 311 | item = super().get(i) 312 | if self.mask is None: return TSItem(To2dTensor(item)) 313 | else: return[TSItem(To2dTensor(item[m])) for m in self.mask] 314 | 315 | 316 | def show_xys(self, xs, ys, figsize=(10, 10), **kwargs): 317 | "Show the `xs` and `ys` on a figure of `figsize`. `kwargs` are passed to the show method." 318 | rows = int(math.sqrt(len(xs))) 319 | fig, axs = plt.subplots(rows, rows, figsize=figsize) 320 | for x, y, ax in zip(xs, ys, axs.flatten()): 321 | x.show(ax=ax, title=str(y), **kwargs) 322 | plt.tight_layout() 323 | plt.show() 324 | 325 | def show_xyzs(self, xs, ys, zs, figsize=(10, 10), **kwargs): 326 | if self._square_show_res: 327 | rows = int(np.ceil(math.sqrt(len(xs)))) 328 | fig, axs = plt.subplots( 329 | rows, 330 | rows, 331 | figsize=figsize, 332 | title='Ground truth\nPredictions', 333 | weight='bold', 334 | size=12) 335 | for x, y, z, ax in zip(xs, ys, zs, axs.flatten()): 336 | x.show(ax=ax, title=f'{str(y)}\n{str(z)}', **kwargs) 337 | else: 338 | fig, axs = plt.subplots( 339 | len(xs), 340 | 2, 341 | figsize=figsize, 342 | title='Ground truth/Predictions', 343 | weight='bold', 344 | size=12) 345 | for i, (x, y, z) in enumerate(zip(xs, ys, zs)): 346 | x.show(ax=axs[i, 0], title=str(y), **kwargs) 347 | x.show(ax=axs[i, 1], title=str(z), **kwargs) 348 | plt.tight_layout() 349 | plt.show() 350 | 351 | @classmethod 352 | def from_array(cls, ts, processor=None, **kwargs): 353 | return cls(ts, processor=processor, **kwargs) 354 | 355 | @classmethod 356 | def from_df(cls, df, path='.', cols=None, feat=None, processor=None, **kwargs) -> 'ItemList': 357 | "Create an `ItemList` in `path` from the inputs in the `cols` of `df`." 358 | if cols is 0: 359 | inputs = df 360 | else: 361 | col_idxs = df_names_to_idx(list(cols), df) 362 | inputs = df.iloc[:, col_idxs] 363 | assert inputs.isna().sum().sum( 364 | ) == 0, f"You have NaN values in column(s) {cols} of your dataframe, please fix it." 365 | inputs = df2array(inputs, feat) 366 | 367 | res = cls( 368 | items=inputs, 369 | path=path, 370 | inner_df=df, 371 | processor=processor, 372 | **kwargs) 373 | return res 374 | 375 | 376 | def df2array(df, feat=None): 377 | if feat is None: 378 | return df.values[:, None] 379 | for i, ch in enumerate(df[feat].unique()): 380 | data_i = df[df[feat] == ch].values[:, None] 381 | if i == 0: data = data_i 382 | else: data = np.concatenate((data, data_i), axis=1) 383 | return data -------------------------------------------------------------------------------- /fastai_timeseries/exp/nb_TSCallbacks.py: -------------------------------------------------------------------------------- 1 | #AUTOGENERATED! DO NOT EDIT! file to edit: ./TSCallbacks.ipynb (unless otherwise specified) 2 | try: from exp.nb_TSBasicData import TSItem 3 | except ImportError: from .nb_TSBasicData import TSItem 4 | 5 | 6 | from fastai.vision import * 7 | from fastai.torch_core import * 8 | from fastai.callback import * 9 | from fastai.callbacks.mixup import * 10 | from fastai.basic_train import Learner, LearnerCallback 11 | 12 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 13 | 14 | 15 | class CutMixCallback(LearnerCallback): 16 | 17 | def __init__(self, learn:Learner, alpha:float=1., alpha2:float=0., stack_y:bool=True, 18 | out:bool=False, mix:bool=False): 19 | 20 | super().__init__(learn) 21 | self.alpha,self.alpha2,self.stack_y,self.out,self.mix = alpha,alpha2,stack_y,out,mix 22 | assert out + mix < 2, 'cutmix out and mix cannot be selected at the same time' 23 | assert alpha2 >= 0 and alpha2 <=1, 'alpha2 must be between 0. and 1.' 24 | if out: self.__name__ = 'CutOutCallBack' 25 | elif mix: self.__name__ = 'CutMixUpCallBack' 26 | else: self.__name__ = 'CutMixCallBack' 27 | 28 | def on_train_begin(self, **kwargs): 29 | if self.stack_y: self.learn.loss_func = MixUpLoss(self.learn.loss_func) 30 | 31 | def on_batch_begin(self, last_input, last_target, train, **kwargs): 32 | "Applies cutmix to `last_input` and `last_target` if `train`." 33 | if not train or self.alpha == 0: return 34 | λ = np.random.beta(self.alpha, self.alpha) 35 | λ = max(λ, 1- λ) 36 | if self.mix: 37 | if self.alpha2 == 0: self.alpha2 = self.alpha 38 | λ2 = np.random.beta(self.alpha2, self.alpha2) 39 | λ2 = λ + (1 - λ) * λ2 40 | λ = λ / λ2 41 | 42 | idx = torch.randperm(last_target.size(0)).to(last_input.device) 43 | 44 | # Create new input 45 | last_input_size = last_input.size() 46 | bbx1, bby1, bbx2, bby2 = rand_bbox(last_input_size, λ) 47 | new_input = last_input.clone() 48 | if self.out: new_input[..., bby1:bby2, bbx1:bbx2] = 0 49 | elif self.mix: new_input[..., bby1:bby2, bbx1:bbx2] = λ2 * last_input[..., bby1:bby2, bbx1:bbx2] + \ 50 | (1 - λ2) * last_input[idx][..., bby1:bby2, bbx1:bbx2] 51 | else: new_input[..., bby1:bby2, bbx1:bbx2] = last_input[idx][..., bby1:bby2, bbx1:bbx2] 52 | λ = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (last_input_size[-1] * last_input_size[-2])) 53 | if self.mix: λ = λ * λ2 54 | λ = last_input.new([λ]) 55 | 56 | # Modify last target 57 | if self.stack_y: 58 | new_target = torch.cat([last_target.unsqueeze(1).float(), last_target[idx].unsqueeze(1).float(), 59 | λ.repeat(last_input_size[0]).unsqueeze(1).float()], 1) 60 | else: 61 | if len(last_target.shape) == 2: 62 | λ = λ.unsqueeze(1).float() 63 | new_target = last_target.float() * λ + last_target[idx].float() * (1-λ) 64 | 65 | return {'last_input': new_input, 'last_target': new_target} 66 | 67 | def on_train_end(self, **kwargs): 68 | if self.stack_y: self.learn.loss_func = self.learn.loss_func.get_old() 69 | 70 | 71 | 72 | def rand_bbox(last_input_size, λ): 73 | '''lambd is always between .5 and 1''' 74 | 75 | W = last_input_size[-1] 76 | H = last_input_size[-2] 77 | cut_rat = np.sqrt(1. - λ) # 0. - .707 78 | cut_w = np.int(W * cut_rat) 79 | cut_h = np.int(H * cut_rat) 80 | 81 | # uniform 82 | cx = np.random.randint(W) 83 | cy = np.random.randint(H) 84 | 85 | bbx1 = np.clip(cx - cut_w // 2, 0, W) 86 | bbx2 = np.clip(cx + cut_w // 2, 0, W) 87 | if len(last_input_size) == 4: 88 | bby1 = np.clip(cy - cut_h // 2, 0, H) 89 | bby2 = np.clip(cy + cut_h // 2, 0, H) 90 | else: 91 | bby1 = 0 92 | bby2 = last_input_size[1] 93 | 94 | return bbx1, bby1, bbx2, bby2 95 | 96 | 97 | def cutmix(learn:Learner, alpha:float=1., stack_y:bool=True) -> Learner: 98 | "Add cutmix https://arxiv.org/pdf/1905.04899 to `learn`." 99 | learn.callback_fns.append(partial(CutMixCallback, alpha=alpha, stack_y=stack_y)) 100 | return learn 101 | 102 | def cutout(learn:Learner, alpha:float=1., stack_y:bool=True, out:bool=True) -> Learner: 103 | "Add cutout https://arxiv.org/abs/1708.04552 to `learn`." 104 | learn.callback_fns.append(partial(CutMixCallback, alpha=alpha, stack_y=stack_y, out=out)) 105 | return learn 106 | 107 | def cutmixup(learn:Learner, alpha:float=1., alpha2:float=1., stack_y:bool=True, mix:bool=True) -> Learner: 108 | "Add cutmixup to `learn`." 109 | learn.callback_fns.append(partial(CutMixCallback, alpha=alpha, alpha2=alpha2, stack_y=stack_y, mix=mix)) 110 | return learn 111 | 112 | setattr(cutmix, 'cb_fn', CutMixCallback) 113 | Learner.cutmix = cutmix 114 | 115 | setattr(cutout, 'cb_fn', CutMixCallback) 116 | Learner.cutout = cutout 117 | 118 | setattr(cutmixup, 'cb_fn', CutMixCallback) 119 | Learner.cutmixup = cutmixup 120 | 121 | from fastai.train import mixup 122 | setattr(mixup, 'cb_fn', MixUpCallback) 123 | 124 | 125 | class RicapLoss(nn.Module): 126 | "Adapt the loss function `crit` to go with ricap data augmentations." 127 | 128 | def __init__(self, crit, reduction='mean'): 129 | super().__init__() 130 | if hasattr(crit, 'reduction'): 131 | self.crit = crit 132 | self.old_red = crit.reduction 133 | setattr(self.crit, 'reduction', 'none') 134 | else: 135 | self.crit = partial(crit, reduction='none') 136 | self.old_crit = crit 137 | self.reduction = reduction 138 | 139 | def forward(self, output, target): 140 | if target.ndim == 2: 141 | c_ = target[:, 1:5] 142 | W_ = target[:, 5:] 143 | loss = [W_[:, k] * self.crit(output, c_[:, k].long()) for k in range(4)] 144 | d = torch.mean(torch.stack(loss)) 145 | else: d = self.crit(output, target) 146 | if self.reduction == 'mean': return d.mean() 147 | elif self.reduction == 'sum': return d.sum() 148 | return d 149 | 150 | def get_old(self): 151 | if hasattr(self, 'old_crit'): return self.old_crit 152 | elif hasattr(self, 'old_red'): 153 | setattr(self.crit, 'reduction', self.old_red) 154 | return self.crit 155 | 156 | class RicapCallback(LearnerCallback): 157 | '''Adapted from : 158 | paper: https://arxiv.org/abs/1811.09030 159 | github: https://github.com/4uiiurz1/pytorch-ricap 160 | and mixup in the fastai library.''' 161 | def __init__(self, learn:Learner, beta:float=.3, stack_y:bool=True): 162 | super().__init__(learn) 163 | self.beta,self.stack_y = beta,stack_y 164 | 165 | def on_train_begin(self, **kwargs): 166 | if self.stack_y: self.learn.loss_func = RicapLoss(self.learn.loss_func) 167 | 168 | def on_batch_begin(self, last_input, last_target, train, **kwargs): 169 | "Applies ricap to `last_input` and `last_target` if `train`." 170 | if not train or self.beta == 0: return 171 | 172 | # get the image size 173 | I_x, I_y = last_input.size()[2:] 174 | 175 | # draw a boundary position (w, h) 176 | w = int(np.round(I_x * np.random.beta(self.beta, self.beta))) 177 | h = int(np.round(I_y * np.random.beta(self.beta, self.beta))) 178 | w_ = [w, I_x - w, w, I_x - w] 179 | h_ = [h, h, I_y - h, I_y - h] 180 | 181 | # select and crop four images 182 | cropped_images = {} 183 | bs = last_input.size(0) 184 | c_ = torch.zeros((bs, 4)).float().to(last_input.device) 185 | W_ = torch.zeros(4).float().to(last_input.device) 186 | for k in range(4): 187 | idx = torch.randperm(bs).to(last_input.device) 188 | x_k = np.random.randint(0, I_x - w_[k] + 1) 189 | y_k = np.random.randint(0, I_y - h_[k] + 1) 190 | cropped_images[k] = last_input[idx][:, :, x_k:x_k + w_[k], y_k:y_k + h_[k]] 191 | c_[:, k] = last_target[idx].float() 192 | W_[k] = w_[k] * h_[k] / (I_x * I_y) 193 | 194 | # patch cropped images 195 | patched_images = torch.cat( 196 | (torch.cat((cropped_images[0], cropped_images[1]), 2), 197 | torch.cat((cropped_images[2], cropped_images[3]), 2)), 3).to(last_input.device) 198 | 199 | # modify last target 200 | if self.stack_y: 201 | new_target = torch.cat((last_target[:,None].float(), c_, 202 | W_[None].repeat(last_target.size(0), 1)), dim=1) 203 | else: 204 | new_target = c_ * W_ 205 | 206 | return {'last_input': patched_images, 'last_target': new_target} 207 | 208 | def on_train_end(self, **kwargs): 209 | if self.stack_y: self.learn.loss_func = self.learn.loss_func.get_old() 210 | 211 | 212 | def ricap(learn:Learner, beta:float=.3, stack_y:bool=True) -> Learner: 213 | "Add ricap https://arxiv.org/pdf/1811.09030.pdf to `learn`." 214 | learn.callback_fns.append(partial(RicapCallback, beta=beta, stack_y=stack_y)) 215 | return learn 216 | 217 | setattr(ricap, 'cb_fn', RicapCallback) 218 | Learner.ricap = ricap 219 | 220 | 221 | def get_fn(a): 222 | while True: 223 | if hasattr(a, 'func'): a = a.func 224 | else: break 225 | return a 226 | 227 | def show_tfms(learn, rows=3, cols=3, figsize=(8, 8)): 228 | xb, yb = learn.data.one_batch() 229 | xb = xb.to('cpu') 230 | yb = yb.to('cpu') 231 | rand_int = np.random.randint(len(xb)) 232 | ndim = xb.ndim 233 | tfms = learn.data.train_ds.tfms 234 | 235 | if ndim == 4: 236 | rand_item = Image(xb[rand_int]) 237 | #print('\noriginal image:') 238 | #display(rand_item) 239 | for i in range(len(xb)): xb[i] = Image(xb[i]).apply_tfms(tfms).data.to(device) 240 | cb_tfms = 0 241 | for cb in learn.callback_fns: 242 | if get_fn(cb).__name__ == 'Recorder': continue 243 | if hasattr(cb, 'keywords') and hasattr(get_fn(cb), 'on_batch_begin'): 244 | cb_fn = partial(get_fn(cb), **cb.keywords) 245 | try: 246 | fig = plt.subplots(rows, cols, figsize=figsize, sharex=True, sharey=True)[1].flatten() 247 | plt.suptitle(get_fn(cb).__name__, cb.keywords, size=14) 248 | [rand_item.show(ax=ax) if i == 0 else (Image(Tensor2ImgTensor(cb_fn(learn).on_batch_begin(xb, yb, True)['last_input'][rand_int])) 249 | .apply_tfms(tfms).show(ax=ax)) for i, ax in enumerate(fig)] 250 | #print(get_fn(cb).__name__, cb.keywords) 251 | fig[0].set_title('original') 252 | plt.show() 253 | cb_tfms += 1 254 | break 255 | except: 256 | plt.close('all') 257 | 258 | elif ndim == 3: 259 | rand_item = TSItem(xb[rand_int]) 260 | cb_tfms = 0 261 | for cb in learn.callback_fns: 262 | if get_fn(cb).__name__ == 'Recorder': continue 263 | if hasattr(cb, 'keywords') and hasattr(get_fn(cb), 'on_batch_begin'): 264 | cb_fn = partial(get_fn(cb), **cb.keywords) 265 | try: 266 | fig = plt.subplots(rows, cols, figsize=figsize, sharex=True, sharey=True)[1].flatten() 267 | plt.suptitle(get_fn(cb).__name__ , size=14) 268 | [rand_item.show(ax=ax) if i == 0 else (TSItem(cb_fn(learn).on_batch_begin(xb, yb, True)['last_input'][rand_int]) 269 | .apply_tfms(tfms).show(ax=ax)) for i, ax in enumerate(fig)] 270 | fig[0].set_title('original') 271 | plt.show() 272 | cb_tfms += 1 273 | break 274 | except: 275 | plt.close('all') 276 | 277 | 278 | if tfms is not None: 279 | fig = plt.subplots(rows, cols, figsize=figsize, sharex=True, sharey=True)[1].flatten() 280 | [rand_item.show(ax=ax) if i == 0 else rand_item.apply_tfms(tfms).show(ax=ax) for i, ax in enumerate(fig)] 281 | fig[0].set_title('original') 282 | try: 283 | t_ = [] 284 | for t in learn.data.train_ds.tfms: t_.append(get_fn(t).__name__) 285 | title = f"{str(t_)[1:-1]} transforms applied" 286 | plt.suptitle(title, size=14) 287 | except: pass 288 | plt.show() 289 | elif cb_tfms == 0: 290 | print('No transformation has been applied') 291 | rand_item.show() 292 | 293 | return learn 294 | 295 | Learner.show_tfms = show_tfms 296 | 297 | def show_tfms_db(data, rows=3, cols=3, figsize=(8, 8)): 298 | xb, yb = data.one_batch() 299 | xb = xb.to('cpu') 300 | yb = yb.to('cpu') 301 | rand_int = np.random.randint(len(xb)) 302 | ndim = xb.ndim 303 | tfms = data.train_ds.tfms 304 | 305 | if ndim == 4: rand_item = Image(xb[rand_int]) 306 | elif ndim == 3:rand_item = TSItem(xb[rand_int]) 307 | if tfms is not None: 308 | fig = plt.subplots(rows, cols, figsize=figsize, sharex=True, sharey=True)[1].flatten() 309 | [rand_item.show(ax=ax) if i == 0 else rand_item.apply_tfms(tfms).show(ax=ax) for i, ax in enumerate(fig)] 310 | fig[0].set_title('original') 311 | try: 312 | t_ = [] 313 | for t in learn.data.train_ds.tfms: t_.append(get_fn(t).__name__) 314 | title = f"{str(t_)[1:-1]} transforms applied" 315 | plt.suptitle(title, size=14) 316 | except: pass 317 | plt.show() 318 | else: 319 | print('No transformation has been applied') 320 | rand_item.show() 321 | return 322 | 323 | DataBunch.show_tfms = show_tfms_db 324 | 325 | from torch.utils.data.sampler import WeightedRandomSampler 326 | 327 | class OverSamplingCallback(LearnerCallback): 328 | def __init__(self,learn:Learner,weights:torch.Tensor=None): 329 | super().__init__(learn) 330 | self.weights = weights 331 | 332 | def on_train_begin(self, **kwargs): 333 | ds,dl = self.data.train_ds,self.data.train_dl 334 | self.labels = ds.y.items 335 | assert np.issubdtype(self.labels.dtype, np.integer), "Can only oversample integer values" 336 | _,self.label_counts = np.unique(self.labels,return_counts=True) 337 | if self.weights is None: self.weights = torch.DoubleTensor((1/self.label_counts)[self.labels]) 338 | self.total_len_oversample = int(self.data.c*np.max(self.label_counts)) 339 | sampler = WeightedRandomSampler(self.weights, self.total_len_oversample) 340 | self.data.train_dl = dl.new(shuffle=False, sampler=sampler) 341 | 342 | def oversampling(learn: Learner) -> Learner: 343 | learn.callback_fns.append(OverSamplingCallback) 344 | return learn 345 | 346 | 347 | Learner.oversampling = oversampling 348 | Learner.os = oversampling 349 | 350 | 351 | from fastai.callbacks.tracker import TrackerCallback 352 | class ReduceLROnPlateau(TrackerCallback): 353 | "A `TrackerCallback` that reduces learning rate when a metric has stopped improving." 354 | def __init__(self, learn:Learner, monitor:str='valid_loss', mode:str='auto', 355 | patience:int=0, factor:float=0.2, min_delta:int=0, min_lr:float=1e-6, verbose=False): 356 | super().__init__(learn, monitor=monitor, mode=mode) 357 | self.patience,self.factor,self.min_delta,self.min_lr,self.verbose = patience,factor,min_delta,min_lr,verbose 358 | if self.operator == np.less: self.min_delta *= -1 359 | 360 | def on_train_begin(self, **kwargs:Any)->None: 361 | "Initialize inner arguments." 362 | self.wait, self.opt = 0, self.learn.opt 363 | super().on_train_begin(**kwargs) 364 | 365 | def on_epoch_end(self, epoch, **kwargs:Any)->None: 366 | "Compare the value monitored to its best and maybe reduce lr." 367 | current = self.get_monitor_value() 368 | if current is None: return 369 | if self.operator(current - self.min_delta, self.best): self.best,self.wait = current,0 370 | elif self.opt.lr > self.min_lr: 371 | self.wait += 1 372 | if self.wait > self.patience: 373 | self.opt.lr = max(self.min_lr, self.opt.lr * self.factor) 374 | self.wait = 0 375 | if self.verbose: print(f'Epoch {epoch}: reducing lr to {self.opt.lr}') 376 | 377 | 378 | def reduce_lr_on_plateau(learn: Learner, monitor='valid_loss', mode='auto', 379 | patience=0, factor=0.2, min_delta=0, min_lr:float=1e-6, verbose=False) -> Learner: 380 | 381 | learn.callback_fns.append( 382 | partial(ReduceLROnPlateau, monitor=monitor, mode=mode, 383 | patience=patience, factor=factor, min_delta=min_delta, min_lr=min_lr, verbose=verbose)) 384 | return learn 385 | 386 | Learner.plateau = reduce_lr_on_plateau 387 | 388 | 389 | import math 390 | 391 | class TfmScheduler(LearnerCallback): 392 | 393 | def __init__(self, 394 | learn: Learner, 395 | tfm_fn: Callable, 396 | sch_param: Union[str, StrList], 397 | sch_val: Union[StartOptEnd, List], 398 | sch_iter: Optional[StartOptEnd] = None, 399 | sch_func: Optional[AnnealFunc] = None, 400 | plot: bool = False, 401 | test: bool = False, 402 | **kwargs: Any): 403 | 404 | super().__init__(learn) 405 | self.learn = learn 406 | self.batches = math.ceil(len(learn.data.train_ds)/learn.data.train_dl.batch_size) 407 | sch_param = listify(sch_param) 408 | if isinstance(sch_param, (float, int)): sch_val = (0, sch_val) 409 | sch_val = tuplify(sch_val) 410 | if len(sch_param) != len(sch_val): sch_val = sch_val * len(sch_param) 411 | assert len(sch_param) == len(sch_val) 412 | if sch_iter is None: sch_iter = (0., 1.) 413 | sch_iter = tuplify(sch_iter) 414 | if len(sch_param) != len(sch_iter): sch_iter = sch_iter * len(sch_param) 415 | assert len(sch_param) == len(sch_iter) 416 | self.tfm_fn,self.sch_param,self.sch_val,self.sch_iter,self.test = tfm_fn,sch_param,sch_val,sch_iter,test 417 | if sch_func is None: sch_func = annealing_linear 418 | sch_func = listify(sch_func) 419 | if len(sch_param) != len(sch_func): sch_func = sch_func * len(sch_param) 420 | assert len(sch_param) == len(sch_func) 421 | self.sch_func = sch_func 422 | self.plot = plot 423 | if not isinstance(self.tfm_fn, functools.partial): self.tfm_fn = partial(self.tfm_fn) 424 | self.fn = get_fn(self.tfm_fn) 425 | if hasattr(self.fn, 'cb_fn'): self.fn = self.fn.cb_fn 426 | if hasattr(self.fn, 'on_batch_begin'): self.cb = True 427 | else: self.cb = False 428 | 429 | 430 | def on_train_begin(self, n_epochs: int, epoch: int, **kwargs: Any): 431 | if self.cb: self.fn(self.learn).on_train_begin() 432 | total_iters = n_epochs * self.batches 433 | self.scheduler = [None] * len(self.sch_param) 434 | for i in range(len(self.sch_param)): 435 | p = self.sch_param[i] 436 | v = self.sch_val[i] 437 | iters = self.sch_iter[i] 438 | func = self.sch_func[i] 439 | self.scheduler[i] = MyScheduler(total_iters, v, sch_iter=iters, sch_func=func) 440 | s = self.scheduler[i] 441 | a = s.start_val 442 | a_ = [] 443 | first_iter = -1 444 | last_iter = 1 445 | for i in range(total_iters): 446 | a = s.step() 447 | if i > 0 and first_iter == -1 and a != a_[-1]: first_iter = (i - 1) / total_iters 448 | elif first_iter != -1 and last_iter == 1 and a == a_[-1]: last_iter = i / total_iters 449 | a_.append(a) 450 | s.restart() 451 | text = '{} between {} and {} in iters {:.2f} to {:.2f}'.format( 452 | p, round(min(a_), 5), round(max(a_), 5), first_iter, last_iter) 453 | print('\n',text) 454 | if self.plot: 455 | plt.plot(a_) 456 | plt.title(text) 457 | plt.show() 458 | 459 | def on_batch_begin(self, last_input, last_target, train, **kwargs): 460 | if self.test: return {'stop_epoch': True, 'stop_training': True, 'skip_validate': True} 461 | if train: 462 | for i, (p, v) in enumerate(zip(self.sch_param, self.sch_val)): 463 | new_v = self.scheduler[i].step() 464 | self.tfm_fn.keywords[p] = new_v 465 | kw = self.tfm_fn.keywords 466 | if self.cb: 467 | return self.fn(self.learn, **kw).on_batch_begin( 468 | last_input=last_input,last_target=last_target,train=train) 469 | else: 470 | new_input = self.fn(last_input, **kw) 471 | return {'last_input': new_input, 'last_target': last_target} 472 | else: return 473 | 474 | def on_train_end(self, **kwargs): 475 | if self.cb: self.fn(self.learn).on_train_end() 476 | 477 | 478 | class MyScheduler(): 479 | "Used to \"step\" from start,end (`vals`) over `n_iter` iterations on a schedule defined by `func`" 480 | def __init__(self, total_iters:int, sch_val:StartOptEnd, sch_iter:Optional[StartOptEnd]=None, 481 | sch_func:Optional[AnnealFunc]=None): 482 | self.total_iters = total_iters 483 | self.start_val,self.end_val = (sch_val[0],sch_val[1]) if is_tuple(sch_val) else (0, sch_val) 484 | if sch_iter is None: self.start_iter,self.end_iter = (0, total_iters) 485 | else: 486 | self.start_iter,self.end_iter = (sch_iter[0],sch_iter[1]) if is_tuple(sch_iter) else (0, sch_iter) 487 | if self.start_iter == 1 or isinstance(self.start_iter, float): 488 | self.start_iter = int(self.start_iter * total_iters) 489 | if self.end_iter == 1 or isinstance(self.end_iter, float): 490 | self.end_iter = int(self.end_iter * total_iters) 491 | self.eff_iters = self.end_iter - self.start_iter 492 | if sch_func is None: self.sch_func = annealing_linear 493 | else: self.sch_func = sch_func 494 | self.n = 0 495 | 496 | def restart(self): self.n = 0 497 | 498 | def step(self)->Number: 499 | "Return next value along annealed schedule." 500 | self.eff_n = min(max(0, self.n - self.start_iter), self.eff_iters) 501 | out = self.sch_func(self.start_val, self.end_val, min(1, self.eff_n/(self.eff_iters - 1))) 502 | self.n += 1 503 | return out 504 | 505 | 506 | def cosine_annealing(start:Number, end:Number, pct:float, pct_start=.3, **kwargs)->Number: 507 | "Cosine anneal from `start` to `end` as pct goes from 0.0 to 1.0." 508 | if pct <= pct_start: 509 | return annealing_cos(start, end, pct/pct_start) 510 | else: 511 | return annealing_cos(end, start, (pct - pct_start)/(1 - pct_start)) 512 | 513 | def inv_annealing_poly(start:Number, end:Number, pct:float, degree:Number, **kwargs)->Number: 514 | "Helper function for `inv_anneal_poly`." 515 | return start + (end - start) * (pct)**degree 516 | 517 | def inv_annealing_cos(start:Number, end:Number, pct:float, **kwargs)->Number: 518 | "Cosine anneal from `start` to `end` as pct goes from 0.0 to 1.0." 519 | cos_out = np.cos(np.pi * pct) + 1 520 | return start + (end - start)/2 * cos_out 521 | 522 | def tuplify(a): 523 | if not isinstance(a, list): a = [a] 524 | for i, x in enumerate(a): 525 | if not isinstance(x, tuple): a[i] = (0, x) 526 | return a 527 | 528 | def get_fn(a): 529 | while True: 530 | if hasattr(a, 'func'): a = a.func 531 | else: break 532 | return a -------------------------------------------------------------------------------- /fastai_timeseries/exp/nb_TSCharts.py: -------------------------------------------------------------------------------- 1 | #AUTOGENERATED! DO NOT EDIT! file to edit: ./TSCharts.ipynb (unless otherwise specified) 2 | 3 | from fastai.vision import Learner, Tensors, Rank0Tensor, flatten_model, listify 4 | from torch import nn 5 | import torch 6 | from fastai.metrics import error_rate 7 | from fastai.callbacks import HookCallback 8 | import numpy as np 9 | import math 10 | import matplotlib.pyplot as plt 11 | 12 | def splitAtFirstParenthesis(s,showDetails,shapeData): 13 | pos=len(s.split('(')[0]) 14 | ret = s[:pos] 15 | if (showDetails): 16 | ret += shapeData + '\n' + s[pos:] 17 | return ret 18 | 19 | class ActivationsHistogram(HookCallback): 20 | "Callback that record histogram of activations." 21 | "NOTE: learner property name will be 'activations_histogram'" 22 | 23 | def __init__(self, learn:Learner, do_remove:bool=True, 24 | hMin=-1, 25 | hMax=1, 26 | nBins=100, 27 | useClasses=False, # if true compute histogram of classes in the last layer 28 | liveChart = True, # show live chart of last layer 29 | modulesId=-1 # array of modules to keep 30 | ): 31 | self.hMin = hMin or (-hMax) 32 | self.hMax = hMax 33 | self.nBins = nBins 34 | self.liveChart = liveChart 35 | self.allModules = [m for m in flatten_model(learn.model)] 36 | self.useClasses = useClasses 37 | modules = self.allModules 38 | if modulesId: 39 | modules = [self.allModules[i] for i in listify(modulesId)] 40 | self.allModules = modules if modules else self.allModules 41 | self.c = learn.data.c # Number of Calsses 42 | super().__init__(learn,modules,do_remove) 43 | 44 | def mkHist(self, x, useClasses): 45 | ret = x.clone().detach().cpu() # Need this for Pytorch 1.0 46 | #ret = x.clone().detach() #Pytorch 1.1 # WARNING: so bad performance on GPU! (x10) 47 | if useClasses: 48 | ret = torch.stack([ret[:,i].histc(self.nBins, self.hMin, self.hMax) for i in range(ret.shape[1])],dim=1) #histogram by class... 49 | else: 50 | ret = ret.histc(self.nBins, self.hMin, self.hMax).unsqueeze(1) # histogram by activation 51 | return ret 52 | 53 | def on_train_begin(self, **kwargs): 54 | "Initialize stats." 55 | super().on_train_begin(**kwargs) 56 | self.stats_hist = None # empty at start 57 | self.stats_valid_hist = None # empty at start 58 | self.stats_epoch = [] 59 | self.cur_epoch = -1 60 | self.cur_train_batch = -1 61 | self.stats_valid_epoch = [] 62 | self.shape_out={} 63 | 64 | def on_epoch_begin(self, **kwargs): 65 | self.cur_epoch += 1 66 | 67 | def on_batch_begin(self, train, **kwargs): 68 | if train: 69 | self.cur_train_batch += 1 70 | 71 | def hook(self, m:nn.Module, i:Tensors, o:Tensors)->Rank0Tensor: 72 | if (isinstance(o,torch.Tensor)) and (m not in self.shape_out): 73 | outShape = o.shape; 74 | self.shape_out[m]=outShape; 75 | return self.mkHist(o,self.useClasses) 76 | 77 | def on_batch_end(self, train, **kwargs): 78 | "Take the stored results and puts it in `self.stats_hist`" 79 | hasValues = True if ((len(self.hooks.stored)>0) and (not (self.hooks.stored[0] is None))) else False 80 | stacked = torch.stack(self.hooks.stored).unsqueeze(1) if hasValues else None 81 | if train and hasValues: 82 | if self.stats_hist is None: self.stats_hist = stacked #start 83 | else: self.stats_hist = torch.cat([self.stats_hist,stacked],dim=1) #cat 84 | if (not train) and hasValues: 85 | if self.stats_valid_hist is None: self.stats_valid_hist = stacked #start 86 | else: self.stats_valid_hist = torch.cat([self.stats_valid_hist,stacked],dim=1) #cat 87 | 88 | def on_epoch_end(self, **kwargs): 89 | self.stats_epoch.append(self.cur_train_batch) 90 | start = 0 if 1==len(self.stats_epoch) else self.stats_epoch[-2] 91 | end = self.stats_epoch[-1] 92 | startValid = 0 if 0==len(self.stats_valid_epoch) else self.stats_valid_epoch[-1] 93 | if not(self.stats_hist is None): 94 | hScale = 1 95 | domain = self.stats_hist[-1][start:end] 96 | yy = np.arange(self.hMin,self.hMax,(self.hMax-self.hMin)/self.nBins) 97 | xx,_ = self.computeXY(domain,hScale,.25,start) # average on last quarter of epoch 98 | xx = xx.sum(-1) # useClasses support 99 | toDisplay = [(xx,yy)] 100 | 101 | if not (self.stats_valid_hist is None): 102 | domainValid = self.stats_valid_hist[-1][startValid:] #till end 103 | xxv,_ = self.computeXY(domainValid,hScale,1,start) # validation average all available data 104 | xxv = xxv.sum(-1) # useClasses support 105 | toDisplay += [(xxv,yy)] 106 | self.stats_valid_epoch += [self.stats_valid_hist.shape[1]] 107 | if self.liveChart: 108 | rec = rec = self.learn.recorder 109 | rec.pbar.update_graph(toDisplay) 110 | 111 | def on_train_end(self, **kwargs): 112 | "Polish the final result." 113 | super().on_train_end(**kwargs) 114 | 115 | @staticmethod 116 | def get_color_value_from_map(idx:int, cmap='Reds', scale=1): 117 | return plt.get_cmap(cmap)(idx*scale) 118 | 119 | @staticmethod 120 | def getHistImg(act,useClasses): 121 | dd = act.squeeze(2) if not useClasses else act.sum(dim=2) # Reassemble... 122 | dd = dd.log() # Scale for visualizaion 123 | dd = dd.t() # rotate 124 | return dd 125 | 126 | @staticmethod 127 | def computeXY(l,hscale,perc,hshift=0): 128 | start = int(l.shape[0]*(1-perc)) 129 | end = l.shape[0] 130 | m = l[start:end].mean(dim=0) # all data mean 131 | xx = hshift + m*hscale 132 | yy = +np.array(range(l.shape[1])) 133 | return xx,yy 134 | 135 | @staticmethod 136 | def plotPerc(ax,l,hscale,perc,hshift=0,colorById=False,linewidth=1,addLabel=False): 137 | xx,yy = ActivationsHistogram.computeXY(l,hscale,perc,hshift) 138 | if colorById: 139 | classes = xx.shape[1] 140 | for i in range(classes): 141 | xx_cur = xx[:,i] 142 | color = ActivationsHistogram.get_color_value_from_map(i/classes, cmap='rainbow') 143 | label = i if addLabel else None 144 | ax.plot(xx_cur,yy,linewidth=linewidth, color=color, label=label); 145 | else: 146 | color = [1-perc,1-perc,1-perc] 147 | ax.plot(xx,yy,linewidth=linewidth, color=color); 148 | 149 | def plotActsHist(self, cols=3, figsize=(20,10), toDisplay=None, hScale = .05, showEpochs=True, showLayerInfo=False, aspectAuto=True, showImage=True): 150 | histsTensor = self.activations_histogram.stats_hist 151 | hists = [histsTensor[i] for i in range(histsTensor.shape[0])] 152 | if toDisplay: hists = [hists[i] for i in listify(toDisplay)] # optionally focus 153 | 154 | n=len(hists) 155 | cols = cols or 3 156 | cols = min(cols,n) 157 | rows = int(math.ceil(n/cols)) 158 | fig = plt.figure(figsize=figsize) 159 | grid = plt.GridSpec(rows, cols, figure=fig) 160 | 161 | for i,l in enumerate(hists): 162 | img=self.getHistImg(l,self.useClasses) 163 | cr = math.floor(i/cols) 164 | cc = i%cols 165 | main_ax = fig.add_subplot(grid[cr,cc]) 166 | if showImage: main_ax.imshow(img); 167 | layerId = listify(toDisplay)[i] if toDisplay else i 168 | m = self.allModules[layerId] 169 | outShapeText = f' (out: {list(self.shape_out[m])})' if (m in self.shape_out) else '' 170 | title = f'L:{layerId}' + '\n' + splitAtFirstParenthesis(str(m),showLayerInfo,outShapeText) 171 | main_ax.set_title(title) 172 | imgH=img.shape[0] 173 | main_ax.set_yticks([]) 174 | main_ax.set_ylabel(str(self.hMin) + " : " + str(self.hMax)) 175 | if aspectAuto: main_ax.set_aspect('auto') 176 | imgW=img.shape[1] 177 | imgH=img.shape[0] 178 | ratioH=-self.hMin/(self.hMax-self.hMin) 179 | zeroPosH = imgH*ratioH 180 | main_ax.plot([0,imgW],[zeroPosH,zeroPosH],'r') # X Axis 181 | if (showEpochs): 182 | start = 0 183 | nEpochs = len(self.activations_histogram.stats_epoch) 184 | for i,hh in enumerate(self.activations_histogram.stats_epoch): 185 | if(i<(nEpochs-1)): main_ax.plot([hh,hh],[0,imgH],color=[0,0,1]) 186 | end = hh # rolling 187 | domain = l[start:end] 188 | domain_mean = domain.mean(-1) # mean on classes 189 | if self.useClasses: 190 | self.plotPerc(main_ax,domain,hScale,1,start,colorById=True,addLabel=(0==i)) #plot all 191 | main_ax.legend(loc='upper left') 192 | else: 193 | self.plotPerc(main_ax,domain_mean,hScale,.5,start) 194 | self.plotPerc(main_ax,domain_mean,hScale,1,start,linewidth=1.5) 195 | start = hh 196 | main_ax.set_xlim([0,imgW]) 197 | main_ax.set_ylim([0,imgH]) -------------------------------------------------------------------------------- /fastai_timeseries/exp/nb_TSDataAugmentation.py: -------------------------------------------------------------------------------- 1 | #AUTOGENERATED! DO NOT EDIT! file to edit: ./TSDataAugmentation.ipynb (unless otherwise specified) 2 | 3 | import copy 4 | import numpy as np 5 | import random 6 | from functools import partial 7 | 8 | 9 | try: from exp.nb_TSUtilities import * 10 | except ImportError: from .nb_TSUtilities import * 11 | 12 | try: from exp.nb_TSBasicData import * 13 | except ImportError: from .nb_TSBasicData import * 14 | 15 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 16 | 17 | 18 | def shuffle_HLs(ts, **kwargs): 19 | line = copy(ts) 20 | pos_rand_list = np.random.choice( 21 | np.arange(ts.shape[-1] // 4), 22 | size=random.randint(0, ts.shape[-1] // 4), 23 | replace=False) 24 | rand_list = pos_rand_list * 4 25 | highs = rand_list + 1 26 | lows = highs + 1 27 | a = np.vstack([highs, lows]).flatten('F') 28 | b = np.vstack([lows, highs]).flatten('F') 29 | line[..., a] = line[..., b] 30 | return line 31 | 32 | setattr(shuffle_HLs, 'use_on_y', False) 33 | 34 | def get_diff(a): 35 | return np.concatenate((np.zeros(a.shape[-2])[:, None], np.diff(a)), axis=1).astype(np.float32) 36 | 37 | setattr(get_diff, 'use_on_y', False) 38 | 39 | 40 | class TSTransform(): 41 | "Utility class for adding probability and wrapping support to transform `func`." 42 | _wrap=None 43 | order=0 44 | def __init__(self, func:Callable, order:Optional[int]=None): 45 | "Create a transform for `func` and assign it an priority `order`, attach to `TS` class." 46 | if order is not None: self.order=order 47 | self.func=func 48 | self.func.__name__ = func.__name__[1:] #To remove the _ that begins every transform function. 49 | functools.update_wrapper(self, self.func) 50 | self.func.__annotations__['return'] = TSItem 51 | setattr(TSItem, func.__name__, lambda x, *args, **kwargs: self.calc(x, *args, **kwargs)) 52 | 53 | def __call__(self, *args:Any, p:float=1., is_random:bool=True, use_on_y:bool=False, **kwargs:Any)->TSItem: 54 | "Calc now if `args` passed; else create a transform called prob `p` if `random`." 55 | if args: return self.calc(*args, **kwargs) 56 | else: return TSRandTransform(self, kwargs=kwargs, is_random=is_random, use_on_y=use_on_y, p=p) 57 | 58 | def calc(self, x:TSItem, *args:Any, **kwargs:Any)->Image: 59 | "Apply to image `x`, wrapping it if necessary." 60 | if self._wrap: return getattr(x, self._wrap)(self.func, *args, **kwargs) 61 | else: return self.func(x, *args, **kwargs) 62 | 63 | @property 64 | def name(self)->str: return self.__class__.__name__ 65 | 66 | def __repr__(self)->str: return f'{self.name}({self.func.__name__})' 67 | 68 | 69 | @dataclass 70 | class TSRandTransform(): 71 | "Wrap `Transform` to add randomized execution." 72 | tfm: Transform 73 | kwargs: dict 74 | p: float = 1.0 75 | do_run: bool = True 76 | is_random: bool = True 77 | use_on_y: bool = False 78 | 79 | # def __post_init__(self): 80 | # functools.update_wrapper(self, self.tfm) 81 | 82 | def __call__(self, x:TSItem, *args, **kwargs) -> TSItem: 83 | "Randomly execute our tfm on `x`." 84 | self.do_run = rand_bool(self.p) 85 | return self.tfm(x, *args, **kwargs) if self.do_run else x 86 | 87 | 88 | # 1) Those that slightly modify the time series in the x and/ or y-axes: 89 | # - TSmagnoise == TSjittering (1) 90 | # - TSmagscale (1) 91 | # - TSdimmagscale (1) 92 | # - TSmagwarp (1) 93 | # - TStimenoise (1) 94 | # - TStimewarp (1) 95 | # - TSzoomin (3) 96 | # - TSrandtimesteps 97 | # - TSzoomout (3) 98 | # - TSrandomzoom 99 | 100 | 101 | # 2) And those that hide certain part of the time series, setting the values to zero or just removing them: 102 | # - TSlookback 103 | # - TStimestepsout (1) * modifies the output shape 104 | # - TStimestepszero (1) 105 | # - TSdimout = TSchannelout 106 | # - TScutout (2) 107 | # - TScrop (1) 108 | # - TSrandomcrop (3) 109 | # - TScentercrop (3) 110 | # - TSmaskout 111 | 112 | 113 | # All of them can be used independently or in combination. In this section, you'll see how these transforms work. 114 | 115 | # (1) Adapted from/ inspired by Um, T. T., Pfister, F. M. J., Pichler, D., Endo, S., Lang, M., Hirche, S., ... & Kulić, D. (2017). 116 | # Data augmentation of wearable sensor data for parkinson's disease monitoring using convolutional neural networks. arXiv preprint arXiv:1706.00527. (includes: Jittering, Scaling, Magnitude-Warping, Time-warping, Random Sampling, among others. 117 | 118 | # (2) Inspired by DeVries, T., & Taylor, G. W. (2017). 119 | # Improved regularization of convolutional neural networks with cutout. arXiv preprint arXiv:1708.04552. 120 | 121 | # (3) Inspired by Buslaev, A., Parinov, A., Khvedchenya, E., Iglovikov, V. I., & Kalinin, A. A. (2018). 122 | # Albumentations: fast and flexible image augmentations. arXiv preprint arXiv:1809.06839. 123 | 124 | 125 | from scipy.interpolate import CubicSpline 126 | def random_curve_generator(ts, magnitude=.1, order=4, noise=None): 127 | seq_len = ts.shape[-1] 128 | x = np.linspace(-seq_len, 2 * seq_len - 1, 3 * (order - 1) + 1, dtype=int) 129 | x2 = np.random.normal(loc=1.0, scale=magnitude, size=len(x)) 130 | f = CubicSpline(x, x2, axis=-1) 131 | return f(np.arange(seq_len)) 132 | 133 | 134 | def random_cum_curve_generator(ts, magnitude=.1, order=4, noise=None): 135 | x = random_curve_generator(ts, magnitude=magnitude, order=order, noise=noise).cumsum() 136 | x -= x[0] 137 | x /= x[-1] 138 | x = np.clip(x, 0, 1) 139 | return x * (ts.shape[-1] - 1) 140 | 141 | 142 | def random_cum_noise_generator(ts, magnitude=.1, noise=None): 143 | seq_len = ts.shape[-1] 144 | x = (np.ones(seq_len) + np.random.normal(loc=0, scale=magnitude, size=seq_len)).cumsum() 145 | x -= x[0] 146 | x /= x[-1] 147 | x = np.clip(x, 0, 1) 148 | return x * (ts.shape[-1] - 1) 149 | 150 | 151 | def _magnoise(x, magnitude=.1, add=True): 152 | if magnitude <= 0: return x 153 | y = None 154 | if isinstance(x, list): 155 | y = x[1] 156 | x = x[0] 157 | seq_len = x.shape[-1] 158 | noise = torch.normal(0, magnitude, (1, seq_len), dtype=x.dtype, device=x.device) 159 | if add: 160 | output = x + noise 161 | return output if y is None else [output, y] 162 | else: 163 | output = x * (1 + noise) 164 | return output if y is None else [output, y] 165 | 166 | TSmagnoise = TSTransform(_magnoise) 167 | TSjittering = TSTransform(_magnoise) 168 | 169 | 170 | from scipy.interpolate import CubicSpline 171 | def _timewarp(x, magnitude=.1, order=4): 172 | '''This is a slow batch tfm''' 173 | if magnitude <= 0: return x 174 | y = None 175 | if isinstance(x, list): 176 | y = x[1] 177 | x = x[0] 178 | seq_len = x.shape[-1] 179 | 180 | f = CubicSpline(np.arange(seq_len), x, axis=-1) 181 | new_x = random_cum_curve_generator(x, magnitude=magnitude, order=order) 182 | output = x.new(f(new_x)) 183 | return output if y is None else [output, y] 184 | 185 | TStimewarp = TSTransform(_timewarp) 186 | 187 | 188 | def _magwarp(x, magnitude=.1, order=4): 189 | if magnitude <= 0: return x 190 | y = None 191 | if isinstance(x, list): 192 | y = x[1] 193 | x = x[0] 194 | y_mult = random_curve_generator(x, magnitude=magnitude, order=order) 195 | output = x * x.new(y_mult) 196 | return output if y is None else [output, y] 197 | 198 | TSmagwarp = TSTransform(_magwarp) 199 | 200 | 201 | def _magscale(x, magnitude=.1): 202 | if magnitude <= 0: return x 203 | y = None 204 | if isinstance(x, list): 205 | y = x[1] 206 | x = x[0] 207 | scale = 1 + torch.rand(1) * magnitude 208 | if np.random.rand() < .5: scale = 1 / scale 209 | output = x * scale.to(device) 210 | return output if y is None else [output, y] 211 | 212 | TSmagscale = TSTransform(_magscale) 213 | 214 | 215 | def _dimmagscale(x, magnitude=.1): 216 | '''This tfm applies magscale to each dimension independently''' 217 | if magnitude <= 0: return x 218 | y = None 219 | if isinstance(x, list): 220 | y = x[1] 221 | x = x[0] 222 | scale = (1 + torch.rand((x.shape[-2], 1)) * magnitude) 223 | if np.random.rand() < .5: scale = 1 / scale 224 | output = x * scale.to(device) 225 | return output if y is None else [output, y] 226 | 227 | TSdimmagscale = TSTransform(_dimmagscale) 228 | 229 | 230 | from scipy.interpolate import CubicSpline 231 | def _timenoise(x, magnitude=.1): 232 | '''This is a slow batch tfm''' 233 | if magnitude <= 0: return x 234 | y = None 235 | if isinstance(x, list): 236 | y = x[1] 237 | x = x[0] 238 | f = CubicSpline(np.arange(x.shape[-1]), x, axis=-1) 239 | new_x = random_cum_noise_generator(x, magnitude=magnitude) 240 | output = x.new(f(new_x)) 241 | return output if y is None else [output, y] 242 | 243 | TStimenoise = TSTransform(_timenoise) 244 | 245 | 246 | from scipy.interpolate import CubicSpline 247 | def _zoomin(x, magnitude=.2): 248 | '''This is a slow batch tfm''' 249 | if magnitude == 0: return x 250 | y = None 251 | if isinstance(x, list): 252 | y = x[1] 253 | x = x[0] 254 | seq_len = x.shape[-1] 255 | lambd = np.random.beta(magnitude, magnitude) 256 | lambd = max(lambd, 1 - lambd) 257 | win_len = int(seq_len * lambd) 258 | if win_len == seq_len: start = 0 259 | else: start = np.random.randint(0, seq_len - win_len) 260 | x2 = x[..., start : start + win_len] 261 | f = CubicSpline(np.arange(x2.shape[-1]), x2, axis=-1) 262 | output = x.new(f(np.linspace(0, win_len - 1, num=seq_len))) 263 | return output if y is None else [output, y] 264 | 265 | TSzoomin = TSTransform(_zoomin) 266 | 267 | 268 | from scipy.interpolate import CubicSpline 269 | def _zoomout(x, magnitude=.2): 270 | '''This is a slow batch tfm''' 271 | if magnitude == 0: return x 272 | y = None 273 | if isinstance(x, list): 274 | y = x[1] 275 | x = x[0] 276 | seq_len = x.shape[-1] 277 | lambd = np.random.beta(magnitude, magnitude) 278 | lambd = max(lambd, 1 - lambd) 279 | f = CubicSpline(np.arange(x.shape[-1]), x, axis=-1) 280 | new_x = torch.zeros_like(x, dtype=x.dtype, device=x.device) 281 | win_len = int(seq_len * lambd) 282 | new_x[..., -win_len:] = x.new(f(np.linspace(0, seq_len - 1, num=win_len))) 283 | output = new_x 284 | return output if y is None else [output, y] 285 | 286 | TSzoomout = TSTransform(_zoomout) 287 | 288 | 289 | def _randomzoom(x, magnitude=.2): 290 | if magnitude == 0: return x 291 | if np.random.rand() <= .5: return _zoomin(x, magnitude=magnitude) 292 | else: return _zoomout(x, magnitude=magnitude) 293 | 294 | TSrandomzoom = TSTransform(_randomzoom) 295 | 296 | 297 | def _randtimestep(x, magnitude=.1): 298 | if magnitude >= 1 or magnitude <= 0: return x 299 | y = None 300 | if isinstance(x, list): 301 | y = x[1] 302 | x = x[0] 303 | seq_len = x.shape[-1] 304 | new_seq_len = int(seq_len * (1 - magnitude)) 305 | timesteps = np.sort(np.random.choice(np.arange(seq_len), 306 | new_seq_len, 307 | replace=False)) 308 | new_x = x.clone()[..., timesteps] 309 | f = CubicSpline(np.arange(new_x.shape[-1]), new_x, axis=-1) 310 | output = x.new(f(np.linspace(0, new_seq_len - 1, num=seq_len))) 311 | return output if y is None else [output, y] 312 | 313 | TSrandtimestep = TSTransform(_randtimestep) 314 | 315 | 316 | def _lookback(x, magnitude=.2): 317 | if magnitude <= 0: return x 318 | y = None 319 | if isinstance(x, list): 320 | y = x[1] 321 | x = x[0] 322 | seq_len = x.shape[-1] 323 | new_x = x.clone() 324 | lambd = np.random.beta(magnitude, magnitude) 325 | lambd = min(lambd, 1 - lambd) 326 | lookback_per = int(lambd * seq_len) 327 | new_x[:, :lookback_per] = 0 328 | output = new_x 329 | return output if y is None else [output, y] 330 | 331 | TSlookback = TSTransform(_lookback) 332 | 333 | def _dimout(ts, magnitude=.2): 334 | if magnitude <= 0: return x 335 | y = None 336 | if isinstance(ts, list): 337 | y = ts[1] 338 | x = ts[0] 339 | else: x = ts 340 | input_ch = x.shape[0] 341 | if input_ch == 1: return ts 342 | new_x = x.clone() 343 | out_ch = np.random.choice(np.arange(input_ch), 344 | min(input_ch - 1, int(np.random.beta(magnitude, 1) * input_ch)), 345 | replace=False) 346 | new_x[out_ch] = 0 347 | output = new_x 348 | return output if y is None else [output, y] 349 | 350 | TSdimout = TSTransform(_dimout) 351 | TSchannelout = TSTransform(_dimout) 352 | 353 | def _cutout(x, magnitude=.1): 354 | if magnitude >= 1 or magnitude <= 0: return x 355 | y = None 356 | if isinstance(x, list): 357 | y = x[1] 358 | x = x[0] 359 | seq_len = x.shape[-1] 360 | new_x = x.clone() 361 | win_len = int(magnitude * seq_len) 362 | start = np.random.randint(-win_len + 1, seq_len) 363 | end = start + win_len 364 | start = max(0, start) 365 | end = min(end, seq_len) 366 | new_x[..., start:end] = 0 367 | output = new_x 368 | return output if y is None else [output, y] 369 | 370 | TScutout= TSTransform(_cutout) 371 | 372 | 373 | def _timestepout(x, magnitude=.1): 374 | '''This tfm modifies the output size''' 375 | if magnitude >= 1 or magnitude <= 0: return x 376 | y = None 377 | if isinstance(x, list): 378 | y = x[1] 379 | x = x[0] 380 | seq_len = x.shape[-1] 381 | timesteps = np.sort(np.random.choice(np.arange(seq_len), 382 | int(seq_len * (1 - magnitude)), 383 | replace=False)) 384 | new_x = x.clone() 385 | output = new_x[..., timesteps] 386 | return output if y is None else [output, y] 387 | 388 | TStimestepout = TSTransform(_timestepout) 389 | 390 | def _timestepzero(x, magnitude=.1): 391 | if magnitude >= 1 or magnitude <= 0: return x 392 | y = None 393 | if isinstance(x, list): 394 | y = x[1] 395 | x = x[0] 396 | seq_len = x.shape[-1] 397 | timesteps = np.sort(np.random.choice(np.arange(seq_len), 398 | int(seq_len * magnitude), 399 | replace=False)) 400 | new_x = x.clone() 401 | new_x[..., timesteps] = 0 402 | output = new_x 403 | return output if y is None else [output, y] 404 | 405 | TStimestepzero = TSTransform(_timestepzero) 406 | 407 | 408 | def _crop(x, magnitude=.1): 409 | if magnitude >= 1 or magnitude <= 0: return x 410 | y = None 411 | if isinstance(x, list): 412 | y = x[1] 413 | x = x[0] 414 | seq_len = x.shape[-1] 415 | win_len = int(seq_len * (1. - magnitude)) 416 | start = np.random.randint(- win_len//2, seq_len - win_len//2) 417 | end = start + win_len 418 | start = max(0, start) 419 | end = min(end, seq_len) 420 | new_x = torch.zeros_like(x, dtype=x.dtype, device=x.device) 421 | new_x[..., start - end :] = x[..., start : end] 422 | output = new_x 423 | return output if y is None else [output, y] 424 | 425 | TScrop = TSTransform(_crop) 426 | 427 | 428 | def _randomcrop(x, magnitude=.2): 429 | if magnitude >= 1 or magnitude <= 0: return x 430 | y = None 431 | if isinstance(x, list): 432 | y = x[1] 433 | x = x[0] 434 | seq_len = x.shape[-1] 435 | lambd = np.random.beta(magnitude, magnitude) 436 | lambd = max(lambd, 1 - lambd) 437 | win_len = int(seq_len * lambd) 438 | start = np.random.randint(- win_len//2, seq_len - win_len//2) 439 | end = start + win_len 440 | start = max(0, start) 441 | end = min(end, seq_len) 442 | new_x = torch.zeros_like(x, dtype=x.dtype, device=x.device) 443 | new_x[..., start - end :] = x[..., start : end] 444 | output = new_x 445 | return output if y is None else [output, y] 446 | 447 | TSrandomcrop = TSTransform(_randomcrop) 448 | 449 | 450 | def _centercrop(x, magnitude=.2): 451 | if magnitude >= 1 or magnitude <= 0: return x 452 | y = None 453 | if isinstance(x, list): 454 | y = x[1] 455 | x = x[0] 456 | seq_len = x.shape[-1] 457 | lambd = np.random.beta(magnitude, magnitude) 458 | lambd = max(lambd, 1 - lambd) 459 | win_len = int(seq_len * lambd) 460 | start = seq_len // 2 - win_len // 2 461 | end = start + win_len 462 | start = max(0, start) 463 | end = min(end, seq_len) 464 | new_x = torch.zeros_like(x, dtype=x.dtype, device=x.device) 465 | new_x[..., start - end :] = x[..., start : end] 466 | output = new_x 467 | return output if y is None else [output, y] 468 | 469 | TScentercrop = TSTransform(_centercrop) 470 | 471 | 472 | def _maskout(x, magnitude=.1): 473 | if magnitude >= 1 or magnitude <= 0: return x 474 | y = None 475 | if isinstance(x, list): 476 | y = x[1] 477 | x = x[0] 478 | seq_len = x.shape[-1] 479 | mask = torch.rand_like(x) <= magnitude 480 | new_x = x.clone() 481 | new_x[mask] = 0 482 | output = new_x 483 | return output if y is None else [output, y] 484 | 485 | TSmaskout = TSTransform(_maskout) 486 | 487 | 488 | def TS_geometric_tfms(**kwargs): 489 | return [[ 490 | TStimewarp(**kwargs), 491 | TSmagwarp(**kwargs), 492 | TStimenoise(**kwargs), 493 | TSmagnoise(**kwargs), 494 | TSmagscale(**kwargs), 495 | TSdimmagscale(**kwargs), 496 | TSzoomin(**kwargs), 497 | TSzoomout(**kwargs), 498 | TSrandomzoom(**kwargs), 499 | TSrandtimestep(**kwargs), 500 | ], []] 501 | 502 | TS_xy_tfms = TS_geometric_tfms 503 | 504 | 505 | def TS_erasing_tfms(**kwargs): 506 | return [[ 507 | 508 | TSdimout(**kwargs), 509 | TScutout(**kwargs), 510 | TStimestepzero(**kwargs), 511 | TScrop(**kwargs), 512 | TSrandomcrop(**kwargs), 513 | TSmaskout(**kwargs) 514 | ], []] 515 | 516 | TS_zero_tfms = TS_erasing_tfms 517 | 518 | 519 | def TS_tfms(**kwargs): 520 | return [TS_geometric_tfms(**kwargs)[0] + TS_erasing_tfms(**kwargs)[0], []] 521 | 522 | 523 | def all_TS_tfms(**kwargs): 524 | return [TS_tfms(**kwargs)[0] + 525 | [TStimestepout(**kwargs), TSlookback(**kwargs), TScentercrop(**kwargs)], 526 | []] 527 | 528 | 529 | class RandAugment(): 530 | def __init__(self, tfms, N=1, **kwargs): 531 | ''' 532 | tfms: list of tfms to select from 533 | N: number of tfms applied each time 534 | kwargs: tfm kwargs 535 | ''' 536 | tfms = listify(tfms) 537 | if isinstance(tfms[0], list): tfms = tfms[0] 538 | self.tfms = tfms 539 | self.N = N 540 | self.kwargs = kwargs 541 | 542 | def __call__(self, x): 543 | if self.N is None: sel_tfms = self.tfms 544 | else: sel_tfms = np.random.choice(self.tfms, self.N, replace=False) 545 | tfms = [partial(tfm, p=1., **self.kwargs) for tfm in sel_tfms] 546 | for tfm in tfms: x = tfm(x) 547 | return x 548 | 549 | def randaugment(learn:Learner, tfms:list=TS_tfms(), N:int=1, **kwargs)->Learner: 550 | learn.data.train_dl.tfms = RandAugment(tfms, N=N, **kwargs) 551 | return learn 552 | 553 | Learner.randaugment = randaugment -------------------------------------------------------------------------------- /fastai_timeseries/exp/nb_TSDatasets.py: -------------------------------------------------------------------------------- 1 | #AUTOGENERATED! DO NOT EDIT! file to edit: ./TSDatasets.ipynb (unless otherwise specified) 2 | from pathlib import Path 3 | import os 4 | import numpy as np 5 | import pandas as pd 6 | from scipy.io import arff 7 | 8 | 9 | try: from exp.nb_TSUtilities import * 10 | except ImportError: from .nb_TSUtilities import * 11 | 12 | try: from exp.nb_TSBasicData import * 13 | except ImportError: from .nb_TSBasicData import * 14 | 15 | 16 | import os 17 | import tempfile 18 | try: from urllib import urlretrieve 19 | except ImportError: from urllib.request import urlretrieve 20 | import shutil 21 | from pyunpack import Archive 22 | 23 | 24 | def decompress_from_url(url, target_dir=None, verbose=False): 25 | """Downloads a compressed file from its URL and uncompresses it. 26 | 27 | Parameters 28 | ---------- 29 | url : string 30 | URL from which to download. 31 | target_dir : str or None (default: None) 32 | Directory to be used to extract downloaded files. 33 | verbose : bool (default: False) 34 | Whether to print information about the process (cached files used, ...) 35 | 36 | Returns 37 | ------- 38 | str or None 39 | Directory in which the compressed file has been extracted if the process was 40 | successful, None otherwise 41 | """ 42 | try: 43 | fname = os.path.basename(url) 44 | tmpdir = tempfile.mkdtemp() 45 | local_comp_fname = os.path.join(tmpdir, fname) 46 | urlretrieve(url, local_comp_fname) 47 | except: 48 | shutil.rmtree(tmpdir) 49 | if verbose: 50 | sys.stderr.write("Could not download url. Please, check url.\n") 51 | try: 52 | if not os.path.exists(target_dir): os.makedirs(target_dir) 53 | Archive(local_comp_fname).extractall(target_dir) 54 | shutil.rmtree(tmpdir) 55 | if verbose: 56 | print("Successfully extracted file %s to path %s" % 57 | (local_comp_fname, target_dir)) 58 | return target_dir 59 | except: 60 | shutil.rmtree(tmpdir) 61 | if verbose: 62 | sys.stderr.write("Could not uncompress file, aborting.\n") 63 | return None 64 | 65 | 66 | 67 | def get_UCR_univariate_list(): 68 | return sorted([ 69 | 'ACSF1', 'Adiac', 'AllGestureWiimoteX', 'AllGestureWiimoteY', 70 | 'AllGestureWiimoteZ', 'ArrowHead', 'AsphaltObstacles', 'BME', 'Beef', 71 | 'BeetleFly', 'BirdChicken', 'CBF', 'Car', 'Chinatown', 72 | 'ChlorineConcentration', 'CinCECGTorso', 'Coffee', 'Computers', 73 | 'CricketX', 'CricketY', 'CricketZ', 'Crop', 'DiatomSizeReduction', 74 | 'DistalPhalanxOutlineAgeGroup', 'DistalPhalanxOutlineCorrect', 75 | 'DistalPhalanxTW', 'DodgerLoopDay', 'DodgerLoopGame', 76 | 'DodgerLoopWeekend', 'ECG200', 'ECG5000', 'ECGFiveDays', 77 | 'EOGHorizontalSignal', 'EOGVerticalSignal', 'Earthquakes', 78 | 'ElectricDevices', 'EthanolLevel', 'FaceAll', 'FaceFour', 'FacesUCR', 79 | 'FiftyWords', 'Fish', 'FordA', 'FordB', 'FreezerRegularTrain', 80 | 'FreezerSmallTrain', 'Fungi', 'GestureMidAirD1', 'GestureMidAirD2', 81 | 'GestureMidAirD3', 'GesturePebbleZ1', 'GesturePebbleZ2', 'GunPoint', 82 | 'GunPointAgeSpan', 'GunPointMaleVersusFemale', 83 | 'GunPointOldVersusYoung', 'Ham', 'HandOutlines', 'Haptics', 'Herring', 84 | 'HouseTwenty', 'InlineSkate', 'InsectEPGRegularTrain', 85 | 'InsectEPGSmallTrain', 'InsectWingbeatSound', 'ItalyPowerDemand', 86 | 'LargeKitchenAppliances', 'Lightning2', 'Lightning7', 'Mallat', 'Meat', 87 | 'MedicalImages', 'MelbournePedestrian', 'MiddlePhalanxOutlineAgeGroup', 88 | 'MiddlePhalanxOutlineCorrect', 'MiddlePhalanxTW', 89 | 'MixedShapesRegularTrain', 'MixedShapesSmallTrain', 'MoteStrain', 90 | 'NonInvasiveFetalECGThorax1', 'NonInvasiveFetalECGThorax2', 'OSULeaf', 91 | 'OliveOil', 'PLAID', 'PhalangesOutlinesCorrect', 'Phoneme', 92 | 'PickupGestureWiimoteZ', 'PigAirwayPressure', 'PigArtPressure', 93 | 'PigCVP', 'Plane', 'PowerCons', 'ProximalPhalanxOutlineAgeGroup', 94 | 'ProximalPhalanxOutlineCorrect', 'ProximalPhalanxTW', 95 | 'RefrigerationDevices', 'Rock', 'ScreenType', 'SemgHandGenderCh2', 96 | 'SemgHandMovementCh2', 'SemgHandSubjectCh2', 'ShakeGestureWiimoteZ', 97 | 'ShapeletSim', 'ShapesAll', 'SmallKitchenAppliances', 'SmoothSubspace', 98 | 'SonyAIBORobotSurface1', 'SonyAIBORobotSurface2', 'StarLightCurves', 99 | 'Strawberry', 'SwedishLeaf', 'Symbols', 'SyntheticControl', 100 | 'ToeSegmentation1', 'ToeSegmentation2', 'Trace', 'TwoLeadECG', 101 | 'TwoPatterns', 'UMD', 'UWaveGestureLibraryAll', 'UWaveGestureLibraryX', 102 | 'UWaveGestureLibraryY', 'UWaveGestureLibraryZ', 'Wafer', 'Wine', 103 | 'WordSynonyms', 'Worms', 'WormsTwoClass', 'Yoga' 104 | ]) 105 | 106 | 107 | def get_UCR_multivariate_list(): 108 | return sorted([ 109 | 'ArticularyWordRecognition', 'AtrialFibrillation', 'BasicMotions', 110 | 'CharacterTrajectories', 'Cricket', 'DuckDuckGeese', 'ERing', 111 | 'EigenWorms', 'Epilepsy', 'EthanolConcentration', 'FaceDetection', 112 | 'FingerMovements', 'HandMovementDirection', 'Handwriting', 'Heartbeat', 113 | 'InsectWingbeat', 'JapaneseVowels', 'LSST', 'Libras', 'MotorImagery', 114 | 'NATOPS', 'PEMS-SF', 'PenDigits', 'PhonemeSpectra', 'RacketSports', 115 | 'SelfRegulationSCP1', 'SelfRegulationSCP2', 'SpokenArabicDigits', 116 | 'StandWalkJump', 'UWaveGestureLibrary' 117 | ]) 118 | 119 | 120 | def get_UCR_univariate(sel_dataset, parent_dir='data/UCR', verbose=False, drop_na=False, check=True): 121 | if check and sel_dataset not in get_UCR_univariate_list(): 122 | print('This dataset does not exist. Please select one from this list:') 123 | print(get_UCR_univariate_list()) 124 | return None, None, None, None 125 | if verbose: print('Dataset:', sel_dataset) 126 | src_website = 'http://www.timeseriesclassification.com/Downloads/' 127 | tgt_dir = Path(parent_dir) / sel_dataset 128 | if verbose: print('Downloading and decompressing data...') 129 | if not os.path.isdir(tgt_dir): 130 | decompress_from_url( 131 | src_website + sel_dataset + '.zip', target_dir=tgt_dir, verbose=verbose) 132 | if verbose: print('...data downloaded and decompressed') 133 | fname_train = sel_dataset + "_TRAIN.arff" 134 | fname_test = sel_dataset + "_TEST.arff" 135 | 136 | train_df = pd.DataFrame(arff.loadarff(os.path.join(tgt_dir, fname_train))[0]) 137 | test_df = pd.DataFrame(arff.loadarff(os.path.join(tgt_dir, fname_test))[0]) 138 | unique_cats = train_df.iloc[:, -1].unique() 139 | mapping = dict(zip(unique_cats, np.arange(len(unique_cats)))) 140 | train_df = train_df.replace({train_df.columns.values[-1]: mapping}) 141 | test_df = test_df.replace({test_df.columns.values[-1]: mapping}) 142 | if drop_na: 143 | train_df.dropna(axis=1, inplace=True) 144 | test_df.dropna(axis=1, inplace=True) 145 | 146 | X_train = train_df.iloc[:, :-1].values.astype(np.float32) 147 | X_test = test_df.iloc[:, :-1].values.astype(np.float32) 148 | y_train = train_df.iloc[:, -1].values.astype(int) 149 | y_test = test_df.iloc[:, -1].values.astype(int) 150 | 151 | X_train = To3dArray(X_train) 152 | X_test = To3dArray(X_test) 153 | 154 | if verbose: 155 | print('Successfully extracted dataset\n') 156 | print('X_train:', X_train.shape) 157 | print('y_train:', y_train.shape) 158 | print('X_valid:', X_test.shape) 159 | print('y_valid:', y_test.shape, '\n') 160 | return X_train, y_train, X_test, y_test 161 | 162 | 163 | 164 | def get_UCR_multivariate(sel_dataset, parent_dir='data/UCR', verbose=False, check=True): 165 | if sel_dataset.lower() == 'mphoneme': sel_dataset = 'Phoneme' 166 | if check and sel_dataset not in get_UCR_multivariate_list(): 167 | print('This dataset does not exist. Please select one from this list:') 168 | print(get_UCR_multivariate_list()) 169 | return None, None, None, None 170 | if verbose: print('Dataset:', sel_dataset) 171 | src_website = 'http://www.timeseriesclassification.com/Downloads/' 172 | tgt_dir = Path(parent_dir) / sel_dataset 173 | 174 | if verbose: print('Downloading and decompressing data...') 175 | if not os.path.isdir(tgt_dir): 176 | decompress_from_url( 177 | src_website + sel_dataset + '.zip', target_dir=tgt_dir, verbose=verbose) 178 | if verbose: print('...data downloaded and decompressed') 179 | if verbose: print('Extracting data...') 180 | X_train_ = [] 181 | X_test_ = [] 182 | for i in range(10000): 183 | if not os.path.isfile( 184 | f'{parent_dir}/{sel_dataset}/{sel_dataset}Dimension' 185 | + str(i + 1) + '_TRAIN.arff'): 186 | break 187 | train_df = pd.DataFrame( 188 | arff.loadarff( 189 | f'{parent_dir}/{sel_dataset}/{sel_dataset}Dimension' 190 | + str(i + 1) + '_TRAIN.arff')[0]) 191 | unique_cats = train_df.iloc[:, -1].unique() 192 | mapping = dict(zip(unique_cats, np.arange(len(unique_cats)))) 193 | train_df = train_df.replace({train_df.columns.values[-1]: mapping}) 194 | test_df = pd.DataFrame( 195 | arff.loadarff( 196 | f'{parent_dir}/{sel_dataset}/{sel_dataset}Dimension' 197 | + str(i + 1) + '_TEST.arff')[0]) 198 | test_df = test_df.replace({test_df.columns.values[-1]: mapping}) 199 | X_train_.append(train_df.iloc[:, :-1].values) 200 | X_test_.append(test_df.iloc[:, :-1].values) 201 | 202 | if verbose: print('...extraction complete') 203 | X_train = np.stack(X_train_, axis=-1) 204 | X_test = np.stack(X_test_, axis=-1) 205 | 206 | # In this case we need to rearrange the arrays () 207 | X_train = np.transpose(X_train, (0, 2, 1)) 208 | X_test = np.transpose(X_test, (0, 2, 1)) 209 | 210 | y_train = np.array([int(float(x)) for x in train_df.iloc[:, -1]]) 211 | y_test = np.array([int(float(x)) for x in test_df.iloc[:, -1]]) 212 | 213 | if verbose: 214 | print('Successfully extracted dataset\n') 215 | print('X_train:', X_train.shape) 216 | print('y_train:', y_train.shape) 217 | print('X_valid:', X_test.shape) 218 | print('y_valid:', y_test.shape, '\n') 219 | return X_train, y_train, X_test, y_test 220 | 221 | 222 | def get_UCR_data(dsid, parent_dir='data/UCR', verbose=False, check=True): 223 | if dsid in get_UCR_univariate_list(): 224 | return get_UCR_univariate(dsid, verbose=verbose, check=check) 225 | elif dsid in get_UCR_multivariate_list(): 226 | return get_UCR_multivariate(dsid, verbose=verbose, check=check) 227 | else: 228 | print(f'This {dsid} dataset does not exist. Please select one from these lists:') 229 | print('\nunivariate datasets') 230 | print(get_UCR_univariate_list()) 231 | print('\nmultivariate datasets') 232 | print(get_UCR_multivariate_list(), '\n') 233 | return None, None, None, None 234 | 235 | 236 | def create_UCR_databunch(dsid, bs=64, scale_type='standardize', scale_by_channel=False, 237 | scale_by_sample=False, scale_range =(-1, 1), verbose=False, check=True): 238 | X_train, y_train, X_valid, y_valid = get_UCR_data(dsid, verbose=verbose, check=check) 239 | data = (ItemLists('.', TSList(X_train), TSList(X_valid)).label_from_lists(y_train, y_valid) 240 | .databunch(bs=min(bs, len(X_train)), val_bs=min(bs, len(X_valid))) 241 | .scale(scale_type=scale_type, scale_by_channel=scale_by_channel, 242 | scale_by_sample=scale_by_sample,scale_range=scale_range) 243 | ) 244 | return data 245 | 246 | 247 | def create_seq_optimized(n_samples=1000, seq_len=100, channels=True, seed=1): 248 | np.random.seed(seed) 249 | y = np.random.randint(seq_len, size=n_samples) 250 | X = np.eye(seq_len)[y] 251 | if channels: 252 | X = np.expand_dims(X, 1) 253 | return X, y 254 | 255 | def get_translation_invariance_data(n_samples, seq_len, seed): 256 | X_train, y_train = create_seq_optimized( 257 | n_samples=n_samples, seq_len=seq_len, channels=True, seed=seed) 258 | X_test, y_test = create_seq_optimized( 259 | n_samples=n_samples, seq_len=seq_len, channels=True, seed=seed + 1) 260 | print(X_train.shape, y_train.shape, X_test.shape, y_test.shape, '\n') 261 | return X_train, y_train, X_test, y_test -------------------------------------------------------------------------------- /fastai_timeseries/exp/nb_TSImageData.py: -------------------------------------------------------------------------------- 1 | #AUTOGENERATED! DO NOT EDIT! file to edit: ./TSImageData.ipynb (unless otherwise specified) 2 | 3 | import torch 4 | import fastai 5 | from fastai.core import * 6 | from fastai.vision.data import * 7 | from fastai.vision.image import * 8 | from fastai.data_block import * 9 | 10 | 11 | import numpy as np 12 | import pywt 13 | from scipy import signal 14 | from pyts.image import MarkovTransitionField as MTF 15 | from pyts.image import RecurrencePlot as RP 16 | from pyts.image import GramianAngularField as GAF 17 | from pyts.approximation import PiecewiseAggregateApproximation as PAA 18 | import datetime 19 | device = defaults.device 20 | 21 | 22 | try: from exp.nb_TSUtilities import * 23 | except ImportError: from .nb_TSUtilities import * 24 | try: from exp.nb_TSBasicData import * 25 | except ImportError: from .nb_TSBasicData import * 26 | try: from exp.nb_TSDataAugmentation import * 27 | except ImportError: from .nb_TSDataAugmentation import * 28 | 29 | 30 | ## All encoders can take a 2d tensor or 2d array 31 | ## Output will be a 2d array for gadf, gasf, mtf, rp, ccx, ccy, and 2d tensor for spectro and scalo 32 | # Unify output with ToTensor() 33 | 34 | 35 | def GADF_encoder(ts, size=None, sample_range=None, overlapping=False, 36 | **kwargs): 37 | ts = To2dArray(ts) 38 | assert ts.ndim == 2, 'ts ndim must be 2!' 39 | if size is None: size = ts.shape[-1] 40 | else: size = min(size, ts.shape[-1]) 41 | encoder = GAF( 42 | image_size=size, 43 | sample_range=sample_range, 44 | method='d', 45 | overlapping=overlapping) 46 | output = np.squeeze(encoder.fit_transform(ts), 0) 47 | return (output + 1) / 2 48 | 49 | 50 | def GASF_encoder(ts, size=None, sample_range=None, overlapping=False, 51 | **kwargs): 52 | ts = To2dArray(ts) 53 | assert ts.ndim == 2, 'ts ndim must be 2!' 54 | if size is None: size = ts.shape[-1] 55 | else: size = min(size, ts.shape[-1]) 56 | encoder = GAF( 57 | image_size=size, 58 | sample_range=sample_range, 59 | method='s', 60 | overlapping=overlapping) 61 | output = np.squeeze(encoder.fit_transform(ts), 0) 62 | return (output + 1) / 2 63 | 64 | 65 | def MTF_encoder(ts, 66 | size=None, 67 | n_bins=8, 68 | strategy='quantile', 69 | overlapping=False, 70 | **kwargs): 71 | ts = To2dArray(ts) 72 | assert ts.ndim == 2, 'ts ndim must be 2!' 73 | if size is None: size = ts.shape[-1] 74 | else: size = min(size, ts.shape[-1]) 75 | ts = PAA(window_size=None, output_size=size).fit_transform(ts) 76 | encoder = MTF( 77 | size, n_bins=n_bins, strategy=strategy, overlapping=overlapping) 78 | output = np.squeeze(encoder.fit_transform(ts), 0) 79 | return output 80 | #return norm(output) 81 | 82 | 83 | def RP_encoder(ts, 84 | size=None, 85 | dimension=1, 86 | time_delay=1, 87 | threshold=None, 88 | percentage=10, 89 | norm_output=True, 90 | **kwargs): 91 | ts = To2dArray(ts) 92 | assert ts.ndim == 2, 'ts ndim must be 2!' 93 | if size is None: size = ts.shape[-1] 94 | else: size = min(size, ts.shape[-1]) 95 | ts = PAA(window_size=None, output_size=size).fit_transform(ts) 96 | encoder = RP( 97 | dimension=dimension, 98 | time_delay=time_delay, 99 | threshold=threshold, 100 | percentage=percentage) 101 | output = np.squeeze(encoder.fit_transform(ts), 0) 102 | if norm_output: return norm(output) 103 | else: return output 104 | 105 | 106 | def Spectro_encoder(ts, 107 | size=None, 108 | n_fft=None, 109 | hop_length=None, 110 | win_length=None, 111 | window=None, 112 | center=True, 113 | pad_mode='reflect', 114 | normalized=False, 115 | onesided=True, 116 | amin=1e-5, 117 | spectro_power=1, 118 | **kwargs): 119 | 120 | ts = ToTensor(ts) 121 | assert ts.ndim == 2, 'ts ndim must be 2!' 122 | if size is None: size = ts.shape[-1] 123 | else: size = min(size, ts.shape[-1]) 124 | if n_fft is None: n_fft = size 125 | if hop_length is None: hop_length = 1 126 | Zxx = torch.stft( 127 | ts, 128 | n_fft, 129 | hop_length=hop_length, 130 | win_length=win_length, 131 | window=window, 132 | center=center, 133 | pad_mode=pad_mode, 134 | normalized=normalized, 135 | onesided=onesided)[0, ..., 0] 136 | mag = torch.abs(Zxx) 137 | mag = torch.pow(mag, spectro_power) 138 | mag = torch.clamp_min(mag, amin) 139 | mag = torch.log10(mag) 140 | mag = norm(mag) 141 | return ToImage(mag, cmap='gray', size=size).data[0] 142 | 143 | 144 | def Scalo_encoder(ts, 145 | size=None, 146 | scales=None, 147 | wavelet='morl', 148 | scalo_power=1, 149 | **kwargs): 150 | 151 | # scales is a range 152 | tslen = ts.shape[-1] 153 | if size is None: size = tslen 154 | if scales is None: 155 | n_scales = min(tslen // 4, 100) 156 | scales = np.arange(1, n_scales + 1) 157 | coefs, scales_freq = pywt.cwt(To1dArray(ts), scales, wavelet, 1) 158 | coefs = torch.Tensor(np.array(coefs, dtype=np.float32)).float() 159 | values = torch.abs(coefs) 160 | values = torch.pow(values, scalo_power) 161 | return ToImage(norm(values), cmap='gray', size=size).data[0] 162 | 163 | 164 | def AddCoordConv(arr, **kwargs): 165 | if arr.ndim == 2: arr = arr[None] 166 | assert arr.ndim == 3, 'arr ndim must be 3!' 167 | xsize = arr.shape[-1] 168 | ysize = arr.shape[-2] 169 | cch = np.repeat( 170 | np.linspace(0, 1, xsize, dtype=np.float32).reshape(1, -1), 171 | ysize, 172 | axis=0)[None] 173 | ccv = np.repeat( 174 | np.linspace(0, 1, ysize, dtype=np.float32, axis=-1).reshape(-1, 1), 175 | xsize, 176 | axis=1)[None] 177 | return np.concatenate((arr, cch, ccv)) 178 | 179 | 180 | def get_plot_fig(ts, size, yrange=(-1, 1), dpi=72): 181 | fig = plt.figure(figsize=(size / dpi, size / dpi)) 182 | ax = plt.axes([0, 0, 1, 1], frameon=False) 183 | ax.get_xaxis().set_visible(False) 184 | ax.get_yaxis().set_visible(False) 185 | plt.xlim((0, ts.shape[-1])) 186 | plt.ylim(yrange) 187 | config = plt.gcf() 188 | plt.close('all') 189 | return config 190 | 191 | 192 | def fig2img(fig, size, return_img=True): 193 | fig.canvas.draw() 194 | buf = np.fromstring( 195 | fig.canvas.tostring_rgb(), dtype=np.uint8).reshape(size, size, 3) / 255 196 | if return_img: return Image(ToTensor(buf).permute(2, 0, 1)) 197 | else: return ToTensor(buf).permute(2, 0, 1) 198 | 199 | 200 | def plot(ts, size=None, yrange=(-1, 1), dpi=72, **kwargs): 201 | if size is None: size = ts.shape[-1] 202 | fig = get_plot_fig(ts, size, yrange=yrange, dpi=dpi) 203 | ax = fig.get_axes()[0] 204 | for tsi in ts: 205 | ax.plot(tsi, linewidth=1) 206 | output = fig2img(fig, size).data[0] 207 | plt.close('all') 208 | return output 209 | 210 | 211 | gadf = partial(GADF_encoder) 212 | setattr(gadf, '__name__', 'gadf') 213 | setattr(gadf, '_order', 10) 214 | setattr(gadf, 'cmap', 'spring') 215 | gasf = partial(GASF_encoder) 216 | setattr(gasf, '__name__', 'gasf') 217 | setattr(gasf, '_order', 10) 218 | setattr(gasf, 'cmap', 'summer') 219 | mtf = partial(MTF_encoder) 220 | setattr(mtf, '__name__', 'mtf') 221 | setattr(mtf, '_order', 10) 222 | setattr(mtf, 'cmap', 'autumn') 223 | rp = partial(RP_encoder) 224 | setattr(rp, '__name__', 'rp') 225 | setattr(rp, '_order', 10) 226 | setattr(rp, 'cmap', 'winter') 227 | spectro = partial(Spectro_encoder) 228 | setattr(spectro, '__name__', 'spectro') 229 | setattr(spectro, '_order', 10) 230 | setattr(spectro, 'cmap', 'cool') 231 | scalo = partial(Scalo_encoder) 232 | setattr(scalo, '__name__', 'scalo') 233 | setattr(scalo, '_order', 10) 234 | setattr(scalo, 'cmap', 'jet') 235 | addcc = partial(AddCoordConv) 236 | setattr(addcc, '__name__', 'addcc') 237 | setattr(addcc, '_order', 20) 238 | setattr(addcc, 'cmap', 'viridis') 239 | plot2img = partial(plot, dpi=get_dpi()) 240 | setattr(plot2img, '__name__', 'plot2img') 241 | setattr(plot2img, '_order', 20) 242 | setattr(plot2img, 'cmap', None) 243 | 244 | 245 | def norm(tensor): 246 | return (tensor - tensor.min()) / (tensor.max() - tensor.min()) 247 | 248 | 249 | def apply_cmap(tensor, cmap=None, **kwargs): 250 | if cmap is None: 251 | if tensor.ndim == 2: return tensor[None] 252 | elif tensor.ndim == 3: return tensor 253 | assert tensor.ndim == 2, f'incorrect tensor ndim --> {tensor.ndim}' 254 | return ToTensor(plt.get_cmap(cmap)(tensor))[..., :3].permute(2, 0, 1) 255 | 256 | 257 | def ToImage(tensor, size=None, cmap=None, **kwargs): 258 | tensor = torch.Tensor(tensor) 259 | if tensor.ndim == 1: tensor = tensor[None] 260 | if cmap is None: 261 | if tensor.ndim == 2: tensor = tensor[None] 262 | assert tensor.ndim == 3 or tensor.ndim == 1, f'incorrect tensor ndim --> {tensor.ndim}' 263 | else: 264 | if tensor.ndim == 3: tensor = tensor.squeeze(0) 265 | assert tensor.ndim == 2, f'incorrect tensor ndim --> {tensor.ndim}' 266 | tensor = apply_cmap(tensor, cmap) 267 | if size is None: return Image(tensor) 268 | else: return Image(tensor).resize(size) 269 | 270 | 271 | def resize_tensor(tensor, size): 272 | if tensor.ndim == 2: tensor = tensor[None] 273 | assert tensor.ndim == 3, 'check input tensor ndim' 274 | return Image(tensor).resize(size).data 275 | 276 | 277 | def add_dim(tensor, **kwargs): 278 | return tensor[None] 279 | 280 | 281 | def _repeat_ch(tensor, **kwargs): 282 | if tensor.shape[-3] == 3: return tensor 283 | else: return tensor.repeat(3, 1, 1) 284 | 285 | 286 | repeat_ch = partial(_repeat_ch) 287 | 288 | 289 | def _add_zero_ch(tensor, **kwargs): 290 | if tensor.shape[-3] == 3: return tensor 291 | elif tensor.shape[-3] == 2: 292 | zeros = torch.zeros_like(tensor[0][None]) 293 | output = torch.cat([tensor, zeros]) 294 | return output 295 | else: 296 | zeros = torch.zeros_like(tensor) 297 | output = torch.cat([tensor, zeros, zeros]) 298 | return output 299 | 300 | 301 | add_zero_ch = partial(_add_zero_ch) 302 | 303 | 304 | 305 | class TS2Image(): 306 | use_on_y = False 307 | 308 | def __init__(self, 309 | encoders=None, 310 | pre=None, 311 | post=None, 312 | xy_aug=None, 313 | add_cc=False, 314 | size=224, 315 | margin_perc=.1, 316 | concatenate=False, 317 | apply_colormap=True, 318 | repeat_channel=False, 319 | **kwargs): 320 | 321 | self.encoders = listify(encoders) 322 | self.n_encoders = len(self.encoders) 323 | self.pre = pre 324 | self.post = post 325 | self.xy_aug = xy_aug 326 | self.add_cc = add_cc 327 | self.size = size 328 | self.margin_perc = margin_perc 329 | self.concatenate = concatenate 330 | self.apply_colormap = apply_colormap 331 | self.kwargs = kwargs 332 | funcs = [] 333 | if self.n_encoders > 1 and self.apply_colormap: 334 | assert self.concatenate == True, 'turn concatenate on to apply colormap' 335 | for i, encoder in enumerate(self.encoders): 336 | aug = [partial(To2dArray)] 337 | aug += [partial(compose(self.pre), **self.kwargs)] 338 | aug += [partial(encoder, size=self.size, **self.kwargs)] 339 | if xy_aug: aug += [partial(compose(xyaug), **self.kwargs)] 340 | if isinstance(apply_colormap, str): aug += [partial(apply_cmap, cmap=apply_colormap)] 341 | elif apply_colormap: aug += [partial(apply_cmap, cmap=encoder.cmap)] 342 | elif add_cc: aug += [partial(addcc)] 343 | else: aug += [partial(add_dim)] 344 | aug += [partial(ToTensor)] 345 | funcs.append(compose(aug)) 346 | if repeat_channel: self.outsh = partial(repeat_ch) 347 | else: self.outsh = partial(add_zero_ch) 348 | self.funcs = funcs 349 | 350 | def __call__(self, ts): 351 | assert self.n_encoders > 0, 'You need to select at least 1 encoder' 352 | output = None 353 | for ch in range(ts.shape[-2]): 354 | for i in range(self.n_encoders): 355 | func = self.funcs[i] 356 | tensor = func(ts[ch]) 357 | if output is None: 358 | output = tensor 359 | rs = output.shape 360 | elif self.concatenate: 361 | output = torch.cat( 362 | (output, torch.zeros(rs[0], int(rs[1] * self.margin_perc), 363 | rs[2]).float()), 1) 364 | output = torch.cat((output, tensor), dim=1) 365 | else: 366 | output = torch.cat((output, tensor), dim=0) 367 | return self.outsh(output) 368 | 369 | 370 | def get_fill_between_fig(data, size=224, yrange=(-1, 1), dpi=72): 371 | fig = plt.figure(figsize=(size / dpi, size / dpi)) 372 | ax = plt.axes([0, 0, 1, 1], frameon=True) 373 | ax.get_xaxis().set_visible(False) 374 | ax.get_yaxis().set_visible(False) 375 | ax.set_facecolor('black') 376 | plt.xlim((0, data.shape[-1])) 377 | plt.ylim(yrange) 378 | config = plt.gcf() 379 | plt.close('all') 380 | return config 381 | 382 | def plot_fill_between(data, size=224, yrange=(-1, 1), dpi=DPI, return_img=True): 383 | data = ToArray(data) 384 | channels = data.shape[-2] 385 | assert channels in [2, 3], 'check sel_channels' 386 | x = np.arange(data.shape[-1]) 387 | fig = get_fill_between_fig(data, size, yrange=yrange, dpi=dpi) 388 | ax = fig.get_axes()[0] 389 | if channels == 3: 390 | ax.fill_between( 391 | x, 392 | data[1], 393 | data[2], 394 | where=data[1] > data[2], 395 | facecolor='#0814ff', 396 | interpolate=True) 397 | ax.fill_between( 398 | x, 399 | data[1], 400 | data[2], 401 | where=data[1] < data[2], 402 | facecolor='#fff308', 403 | interpolate=True) 404 | ax.fill_between( 405 | x, 406 | data[0], 407 | data[1], 408 | where=data[0] > data[1], 409 | facecolor='#0aff2f', 410 | interpolate=True) 411 | ax.fill_between( 412 | x, 413 | data[0], 414 | data[1], 415 | where=data[0] < data[1], 416 | facecolor='#ff0ada', 417 | interpolate=True) 418 | output = fig2img(fig, size, return_img) 419 | plt.close('all') 420 | return output 421 | 422 | def get_fill_between_plot(data, sel_TCs=None, sel_channels=None, size=None, return_img=True): 423 | if size is None: size = data.shape[-1] 424 | tfs = len(sel_TCs) 425 | feats = len(sel_channels) 426 | assert data.shape[-2] == tfs * feats, 'check sel_TCs and sel_channels' 427 | for j in range(tfs): 428 | if j == 0: 429 | ImgData = plot_fill_between(data[j * feats:(j + 1) * feats], size, return_img=False) 430 | else: 431 | ImgData = np.concatenate((ImgData, 432 | torch.ones(3, int(size * .05), size), 433 | plot_fill_between(data[j * feats:(j + 1) * feats], size, return_img=False)), axis=1) 434 | 435 | if return_img: return Image(ToTensor(ImgData)) 436 | else: return ToTensor(ImgData) 437 | 438 | 439 | 440 | class TS2ImageList(ItemList): 441 | "`ItemList` suitable for time series" 442 | _bunch, _square_show, _square_show_res = ImageDataBunch, True, True 443 | 444 | def __init__(self, items, *args, **kwargs): 445 | items = To3dTensor(items) 446 | super().__init__(items, *args, **kwargs) 447 | self.c, self.size = 3, {} 448 | self.channels = items.shape[-2] 449 | self.seq_len = items.shape[-1] 450 | 451 | def get(self, i): 452 | item = super().get(i) 453 | return TimeSeriesItem(To2dTensor(item)) 454 | 455 | @classmethod 456 | def from_array(cls, ts, **kwargs): 457 | return cls(ts) 458 | 459 | @classmethod 460 | def from_df(cls, df, path='.', cols=0, processor=None, 461 | **kwargs) -> 'ItemList': 462 | "Create an `ItemList` in `path` from the inputs in the `cols` of `df`." 463 | if cols is 0: 464 | inputs = df 465 | else: 466 | col_idxs = df_names_to_idx(list(cols), df) 467 | inputs = df.iloc[:, col_idxs] 468 | assert inputs.isna().sum().sum( 469 | ) == 0, f"You have NaN values in column(s) {cols} of your dataframe, please fix it." 470 | res = cls( 471 | items=inputs.values, 472 | path=path, 473 | inner_df=df, 474 | processor=processor, 475 | **kwargs) 476 | return res 477 | 478 | @classmethod 479 | def from_csv(cls, path, csv_name, header='infer', **kwargs) -> 'ItemList': 480 | "Get the filenames in `path/csv_name` opened with `header`." 481 | path = Path(path) 482 | df = pd.read_csv(path / csv_name, header=header) 483 | return cls.from_df(df, path=path, **kwargs) 484 | 485 | def reconstruct(self, t): 486 | return Image(t.float().clamp(min=0, max=1)) 487 | 488 | def show_xys(self, xs, ys, imgsize=4, figsize=None, **kwargs): 489 | "Show the `xs` (inputs) and `ys` (targets) on a figure of `figsize`." 490 | rows = int(np.ceil(math.sqrt(len(xs)))) 491 | axs = subplots(rows, rows, imgsize=imgsize, figsize=figsize) 492 | for x, y, ax in zip(xs, ys, axs.flatten()): 493 | x.show(ax=ax, y=y, **kwargs) 494 | for ax in axs.flatten()[len(xs):]: 495 | ax.axis('off') 496 | plt.tight_layout() 497 | 498 | def show_xyzs(self, xs, ys, zs, imgsize=4, figsize=None, **kwargs): 499 | "Show `xs` (inputs), `ys` (targets) and `zs` (predictions) on a figure of `figsize`." 500 | if self._square_show_res: 501 | title = 'Ground truth\nPredictions' 502 | rows = int(np.ceil(math.sqrt(len(xs)))) 503 | axs = subplots( 504 | rows, 505 | rows, 506 | imgsize=imgsize, 507 | figsize=figsize, 508 | title=title, 509 | weight='bold', 510 | size=12) 511 | for x, y, z, ax in zip(xs, ys, zs, axs.flatten()): 512 | x.show(ax=ax, title=f'{str(y)}\n{str(z)}', **kwargs) 513 | for ax in axs.flatten()[len(xs):]: 514 | ax.axis('off') 515 | else: 516 | title = 'Ground truth/Predictions' 517 | axs = subplots( 518 | len(xs), 519 | 2, 520 | imgsize=imgsize, 521 | figsize=figsize, 522 | title=title, 523 | weight='bold', 524 | size=14) 525 | for i, (x, y, z) in enumerate(zip(xs, ys, zs)): 526 | x.show(ax=axs[i, 0], y=y, **kwargs) 527 | x.show(ax=axs[i, 1], y=z, **kwargs) -------------------------------------------------------------------------------- /fastai_timeseries/exp/nb_TSTrain.py: -------------------------------------------------------------------------------- 1 | #AUTOGENERATED! DO NOT EDIT! file to edit: ./TSTrain.ipynb (unless otherwise specified) 2 | 3 | from fastai.callbacks import * 4 | 5 | try: 6 | from exp.nb_TSUtilities import * 7 | from exp.nb_TSBasicData import * 8 | from exp.nb_TSDatasets import * 9 | 10 | except ImportError: 11 | from .nb_TSUtilities import * 12 | from .nb_TSBasicData import * 13 | from .nb_TSDatasets import * 14 | 15 | import time 16 | 17 | 18 | def run_UCR_test(iters, epochs, datasets, arch, 19 | bs=64, max_lr=3e-3, pct_start=.7, warmup=False, wd=1e-2, 20 | metrics=[accuracy], mixup=False, 21 | scale_type ='standardize', scale_subtype='per_channel', scale_range=(-1, 1), 22 | opt_func=functools.partial(torch.optim.Adam, betas=(0.9, 0.99)), 23 | loss_func=None, **arch_kwargs): 24 | ds_, acc_, acces_, accmax_, iter_, time_, epochs_, loss_, val_loss_ = [], [], [], [], [], [], [], [], [] 25 | datasets = listify(datasets) 26 | for ds in datasets: 27 | db = create_UCR_databunch(ds) 28 | for i in range(iters): 29 | print('\n', ds, i) 30 | ds_.append(ds) 31 | iter_.append(i) 32 | epochs_.append(epochs) 33 | model = arch(db.features, db.c, **arch_kwargs).to(defaults.device) 34 | learn = Learner(db, model, opt_func=opt_func, loss_func=loss_func) 35 | if mixup: learn.mixup() 36 | learn.metrics = metrics 37 | start_time = time.time() 38 | learn.fit_one_cycle(epochs, max_lr=max_lr, pct_start=pct_start, moms=(.95, .85) if warmup else (.95, .95), 39 | div_factor=25.0 if warmup else 1., wd=wd) 40 | duration = time.time() - start_time 41 | time_.append('{:.0f}'.format(duration)) 42 | early_stop = math.ceil(np.argmin(learn.recorder.losses) / len(learn.data.train_dl)) 43 | acc_.append(learn.recorder.metrics[-1][0].item()) 44 | acces_.append(learn.recorder.metrics[early_stop - 1][0].item()) 45 | accmax_.append(np.max(learn.recorder.metrics)) 46 | loss_.append(learn.recorder.losses[-1].item()) 47 | val_loss_.append(learn.recorder.val_losses[-1].item()) 48 | if len(datasets) * iters >1: clear_output() 49 | df = (pd.DataFrame(np.stack((ds_, iter_, epochs_, loss_, val_loss_ ,acc_, acces_, accmax_, time_)).T, 50 | columns=['dataset', 'iter', 'epochs', 'loss', 'val_loss', 51 | 'accuracy', 'accuracy_ts', 52 | 'max_accuracy', 'time (s)']) 53 | ) 54 | df = df.astype({'loss': float, 'val_loss': float, 'accuracy': float, 55 | 'accuracy_ts': float, 'max_accuracy': float}) 56 | display(df) 57 | return learn, df 58 | 59 | 60 | 61 | def FlatCosAnnealScheduler(learn, lr:float=4e-3, epochs:int=1, moms:Floats=(0.85,0.95), 62 | start_pct:float=0.7, curve='cosine'): 63 | "Manage FCFit trainnig as found in the ImageNette experiments" 64 | #n = len(learn.data.train_dl) 65 | import math 66 | n = math.ceil(len(learn.data.train_ds)/learn.data.train_dl.batch_size) 67 | anneal_start = int(n * epochs * start_pct) 68 | batch_finish = ((n * epochs) - anneal_start) 69 | if curve=="cosine": curve_type=annealing_cos 70 | elif curve=="linear": curve_type=annealing_linear 71 | elif curve=="exponential": curve_type=annealing_exp 72 | else: raiseValueError(f"annealing type not supported {curve}") 73 | phase0 = TrainingPhase(anneal_start).schedule_hp('lr', lr).schedule_hp('mom', moms[0]) 74 | phase1 = (TrainingPhase(batch_finish) 75 | .schedule_hp('lr', lr, anneal=curve_type) 76 | .schedule_hp('mom', (moms[0],moms[1]), anneal=curve_type) 77 | ) 78 | phases = [phase0, phase1] 79 | return GeneralScheduler(learn, phases) 80 | 81 | def fit_fc(learn:Learner, epochs:int, lr:float=4e-3, 82 | moms:Tuple[float,float]=(0.85,0.95), start_pct:float=0.7, 83 | wd:float=None, callbacks:Optional[CallbackList]=None, 84 | show_curve:bool=False)->None: 85 | "Fit a model with Flat Cosine Annealing" 86 | max_lr = learn.lr_range(lr) 87 | callbacks = listify(callbacks) 88 | callbacks.append(FlatCosAnnealScheduler(learn, lr, moms=moms, start_pct=start_pct, epochs=epochs)) 89 | learn.fit(epochs, max_lr, wd=wd, callbacks=callbacks) 90 | if show_curve: learn.recorder.plot_lr() -------------------------------------------------------------------------------- /fastai_timeseries/exp/nb_TSUtilities.py: -------------------------------------------------------------------------------- 1 | #AUTOGENERATED! DO NOT EDIT! file to edit: ./TSUtilities.ipynb (unless otherwise specified) 2 | 3 | import fastai 4 | from fastai.basics import * 5 | from fastai.vision import * 6 | from fastai.tabular import * 7 | 8 | 9 | import numpy as np 10 | import pandas as pd 11 | import scipy as sp 12 | import matplotlib.pyplot as plt 13 | import torch 14 | import torchvision 15 | import os 16 | from pathlib import Path 17 | import warnings 18 | warnings.filterwarnings("ignore") 19 | import datetime 20 | import pprint 21 | from scipy.stats import ttest_ind 22 | import sklearn 23 | from sklearn import metrics 24 | from sklearn.metrics import accuracy_score, precision_score, recall_score, matthews_corrcoef, f1_score 25 | path = Path(os.getcwd()) 26 | from numbers import Integral 27 | from IPython.display import display, HTML, clear_output 28 | display(HTML("")) 29 | 30 | ts2img_kwargs = {} 31 | 32 | 33 | 34 | def get_dpi(): 35 | plt.plot([0, 1]) 36 | plt.close() 37 | DPI = plt.gcf().get_dpi() 38 | plt.close() 39 | return int(DPI) 40 | 41 | 42 | DPI = get_dpi() 43 | defaults.dpi = DPI 44 | device = defaults.device 45 | cpus = defaults.cpus 46 | 47 | 48 | def get_elements(arr, idx): 49 | from operator import itemgetter 50 | return itemgetter(*idx)(arr) 51 | 52 | def cloning(list1): 53 | li_copy = list1[:] 54 | return li_copy 55 | 56 | 57 | 58 | def scale(arr, 59 | train_stats=None, 60 | scaling_type='normalization', 61 | scaling_subtype='per_channel', 62 | scale_range=(-1, 1), 63 | **kwargs): 64 | 65 | if scaling_type is None: return arr, None 66 | if arr is None: return None, None 67 | if arr.ndim != 3: 68 | arr = To3dArray(arr) 69 | if train_stats is None: 70 | # Get train stats 71 | train = arr 72 | if scaling_type == 'normalization': 73 | if train.ndim == 2: 74 | if scaling_subtype == 'all_samples' or scaling_subtype == 'per_sample': 75 | train_min = train.min(keepdims=True) 76 | train_max = train.max(keepdims=True) 77 | elif scaling_subtype == 'per_channel': 78 | train_min = train.min(axis=(1), keepdims=True) 79 | train_max = train.max(axis=(1), keepdims=True) 80 | else: 81 | print('***** Please, select a valid scaling_subtype *****') 82 | return 83 | elif train.ndim == 3: 84 | if scaling_subtype == 'all_samples': 85 | train_min = train.min(keepdims=True) 86 | train_max = train.max(keepdims=True) 87 | elif scaling_subtype == 'per_sample': 88 | train_min = train.min(axis=(1, 2), keepdims=True) 89 | train_max = train.max(axis=(1, 2), keepdims=True) 90 | elif scaling_subtype == 'per_channel': 91 | train_min = train.min(axis=(0, 2), keepdims=True) 92 | train_max = train.max(axis=(0, 2), keepdims=True) 93 | else: 94 | print('***** Please, select a valid scaling_subtype *****') 95 | return 96 | train_stats = (train_min, train_max) 97 | elif scaling_type == 'standardization': 98 | if train.ndim == 2: 99 | if scaling_subtype == 'all_samples' or scaling_subtype == 'per_sample': 100 | train_mean = train.mean(keepdims=True) 101 | train_std = train.std(keepdims=True) 102 | elif scaling_subtype == 'per_channel': 103 | train_mean = train.mean(axis=(1), keepdims=True) 104 | train_std = train.std(axis=(1), keepdims=True) 105 | else: 106 | print('***** Please, select a valid scaling_subtype *****') 107 | return 108 | elif train.ndim == 3: 109 | if scaling_subtype == 'all_samples': 110 | train_mean = train.mean(keepdims=True) 111 | train_std = train.std(keepdims=True) 112 | elif scaling_subtype == 'per_sample': 113 | train_mean = train.mean(axis=(1, 2), keepdims=True) 114 | train_std = train.std(axis=(1, 2), keepdims=True) 115 | elif scaling_subtype == 'per_channel': 116 | train_mean = train.mean(axis=(0, 2), keepdims=True) 117 | train_std = train.std(axis=(0, 2), keepdims=True) 118 | else: 119 | print('***** Please, select a valid scaling_subtype *****') 120 | return 121 | train_stats = (train_mean, train_std) 122 | else: 123 | print('***** Please, select a valid scaling_type *****') 124 | return 125 | 126 | # Calculate 127 | if scaling_type == 'normalization': 128 | train_min, train_max = train_stats 129 | if scaling_subtype == 'per_sample': 130 | return ((((arr - train_min)) / (train_max - train_min)) * ( 131 | scale_range[1] - scale_range[0]) + scale_range[0], None) 132 | else: 133 | return ((((arr - train_min)) / (train_max - train_min)) * ( 134 | scale_range[1] - scale_range[0]) + scale_range[0], (train_min, train_max)) 135 | #return (To3dTensor((((arr - train_min)) / (train_max - train_min)) * (scale_range[1] - scale_range[0]) + scale_range[0]), (train_min, train_max)) 136 | elif scaling_type == 'standardization': 137 | mean, std = train_stats 138 | if scaling_subtype == 'per_sample': 139 | return (np.nan_to_num((arr - mean) / std), None) 140 | else: 141 | return (np.nan_to_num((arr - mean) / std), (mean, std)) 142 | #return (To3dTensor(np.nan_to_num((arr - mean) / std)), (mean, std)) 143 | else: 144 | return 145 | 146 | def scale_data(X_train, 147 | X_valid, 148 | X_test=None, 149 | scaling_type='normalization', 150 | scaling_subtype='per_channel', 151 | scale_range=(-1, 1)): 152 | 153 | X_train_sc, train_stats = scale( 154 | X_train, 155 | train_stats=None, 156 | scaling_type=scaling_type, 157 | scaling_subtype=scaling_subtype, 158 | scale_range=scale_range) 159 | 160 | X_valid_sc, _ = scale( 161 | X_valid, 162 | train_stats=train_stats, 163 | scaling_type=scaling_type, 164 | scaling_subtype=scaling_subtype, 165 | scale_range=scale_range) 166 | 167 | X_test_sc, _ = scale( 168 | X_test, 169 | train_stats=train_stats, 170 | scaling_type=scaling_type, 171 | scaling_subtype=scaling_subtype, 172 | scale_range=scale_range) 173 | 174 | if scaling_type == 'normalization': 175 | X_valid_sc = np.clip(X_valid_sc, scale_range[0], scale_range[1]) 176 | if X_test is not None: 177 | X_test_sc = np.clip(X_test_sc, scale_range[0], scale_range[1]) 178 | 179 | return X_train_sc, X_valid_sc, X_test_sc 180 | 181 | 182 | 183 | def cap_outliers(y, lower=None, verbose=False): 184 | q25, q75 = np.percentile(y, 25), np.percentile(y, 75) 185 | iqr = q75 - q25 186 | cut_off = iqr * 1.5 187 | if lower is None: 188 | lower = q25 - cut_off 189 | upper = q75 + cut_off 190 | outliers = sorted([x for x in y if x < lower or x > upper]) 191 | if verbose: print('outliers capped:', outliers, (lower, upper)) 192 | y = np.array([min(max(x, lower), upper) for x in y]) 193 | return y 194 | 195 | 196 | def get_y_range(y, problem_type): 197 | if problem_type == 'regression': 198 | y_range = (int((y.min() - .5) * 2) / 2, math.ceil( 199 | (y.max() + .5) * 2) / 2) 200 | print('y_range:', y_range) 201 | else: 202 | y_range = None 203 | return y_range 204 | 205 | 206 | 207 | def get_stratified_train_val_test_idxs(y, 208 | n_folds, 209 | test_fold=False, 210 | y_add=None, 211 | add_train_folds=None, 212 | oversample=False, 213 | seed=1): 214 | from sklearn.model_selection import StratifiedKFold 215 | if isinstance(y, np.ndarray): y = torch.Tensor(y).to(dtype=torch.int64) 216 | if n_folds == 1: 217 | folds = 5 218 | else: 219 | folds = n_folds 220 | if y_add: 221 | if add_train_folds: 222 | # Augmented dataset 223 | train_add_idx = list( 224 | zip(*list( 225 | KFold( 226 | n_splits=n_folds, shuffle=True, random_state=cv_seed). 227 | split(np.zeros(len(y_add)), Y_train_add))))[0] 228 | else: 229 | train_add_idx = np.arange(len(y_add)) 230 | else: 231 | train_add_idx = None 232 | if test_fold: 233 | outer_folds = list( 234 | StratifiedKFold( 235 | n_splits=folds + 1, shuffle=True, random_state=seed).split( 236 | np.zeros(len(y)), y)) 237 | test_idx = outer_folds[0][1] 238 | inner_idxs = outer_folds[0][0] 239 | inner_folds = StratifiedKFold( 240 | n_splits=folds, shuffle=True, random_state=seed).split( 241 | np.zeros(len(inner_idxs)), y[inner_idxs]) 242 | train_idx = [] 243 | val_idx = [] 244 | for train, val in inner_folds: 245 | if oversample: 246 | train = oversampled_idxs(y[inner_idxs], train, seed=seed) 247 | train_idx.append(inner_idxs[train]) 248 | val_idx.append(inner_idxs[val]) 249 | if n_folds == 1: 250 | return [train_idx[0]], [val_idx[0]], test_idx 251 | return train_idx, val_idx, test_idx, train_add_idx 252 | else: 253 | inner_folds = StratifiedKFold( 254 | n_splits=folds, shuffle=True, random_state=seed).split( 255 | np.zeros(len(y)), y) 256 | train_idx = [] 257 | val_idx = [] 258 | for train, val in inner_folds: 259 | if oversample: 260 | train = oversampled_idxs(y, train, seed=seed) 261 | train_idx.append(train) 262 | val_idx.append(val) 263 | if n_folds == 1: 264 | return [train_idx[0]], [val_idx[0]], None 265 | return train_idx, val_idx, None, train_add_idx 266 | 267 | 268 | def check_overlap(a, b): 269 | overlap = [i for i in a if i in b] 270 | if overlap == []: 271 | return 272 | return overlap 273 | 274 | 275 | def leakage_finder(train, val, test=None): 276 | if check_overlap(train, val) is not None: 277 | print('train-val leakage!') 278 | return check_overlap(train, val) 279 | if test is not None: 280 | if check_overlap(train, test) is not None: 281 | print('train-test leakage!') 282 | return check_overlap(train, test) 283 | if check_overlap(val, test) is not None: 284 | print('val-test leakage!') 285 | return check_overlap(val, test) 286 | return 287 | 288 | 289 | def oversampled_idxs(y, idx, seed=1, verbose=False): 290 | from imblearn.over_sampling import RandomOverSampler 291 | from collections import Counter 292 | ros = RandomOverSampler(random_state=seed) 293 | resampled_idxs, y_resampled = ros.fit_resample(idx.reshape(-1, 1), y[idx]) 294 | if verbose: 295 | print('classes:', count_classes(y_resampled)) 296 | return np.sort(resampled_idxs.ravel()) 297 | 298 | 299 | def split_data(X, y, train_idx, valid_idx, test_idx, train_add_idx=None): 300 | X_train, y_train = X[train_idx[0]], y[train_idx[0]] 301 | X_valid, y_valid = X[valid_idx[0]], y[valid_idx[0]] 302 | X_test, y_test = X[test_idx], y[test_idx] 303 | print('X_train:', X_train.shape, 'X_valid:', X_valid.shape, 'X_test:', 304 | X_test.shape) 305 | print('y_train:', y_train.shape, 'y_valid:', y_valid.shape, 'y_test:', 306 | y_test.shape) 307 | if train_add_idx is not None: 308 | X_train_add, y_train_add = X[train_add_idx[0]], y[train_add_idx[0]] 309 | print('X_train_add:', X_train_add.shape, 'y_train_add:', 310 | y_train_add.shape) 311 | else: 312 | X_train_add, y_train_add = None, None 313 | return X_train, y_train, X_valid, y_valid, X_test, y_test, X_train_add, y_train_add 314 | 315 | 316 | def count_classes(y): 317 | return dict(sorted(Counter(y).items())) 318 | 319 | 320 | def get_class_weights(target): 321 | if isinstance(target, np.ndarray): target = torch.Tensor(target).to(dtype=torch.int64) 322 | # Compute samples weight (each sample should get its own weight) 323 | class_sample_count = torch.tensor( 324 | [(target == t).sum() for t in torch.unique(target, sorted=True)]) 325 | weights = 1. / class_sample_count.float() 326 | return (weights / weights.sum()).to(device) 327 | 328 | def get_weighted_sampler(target): 329 | from torch.utils.data.sampler import WeightedRandomSampler 330 | weight = get_class_weights(target) 331 | samples_weight = torch.tensor([weight[t] for t in target]) 332 | # Create sampler, dataset, loader 333 | return WeightedRandomSampler(samples_weight, len(samples_weight)) 334 | 335 | def history_output(learn, max_lr, epochs, t0, t1): 336 | 337 | print('\ndataset :', learn.data.dsid) 338 | #print('shape :', learn.data.train_ds[0][0].data.shape[-2:]) 339 | print('model :', learn.model.__name__) 340 | print('epochs :', epochs) 341 | print('batch size :', learn.data.train_dl.batch_size) 342 | print('max_lr :', max_lr) 343 | print('wd :', learn.wd) 344 | print('time :', 345 | str(datetime.timedelta(seconds=(t1 - t0).seconds))) 346 | metrics = np.array(learn.recorder.metrics) 347 | metrics_names = learn.recorder.metrics_names 348 | train_loss = learn.recorder.losses 349 | best_train_epoch = np.nanargmin(train_loss) 350 | val_loss = learn.recorder.val_losses 351 | best_val_epoch = np.nanargmin(val_loss) 352 | epochs10p = min(5, max(1, int(.1 * epochs))) 353 | n_batches = len(learn.data.train_dl) 354 | b_per_eopch = math.ceil(len(learn.data.train_ds) / learn.data.train_dl.batch_size) 355 | 356 | 357 | print('\ntrain loss:') 358 | print('min train loss : {:.5f} epoch: {:}'.format( 359 | np.min(train_loss), (best_train_epoch + 1) // n_batches)) 360 | print('final loss : {:.5f} epoch: {:}'.format( 361 | train_loss[-1], len(train_loss) // n_batches)) 362 | print('final avg loss : {:.5f} +/- {:.5f} in last {:} epochs'.format( 363 | np.mean(train_loss[-epochs10p*n_batches:]), np.std(train_loss[-epochs10p*n_batches:]), epochs10p)) 364 | print('\nval loss:') 365 | print('min val loss : {:.5f} epoch: {:}'.format( 366 | np.min(val_loss), best_val_epoch + 1)) 367 | print('final loss : {:.5f} epoch: {:}'.format( 368 | val_loss[-1], len(val_loss))) 369 | print('final avg loss : {:.5f} +/- {:.5f} in last {:} epochs'.format( 370 | np.mean(val_loss[-epochs10p:]), np.std(val_loss[-epochs10p:]), epochs10p)) 371 | if len(metrics) > 0: 372 | for i in range(0, metrics.shape[1]): 373 | metric = metrics[:, i] 374 | print() 375 | if not np.isnan(np.nanmax(metric)): 376 | print(metrics_names[i]) 377 | print('highest metric : {:.5f} epoch: {:}'.format( 378 | np.nanmax(metric), 379 | np.nanargmax(metric) + 1)) 380 | print('early stopping metric : {:.5f} epoch: {:}'.format( 381 | metric[best_train_epoch // b_per_eopch], best_train_epoch // b_per_eopch + 1)) 382 | print('final metric : {:.5f} epoch: {:}'.format( 383 | metric[-1], len(metric))) 384 | print('final avg metric : {:.5f} +/- {:.5f} in last {:} epochs'.format( 385 | np.mean(metric[-epochs10p:]), np.std(metric[-epochs10p:]), epochs10p)) 386 | 387 | learn.recorder.plot_lr() 388 | learn.recorder.plot_losses() 389 | if len(metrics) > 0: learn.recorder.plot_metrics() 390 | #try: 391 | # interp = ClassificationInterpretation.from_learner(learn) 392 | # interp.plot_confusion_matrix(figsize=(6,6), cmap='Greens') 393 | #except: pass 394 | return 395 | 396 | 397 | def model_summary(model, data, find_all=False, print_mod=False): 398 | xb, yb = get_batch(data.valid_dl, model) 399 | mods = find_modules(model, is_lin_layer) if find_all else model.children() 400 | f = lambda hook,mod,inp,out: print(f"====\n{mod}\n" if print_mod else "", out.shape) 401 | with Hooks(mods, f) as hooks: 402 | learn.model(xb) 403 | 404 | 405 | def get_batch(dl, learn): 406 | learn.xb, learn.yb = next(iter(dl)) 407 | learn.do_begin_fit(0) 408 | learn('begin_batch') 409 | learn('after_fit') 410 | return learn.xb, learn.yb 411 | 412 | def conv(ni, nf, ks=3, stride=1, bias=False): 413 | return nn.Conv2d(ni, nf, kernel_size=ks, stride=stride, padding=ks//2, bias=bias) 414 | 415 | def noopr(x, **kwargs): 416 | return x 417 | 418 | 419 | 420 | def ToTensor(arr, **kwargs): 421 | if isinstance(arr, np.ndarray): 422 | arr = torch.from_numpy(arr) 423 | elif not isinstance(arr, torch.Tensor): 424 | print(f"Can't convert {type(arr)} to torch.Tensor") 425 | return arr.float() 426 | 427 | 428 | def ToArray(arr): 429 | if isinstance(arr, torch.Tensor): 430 | arr = np.array(arr) 431 | elif not isinstance(arr, np.ndarray): 432 | print(f"Can't convert {type(arr)} to np.array") 433 | if arr.dtype == 'O': arr = np.array(arr, dtype=np.float32) 434 | return arr 435 | 436 | 437 | def To3dTensor(arr): 438 | if arr.dtype == 'O': arr = np.array(arr, dtype=np.float32) 439 | arr = ToTensor(arr) 440 | if arr.ndim == 1: arr = arr[None, None] 441 | elif arr.ndim == 2: arr = arr[:, None] 442 | elif arr.ndim == 4: arr = arr[0] 443 | assert arr.ndim == 3, 'Please, review input dimensions' 444 | return arr 445 | 446 | 447 | def To2dTensor(arr): 448 | if arr.dtype == 'O': arr = np.array(arr, dtype=np.float32) 449 | arr = ToTensor(arr) 450 | if arr.ndim == 1: arr = arr[None] 451 | elif arr.ndim == 3: arr = torch.squeeze(arr, 0) 452 | assert arr.ndim == 2, 'Please, review input dimensions' 453 | return arr 454 | 455 | 456 | def To1dTensor(arr): 457 | if arr.dtype == 'O': arr = np.array(arr, dtype=np.float32) 458 | arr = ToTensor(arr) 459 | if arr.ndim == 3: arr = torch.squeeze(arr, 1) 460 | if arr.ndim == 2: arr = torch.squeeze(arr, 0) 461 | assert arr.ndim == 1, 'Please, review input dimensions' 462 | return arr 463 | 464 | 465 | def To3dArray(arr): 466 | arr = ToArray(arr) 467 | if arr.ndim == 1: arr = arr[None, None] 468 | elif arr.ndim == 2: arr = arr[:, None] 469 | elif arr.ndim == 4: arr = arr[0] 470 | assert arr.ndim == 3, 'Please, review input dimensions' 471 | return np.array(arr) 472 | 473 | 474 | def To2dArray(arr): 475 | arr = ToArray(arr) 476 | if arr.ndim == 1: arr = arr[None] 477 | if arr.ndim == 3: arr = np.squeeze(arr, 0) 478 | assert arr.ndim == 2, 'Please, review input dimensions' 479 | return np.array(arr) 480 | 481 | 482 | def To1dArray(arr): 483 | arr = ToArray(arr) 484 | if arr.ndim == 3: arr = np.squeeze(arr, 1) 485 | if arr.ndim == 2: arr = np.squeeze(arr, 0) 486 | assert arr.ndim == 1, 'Please, review input dimensions' 487 | return np.array(arr) 488 | 489 | 490 | def ToDevice(ts, **kwargs): 491 | if isinstance(ts, torch.Tensor): 492 | return ts.type(torch.FloatTensor) 493 | return torch.from_numpy(ts).type(torch.FloatTensor).to(device) 494 | if defaults.device.type == 'cpu': 495 | ts = torch.from_numpy(ts).double() #.type(torch.FloatTensor) 496 | return ts 497 | else: 498 | ts = torch.from_numpy(ts).double().to( 499 | device) #.type(torch.FloatTensor) 500 | return ts 501 | 502 | 503 | cudify = partial(ToDevice) 504 | 505 | 506 | 507 | def mape(pred, targ): 508 | "Mean absolute error between `pred` and `targ`." 509 | pred, targ = flatten_check(pred, targ) 510 | return (torch.abs(targ - pred) / targ).mean() 511 | 512 | 513 | from fastai.metrics import RegMetrics 514 | class MAPE(RegMetrics): 515 | "Compute the root mean absolute percentage error." 516 | def on_epoch_end(self, last_metrics, **kwargs): 517 | return add_metrics(last_metrics, mape(self.preds, self.targs)) 518 | 519 | 520 | from fastai.metrics import CMScores 521 | 522 | class BPR(CMScores): 523 | 'Balanced Precision Recall' 524 | _order = -20 525 | 526 | def __init__(self, alpha=1, beta=1): 527 | super().__init__() 528 | self.alpha, self.beta = alpha, beta 529 | 530 | def on_epoch_end(self, last_metrics, **kwargs): 531 | bpr = (min(self._precision(), self._recall()) ** self.alpha) * (max(self._precision(), self._recall()) ** self.beta) 532 | return add_metrics(last_metrics, bpr ** (1 / (self.alpha + self.beta))) 533 | 534 | 535 | class FocalLoss(nn.Module): 536 | def __init__(self, α=.25, γ=2., weight=None): 537 | super().__init__() 538 | self.α = α 539 | self.γ = γ 540 | self.weight = weight 541 | 542 | def forward(self, inputs, targets, **kwargs): 543 | CE_loss = nn.CrossEntropyLoss(weight=self.weight, reduction='sum')(inputs, targets) 544 | pt = torch.exp(-CE_loss) 545 | F_loss = self.α * ((1 - pt)**self.γ) * CE_loss 546 | return F_loss.mean() 547 | 548 | 549 | def get_model_hp(tsmodel, kwargs=[{}]): 550 | all_args = inspect.getargspec(tsmodel.__dict__['__init__']) 551 | if all_args.defaults: 552 | tsmodel_dict = dict(zip(all_args.args[-len(all_args.defaults):], all_args.defaults)) 553 | if kwargs != [{}]: 554 | for i in range(len(listify(kwargs))): 555 | for k,v in kwargs[i].items(): tsmodel_dict[k] = v 556 | return tsmodel_dict 557 | else: return {} 558 | 559 | 560 | def get_outcome_stats (learn, y_outcome, problem_type, train, valid, test=None, thr=None): 561 | train_pred, train_true = learn.get_preds(DatasetType.Fix) 562 | train_true = train_true.numpy() #.astype(int) 563 | if problem_type == 'regression': 564 | train_preds = train_pred.numpy().ravel() #.astype(int) 565 | else: 566 | train_preds = train_pred[:, 1].numpy() 567 | 568 | if thr is not None: max_thr = thr 569 | elif problem_type == 'regression': 570 | max_thr = -1000 571 | max_sum = -1000 572 | sum_ = [] 573 | xrange = np.arange( 574 | int(np.min(train_preds) // .001) * .001, 575 | int(1 + np.max(train_preds) // .001) * .001, .001) 576 | for i in xrange: 577 | thr = i 578 | y_true_train = y_outcome[train] 579 | train_preds_sum = y_true_train[train_preds >= thr].sum() 580 | sum_.append(train_preds_sum) 581 | if train_preds_sum > max_sum: 582 | max_thr = thr 583 | max_sum = train_preds_sum 584 | else: max_thr = .5 585 | y_true_train = y_outcome[train] 586 | pred_train_trades = y_true_train[train_preds >= max_thr] 587 | 588 | valid_pred, valid_true = learn.get_preds(DatasetType.Valid) 589 | if problem_type == 'regression': 590 | valid_preds = np.array(valid_pred).ravel() #.astype(int) 591 | else: 592 | valid_preds = valid_pred[:, 1].numpy() 593 | y_true_valid = y_outcome[valid] 594 | pred_val_trades = y_true_valid[valid_preds >= max_thr] 595 | 596 | try: 597 | test_pred, test_true = learn.get_preds(DatasetType.Test) 598 | except: 599 | test_pred = [] 600 | if problem_type == 'regression': 601 | test_preds = np.array(test_pred).ravel() #.astype(int) 602 | else: 603 | test_preds = test_pred[:, 1].numpy() 604 | y_true_test = y_outcome[test] 605 | pred_test_trades = y_true_test[test_preds >= max_thr] 606 | 607 | if thr is None and problem_type == 'regression': 608 | plt.plot(xrange, sum_) 609 | plt.xticks(np.arange(int(np.min(xrange) // .05) * .05, np.max(xrange)+.2, .2)) 610 | plt.show() 611 | 612 | 613 | print('Thr : {:.2f}'.format(max_thr)) 614 | print('TRAIN: %trades: {:3.1%} avgR: {:+1.3f}({:+1.3f}) win%: {:2.1%}'.format( 615 | len(pred_train_trades) / len(y_true_train), pred_train_trades.mean(), 616 | y_true_train.mean(), (pred_train_trades > 0).mean())) 617 | 618 | print('VALID: %trades: {:3.1%} avgR: {:+1.3f}({:+1.3f}) win%: {:2.1%}'.format( 619 | len(pred_val_trades) / len(y_true_valid), pred_val_trades.mean(), 620 | y_true_valid.mean(), (pred_val_trades > 0).mean())) 621 | 622 | if test_pred != []: 623 | print('TEST : %trades: {:3.1%} avgR: {:+1.3f}({:+1.3f}) win%: {:2.1%}'.format( 624 | len(pred_test_trades) / len(y_true_test), pred_test_trades.mean(), 625 | y_true_test.mean(), (pred_test_trades > 0).mean())) 626 | 627 | 628 | return 629 | 630 | 631 | def get_last_pos(arr, val, ndigits=5): 632 | if round(arr[-1], ndigits) != val: return 0 633 | for i,v in enumerate(arr[::-1]): 634 | if round(v, ndigits) != val: return i-1 635 | 636 | def plot_weights(learn): 637 | w1 = np.sort(list(learn.model.parameters())[0].data.flatten()) 638 | plt.hist(w1, 50, color='blue') 639 | plt.xlim([-1.75, 1.75]) 640 | plt.title('Layer init weights mean: {:.5f} std: {:.5f}'.format( 641 | w1.mean(), w1.std())) 642 | plt.grid() 643 | plt.show() 644 | 645 | def load_params(m, path): 646 | m.load_state_dict(torch.load(path)) 647 | return m.to(device) 648 | 649 | 650 | def noop(x): return x 651 | 652 | def get_layers(model, cond=noop): 653 | if isinstance(model, Learner): model=model.model 654 | return [m for m in flatten_model(model) if any([c(m) for c in listify(cond)])] 655 | 656 | def count_params(model): 657 | if isinstance(model, Learner): model = model.model 658 | count = 0 659 | for l in get_layers(model): 660 | for i in range(len(list(l.parameters()))): 661 | count += len(list(l.parameters())[i].data.flatten()) 662 | return count 663 | 664 | 665 | def nb_auto_export(): 666 | from IPython.display import display, Javascript 667 | import time 668 | from notebook2script import notebook2script 669 | display(Javascript('IPython.notebook.save_notebook()')) 670 | time.sleep(1) 671 | notebook2script() 672 | display(Javascript('IPython.notebook.save_checkpoint()')) -------------------------------------------------------------------------------- /fastai_timeseries/exp/rocket_functions.py: -------------------------------------------------------------------------------- 1 | # Angus Dempster, Francois Petitjean, Geoff Webb 2 | 3 | # Dempster A, Petitjean F, Webb GI (2019) ROCKET: Exceptionally fast and 4 | # accurate time series classification using random convolutional kernels. 5 | # arXiv:1910.13051 6 | 7 | # changes: 8 | # - added kss parameter to generate_kernels 9 | # - convert X to np.float64 10 | 11 | from numba import njit, prange 12 | import numpy as np 13 | 14 | @njit 15 | def generate_kernels(input_length, num_kernels, kss=[7, 9, 11], pad=True, dilate=True): 16 | candidate_lengths = np.array((kss)) 17 | # initialise kernel parameters 18 | weights = np.zeros((num_kernels, candidate_lengths.max())) # see note 19 | lengths = np.zeros(num_kernels, dtype = np.int32) # see note 20 | biases = np.zeros(num_kernels) 21 | dilations = np.zeros(num_kernels, dtype = np.int32) 22 | paddings = np.zeros(num_kernels, dtype = np.int32) 23 | # note: only the first *lengths[i]* values of *weights[i]* are used 24 | for i in range(num_kernels): 25 | length = np.random.choice(candidate_lengths) 26 | _weights = np.random.normal(0, 1, length) 27 | bias = np.random.uniform(-1, 1) 28 | if dilate: dilation = 2 ** np.random.uniform(0, np.log2((input_length - 1) // (length - 1))) 29 | else: dilation = 1 30 | if pad: padding = ((length - 1) * dilation) // 2 if np.random.randint(2) == 1 else 0 31 | else: padding = 0 32 | weights[i, :length] = _weights - _weights.mean() 33 | lengths[i], biases[i], dilations[i], paddings[i] = length, bias, dilation, padding 34 | return weights, lengths, biases, dilations, paddings 35 | 36 | @njit(fastmath = True) 37 | def apply_kernel(X, weights, length, bias, dilation, padding): 38 | # zero padding 39 | if padding > 0: 40 | _input_length = len(X) 41 | _X = np.zeros(_input_length + (2 * padding)) 42 | _X[padding:(padding + _input_length)] = X 43 | X = _X 44 | input_length = len(X) 45 | output_length = input_length - ((length - 1) * dilation) 46 | _ppv = 0 # "proportion of positive values" 47 | _max = np.NINF 48 | for i in range(output_length): 49 | _sum = bias 50 | for j in range(length): 51 | _sum += weights[j] * X[i + (j * dilation)] 52 | if _sum > 0: 53 | _ppv += 1 54 | if _sum > _max: 55 | _max = _sum 56 | return _ppv / output_length, _max 57 | 58 | @njit(parallel = True, fastmath = True) 59 | def apply_kernels(X, kernels): 60 | X = X.astype(np.float64) 61 | weights, lengths, biases, dilations, paddings = kernels 62 | num_examples = len(X) 63 | num_kernels = len(weights) 64 | # initialise output 65 | _X = np.zeros((num_examples, num_kernels * 2)) # 2 features per kernel 66 | for i in prange(num_examples): 67 | for j in range(num_kernels): 68 | _X[i, (j * 2):((j * 2) + 2)] = \ 69 | apply_kernel(X[i], weights[j][:lengths[j]], lengths[j], biases[j], dilations[j], paddings[j]) 70 | return _X -------------------------------------------------------------------------------- /images/LSST_scalling.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timeseriesAI/timeseriesAI1/8c0ff06737ab0b15938372de3d298b8a6083b120/images/LSST_scalling.jpg -------------------------------------------------------------------------------- /images/LSST_scalling2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timeseriesAI/timeseriesAI1/8c0ff06737ab0b15938372de3d298b8a6083b120/images/LSST_scalling2.jpg -------------------------------------------------------------------------------- /images/MixMatch_comparison_performance.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timeseriesAI/timeseriesAI1/8c0ff06737ab0b15938372de3d298b8a6083b120/images/MixMatch_comparison_performance.jpg -------------------------------------------------------------------------------- /images/MixMatch_time_comparison.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timeseriesAI/timeseriesAI1/8c0ff06737ab0b15938372de3d298b8a6083b120/images/MixMatch_time_comparison.jpg -------------------------------------------------------------------------------- /images/ROCKET_InceptionTime_multi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timeseriesAI/timeseriesAI1/8c0ff06737ab0b15938372de3d298b8a6083b120/images/ROCKET_InceptionTime_multi.png -------------------------------------------------------------------------------- /images/TSAugmentation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timeseriesAI/timeseriesAI1/8c0ff06737ab0b15938372de3d298b8a6083b120/images/TSAugmentation.png -------------------------------------------------------------------------------- /images/Time_ROCKET_comp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timeseriesAI/timeseriesAI1/8c0ff06737ab0b15938372de3d298b8a6083b120/images/Time_ROCKET_comp.png -------------------------------------------------------------------------------- /images/accuracy_blf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timeseriesAI/timeseriesAI1/8c0ff06737ab0b15938372de3d298b8a6083b120/images/accuracy_blf.jpg -------------------------------------------------------------------------------- /images/beef_mixup.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timeseriesAI/timeseriesAI1/8c0ff06737ab0b15938372de3d298b8a6083b120/images/beef_mixup.jpg -------------------------------------------------------------------------------- /images/sel_samples_blf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timeseriesAI/timeseriesAI1/8c0ff06737ab0b15938372de3d298b8a6083b120/images/sel_samples_blf.jpg -------------------------------------------------------------------------------- /images/time_blf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timeseriesAI/timeseriesAI1/8c0ff06737ab0b15938372de3d298b8a6083b120/images/time_blf.jpg -------------------------------------------------------------------------------- /images/tweet_blf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timeseriesAI/timeseriesAI1/8c0ff06737ab0b15938372de3d298b8a6083b120/images/tweet_blf.jpg -------------------------------------------------------------------------------- /images/valid_loss_blf.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/timeseriesAI/timeseriesAI1/8c0ff06737ab0b15938372de3d298b8a6083b120/images/valid_loss_blf.jpg -------------------------------------------------------------------------------- /old/nb_TSDataAugmentation.py: -------------------------------------------------------------------------------- 1 | #AUTOGENERATED! DO NOT EDIT! file to edit: ./TSDataAugmentation.ipynb (unless otherwise specified) 2 | 3 | import copy 4 | import numpy as np 5 | import random 6 | from functools import partial 7 | 8 | 9 | try: from exp.nb_TSUtilities import * 10 | except ImportError: from .nb_TSUtilities import * 11 | 12 | try: from exp.nb_TSBasicData import * 13 | except ImportError: from .nb_TSBasicData import * 14 | 15 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 16 | 17 | 18 | def shuffle_HLs(ts, **kwargs): 19 | line = copy(ts) 20 | pos_rand_list = np.random.choice( 21 | np.arange(ts.shape[-1] // 4), 22 | size=random.randint(0, ts.shape[-1] // 4), 23 | replace=False) 24 | rand_list = pos_rand_list * 4 25 | highs = rand_list + 1 26 | lows = highs + 1 27 | a = np.vstack([highs, lows]).flatten('F') 28 | b = np.vstack([lows, highs]).flatten('F') 29 | line[..., a] = line[..., b] 30 | return line 31 | 32 | setattr(shuffle_HLs, 'use_on_y', False) 33 | 34 | 35 | def get_diff(a): 36 | return np.concatenate((np.zeros(a.shape[-2])[:, None], np.diff(a)), axis=1).astype(np.float32) 37 | 38 | setattr(get_diff, 'use_on_y', False) 39 | 40 | 41 | class TSTransform(): 42 | "Utility class for adding probability and wrapping support to transform `func`." 43 | _wrap=None 44 | order=0 45 | def __init__(self, func:Callable, order:Optional[int]=None): 46 | "Create a transform for `func` and assign it an priority `order`, attach to `TS` class." 47 | if order is not None: self.order=order 48 | self.func=func 49 | self.func.__name__ = func.__name__[1:] #To remove the _ that begins every transform function. 50 | functools.update_wrapper(self, self.func) 51 | self.func.__annotations__['return'] = TSItem 52 | self.params = copy(func.__annotations__) 53 | self.def_args = _get_default_args(func) 54 | setattr(TSItem, func.__name__, lambda x, *args, **kwargs: self.calc(x, *args, **kwargs)) 55 | 56 | def __call__(self, *args:Any, p:float=1., is_random:bool=True, use_on_y:bool=False, **kwargs:Any)->TSItem: 57 | "Calc now if `args` passed; else create a transform called prob `p` if `random`." 58 | if args: return self.calc(*args, **kwargs) 59 | else: return RandTransform(self, kwargs=kwargs, is_random=is_random, use_on_y=use_on_y, p=p) 60 | 61 | def calc(self, x:TSItem, *args:Any, **kwargs:Any)->Image: 62 | "Apply to image `x`, wrapping it if necessary." 63 | if self._wrap: return getattr(x, self._wrap)(self.func, *args, **kwargs) 64 | else: return self.func(x, *args, **kwargs) 65 | 66 | @property 67 | def name(self)->str: return self.__class__.__name__ 68 | 69 | def __repr__(self)->str: return f'{self.name} ({self.func.__name__})' 70 | 71 | 72 | def _get_default_args(func:Callable): 73 | return {k: v.default 74 | for k, v in inspect.signature(func).parameters.items() 75 | if v.default is not inspect.Parameter.empty} 76 | 77 | 78 | # 1) Those that slightly modify the time series in the x and/ or y-axes: 79 | # - TSmagnoise == TSjittering (1) 80 | # - TSmagscale (1) 81 | # - TSmagwarp (1) 82 | # - TStimenoise (1) 83 | # - TStimewarp (1) 84 | 85 | # 2) And those that remove certain part of the time series (a section or channel): 86 | # - TSlookback 87 | # - TStimestepsout (1) 88 | # - TSchannelout 89 | # - TScutout (2) 90 | # - TScrop (1) 91 | # - TSwindowslice (3) 92 | # - TSzoom (1) 93 | 94 | # All of them can be used independently or in combination. In this section, you'll see how these transforms work. 95 | 96 | # (1) Adapted from/ inspired by Um, T. T., Pfister, F. M. J., Pichler, D., Endo, S., Lang, M., Hirche, S., ... & Kulić, D. (2017). Data augmentation of wearable sensor data for parkinson's disease monitoring using convolutional neural networks. arXiv preprint arXiv:1706.00527. (includes: Jittering, Scaling, Magnitude-Warping, Time-warping, Random Sampling, among others. 97 | 98 | # (2) Inspired by DeVries, T., & Taylor, G. W. (2017). Improved regularization of convolutional neural networks with cutout. arXiv preprint arXiv:1708.04552. 99 | 100 | # (3) Inspired by Le Guennec, A., Malinowski, S., & Tavenard, R. (2016, September). Data augmentation for time series classification using convolutional neural networks. 101 | 102 | 103 | 104 | def _ynoise(ts, alpha=.05, add=True): 105 | seq_len = ts.shape[-1] 106 | if add: 107 | noise = torch.normal(0, alpha, (1, ts.shape[-1]), dtype=ts.dtype, device=ts.device) 108 | return ts + noise 109 | else: 110 | scale = torch.ones(seq_len) + torch.normal(0, alpha, (1, ts.shape[-1]), dtype=ts.dtype, device=ts.device) 111 | return ts * scale 112 | 113 | TSynoise = TSTransform(_ynoise) 114 | TSmagnoise = TSTransform(_ynoise) 115 | TSjittering = TSTransform(_ynoise) 116 | 117 | from scipy.interpolate import CubicSpline 118 | def random_curve_generator(ts, alpha=.1, order=4, noise=None): 119 | seq_len = ts.shape[-1] 120 | x = np.linspace(- seq_len, 2 * seq_len - 1, 3 * (order - 1) + 1, dtype=int) 121 | x2 = np.random.normal(loc=1.0, scale=alpha, size=len(x)) 122 | f = CubicSpline(x, x2, axis=1) 123 | return f(np.arange(seq_len)) 124 | 125 | def random_cum_curve_generator(ts, alpha=.1, order=4, noise=None): 126 | x = random_curve_generator(ts, alpha=alpha, order=order, noise=noise).cumsum() 127 | x -= x[0] 128 | x /= x[-1] 129 | x = np.clip(x, 0, 1) 130 | return x * (ts.shape[-1] - 1) 131 | 132 | def random_cum_noise_generator(ts, alpha=.1, noise=None): 133 | seq_len = ts.shape[-1] 134 | x = (np.ones(seq_len) + np.random.normal(loc=0, scale=alpha, size=seq_len)).cumsum() 135 | x -= x[0] 136 | x /= x[-1] 137 | x = np.clip(x, 0, 1) 138 | return x * (ts.shape[-1] - 1) 139 | 140 | from scipy.interpolate import CubicSpline 141 | def _xwarp(ts, alpha=.05, order=4): 142 | f = CubicSpline(np.arange(ts.shape[-1]), ts, axis=1) 143 | new_x = random_cum_curve_generator(ts, alpha=alpha, order=order) 144 | return ts.new(f(new_x)) 145 | 146 | TSxwarp = TSTransform(_xwarp) 147 | TStimewarp = TSTransform(_xwarp) 148 | 149 | from scipy.interpolate import CubicSpline 150 | def _ywarp(ts, alpha=.05, order=4): 151 | f2 = CubicSpline(np.arange(ts.shape[-1]), ts, axis=1) 152 | y_mult = random_curve_generator(ts, alpha=alpha, order=order) 153 | return ts * ts.new(y_mult) 154 | 155 | TSywarp = TSTransform(_ywarp) 156 | TSmagwarp = TSTransform(_ywarp) 157 | 158 | from scipy.interpolate import CubicSpline 159 | def _scale(ts, alpha=.1): 160 | rand = 1 - torch.rand(1)[0] * 2 161 | scale = 1 + torch.abs(rand) * alpha 162 | if rand < 0: scale = 1 / scale 163 | return ts * scale 164 | 165 | TSmagscale = TSTransform(_scale) 166 | 167 | from scipy.interpolate import CubicSpline 168 | def _xnoisewarp(ts, alpha=.1): 169 | f = CubicSpline(np.arange(ts.shape[-1]), ts, axis=1) 170 | new_x = random_cum_noise_generator(ts, alpha=alpha) 171 | return ts.new(f(new_x)) 172 | 173 | TSxnoisewarp = TSTransform(_xnoisewarp) 174 | TStimenoise = TSTransform(_xnoisewarp) 175 | 176 | def get_TS_xy_tfms(): 177 | return [[TStimewarp(p=.5), TSmagwarp(p=.5), TStimenoise(p=.5), TSmagnoise(p=.5)], []] 178 | 179 | 180 | def _rand_lookback(ts, alpha=.2): 181 | new_ts = ts.clone() 182 | lambd = np.random.beta(alpha, alpha) 183 | lambd = min(lambd, 1 - lambd) 184 | lookback_per = int(lambd * new_ts.shape[-1]) 185 | new_ts[:, :lookback_per] = 0 186 | return new_ts 187 | 188 | TSlookback = TSTransform(_rand_lookback) 189 | 190 | def _random_channel_out(ts, alpha=.2): 191 | input_ch = ts.shape[0] 192 | if input_ch == 1: return ts 193 | new_ts = ts.clone() 194 | out_ch = np.random.choice(np.arange(input_ch), 195 | min(input_ch - 1, int(np.random.beta(alpha, 1) * input_ch)), 196 | replace=False) 197 | new_ts[out_ch] = 0 198 | return new_ts 199 | 200 | TSchannelout = TSTransform(_random_channel_out) 201 | 202 | def _cutout(ts, perc=.1): 203 | if perc >= 1 or perc <= 0: return ts 204 | seq_len = ts.shape[-1] 205 | new_ts = ts.clone() 206 | win_len = int(perc * seq_len) 207 | start = np.random.randint(-win_len + 1, seq_len) 208 | end = start + win_len 209 | start = max(0, start) 210 | end = min(end, seq_len) 211 | new_ts[..., start:end] = 0 212 | return new_ts 213 | 214 | TScutout= TSTransform(_cutout) 215 | 216 | def _timesteps_out(ts, perc=.1): 217 | if perc >= 1 or perc <= 0: return ts 218 | seq_len = ts.shape[-1] 219 | timesteps = np.sort(np.random.choice(np.arange(seq_len), int(seq_len * (1 - perc)), replace=False)) 220 | return ts[..., timesteps] 221 | 222 | TStimestepsout = TSTransform(_timesteps_out) 223 | 224 | def _crop(ts, perc=.9): 225 | if perc >= 1 or perc <= 0: return ts 226 | seq_len = ts.shape[-1] 227 | win_len = int(seq_len * perc) 228 | new_ts = torch.zeros((ts.shape[-2], win_len)) 229 | start = np.random.randint(-win_len + 1, seq_len) 230 | end = start + win_len 231 | start = max(0, start) 232 | end = min(end, seq_len) 233 | new_ts[:, - end + start :] = ts[:, start : end] 234 | return new_ts 235 | 236 | TScrop = TSTransform(_crop) 237 | 238 | def _window_slice(ts, perc=.9): 239 | if perc == 1.0 or perc == 0: return ts 240 | seq_len = ts.shape[-1] 241 | win_len = int(seq_len * perc) 242 | start = np.random.randint(0, seq_len - win_len) 243 | return ts[:, start : start + win_len] 244 | 245 | TSwindowslice = TSTransform(_window_slice) 246 | 247 | def _random_zoom(ts, alpha=.2): 248 | if alpha == 1.0: return a 249 | seq_len = ts.shape[-1] 250 | lambd = np.random.beta(alpha, alpha) 251 | lambd = max(lambd, 1 - lambd) 252 | win_len = int(seq_len * lambd) 253 | if win_len == seq_len: start = 0 254 | else: start = np.random.randint(0, seq_len - win_len) 255 | y = ts[:, start : start + win_len] 256 | f = CubicSpline(np.arange(y.shape[-1]), y, axis=1) 257 | return ts.new(f(np.linspace(0, win_len - 1, num=seq_len))) 258 | 259 | TSzoom = TSTransform(_random_zoom) 260 | 261 | def get_TS_remove_tfms(): 262 | return [[TSlookback(p=.5), TStimestepsout(p=.5), TSchannelout(p=.5), TScutout(p=.5), 263 | TScrop(p=.5), TSwindowslice(p=.5), TSzoom(p=.5)], []] -------------------------------------------------------------------------------- /torchtimeseries/models/FCN.py: -------------------------------------------------------------------------------- 1 | # This is an unofficial PyTorch implementation by Ignacio Oguiza - oguiza@gmail.com based on: 2 | 3 | # Wang, Z., Yan, W., & Oates, T. (2017, May). Time series classification from scratch with deep neural networks: A strong baseline. In 2017 international joint conference on neural networks (IJCNN) (pp. 1578-1585). IEEE. 4 | 5 | # Fawaz, H. I., Forestier, G., Weber, J., Idoumghar, L., & Muller, P. A. (2019). Deep learning for time series classification: a review. Data Mining and Knowledge Discovery, 33(4), 917-963. 6 | # FCN TensorFlow implementation: https://github.com/hfawaz/dl-4-tsc 7 | 8 | # 👀 kernel filter size 8 has been replaced by 7 (I believe it's a bug) 9 | 10 | import torch 11 | import torch.nn as nn 12 | from .layers import * 13 | 14 | __all__ = ['FCN'] 15 | 16 | class FCN(nn.Module): 17 | def __init__(self,c_in,c_out,layers=[128,256,128],kss=[7,5,3]): 18 | super().__init__() 19 | self.conv1 = convlayer(c_in,layers[0],kss[0]) 20 | self.conv2 = convlayer(layers[0],layers[1],kss[1]) 21 | self.conv3 = convlayer(layers[1],layers[2],kss[2]) 22 | self.gap = nn.AdaptiveAvgPool1d(1) 23 | self.fc = nn.Linear(layers[-1],c_out) 24 | 25 | def forward(self, x): 26 | x = self.conv1(x) 27 | x = self.conv2(x) 28 | x = self.conv3(x) 29 | x = self.gap(x).squeeze(-1) 30 | return self.fc(x) -------------------------------------------------------------------------------- /torchtimeseries/models/InceptionTime.py: -------------------------------------------------------------------------------- 1 | # This is an unofficial PyTorch implementation by Ignacio Oguiza - oguiza@gmail.com based on: 2 | 3 | # Fawaz, H. I., Lucas, B., Forestier, G., Pelletier, C., Schmidt, D. F., Weber, J., ... & Petitjean, F. (2019). InceptionTime: Finding AlexNet for Time Series Classification. arXiv preprint arXiv:1909.04939. 4 | # Official InceptionTime tensorflow implementation: https://github.com/hfawaz/InceptionTime 5 | 6 | import torch 7 | import torch.nn as nn 8 | 9 | __all__ = ['InceptionTime'] 10 | 11 | def noop(x): 12 | return x 13 | 14 | def shortcut(c_in, c_out): 15 | return nn.Sequential(*[nn.Conv1d(c_in, c_out, kernel_size=1), 16 | nn.BatchNorm1d(c_out)]) 17 | 18 | class Inception(nn.Module): 19 | def __init__(self, c_in, bottleneck=32, ks=40, nb_filters=32): 20 | 21 | super().__init__() 22 | self.bottleneck = nn.Conv1d(c_in, bottleneck, 1) if bottleneck and c_in > 1 else noop 23 | mts_feat = bottleneck or c_in 24 | conv_layers = [] 25 | kss = [ks // (2**i) for i in range(3)] 26 | # ensure odd kss until nn.Conv1d with padding='same' is available in pytorch 1.3 27 | kss = [ksi if ksi % 2 != 0 else ksi - 1 for ksi in kss] 28 | for i in range(len(kss)): 29 | conv_layers.append( 30 | nn.Conv1d(mts_feat, nb_filters, kernel_size=kss[i], padding=kss[i] // 2)) 31 | self.conv_layers = nn.ModuleList(conv_layers) 32 | self.maxpool = nn.MaxPool1d(3, stride=1, padding=1) 33 | self.conv = nn.Conv1d(c_in, nb_filters, kernel_size=1) 34 | self.bn = nn.BatchNorm1d(nb_filters * 4) 35 | self.act = nn.ReLU() 36 | 37 | def forward(self, x): 38 | input_tensor = x 39 | x = self.bottleneck(input_tensor) 40 | for i in range(3): 41 | out_ = self.conv_layers[i](x) 42 | if i == 0: out = out_ 43 | else: out = torch.cat((out, out_), 1) 44 | mp = self.conv(self.maxpool(input_tensor)) 45 | inc_out = torch.cat((out, mp), 1) 46 | return self.act(self.bn(inc_out)) 47 | 48 | 49 | class InceptionBlock(nn.Module): 50 | def __init__(self,c_in,bottleneck=32,ks=40,nb_filters=32,residual=True,depth=6): 51 | 52 | super().__init__() 53 | 54 | self.residual = residual 55 | self.depth = depth 56 | 57 | #inception & residual layers 58 | inc_mods = [] 59 | res_layers = [] 60 | res = 0 61 | for d in range(depth): 62 | inc_mods.append( 63 | Inception(c_in if d == 0 else nb_filters * 4, bottleneck=bottleneck if d > 0 else 0,ks=ks, 64 | nb_filters=nb_filters)) 65 | if self.residual and d % 3 == 2: 66 | res_layers.append(shortcut(c_in if res == 0 else nb_filters * 4, nb_filters * 4)) 67 | res += 1 68 | else: res_layer = res_layers.append(None) 69 | self.inc_mods = nn.ModuleList(inc_mods) 70 | self.res_layers = nn.ModuleList(res_layers) 71 | self.act = nn.ReLU() 72 | 73 | def forward(self, x): 74 | res = x 75 | for d, l in enumerate(range(self.depth)): 76 | x = self.inc_mods[d](x) 77 | if self.residual and d % 3 == 2: 78 | res = self.res_layers[d](res) 79 | x += res 80 | res = x 81 | x = self.act(x) 82 | return x 83 | 84 | class InceptionTime(nn.Module): 85 | def __init__(self,c_in,c_out,bottleneck=32,ks=40,nb_filters=32,residual=True,depth=6): 86 | super().__init__() 87 | self.block = InceptionBlock(c_in,bottleneck=bottleneck,ks=ks,nb_filters=nb_filters, 88 | residual=residual,depth=depth) 89 | self.gap = nn.AdaptiveAvgPool1d(1) 90 | self.fc = nn.Linear(nb_filters * 4, c_out) 91 | 92 | def forward(self, x): 93 | x = self.block(x) 94 | x = self.gap(x).squeeze(-1) 95 | x = self.fc(x) 96 | return x -------------------------------------------------------------------------------- /torchtimeseries/models/ROCKET.py: -------------------------------------------------------------------------------- 1 | # This is an unofficial Multivariate, GPU (PyTorch) implementation by Ignacio Oguiza - oguiza@gmail.com based on: 2 | 3 | # Dempster, A., Petitjean, F., & Webb, G. I. (2019). ROCKET: Exceptionally fast and accurate time series classification using random convolutional kernels. arXiv preprint arXiv:1910.13051. 4 | # GitHub code: https://github.com/angus924/rocket 5 | 6 | import torch 7 | import torch.nn as nn 8 | import numpy as np 9 | 10 | class ROCKET(nn.Module): 11 | def __init__(self, c_in, seq_len, n_kernels=10000, kss=[7, 9, 11]): 12 | 13 | ''' 14 | ROCKET is a GPU Pytorch implementation of the original ROCKET methods generate_kernels and apply_kernels that can be used with univariate and multivariate time series. 15 | Input: is a 3d torch tensor of type torch.float32. When used with univariate TS, make sure you transform the 2d to 3d by adding unsqueeze(1) 16 | c_in: number of channels in (features). For univariate c_in is 1. 17 | seq_len: sequence length (is the last dimension of the input) 18 | ''' 19 | super().__init__() 20 | kss = [ks for ks in kss if ks < seq_len] 21 | convs = nn.ModuleList() 22 | for i in range(n_kernels): 23 | ks = np.random.choice(kss) 24 | dilation = 2**np.random.uniform(0, np.log2((seq_len - 1) // (ks - 1))) 25 | padding = int((ks - 1) * dilation // 2) if np.random.randint(2) == 1 else 0 26 | weight = torch.normal(0, 1, (1, c_in, ks)) 27 | weight -= weight.mean() 28 | bias = 2 * (torch.rand(1) - .5) 29 | layer = nn.Conv1d(c_in, 1, ks, padding=2 * padding, dilation=int(dilation), bias=True) 30 | layer.weight = torch.nn.Parameter(weight, requires_grad=False) 31 | layer.bias = torch.nn.Parameter(bias, requires_grad=False) 32 | convs.append(layer) 33 | self.convs = convs 34 | self.n_kernels = n_kernels 35 | self.kss = kss 36 | 37 | def forward(self, x): 38 | for i in range(self.n_kernels): 39 | out = self.convs[i](x) 40 | _max = out.max(dim=-1).values 41 | _ppv = torch.gt(out, 0).sum(dim=-1).float() / out.shape[-1] 42 | cat = torch.cat((_max, _ppv), dim=-1) 43 | output = cat if i == 0 else torch.cat((output, cat), dim=-1) 44 | return output -------------------------------------------------------------------------------- /torchtimeseries/models/ResCNN.py: -------------------------------------------------------------------------------- 1 | # This is an unofficial PyTorch implementation by Ignacio Oguiza - oguiza@gmail.com based on: 2 | 3 | # Zou, X., Wang, Z., Li, Q., & Sheng, W. (2019). Integration of residual network and convolutional neural network along with various activation functions and global pooling for time series classification. Neurocomputing. 4 | # No official implementation found 5 | 6 | 7 | import torch 8 | import torch.nn as nn 9 | from .layers import * 10 | 11 | __all__ = ['ResCNN'] 12 | 13 | 14 | class Block(nn.Module): 15 | def __init__(self, ni, nf, ks=[7, 5, 3], act_fn='relu'): 16 | super().__init__() 17 | self.conv1 = convlayer(ni, nf, ks[0], act_fn=act_fn) 18 | self.conv2 = convlayer(nf, nf, ks[1], act_fn=act_fn) 19 | self.conv3 = convlayer(nf, nf, ks[2], act_fn=False) 20 | 21 | # expand channels for the sum if necessary 22 | self.shortcut = noop if ni == nf else convlayer(ni, nf, ks=1, act_fn=False) 23 | self.act_fn = get_act_layer(act_fn) 24 | 25 | def forward(self, x): 26 | res = x 27 | x = self.conv1(x) 28 | x = self.conv2(x) 29 | x = self.conv3(x) 30 | sc = self.shortcut(res) 31 | x += sc 32 | x = self.act_fn(x) 33 | return x 34 | 35 | 36 | class ResCNN(nn.Module): 37 | def __init__(self, c_in, c_out): 38 | nf = 64 39 | super().__init__() 40 | self.block = Block(c_in, nf, ks=[7, 5, 3], act_fn='relu') 41 | self.conv1 = convlayer(nf, nf * 2, ks=3, act_fn='leakyrelu', negative_slope=.2) 42 | self.conv2 = convlayer(nf * 2, nf * 4, ks=3, act_fn='prelu') 43 | self.conv3 = convlayer(nf * 4, nf * 2, ks=3, act_fn='elu', alpha=.3) 44 | self.gap = nn.AdaptiveAvgPool1d(1) 45 | self.lin = nn.Linear(nf * 2, c_out) 46 | 47 | def forward(self, x): 48 | x = self.block(x) 49 | x = self.conv1(x) 50 | x = self.conv2(x) 51 | x = self.conv3(x) 52 | x = self.gap(x).squeeze(-1) 53 | return self.lin(x) -------------------------------------------------------------------------------- /torchtimeseries/models/ResNet.py: -------------------------------------------------------------------------------- 1 | # This is an unofficial PyTorch implementation by Ignacio Oguiza - oguiza@gmail.com based on: 2 | 3 | # Wang, Z., Yan, W., & Oates, T. (2017, May). Time series classification from scratch with deep neural networks: A strong baseline. In 2017 international joint conference on neural networks (IJCNN) (pp. 1578-1585). IEEE. 4 | 5 | # Fawaz, H. I., Forestier, G., Weber, J., Idoumghar, L., & Muller, P. A. (2019). Deep learning for time series classification: a review. Data Mining and Knowledge Discovery, 33(4), 917-963. 6 | # Official ResNet TensorFlow implementation: https://github.com/hfawaz/dl-4-tsc 7 | 8 | # 👀 kernel filter size 8 has been replaced by 7 (I believe it's a bug) 9 | 10 | 11 | import torch 12 | import torch.nn as nn 13 | from .layers import * 14 | 15 | __all__ = ['ResNet'] 16 | 17 | class ResBlock(nn.Module): 18 | def __init__(self, ni, nf, ks=[7, 5, 3], act_fn='relu'): 19 | super().__init__() 20 | self.conv1 = convlayer(ni, nf, ks[0], act_fn=act_fn) 21 | self.conv2 = convlayer(nf, nf, ks[1], act_fn=act_fn) 22 | self.conv3 = convlayer(nf, nf, ks[2], act_fn=False) 23 | 24 | # expand channels for the sum if necessary 25 | self.shortcut = noop if ni == nf else convlayer(ni, nf, ks=1, act_fn=False) 26 | self.act_fn = get_act_layer(act_fn) 27 | 28 | def forward(self, x): 29 | res = x 30 | x = self.conv1(x) 31 | x = self.conv2(x) 32 | x = self.conv3(x) 33 | sc = self.shortcut(res) 34 | x += sc 35 | x = self.act_fn(x) 36 | return x 37 | 38 | class ResNet(nn.Module): 39 | def __init__(self,c_in, c_out): 40 | super().__init__() 41 | nf = 64 42 | 43 | self.block1 = ResBlock(c_in, nf, ks=[7, 5, 3], act_fn='relu') 44 | self.block2 = ResBlock(nf, nf * 2, ks=[7, 5, 3], act_fn='relu') 45 | self.block3 = ResBlock(nf * 2, nf * 2, ks=[7, 5, 3], act_fn='relu') 46 | self.gap = nn.AdaptiveAvgPool1d(1) 47 | self.fc = nn.Linear(nf * 2, c_out) 48 | 49 | def forward(self, x): 50 | x = self.block1(x) 51 | x = self.block2(x) 52 | x = self.block3(x) 53 | x = self.gap(x).squeeze(-1) 54 | return self.fc(x) -------------------------------------------------------------------------------- /torchtimeseries/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .FCN import * 2 | from .ResNet import * 3 | from .ResCNN import * 4 | from .InceptionTime import * 5 | from .layers import * 6 | from .ROCKET import * -------------------------------------------------------------------------------- /torchtimeseries/models/layers.py: -------------------------------------------------------------------------------- 1 | 2 | #original FTSwish = https://arxiv.org/abs/1812.06247 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | from fastai.torch_core import Module 8 | from typing import Callable 9 | from functools import partial 10 | from . import * 11 | #from . import FCN, ResNet, Res2Net, InceptionTime, InceptionTimePlus, ResNetPlus2 12 | 13 | 14 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 15 | 16 | def noop(x): return x 17 | 18 | class LambdaPlus(Module): 19 | def __init__(self, func, *args, **kwargs): self.func,self.args,self.kwargs=func,args,kwargs 20 | def forward(self, x): return self.func(x, *self.args, **self.kwargs) 21 | 22 | 23 | def get_act_layer(act_fn, **kwargs): 24 | act_fn = act_fn.lower() 25 | assert act_fn in ['relu', 'leakyrelu', 'prelu', 'elu', 26 | 'mish', 'swish'], 'incorrect act_fn' 27 | if act_fn == 'relu': return nn.ReLU() 28 | elif act_fn == 'leakyrelu': return nn.LeakyReLU(**kwargs) 29 | elif act_fn == 'prelu': return nn.PReLU(**kwargs) 30 | elif act_fn == 'elu': return nn.ELU(**kwargs) 31 | elif act_fn == 'mish': return Mish() 32 | elif act_fn == 'swish': return Swish() 33 | 34 | 35 | def same_padding1d(seq_len,ks,stride=1,dilation=1): 36 | assert stride > 0 37 | assert dilation >= 1 38 | effective_ks = (ks - 1) * dilation + 1 39 | out_dim = (seq_len + stride - 1) // stride 40 | p = max(0, (out_dim - 1) * stride + effective_ks - seq_len) 41 | padding_before = p // 2 42 | padding_after = p - padding_before 43 | return padding_before, padding_after 44 | 45 | class ZeroPad1d(nn.ConstantPad1d): 46 | def __init__(self, padding): 47 | super().__init__(padding, 0.) 48 | 49 | class ConvSP1d(nn.Module): 50 | "Conv1d padding='same'" 51 | def __init__(self,c_in,c_out,ks,stride=1,padding='same',dilation=1,bias=True): 52 | super().__init__() 53 | self.ks, self.stride, self.dilation = ks, stride, dilation 54 | self.conv = nn.Conv1d(c_in,c_out,ks,stride=stride,padding=0,dilation=dilation,bias=bias) 55 | self.zeropad = ZeroPad1d 56 | self.weight = self.conv.weight 57 | self.bias = self.conv.bias 58 | 59 | def forward(self, x): 60 | padding = same_padding1d(x.shape[-1],self.ks,stride=self.stride,dilation=self.dilation) 61 | return self.conv(self.zeropad(padding)(x)) 62 | 63 | def convlayer(c_in,c_out,ks=3,padding='same',bias=True,stride=1, 64 | bn_init=False,zero_bn=False,bn_before=True, 65 | act_fn='relu', **kwargs): 66 | '''conv layer (padding="same") + bn + act''' 67 | if ks % 2 == 1 and padding == 'same': padding = ks // 2 68 | layers = [ConvSP1d(c_in,c_out, ks, bias=bias, stride=stride) if padding == 'same' else \ 69 | nn.Conv1d(c_in,c_out, ks, stride=stride, padding=padding, bias=bias)] 70 | bn = nn.BatchNorm1d(c_out) 71 | if bn_init: nn.init.constant_(bn.weight, 0. if zero_bn else 1.) 72 | if bn_before: layers.append(bn) 73 | if act_fn: layers.append(get_act_layer(act_fn, **kwargs)) 74 | if not bn_before: layers.append(bn) 75 | return nn.Sequential(*layers) 76 | 77 | 78 | class Flatten(nn.Module): 79 | def forward(self, x): return x.view(x.size(0), -1) 80 | 81 | class Squeeze(nn.Module): 82 | def __init__(self, dim=-1): 83 | super().__init__() 84 | self.dim = dim 85 | def forward(self, x): return x.squeeze(dim=self.dim) 86 | 87 | class Unsqueeze(nn.Module): 88 | def __init__(self, dim=-1): 89 | super().__init__() 90 | self.dim = dim 91 | def forward(self, x): return x.unsqueeze(dim=self.dim) 92 | 93 | class YRange(nn.Module): 94 | def __init__(self, y_range:tuple): 95 | super().__init__() 96 | self.y_range = y_range 97 | def forward(self, x): 98 | x = F.sigmoid(x) 99 | x = x * (self.y_range[1] - self.y_range[0]) 100 | return x + self.y_range[0] 101 | 102 | 103 | class Mult(nn.Module): 104 | def __init__(self, mult, trainable=True): 105 | self.mult,self.trainable = mult,trainable 106 | super().__init__() 107 | self.weight = nn.Parameter(torch.Tensor(1).fill_(self.mult).to(device), requires_grad=self.trainable) 108 | def forward(self, x): 109 | return x.mul_(self.weight) 110 | 111 | 112 | class Exp(nn.Module): 113 | def __init__(self, exp, trainable=True): 114 | self.exp,self.trainable = exp,trainable 115 | super().__init__() 116 | self.weight = nn.Parameter(torch.Tensor(1).fill_(self.exp).to(device), requires_grad=self.trainable) 117 | def forward(self, x): 118 | return x**self.weight 119 | 120 | 121 | def get_cls(model, c_in, seq_len, c_out, **kwargs): 122 | if model.lower() == 'fcn': return FCN(c_in, c_out, **kwargs) 123 | elif model.lower() == 'resnet': return ResNet.ResNet(c_in, c_out, **kwargs) 124 | elif model.lower() == 'inceptiontime': return InceptionTime(c_in, c_out, **kwargs) 125 | elif model.lower() == 'rescnn': return ResCNN(c_in, c_out, **kwargs) 126 | else: print('Model not available!!') 127 | 128 | 129 | # ACTIVATION LAYERS 130 | class FTSwishPlus(nn.Module): 131 | def __init__(self, threshold=-.25, sub=-.1, **kwargs): 132 | super().__init__() 133 | self.threshold,self.sub = threshold,sub 134 | 135 | def forward(self, x): 136 | x = F.relu(x) * torch.sigmoid(x) + self.threshold 137 | if self.sub is not None: x.sub_(self.sub) 138 | return x 139 | 140 | 141 | class Swish(nn.Module): 142 | def __init__(self, inplace=False): 143 | super().__init__() 144 | self.inplace = inplace 145 | 146 | def forward(self, x): 147 | if self.inplace: 148 | x.mul_(F.sigmoid(x)) 149 | return x 150 | else: 151 | return x * F.sigmoid(x) 152 | 153 | 154 | class GeneralRelu(nn.Module): 155 | def __init__(self, leak=None, sub=0., maxv=None, **kwargs): 156 | super().__init__() 157 | self.leak,self.sub,self.maxv = leak,sub,maxv 158 | 159 | def forward(self, x): 160 | x = F.leaky_relu(x,self.leak) if self.leak is not None else F.relu(x) 161 | x = x.sub_(self.sub) 162 | if self.maxv is not None: x.clamp_max_(self.maxv) 163 | return x 164 | 165 | 166 | #Mish - "Mish: A Self Regularized Non-Monotonic Neural Activation Function" 167 | #https://arxiv.org/abs/1908.08681v1 168 | #implemented for PyTorch / FastAI by lessw2020 169 | #github: https://github.com/lessw2020/mish 170 | class Mish(nn.Module): 171 | def __init__(self): 172 | super().__init__() 173 | def forward(self, x): 174 | x = x *( torch.tanh(F.softplus(x))) 175 | return x 176 | 177 | def get_act_fn_norm(act_fn): 178 | x = torch.randn(1000000) 179 | x = act_fn(x) 180 | x1 = x / x.std() 181 | x2 = x1- x1.mean() 182 | return 1/x.std(), x1.mean() 183 | 184 | 185 | class AFN(nn.Module): 186 | def __init__(self, act_fn=F.relu, trainable=True, **kwargs): 187 | super().__init__() 188 | self.act_fn = partial(act_fn,**kwargs) 189 | mul, add = get_act_fn_norm(self.act_fn) 190 | self.weight = nn.Parameter(torch.Tensor(1).fill_(mul).to(device), requires_grad=trainable) 191 | self.bias = nn.Parameter(torch.Tensor(1).fill_(add).to(device), requires_grad=trainable) 192 | 193 | def forward(self, x): 194 | x = self.act_fn(x) 195 | x.mul_(self.weight) 196 | x.add_(self.bias) 197 | return x 198 | --------------------------------------------------------------------------------