├── README.md ├── data ├── losses_adam.csv ├── losses_adam2.csv ├── losses_adam_kfac.csv ├── losses_adam_kfac2.csv ├── losses_exact_sgd.csv ├── losses_longadam.csv ├── losses_longsgd.csv ├── losses_longsgd2.csv ├── losses_sgd.csv ├── losses_sgd2.csv ├── losses_sgd_kfac.csv ├── losses_sgd_kfac2.csv ├── losses_stochastic_sgd.csv ├── vlosses_adam.csv ├── vlosses_adam2.csv ├── vlosses_adam_kfac.csv ├── vlosses_adam_kfac2.csv ├── vlosses_exact_sgd.csv ├── vlosses_longadam.csv ├── vlosses_longsgd.csv ├── vlosses_longsgd2.csv ├── vlosses_sgd.csv ├── vlosses_sgd2.csv ├── vlosses_sgd_kfac.csv ├── vlosses_sgd_kfac2.csv └── vlosses_stochastic_sgd.csv ├── deep_autoencoder.ipynb ├── derivation.pdf ├── derivation.tex ├── kfac_pytorch.py ├── run_experiments.py └── util.py /README.md: -------------------------------------------------------------------------------- 1 | # kfac_pytorch 2 | 3 | Requirements: MKL, TensorFlow (for fetching MNIST), CUDA, PyTorch 4 | 5 | Usage: 6 | ``` 7 | python kfac_pytorch.py 8 | 9 | Using MKL 10 | Step 0 loss 97.542419434 11 | Step 1 loss 62.339828491 12 | Step 2 loss 44.860393524 13 | Step 3 loss 79.031013489 14 | Step 4 loss 56.055324554 15 | Step 5 loss 48.206447601 16 | Step 6 loss 43.934066772 17 | Step 7 loss 40.302700043 18 | Step 8 loss 38.371196747 19 | Step 9 loss 38.781795502 20 | Times: min: 388.66, median: 400.81, mean: 2198.33 21 | 22 | ``` 23 | - Write-up: [Optimizing deeper networks with KFAC in PyTorch.](https://medium.com/@yaroslavvb/optimizing-deeper-networks-with-kfac-in-pytorch-4004adcba1b0) 24 | - Experiments: [deep_autoencoder.ipynb](https://github.com/yaroslavvb/kfac_pytorch/blob/master/deep_autoencoder.ipynb) 25 | 26 | ![test losses](https://i.stack.imgur.com/rvSgt.png) 27 | -------------------------------------------------------------------------------- /data/losses_adam2.csv: -------------------------------------------------------------------------------- 1 | 4.324058151245117188e+01 2 | 4.250683975219726562e+01 3 | 4.151544189453125000e+01 4 | 4.029293823242187500e+01 5 | 3.890570068359375000e+01 6 | 3.747525024414062500e+01 7 | 3.620365524291992188e+01 8 | 3.536770248413085938e+01 9 | 3.489752578735351562e+01 10 | 3.423884963989257812e+01 11 | 3.329550933837890625e+01 12 | 3.233608245849609375e+01 13 | 3.157316780090332031e+01 14 | 3.104348373413085938e+01 15 | 3.065035247802734375e+01 16 | 3.028688049316406250e+01 17 | 2.992195510864257812e+01 18 | 2.957456207275390625e+01 19 | 2.929048347473144531e+01 20 | 2.907987022399902344e+01 21 | 2.888668060302734375e+01 22 | 2.866348648071289062e+01 23 | 2.841987800598144531e+01 24 | 2.819714927673339844e+01 25 | 2.801589965820312500e+01 26 | 2.786045646667480469e+01 27 | 2.770471191406250000e+01 28 | 2.753543663024902344e+01 29 | 2.735826492309570312e+01 30 | 2.719073104858398438e+01 31 | 2.703315925598144531e+01 32 | 2.686620521545410156e+01 33 | 2.667484283447265625e+01 34 | 2.646753311157226562e+01 35 | 2.626109313964843750e+01 36 | 2.605618286132812500e+01 37 | 2.584267807006835938e+01 38 | 2.561741828918457031e+01 39 | 2.538932418823242188e+01 40 | 2.516392517089843750e+01 41 | 2.493379211425781250e+01 42 | 2.469193649291992188e+01 43 | 2.444276046752929688e+01 44 | 2.419208908081054688e+01 45 | 2.393704795837402344e+01 46 | 2.367146110534667969e+01 47 | 2.339851570129394531e+01 48 | 2.312008476257324219e+01 49 | 2.283004760742187500e+01 50 | 2.252490615844726562e+01 51 | 2.220940780639648438e+01 52 | 2.188535308837890625e+01 53 | 2.155323028564453125e+01 54 | 2.122706985473632812e+01 55 | 2.091425704956054688e+01 56 | 2.060881423950195312e+01 57 | 2.029790687561035156e+01 58 | 1.998233795166015625e+01 59 | 1.967707633972167969e+01 60 | 1.937967109680175781e+01 61 | 1.908118438720703125e+01 62 | 1.877928161621093750e+01 63 | 1.848262405395507812e+01 64 | 1.819360351562500000e+01 65 | 1.789909744262695312e+01 66 | 1.759049606323242188e+01 67 | 1.727243614196777344e+01 68 | 1.696795082092285156e+01 69 | 1.670005035400390625e+01 70 | 1.645053291320800781e+01 71 | 1.619706153869628906e+01 72 | 1.595493221282958984e+01 73 | 1.572466087341308594e+01 74 | 1.549680233001708984e+01 75 | 1.527518558502197266e+01 76 | 1.508334064483642578e+01 77 | 1.494279003143310547e+01 78 | 1.470914936065673828e+01 79 | 1.441867637634277344e+01 80 | 1.430942440032958984e+01 81 | 1.415047454833984375e+01 82 | 1.389111518859863281e+01 83 | 1.378555583953857422e+01 84 | 1.365938377380371094e+01 85 | 1.342632961273193359e+01 86 | 1.330322456359863281e+01 87 | 1.318258762359619141e+01 88 | 1.297729682922363281e+01 89 | 1.284970951080322266e+01 90 | 1.274849319458007812e+01 91 | 1.257691192626953125e+01 92 | 1.242516040802001953e+01 93 | 1.233615970611572266e+01 94 | 1.223425865173339844e+01 95 | 1.211550903320312500e+01 96 | 1.206421566009521484e+01 97 | 1.209486961364746094e+01 98 | 1.190659427642822266e+01 99 | 1.168295955657958984e+01 100 | 1.175089931488037109e+01 101 | 1.155706977844238281e+01 102 | 1.140065956115722656e+01 103 | 1.142056274414062500e+01 104 | 1.121418285369873047e+01 105 | 1.120101737976074219e+01 106 | 1.111533546447753906e+01 107 | 1.095566558837890625e+01 108 | 1.096983718872070312e+01 109 | 1.082747364044189453e+01 110 | 1.075840950012207031e+01 111 | 1.071420764923095703e+01 112 | 1.057998371124267578e+01 113 | 1.055279731750488281e+01 114 | 1.047194385528564453e+01 115 | 1.038354396820068359e+01 116 | 1.036488437652587891e+01 117 | 1.030164337158203125e+01 118 | 1.030677032470703125e+01 119 | 1.039491653442382812e+01 120 | 1.041990852355957031e+01 121 | 1.041514205932617188e+01 122 | 1.024741172790527344e+01 123 | 1.013701915740966797e+01 124 | 1.005723476409912109e+01 125 | 1.002372741699218750e+01 126 | 9.947621345520019531e+00 127 | 9.823193550109863281e+00 128 | 9.872388839721679688e+00 129 | 9.693023681640625000e+00 130 | 9.714738845825195312e+00 131 | 9.642345428466796875e+00 132 | 9.564180374145507812e+00 133 | 9.575853347778320312e+00 134 | 9.459963798522949219e+00 135 | 9.460459709167480469e+00 136 | 9.388738632202148438e+00 137 | 9.353817939758300781e+00 138 | 9.301233291625976562e+00 139 | 9.265819549560546875e+00 140 | 9.212225914001464844e+00 141 | 9.171435356140136719e+00 142 | 9.131985664367675781e+00 143 | 9.089088439941406250e+00 144 | 9.045456886291503906e+00 145 | 9.006560325622558594e+00 146 | 8.980271339416503906e+00 147 | 8.926061630249023438e+00 148 | 8.908781051635742188e+00 149 | 8.906017303466796875e+00 150 | 8.930100440979003906e+00 151 | 9.055032730102539062e+00 152 | 9.156416893005371094e+00 153 | 9.049052238464355469e+00 154 | 8.748218536376953125e+00 155 | 8.727972030639648438e+00 156 | 8.858580589294433594e+00 157 | 8.725849151611328125e+00 158 | 8.601355552673339844e+00 159 | 8.669062614440917969e+00 160 | 8.616147994995117188e+00 161 | 8.503302574157714844e+00 162 | 8.543712615966796875e+00 163 | 8.510173797607421875e+00 164 | 8.405147552490234375e+00 165 | 8.428027153015136719e+00 166 | 8.412925720214843750e+00 167 | 8.327095031738281250e+00 168 | 8.317353248596191406e+00 169 | 8.309118270874023438e+00 170 | 8.253587722778320312e+00 171 | 8.226222038269042969e+00 172 | 8.218986511230468750e+00 173 | 8.174874305725097656e+00 174 | 8.132468223571777344e+00 175 | 8.126475334167480469e+00 176 | 8.106541633605957031e+00 177 | 8.064621925354003906e+00 178 | 8.038747787475585938e+00 179 | 8.029648780822753906e+00 180 | 8.013486862182617188e+00 181 | 7.992475509643554688e+00 182 | 8.028777122497558594e+00 183 | 8.195149421691894531e+00 184 | 8.509519577026367188e+00 185 | 8.579612731933593750e+00 186 | 8.151226997375488281e+00 187 | 8.034217834472656250e+00 188 | 8.286774635314941406e+00 189 | 8.027946472167968750e+00 190 | 7.884156703948974609e+00 191 | 8.100975990295410156e+00 192 | 7.853950500488281250e+00 193 | 7.913415431976318359e+00 194 | 7.876739501953125000e+00 195 | 7.752937316894531250e+00 196 | 7.871239662170410156e+00 197 | 7.701020240783691406e+00 198 | 7.775952339172363281e+00 199 | 7.698779582977294922e+00 200 | 7.688792705535888672e+00 201 | 7.684999942779541016e+00 202 | 7.616929531097412109e+00 203 | 7.645935058593750000e+00 204 | 7.579819202423095703e+00 205 | 7.597191810607910156e+00 206 | 7.556252956390380859e+00 207 | 7.533626556396484375e+00 208 | 7.528030395507812500e+00 209 | 7.488761425018310547e+00 210 | 7.479713916778564453e+00 211 | 7.437904357910156250e+00 212 | 7.401632785797119141e+00 213 | 7.396276473999023438e+00 214 | 7.362594127655029297e+00 215 | 7.342128753662109375e+00 216 | 7.316802978515625000e+00 217 | 7.293880939483642578e+00 218 | 7.275528907775878906e+00 219 | 7.251736640930175781e+00 220 | 7.233085155487060547e+00 221 | 7.219208240509033203e+00 222 | 7.208866119384765625e+00 223 | 7.219177722930908203e+00 224 | 7.253714084625244141e+00 225 | 7.345815658569335938e+00 226 | 7.486672401428222656e+00 227 | 7.623248100280761719e+00 228 | 7.439550399780273438e+00 229 | 7.141987323760986328e+00 230 | 7.094795227050781250e+00 231 | 7.267185688018798828e+00 232 | 7.246438980102539062e+00 233 | 7.036241054534912109e+00 234 | 7.068731784820556641e+00 235 | 7.166643619537353516e+00 236 | 7.038798809051513672e+00 237 | 6.975800037384033203e+00 238 | 7.055510044097900391e+00 239 | 7.012034893035888672e+00 240 | 6.926672458648681641e+00 241 | 6.965366363525390625e+00 242 | 6.966619968414306641e+00 243 | 6.893670082092285156e+00 244 | 6.895497322082519531e+00 245 | 6.915696620941162109e+00 246 | 6.866374969482421875e+00 247 | 6.841480255126953125e+00 248 | 6.860698223114013672e+00 249 | 6.838494300842285156e+00 250 | 6.800734996795654297e+00 251 | 6.805276393890380859e+00 252 | 6.806910991668701172e+00 253 | 6.776175498962402344e+00 254 | 6.762291431427001953e+00 255 | 6.778370857238769531e+00 256 | 6.792153835296630859e+00 257 | 6.823955059051513672e+00 258 | 6.926388263702392578e+00 259 | 7.113663196563720703e+00 260 | 7.205359935760498047e+00 261 | 6.987277030944824219e+00 262 | 6.750251293182373047e+00 263 | 6.828375816345214844e+00 264 | 6.966853141784667969e+00 265 | 6.819936752319335938e+00 266 | 6.699828147888183594e+00 267 | 6.785497665405273438e+00 268 | 6.744782447814941406e+00 269 | 6.655863285064697266e+00 270 | 6.704824924468994141e+00 271 | 6.680873870849609375e+00 272 | 6.622713088989257812e+00 273 | 6.652544498443603516e+00 274 | 6.621819972991943359e+00 275 | 6.586152076721191406e+00 276 | 6.609792709350585938e+00 277 | 6.573899745941162109e+00 278 | 6.547862052917480469e+00 279 | 6.571336746215820312e+00 280 | 6.539628505706787109e+00 281 | 6.514384269714355469e+00 282 | 6.534008026123046875e+00 283 | 6.509088516235351562e+00 284 | 6.478007316589355469e+00 285 | 6.491768360137939453e+00 286 | 6.481903553009033203e+00 287 | 6.452835083007812500e+00 288 | 6.455957889556884766e+00 289 | 6.458556175231933594e+00 290 | 6.445206642150878906e+00 291 | 6.455742359161376953e+00 292 | 6.499244689941406250e+00 293 | 6.588602066040039062e+00 294 | 6.796399593353271484e+00 295 | 7.091689109802246094e+00 296 | 7.098243713378906250e+00 297 | 6.662360668182373047e+00 298 | 6.475848674774169922e+00 299 | 6.746299266815185547e+00 300 | 6.671047687530517578e+00 301 | 6.385641574859619141e+00 302 | 6.556565284729003906e+00 303 | 6.572278976440429688e+00 304 | 6.368388175964355469e+00 305 | 6.490806102752685547e+00 306 | 6.451303958892822266e+00 307 | 6.364757061004638672e+00 308 | 6.442392349243164062e+00 309 | 6.352243423461914062e+00 310 | 6.362600326538085938e+00 311 | 6.376090049743652344e+00 312 | 6.303322315216064453e+00 313 | 6.341582775115966797e+00 314 | 6.306967258453369141e+00 315 | 6.289905548095703125e+00 316 | 6.302726745605468750e+00 317 | 6.260151386260986328e+00 318 | 6.276049613952636719e+00 319 | 6.260567188262939453e+00 320 | 6.231240749359130859e+00 321 | 6.250241756439208984e+00 322 | 6.225442886352539062e+00 323 | 6.212441444396972656e+00 324 | 6.220066547393798828e+00 325 | 6.192436218261718750e+00 326 | 6.190607070922851562e+00 327 | 6.193387508392333984e+00 328 | 6.168448925018310547e+00 329 | 6.165446758270263672e+00 330 | 6.164012432098388672e+00 331 | 6.145906925201416016e+00 332 | 6.143323898315429688e+00 333 | 6.141300201416015625e+00 334 | 6.126543045043945312e+00 335 | 6.120718002319335938e+00 336 | 6.119143962860107422e+00 337 | 6.112458229064941406e+00 338 | 6.119690895080566406e+00 339 | 6.160902500152587891e+00 340 | 6.281438350677490234e+00 341 | 6.605006217956542969e+00 342 | 7.249194145202636719e+00 343 | 7.324826240539550781e+00 344 | 6.473043441772460938e+00 345 | 6.228675842285156250e+00 346 | 6.767178535461425781e+00 347 | 6.436766147613525391e+00 348 | 6.168076038360595703e+00 349 | 6.572916984558105469e+00 350 | 6.168668746948242188e+00 351 | 6.306997299194335938e+00 352 | 6.292124271392822266e+00 353 | 6.103813648223876953e+00 354 | 6.305202960968017578e+00 355 | 6.068061828613281250e+00 356 | 6.233975887298583984e+00 357 | 6.068376064300537109e+00 358 | 6.152287006378173828e+00 359 | 6.081074237823486328e+00 360 | 6.081728458404541016e+00 361 | 6.087399482727050781e+00 362 | 6.036642551422119141e+00 363 | 6.077378273010253906e+00 364 | 6.003964900970458984e+00 365 | 6.059075832366943359e+00 366 | 5.988464832305908203e+00 367 | 6.030076980590820312e+00 368 | 5.977003097534179688e+00 369 | 6.002802848815917969e+00 370 | 5.971007823944091797e+00 371 | 5.972235202789306641e+00 372 | 5.965801239013671875e+00 373 | 5.946963787078857422e+00 374 | 5.957467555999755859e+00 375 | 5.927857398986816406e+00 376 | 5.940391063690185547e+00 377 | 5.918239116668701172e+00 378 | 5.921404361724853516e+00 379 | 5.911398887634277344e+00 380 | 5.900965690612792969e+00 381 | 5.902344226837158203e+00 382 | 5.886303424835205078e+00 383 | 5.887813568115234375e+00 384 | 5.878069877624511719e+00 385 | 5.872107028961181641e+00 386 | 5.869586467742919922e+00 387 | 5.858539104461669922e+00 388 | 5.856192588806152344e+00 389 | 5.849102020263671875e+00 390 | 5.842231273651123047e+00 391 | 5.838991641998291016e+00 392 | 5.831137657165527344e+00 393 | 5.826461791992187500e+00 394 | 5.821566104888916016e+00 395 | 5.814816951751708984e+00 396 | 5.810883045196533203e+00 397 | 5.805781841278076172e+00 398 | 5.801361560821533203e+00 399 | 5.800670146942138672e+00 400 | 5.805809020996093750e+00 401 | 5.832085609436035156e+00 402 | 5.919252395629882812e+00 403 | 6.183970451354980469e+00 404 | 6.729770183563232422e+00 405 | 7.427816867828369141e+00 406 | 6.713530063629150391e+00 407 | 5.856919765472412109e+00 408 | 6.478962898254394531e+00 409 | 6.434874057769775391e+00 410 | 5.858128547668457031e+00 411 | 6.329310894012451172e+00 412 | 5.981172084808349609e+00 413 | 6.021009922027587891e+00 414 | 6.063569545745849609e+00 415 | 5.882217884063720703e+00 416 | 6.038293361663818359e+00 417 | 5.825654983520507812e+00 418 | 5.984562397003173828e+00 419 | 5.800269603729248047e+00 420 | 5.922268390655517578e+00 421 | 5.803063869476318359e+00 422 | 5.865536689758300781e+00 423 | 5.792100906372070312e+00 424 | 5.826158046722412109e+00 425 | 5.778805255889892578e+00 426 | 5.797509670257568359e+00 427 | 5.765366554260253906e+00 428 | 5.770214557647705078e+00 429 | 5.747861385345458984e+00 430 | 5.749607086181640625e+00 431 | 5.735608577728271484e+00 432 | 5.728995323181152344e+00 433 | 5.720441818237304688e+00 434 | 5.713536739349365234e+00 435 | 5.705840587615966797e+00 436 | 5.697722911834716797e+00 437 | 5.693718433380126953e+00 438 | 5.684025764465332031e+00 439 | 5.679753303527832031e+00 440 | 5.672437191009521484e+00 441 | 5.667181015014648438e+00 442 | 5.660928249359130859e+00 443 | 5.656557559967041016e+00 444 | 5.648064136505126953e+00 445 | 5.646885871887207031e+00 446 | 5.638329505920410156e+00 447 | 5.634810924530029297e+00 448 | 5.629347324371337891e+00 449 | 5.624269008636474609e+00 450 | 5.620038032531738281e+00 451 | 5.614400386810302734e+00 452 | 5.611459732055664062e+00 453 | 5.604726791381835938e+00 454 | 5.601946830749511719e+00 455 | 5.596786022186279297e+00 456 | 5.592133045196533203e+00 457 | 5.588812828063964844e+00 458 | 5.583195686340332031e+00 459 | 5.580494880676269531e+00 460 | 5.575722694396972656e+00 461 | 5.571234703063964844e+00 462 | 5.568482398986816406e+00 463 | 5.563533782958984375e+00 464 | 5.560338020324707031e+00 465 | 5.557155132293701172e+00 466 | 5.553660869598388672e+00 467 | 5.552538871765136719e+00 468 | 5.553236961364746094e+00 469 | 5.560789108276367188e+00 470 | 5.582906246185302734e+00 471 | 5.642524242401123047e+00 472 | 5.772476196289062500e+00 473 | 6.064939975738525391e+00 474 | 6.314712524414062500e+00 475 | 6.417964458465576172e+00 476 | 5.795729160308837891e+00 477 | 5.587249755859375000e+00 478 | 5.940947055816650391e+00 479 | 5.841278553009033203e+00 480 | 5.542412757873535156e+00 481 | 5.700677394866943359e+00 482 | 5.710066795349121094e+00 483 | 5.530952930450439453e+00 484 | 5.646296501159667969e+00 485 | 5.603327751159667969e+00 486 | 5.515361309051513672e+00 487 | 5.608925819396972656e+00 488 | 5.534748554229736328e+00 489 | 5.527986049652099609e+00 490 | 5.563326835632324219e+00 491 | 5.494839668273925781e+00 492 | 5.536859512329101562e+00 493 | 5.519761085510253906e+00 494 | 5.486723899841308594e+00 495 | 5.522447586059570312e+00 496 | 5.483800411224365234e+00 497 | 5.488787174224853516e+00 498 | 5.496134281158447266e+00 499 | 5.464539527893066406e+00 500 | 5.480674266815185547e+00 501 | 5.469000339508056641e+00 502 | 5.456308364868164062e+00 503 | 5.468143463134765625e+00 504 | 5.449047088623046875e+00 505 | 5.447082996368408203e+00 506 | 5.451321125030517578e+00 507 | 5.435705661773681641e+00 508 | 5.438585281372070312e+00 509 | 5.436822414398193359e+00 510 | 5.424374103546142578e+00 511 | 5.427390575408935547e+00 512 | 5.423396110534667969e+00 513 | 5.413749694824218750e+00 514 | 5.415512561798095703e+00 515 | 5.411659240722656250e+00 516 | 5.404067516326904297e+00 517 | 5.404829978942871094e+00 518 | 5.401827812194824219e+00 519 | 5.394987583160400391e+00 520 | 5.394587993621826172e+00 521 | 5.393599510192871094e+00 522 | 5.389215469360351562e+00 523 | 5.390177249908447266e+00 524 | 5.396914005279541016e+00 525 | 5.408484935760498047e+00 526 | 5.441784858703613281e+00 527 | 5.518654823303222656e+00 528 | 5.686267852783203125e+00 529 | 5.944321155548095703e+00 530 | 6.286396503448486328e+00 531 | 6.136970043182373047e+00 532 | 5.646551609039306641e+00 533 | 5.378326416015625000e+00 534 | 5.649526596069335938e+00 535 | 5.781669139862060547e+00 536 | 5.430076599121093750e+00 537 | 5.460660457611083984e+00 538 | 5.653326511383056641e+00 539 | 5.423937797546386719e+00 540 | 5.423806667327880859e+00 541 | 5.559529781341552734e+00 542 | 5.384108543395996094e+00 543 | 5.425010204315185547e+00 544 | 5.486309528350830078e+00 545 | 5.353106021881103516e+00 546 | 5.432452201843261719e+00 547 | 5.419260501861572266e+00 548 | 5.345218181610107422e+00 549 | 5.422626018524169922e+00 550 | 5.363014698028564453e+00 551 | 5.351916313171386719e+00 552 | 5.391334056854248047e+00 553 | 5.328764915466308594e+00 554 | 5.354629516601562500e+00 555 | 5.352302074432373047e+00 556 | 5.316259860992431641e+00 557 | 5.345512866973876953e+00 558 | 5.320917606353759766e+00 559 | 5.311408042907714844e+00 560 | 5.328930377960205078e+00 561 | 5.301919460296630859e+00 562 | 5.306020736694335938e+00 563 | 5.309988498687744141e+00 564 | 5.289872169494628906e+00 565 | 5.298350811004638672e+00 566 | 5.294495105743408203e+00 567 | 5.279844284057617188e+00 568 | 5.287860870361328125e+00 569 | 5.282197475433349609e+00 570 | 5.271551609039306641e+00 571 | 5.277034759521484375e+00 572 | 5.271572113037109375e+00 573 | 5.262728214263916016e+00 574 | 5.265968322753906250e+00 575 | 5.262249946594238281e+00 576 | 5.254526615142822266e+00 577 | 5.255789756774902344e+00 578 | 5.253626346588134766e+00 579 | 5.246588706970214844e+00 580 | 5.245701313018798828e+00 581 | 5.245043754577636719e+00 582 | 5.239593982696533203e+00 583 | 5.236832618713378906e+00 584 | 5.236853599548339844e+00 585 | 5.234477996826171875e+00 586 | 5.233205318450927734e+00 587 | 5.238489151000976562e+00 588 | 5.253405570983886719e+00 589 | 5.293784141540527344e+00 590 | 5.407704353332519531e+00 591 | 5.681974887847900391e+00 592 | 6.248499393463134766e+00 593 | 6.562948226928710938e+00 594 | 6.218940258026123047e+00 595 | 5.376243114471435547e+00 596 | 5.787705898284912109e+00 597 | 6.033855915069580078e+00 598 | 5.308135986328125000e+00 599 | 5.696813106536865234e+00 600 | 5.700057983398437500e+00 601 | 5.322164058685302734e+00 602 | 5.700682163238525391e+00 603 | 5.316306114196777344e+00 604 | 5.511093616485595703e+00 605 | 5.357596874237060547e+00 606 | 5.370792388916015625e+00 607 | 5.397292613983154297e+00 608 | 5.288625240325927734e+00 609 | 5.391851902008056641e+00 610 | 5.250884532928466797e+00 611 | 5.360389709472656250e+00 612 | 5.237993240356445312e+00 613 | 5.333380222320556641e+00 614 | 5.223283290863037109e+00 615 | 5.302264213562011719e+00 616 | 5.218310832977294922e+00 617 | 5.275022983551025391e+00 618 | 5.216609954833984375e+00 619 | 5.247957229614257812e+00 620 | 5.210739135742187500e+00 621 | 5.227724075317382812e+00 622 | 5.206731319427490234e+00 623 | 5.208325862884521484e+00 624 | 5.202112197875976562e+00 625 | 5.191302776336669922e+00 626 | 5.197045803070068359e+00 627 | 5.177798271179199219e+00 628 | 5.190234661102294922e+00 629 | 5.168127059936523438e+00 630 | 5.180487632751464844e+00 631 | 5.161425590515136719e+00 632 | 5.170014858245849609e+00 633 | 5.158395767211914062e+00 634 | 5.157358646392822266e+00 635 | 5.155265331268310547e+00 636 | 5.147437572479248047e+00 637 | 5.150834083557128906e+00 638 | 5.140191078186035156e+00 639 | 5.143262863159179688e+00 640 | 5.136710643768310547e+00 641 | 5.134835720062255859e+00 642 | 5.133208751678466797e+00 643 | 5.127100467681884766e+00 644 | 5.128325462341308594e+00 645 | 5.122685432434082031e+00 646 | 5.121356487274169922e+00 647 | 5.119188308715820312e+00 648 | 5.115041255950927734e+00 649 | 5.114619731903076172e+00 650 | 5.110557079315185547e+00 651 | 5.108552932739257812e+00 652 | 5.106893062591552734e+00 653 | 5.103369235992431641e+00 654 | 5.102087497711181641e+00 655 | 5.099292755126953125e+00 656 | 5.096667289733886719e+00 657 | 5.095188617706298828e+00 658 | 5.092422008514404297e+00 659 | 5.090414524078369141e+00 660 | 5.088457107543945312e+00 661 | 5.085930347442626953e+00 662 | 5.084126472473144531e+00 663 | 5.082138538360595703e+00 664 | 5.080036640167236328e+00 665 | 5.078577041625976562e+00 666 | 5.077291488647460938e+00 667 | 5.076878547668457031e+00 668 | 5.078744888305664062e+00 669 | 5.085391044616699219e+00 670 | 5.102697849273681641e+00 671 | 5.148271560668945312e+00 672 | 5.246071338653564453e+00 673 | 5.478911876678466797e+00 674 | 5.759899139404296875e+00 675 | 6.128073692321777344e+00 676 | 5.642488956451416016e+00 677 | 5.141901016235351562e+00 678 | 5.221325874328613281e+00 679 | 5.493855953216552734e+00 680 | 5.323047161102294922e+00 681 | 5.089896202087402344e+00 682 | 5.321290493011474609e+00 683 | 5.290571689605712891e+00 684 | 5.082696914672851562e+00 685 | 5.291717052459716797e+00 686 | 5.209405899047851562e+00 687 | 5.089239120483398438e+00 688 | 5.257832527160644531e+00 689 | 5.120509624481201172e+00 690 | 5.122778892517089844e+00 691 | 5.185490608215332031e+00 692 | 5.064347743988037109e+00 693 | 5.144214630126953125e+00 694 | 5.095299720764160156e+00 695 | 5.073166847229003906e+00 696 | 5.116471767425537109e+00 697 | 5.049043655395507812e+00 698 | 5.092794418334960938e+00 699 | 5.066929817199707031e+00 700 | 5.050964355468750000e+00 701 | 5.077766418457031250e+00 702 | 5.035625934600830078e+00 703 | 5.059679508209228516e+00 704 | 5.044126987457275391e+00 705 | 5.031584262847900391e+00 706 | 5.049528598785400391e+00 707 | 5.022660255432128906e+00 708 | 5.034172058105468750e+00 709 | 5.029461860656738281e+00 710 | 5.015813350677490234e+00 711 | 5.027701377868652344e+00 712 | 5.013339042663574219e+00 713 | 5.013519287109375000e+00 714 | 5.016251087188720703e+00 715 | 5.003370761871337891e+00 716 | 5.008355140686035156e+00 717 | 5.004792213439941406e+00 718 | 4.997690200805664062e+00 719 | 5.001789569854736328e+00 720 | 4.995558738708496094e+00 721 | 4.992002010345458984e+00 722 | 4.993927001953125000e+00 723 | 4.988000392913818359e+00 724 | 4.986465454101562500e+00 725 | 4.986962318420410156e+00 726 | 4.981746196746826172e+00 727 | 4.980756759643554688e+00 728 | 4.980599880218505859e+00 729 | 4.975982666015625000e+00 730 | 4.974631309509277344e+00 731 | 4.974081039428710938e+00 732 | 4.970379352569580078e+00 733 | 4.968827247619628906e+00 734 | 4.968247890472412109e+00 735 | 4.965325832366943359e+00 736 | 4.963768005371093750e+00 737 | 4.963751316070556641e+00 738 | 4.962679386138916016e+00 739 | 4.963511943817138672e+00 740 | 4.969151496887207031e+00 741 | 4.982368469238281250e+00 742 | 5.012811183929443359e+00 743 | 5.089853286743164062e+00 744 | 5.233788013458251953e+00 745 | 5.531733512878417969e+00 746 | 5.719130516052246094e+00 747 | 5.695899486541748047e+00 748 | 5.212698936462402344e+00 749 | 4.985279560089111328e+00 750 | 5.246255397796630859e+00 751 | 5.315448284149169922e+00 752 | 5.038523197174072266e+00 753 | 5.014715671539306641e+00 754 | 5.190006256103515625e+00 755 | 5.063101768493652344e+00 756 | 4.981350421905517578e+00 757 | 5.123898506164550781e+00 758 | 5.039695739746093750e+00 759 | 4.977601051330566406e+00 760 | 5.078267574310302734e+00 761 | 5.002309799194335938e+00 762 | 4.983974456787109375e+00 763 | 5.042257785797119141e+00 764 | 4.965598106384277344e+00 765 | 4.988893508911132812e+00 766 | 5.002696514129638672e+00 767 | 4.947369575500488281e+00 768 | 4.983894348144531250e+00 769 | 4.966026782989501953e+00 770 | 4.940603733062744141e+00 771 | 4.969603538513183594e+00 772 | 4.940729618072509766e+00 773 | 4.938639163970947266e+00 774 | 4.950467109680175781e+00 775 | 4.927175045013427734e+00 776 | 4.935126781463623047e+00 777 | 4.932690143585205078e+00 778 | 4.919647693634033203e+00 779 | 4.927465438842773438e+00 780 | 4.919094085693359375e+00 781 | 4.914157390594482422e+00 782 | 4.918420791625976562e+00 783 | 4.909520626068115234e+00 784 | 4.907695770263671875e+00 785 | 4.909449577331542969e+00 786 | 4.902459621429443359e+00 787 | 4.901503562927246094e+00 788 | 4.901586055755615234e+00 789 | 4.896340847015380859e+00 790 | 4.895009040832519531e+00 791 | 4.894388675689697266e+00 792 | 4.890547752380371094e+00 793 | 4.888991832733154297e+00 794 | 4.888130187988281250e+00 795 | 4.885107040405273438e+00 796 | 4.883303165435791016e+00 797 | 4.882508754730224609e+00 798 | 4.880204677581787109e+00 799 | 4.878113746643066406e+00 800 | 4.877159118652343750e+00 801 | 4.875802993774414062e+00 802 | 4.874305725097656250e+00 803 | 4.874036788940429688e+00 804 | 4.874947547912597656e+00 805 | 4.877903938293457031e+00 806 | 4.886900901794433594e+00 807 | 4.909313678741455078e+00 808 | 4.961289405822753906e+00 809 | 5.073488712310791016e+00 810 | 5.305251121520996094e+00 811 | 5.658383846282958984e+00 812 | 5.915799617767333984e+00 813 | 5.611837863922119141e+00 814 | 4.993884086608886719e+00 815 | 4.994711399078369141e+00 816 | 5.361148834228515625e+00 817 | 5.190062046051025391e+00 818 | 4.893638610839843750e+00 819 | 5.148440361022949219e+00 820 | 5.113555908203125000e+00 821 | 4.899607658386230469e+00 822 | 5.100164413452148438e+00 823 | 4.990114212036132812e+00 824 | 4.923252582550048828e+00 825 | 5.050522327423095703e+00 826 | 4.902982234954833984e+00 827 | 4.962901592254638672e+00 828 | 4.957222938537597656e+00 829 | 4.883706092834472656e+00 830 | 4.964165687561035156e+00 831 | 4.878674030303955078e+00 832 | 4.910594940185546875e+00 833 | 4.908930778503417969e+00 834 | 4.867223262786865234e+00 835 | 4.908992767333984375e+00 836 | 4.860922336578369141e+00 837 | 4.883418083190917969e+00 838 | 4.872795104980468750e+00 839 | 4.854798793792724609e+00 840 | 4.875658512115478516e+00 841 | 4.846621513366699219e+00 842 | 4.859865665435791016e+00 843 | 4.852380752563476562e+00 844 | 4.840646743774414062e+00 845 | 4.852349281311035156e+00 846 | 4.836014747619628906e+00 847 | 4.840105533599853516e+00 848 | 4.837991714477539062e+00 849 | 4.828674316406250000e+00 850 | 4.835464954376220703e+00 851 | 4.826375961303710938e+00 852 | 4.825178146362304688e+00 853 | 4.827070236206054688e+00 854 | 4.818723201751708984e+00 855 | 4.820880413055419922e+00 856 | 4.818891525268554688e+00 857 | 4.813705444335937500e+00 858 | 4.815516948699951172e+00 859 | 4.811895370483398438e+00 860 | 4.809002876281738281e+00 861 | 4.809779644012451172e+00 862 | 4.806209087371826172e+00 863 | 4.804447174072265625e+00 864 | 4.804600238800048828e+00 865 | 4.801337242126464844e+00 866 | 4.799848079681396484e+00 867 | 4.799480915069580078e+00 868 | 4.796713352203369141e+00 869 | 4.795283317565917969e+00 870 | 4.794567584991455078e+00 871 | 4.792347431182861328e+00 872 | 4.790934085845947266e+00 873 | 4.790057182312011719e+00 874 | 4.788136482238769531e+00 875 | 4.786598682403564453e+00 876 | 4.785696029663085938e+00 877 | 4.784096717834472656e+00 878 | 4.782477855682373047e+00 879 | 4.781446456909179688e+00 880 | 4.780118465423583984e+00 881 | 4.778518676757812500e+00 882 | 4.777324199676513672e+00 883 | 4.776231765747070312e+00 884 | 4.774928092956542969e+00 885 | 4.773888111114501953e+00 886 | 4.773469448089599609e+00 887 | 4.774133205413818359e+00 888 | 4.777778625488281250e+00 889 | 4.790203094482421875e+00 890 | 4.824793815612792969e+00 891 | 4.923987865447998047e+00 892 | 5.158408164978027344e+00 893 | 5.739570617675781250e+00 894 | 6.239322662353515625e+00 895 | 6.374514579772949219e+00 896 | 5.067633628845214844e+00 897 | 5.138685226440429688e+00 898 | 5.909677028656005859e+00 899 | 5.037776947021484375e+00 900 | 5.266566276550292969e+00 901 | 5.539655685424804688e+00 902 | 4.936028003692626953e+00 903 | 5.465404510498046875e+00 904 | 4.969741821289062500e+00 905 | 5.194053649902343750e+00 906 | 4.968147754669189453e+00 907 | 5.057561397552490234e+00 908 | 4.974938392639160156e+00 909 | 4.985977649688720703e+00 910 | 4.926303386688232422e+00 911 | 4.958625793457031250e+00 912 | 4.890216827392578125e+00 913 | 4.940626621246337891e+00 914 | 4.852027416229248047e+00 915 | 4.921267032623291016e+00 916 | 4.832937240600585938e+00 917 | 4.899877548217773438e+00 918 | 4.813349246978759766e+00 919 | 4.881453990936279297e+00 920 | 4.801447391510009766e+00 921 | 4.860270023345947266e+00 922 | 4.792502880096435547e+00 923 | 4.841412544250488281e+00 924 | 4.785426139831542969e+00 925 | 4.822609424591064453e+00 926 | 4.780287742614746094e+00 927 | 4.807072639465332031e+00 928 | 4.774031639099121094e+00 929 | 4.793525695800781250e+00 930 | 4.768420696258544922e+00 931 | 4.782433509826660156e+00 932 | 4.762708663940429688e+00 933 | 4.772683143615722656e+00 934 | 4.756617069244384766e+00 935 | 4.765454769134521484e+00 936 | 4.750785350799560547e+00 937 | 4.757923126220703125e+00 938 | 4.746381282806396484e+00 939 | 4.751459121704101562e+00 940 | 4.741441726684570312e+00 941 | 4.745888710021972656e+00 942 | 4.737395763397216797e+00 943 | 4.740209102630615234e+00 944 | 4.733799934387207031e+00 945 | 4.735161781311035156e+00 946 | 4.730532646179199219e+00 947 | 4.729981422424316406e+00 948 | 4.727812767028808594e+00 949 | 4.725301265716552734e+00 950 | 4.724956035614013672e+00 951 | 4.721277713775634766e+00 952 | 4.721692562103271484e+00 953 | 4.718046188354492188e+00 954 | 4.718110084533691406e+00 955 | 4.715487003326416016e+00 956 | 4.714472293853759766e+00 957 | 4.712875843048095703e+00 958 | 4.711149215698242188e+00 959 | 4.710289955139160156e+00 960 | 4.708232879638671875e+00 961 | 4.707456111907958984e+00 962 | 4.705555438995361328e+00 963 | 4.704619407653808594e+00 964 | 4.703102588653564453e+00 965 | 4.701732635498046875e+00 966 | 4.700755596160888672e+00 967 | 4.699089050292968750e+00 968 | 4.698185920715332031e+00 969 | 4.696722030639648438e+00 970 | 4.695510864257812500e+00 971 | 4.694411754608154297e+00 972 | 4.693016052246093750e+00 973 | 4.692011833190917969e+00 974 | 4.690723419189453125e+00 975 | 4.689550399780273438e+00 976 | 4.688440322875976562e+00 977 | 4.687202453613281250e+00 978 | 4.686131477355957031e+00 979 | 4.684942245483398438e+00 980 | 4.683822154998779297e+00 981 | 4.682721138000488281e+00 982 | 4.681550502777099609e+00 983 | 4.680502891540527344e+00 984 | 4.679384231567382812e+00 985 | 4.678305625915527344e+00 986 | 4.677330493927001953e+00 987 | 4.676352500915527344e+00 988 | 4.675551414489746094e+00 989 | 4.675035953521728516e+00 990 | 4.675030708312988281e+00 991 | 4.676233291625976562e+00 992 | 4.679884433746337891e+00 993 | 4.688988208770751953e+00 994 | 4.709637165069580078e+00 995 | 4.756596565246582031e+00 996 | 4.852429866790771484e+00 997 | 5.049119472503662109e+00 998 | 5.322140216827392578e+00 999 | 5.587139606475830078e+00 1000 | 5.379793643951416016e+00 1001 | 4.870262622833251953e+00 1002 | 4.719626426696777344e+00 1003 | 4.980951309204101562e+00 1004 | 5.034140586853027344e+00 1005 | 4.732082843780517578e+00 1006 | 4.790478229522705078e+00 1007 | 4.935858726501464844e+00 1008 | 4.731698513031005859e+00 1009 | 4.754724502563476562e+00 1010 | 4.852752208709716797e+00 1011 | 4.702860355377197266e+00 1012 | 4.752739906311035156e+00 1013 | 4.786714553833007812e+00 1014 | 4.683603286743164062e+00 1015 | 4.753628730773925781e+00 1016 | 4.728340148925781250e+00 1017 | 4.686569213867187500e+00 1018 | 4.737311363220214844e+00 1019 | 4.686340808868408203e+00 1020 | 4.697710990905761719e+00 1021 | 4.706600189208984375e+00 1022 | 4.669054508209228516e+00 1023 | 4.698894023895263672e+00 1024 | 4.677902698516845703e+00 1025 | 4.667406082153320312e+00 1026 | 4.687011718750000000e+00 1027 | 4.660624504089355469e+00 1028 | 4.668252468109130859e+00 1029 | 4.670060634613037109e+00 1030 | 4.654067039489746094e+00 1031 | 4.664825916290283203e+00 1032 | 4.654995441436767578e+00 1033 | 4.651540756225585938e+00 1034 | 4.657682418823242188e+00 1035 | 4.645684242248535156e+00 1036 | 4.648252964019775391e+00 1037 | 4.648875713348388672e+00 1038 | 4.640705585479736328e+00 1039 | 4.644485473632812500e+00 1040 | 4.641164779663085938e+00 1041 | 4.636380195617675781e+00 1042 | 4.639874935150146484e+00 1043 | 4.635764598846435547e+00 1044 | 4.632629394531250000e+00 1045 | 4.634808540344238281e+00 1046 | 4.631164073944091797e+00 1047 | 4.629137516021728516e+00 1048 | 4.630224704742431641e+00 1049 | 4.627079963684082031e+00 1050 | 4.625557899475097656e+00 1051 | 4.626142501831054688e+00 1052 | 4.623522758483886719e+00 1053 | 4.621983528137207031e+00 1054 | 4.622175693511962891e+00 1055 | 4.620184421539306641e+00 1056 | 4.618543624877929688e+00 1057 | 4.618366241455078125e+00 1058 | 4.616953372955322266e+00 1059 | 4.615382194519042969e+00 1060 | 4.614965438842773438e+00 1061 | 4.613921165466308594e+00 1062 | 4.612351417541503906e+00 1063 | 4.611587524414062500e+00 1064 | 4.610811710357666016e+00 1065 | 4.609470367431640625e+00 1066 | 4.608567237854003906e+00 1067 | 4.607992172241210938e+00 1068 | 4.607003688812255859e+00 1069 | 4.606163978576660156e+00 1070 | 4.606058597564697266e+00 1071 | 4.606404781341552734e+00 1072 | 4.607660770416259766e+00 1073 | 4.611604690551757812e+00 1074 | 4.620571136474609375e+00 1075 | 4.640072345733642578e+00 1076 | 4.680336952209472656e+00 1077 | 4.765451908111572266e+00 1078 | 4.918299674987792969e+00 1079 | 5.173166275024414062e+00 1080 | 5.380558013916015625e+00 1081 | 5.333293437957763672e+00 1082 | 4.909926414489746094e+00 1083 | 4.640540122985839844e+00 1084 | 4.835408687591552734e+00 1085 | 4.950118064880371094e+00 1086 | 4.708973884582519531e+00 1087 | 4.640703201293945312e+00 1088 | 4.815250873565673828e+00 1089 | 4.725491523742675781e+00 1090 | 4.624112606048583984e+00 1091 | 4.753805160522460938e+00 1092 | 4.682698726654052734e+00 1093 | 4.624467372894287109e+00 1094 | 4.720615386962890625e+00 1095 | 4.642120361328125000e+00 1096 | 4.631834030151367188e+00 1097 | 4.681624412536621094e+00 1098 | 4.613124847412109375e+00 1099 | 4.640542507171630859e+00 1100 | 4.643214225769042969e+00 1101 | 4.601462364196777344e+00 1102 | 4.638385295867919922e+00 1103 | 4.612655162811279297e+00 1104 | 4.603460311889648438e+00 1105 | 4.623819351196289062e+00 1106 | 4.594528198242187500e+00 1107 | 4.605702400207519531e+00 1108 | 4.606160640716552734e+00 1109 | 4.587594985961914062e+00 1110 | 4.602337360382080078e+00 1111 | 4.591903209686279297e+00 1112 | 4.584988594055175781e+00 1113 | 4.595358848571777344e+00 1114 | 4.583015441894531250e+00 1115 | 4.582385063171386719e+00 1116 | 4.586971759796142578e+00 1117 | 4.577311038970947266e+00 1118 | 4.579677104949951172e+00 1119 | 4.580132484436035156e+00 1120 | 4.572639942169189453e+00 1121 | 4.575327873229980469e+00 1122 | 4.574731826782226562e+00 1123 | 4.569476604461669922e+00 1124 | 4.571159839630126953e+00 1125 | 4.569963932037353516e+00 1126 | 4.565974235534667969e+00 1127 | 4.566886425018310547e+00 1128 | 4.566142082214355469e+00 1129 | 4.562995433807373047e+00 1130 | 4.563086986541748047e+00 1131 | 4.562541961669921875e+00 1132 | 4.559948444366455078e+00 1133 | 4.559488773345947266e+00 1134 | 4.559229373931884766e+00 1135 | 4.557116031646728516e+00 1136 | 4.556090354919433594e+00 1137 | 4.555907726287841797e+00 1138 | 4.554377079010009766e+00 1139 | 4.553026676177978516e+00 1140 | 4.552736759185791016e+00 1141 | 4.551812171936035156e+00 1142 | 4.550328731536865234e+00 1143 | 4.549631595611572266e+00 1144 | 4.549147129058837891e+00 1145 | 4.547992706298828125e+00 1146 | 4.547009944915771484e+00 1147 | 4.546658992767333984e+00 1148 | 4.546193122863769531e+00 1149 | 4.545659065246582031e+00 1150 | 4.546091079711914062e+00 1151 | 4.547938346862792969e+00 1152 | 4.552172183990478516e+00 1153 | 4.561576366424560547e+00 1154 | 4.583691120147705078e+00 1155 | 4.627058029174804688e+00 1156 | 4.726473808288574219e+00 1157 | 4.881966114044189453e+00 1158 | 5.193955898284912109e+00 1159 | 5.311691284179687500e+00 1160 | 5.314049243927001953e+00 1161 | 4.795721530914306641e+00 1162 | 4.587200164794921875e+00 1163 | 4.811932563781738281e+00 1164 | 4.848617553710937500e+00 1165 | 4.626372337341308594e+00 1166 | 4.615892410278320312e+00 1167 | 4.743780136108398438e+00 1168 | 4.636427402496337891e+00 1169 | 4.584567070007324219e+00 1170 | 4.686268806457519531e+00 1171 | 4.609841346740722656e+00 1172 | 4.572381973266601562e+00 1173 | 4.650022029876708984e+00 1174 | 4.581541538238525391e+00 1175 | 4.572867393493652344e+00 1176 | 4.617480278015136719e+00 1177 | 4.557154178619384766e+00 1178 | 4.575489997863769531e+00 1179 | 4.584864616394042969e+00 1180 | 4.543893814086914062e+00 1181 | 4.572582721710205078e+00 1182 | 4.557171344757080078e+00 1183 | 4.541029930114746094e+00 1184 | 4.562928199768066406e+00 1185 | 4.540054798126220703e+00 1186 | 4.540158271789550781e+00 1187 | 4.550103187561035156e+00 1188 | 4.531624317169189453e+00 1189 | 4.538344383239746094e+00 1190 | 4.538006305694580078e+00 1191 | 4.526866436004638672e+00 1192 | 4.534429073333740234e+00 1193 | 4.529394626617431641e+00 1194 | 4.523153781890869141e+00 1195 | 4.529111385345458984e+00 1196 | 4.523484230041503906e+00 1197 | 4.519776344299316406e+00 1198 | 4.523434162139892578e+00 1199 | 4.518871307373046875e+00 1200 | 4.516696453094482422e+00 1201 | 4.518737316131591797e+00 1202 | 4.514765262603759766e+00 1203 | 4.512945175170898438e+00 1204 | 4.514649391174316406e+00 1205 | 4.511826992034912109e+00 1206 | 4.509499549865722656e+00 1207 | 4.510542392730712891e+00 1208 | 4.508991718292236328e+00 1209 | 4.506631851196289062e+00 1210 | 4.506798744201660156e+00 1211 | 4.505993843078613281e+00 1212 | 4.503831386566162109e+00 1213 | 4.503417491912841797e+00 1214 | 4.503247261047363281e+00 1215 | 4.501502990722656250e+00 1216 | 4.500342845916748047e+00 1217 | 4.500267505645751953e+00 1218 | 4.499231815338134766e+00 1219 | 4.497789859771728516e+00 1220 | 4.497432231903076172e+00 1221 | 4.497005939483642578e+00 1222 | 4.495722293853759766e+00 1223 | 4.494951248168945312e+00 1224 | 4.495007514953613281e+00 1225 | 4.494847297668457031e+00 1226 | 4.494936943054199219e+00 1227 | 4.496889591217041016e+00 1228 | 4.501639842987060547e+00 1229 | 4.511447429656982422e+00 1230 | 4.532641887664794922e+00 1231 | 4.577881813049316406e+00 1232 | 4.668240547180175781e+00 1233 | 4.834130287170410156e+00 1234 | 5.073011398315429688e+00 1235 | 5.264469623565673828e+00 1236 | 5.135745048522949219e+00 1237 | 4.701954364776611328e+00 1238 | 4.519597053527832031e+00 1239 | 4.728123664855957031e+00 1240 | 4.820924758911132812e+00 1241 | 4.584256649017333984e+00 1242 | 4.544330120086669922e+00 1243 | 4.700212955474853516e+00 1244 | 4.600119590759277344e+00 1245 | 4.519902706146240234e+00 1246 | 4.630959987640380859e+00 1247 | 4.569007873535156250e+00 1248 | 4.512705326080322266e+00 1249 | 4.594376087188720703e+00 1250 | 4.534668445587158203e+00 1251 | 4.515775203704833984e+00 1252 | 4.566298484802246094e+00 1253 | 4.506889343261718750e+00 1254 | 4.518529891967773438e+00 1255 | 4.540069580078125000e+00 1256 | 4.492202758789062500e+00 1257 | 4.518203258514404297e+00 1258 | 4.513294219970703125e+00 1259 | 4.488668441772460938e+00 1260 | 4.512935638427734375e+00 1261 | 4.494690418243408203e+00 1262 | 4.487631320953369141e+00 1263 | 4.502690315246582031e+00 1264 | 4.484400749206542969e+00 1265 | 4.486971855163574219e+00 1266 | 4.491555213928222656e+00 1267 | 4.477876186370849609e+00 1268 | 4.484475612640380859e+00 1269 | 4.482569217681884766e+00 1270 | 4.474076747894287109e+00 1271 | 4.480343341827392578e+00 1272 | 4.475860595703125000e+00 1273 | 4.470190048217773438e+00 1274 | 4.475702762603759766e+00 1275 | 4.471533298492431641e+00 1276 | 4.467161178588867188e+00 1277 | 4.470685958862304688e+00 1278 | 4.467720031738281250e+00 1279 | 4.464292526245117188e+00 1280 | 4.466413497924804688e+00 1281 | 4.464366436004638672e+00 1282 | 4.461590290069580078e+00 1283 | 4.462858676910400391e+00 1284 | 4.461528301239013672e+00 1285 | 4.458803653717041016e+00 1286 | 4.459333896636962891e+00 1287 | 4.458827972412109375e+00 1288 | 4.456375122070312500e+00 1289 | 4.456057071685791016e+00 1290 | 4.456197738647460938e+00 1291 | 4.454397201538085938e+00 1292 | 4.453234672546386719e+00 1293 | 4.453283309936523438e+00 1294 | 4.452392578125000000e+00 1295 | 4.451039314270019531e+00 1296 | 4.450639247894287109e+00 1297 | 4.450197219848632812e+00 1298 | 4.449120044708251953e+00 1299 | 4.448437690734863281e+00 1300 | 4.448286056518554688e+00 1301 | 4.447868824005126953e+00 1302 | 4.447619438171386719e+00 1303 | 4.448480606079101562e+00 1304 | 4.450766086578369141e+00 1305 | 4.455265998840332031e+00 1306 | 4.465353488922119141e+00 1307 | 4.486018657684326172e+00 1308 | 4.529933452606201172e+00 1309 | 4.610585212707519531e+00 1310 | 4.773552417755126953e+00 1311 | 4.981889247894287109e+00 1312 | 5.241396903991699219e+00 1313 | 5.107152462005615234e+00 1314 | 4.716474056243896484e+00 1315 | 4.462242126464843750e+00 1316 | 4.627721786499023438e+00 1317 | 4.801859378814697266e+00 1318 | 4.561788558959960938e+00 1319 | 4.481115341186523438e+00 1320 | 4.653952598571777344e+00 1321 | 4.558404922485351562e+00 1322 | 4.462439537048339844e+00 1323 | 4.573493480682373047e+00 1324 | 4.518789291381835938e+00 1325 | 4.459180355072021484e+00 1326 | 4.534851074218750000e+00 1327 | 4.483152866363525391e+00 1328 | 4.459378719329833984e+00 1329 | 4.510065555572509766e+00 1330 | 4.457324981689453125e+00 1331 | 4.464812278747558594e+00 1332 | 4.487755298614501953e+00 1333 | 4.443782806396484375e+00 1334 | 4.467321395874023438e+00 1335 | 4.467709541320800781e+00 1336 | 4.439414501190185547e+00 1337 | 4.464454174041748047e+00 1338 | 4.450973510742187500e+00 1339 | 4.439447879791259766e+00 1340 | 4.457181453704833984e+00 1341 | 4.439613342285156250e+00 1342 | 4.438573837280273438e+00 1343 | 4.448392391204833984e+00 1344 | 4.433259963989257812e+00 1345 | 4.436281681060791016e+00 1346 | 4.439587593078613281e+00 1347 | 4.428880214691162109e+00 1348 | 4.433146953582763672e+00 1349 | 4.433387279510498047e+00 1350 | 4.425198078155517578e+00 1351 | 4.428824424743652344e+00 1352 | 4.428720951080322266e+00 1353 | 4.422531604766845703e+00 1354 | 4.424849510192871094e+00 1355 | 4.424910545349121094e+00 1356 | 4.420072555541992188e+00 1357 | 4.420932769775390625e+00 1358 | 4.421307563781738281e+00 1359 | 4.417912483215332031e+00 1360 | 4.417842388153076172e+00 1361 | 4.418271541595458984e+00 1362 | 4.415699005126953125e+00 1363 | 4.414790153503417969e+00 1364 | 4.415363311767578125e+00 1365 | 4.413656234741210938e+00 1366 | 4.412156581878662109e+00 1367 | 4.412496089935302734e+00 1368 | 4.411798477172851562e+00 1369 | 4.410184860229492188e+00 1370 | 4.409796714782714844e+00 1371 | 4.409660816192626953e+00 1372 | 4.408503532409667969e+00 1373 | 4.407634258270263672e+00 1374 | 4.407539844512939453e+00 1375 | 4.407221794128417969e+00 1376 | 4.406834602355957031e+00 1377 | 4.407551288604736328e+00 1378 | 4.409965038299560547e+00 1379 | 4.415055274963378906e+00 1380 | 4.427067756652832031e+00 1381 | 4.454401969909667969e+00 1382 | 4.516469955444335938e+00 1383 | 4.642221927642822266e+00 1384 | 4.903247833251953125e+00 1385 | 5.245216846466064453e+00 1386 | 5.560220241546630859e+00 1387 | 5.139305591583251953e+00 1388 | 4.542347431182861328e+00 1389 | 4.554632663726806641e+00 1390 | 4.875520229339599609e+00 1391 | 4.702998161315917969e+00 1392 | 4.446685314178466797e+00 1393 | 4.695538997650146484e+00 1394 | 4.630419254302978516e+00 1395 | 4.459553718566894531e+00 1396 | 4.652744770050048828e+00 1397 | 4.509754657745361328e+00 1398 | 4.499187469482421875e+00 1399 | 4.587774753570556641e+00 1400 | 4.442557811737060547e+00 1401 | 4.534279346466064453e+00 1402 | 4.480387210845947266e+00 1403 | 4.464177608489990234e+00 1404 | 4.501327037811279297e+00 1405 | 4.431603908538818359e+00 1406 | 4.487092494964599609e+00 1407 | 4.438995838165283203e+00 1408 | 4.449995517730712891e+00 1409 | 4.450054645538330078e+00 1410 | 4.424982547760009766e+00 1411 | 4.450840473175048828e+00 1412 | 4.416114807128906250e+00 1413 | 4.433771610260009766e+00 1414 | 4.421882152557373047e+00 1415 | 4.417332649230957031e+00 1416 | 4.423054218292236328e+00 1417 | 4.404545783996582031e+00 1418 | 4.420232772827148438e+00 1419 | 4.405411720275878906e+00 1420 | 4.405074119567871094e+00 1421 | 4.409525871276855469e+00 1422 | 4.396830558776855469e+00 1423 | 4.405890941619873047e+00 1424 | 4.397150516510009766e+00 1425 | 4.394817829132080078e+00 1426 | 4.399671554565429688e+00 1427 | 4.390581607818603516e+00 1428 | 4.394026279449462891e+00 1429 | 4.392122268676757812e+00 1430 | 4.387025833129882812e+00 1431 | 4.390642642974853516e+00 1432 | 4.386805534362792969e+00 1433 | 4.385265827178955078e+00 1434 | 4.386607646942138672e+00 1435 | 4.382604598999023438e+00 1436 | 4.382800579071044922e+00 1437 | 4.382731914520263672e+00 1438 | 4.379866123199462891e+00 1439 | 4.380428314208984375e+00 1440 | 4.379406929016113281e+00 1441 | 4.377438545227050781e+00 1442 | 4.377772808074951172e+00 1443 | 4.376409530639648438e+00 1444 | 4.375081062316894531e+00 1445 | 4.375287532806396484e+00 1446 | 4.374030590057373047e+00 1447 | 4.373055934906005859e+00 1448 | 4.373022556304931641e+00 1449 | 4.371860980987548828e+00 1450 | 4.371012210845947266e+00 1451 | 4.370834827423095703e+00 1452 | 4.369785785675048828e+00 1453 | 4.368898391723632812e+00 1454 | 4.368644714355468750e+00 1455 | 4.367855548858642578e+00 1456 | 4.367011547088623047e+00 1457 | 4.366689205169677734e+00 1458 | 4.366081237792968750e+00 1459 | 4.365298271179199219e+00 1460 | 4.364947319030761719e+00 1461 | 4.364635467529296875e+00 1462 | 4.364221572875976562e+00 1463 | 4.364346504211425781e+00 1464 | 4.365214824676513672e+00 1465 | 4.367053985595703125e+00 1466 | 4.371425151824951172e+00 1467 | 4.380957126617431641e+00 1468 | 4.401088714599609375e+00 1469 | 4.441564559936523438e+00 1470 | 4.526449680328369141e+00 1471 | 4.676258087158203125e+00 1472 | 4.945591926574707031e+00 1473 | 5.192610263824462891e+00 1474 | 5.278753280639648438e+00 1475 | 4.845747470855712891e+00 1476 | 4.419696807861328125e+00 1477 | 4.514160633087158203e+00 1478 | 4.755589008331298828e+00 1479 | 4.576741695404052734e+00 1480 | 4.385976791381835938e+00 1481 | 4.586089134216308594e+00 1482 | 4.567293643951416016e+00 1483 | 4.383872509002685547e+00 1484 | 4.534636974334716797e+00 1485 | 4.502266883850097656e+00 1486 | 4.391774654388427734e+00 1487 | 4.515244483947753906e+00 1488 | 4.432792663574218750e+00 1489 | 4.408163547515869141e+00 1490 | 4.480687618255615234e+00 1491 | 4.385067939758300781e+00 1492 | 4.428351402282714844e+00 1493 | 4.426909446716308594e+00 1494 | 4.376849174499511719e+00 1495 | 4.428570747375488281e+00 1496 | 4.381678581237792969e+00 1497 | 4.390911579132080078e+00 1498 | 4.403464317321777344e+00 1499 | 4.366338729858398438e+00 1500 | 4.395590305328369141e+00 1501 | 4.375196456909179688e+00 1502 | 4.370644092559814453e+00 1503 | 4.383919239044189453e+00 1504 | 4.359629154205322266e+00 1505 | 4.373805522918701172e+00 1506 | 4.368338584899902344e+00 1507 | 4.356155872344970703e+00 1508 | 4.368763923645019531e+00 1509 | 4.356829643249511719e+00 1510 | 4.356975078582763672e+00 1511 | 4.361022949218750000e+00 1512 | 4.349890708923339844e+00 1513 | 4.355552673339843750e+00 1514 | 4.354362964630126953e+00 1515 | 4.346881866455078125e+00 1516 | 4.352142810821533203e+00 1517 | 4.348342895507812500e+00 1518 | 4.344732761383056641e+00 1519 | 4.348646163940429688e+00 1520 | 4.344466686248779297e+00 1521 | 4.342577934265136719e+00 1522 | 4.344564914703369141e+00 1523 | 4.341154098510742188e+00 1524 | 4.340509414672851562e+00 1525 | 4.341812610626220703e+00 1526 | 4.338718414306640625e+00 1527 | 4.337852954864501953e+00 1528 | 4.338691234588623047e+00 1529 | 4.336496353149414062e+00 1530 | 4.335760593414306641e+00 1531 | 4.336341857910156250e+00 1532 | 4.334637165069580078e+00 1533 | 4.333595752716064453e+00 1534 | 4.333880901336669922e+00 1535 | 4.332666397094726562e+00 1536 | 4.331507205963134766e+00 1537 | 4.331596851348876953e+00 1538 | 4.330915927886962891e+00 1539 | 4.329808235168457031e+00 1540 | 4.329609394073486328e+00 1541 | 4.329210281372070312e+00 1542 | 4.328181743621826172e+00 1543 | 4.327681541442871094e+00 1544 | 4.327448368072509766e+00 1545 | 4.326712131500244141e+00 1546 | 4.326019763946533203e+00 1547 | 4.325725078582763672e+00 1548 | 4.325323104858398438e+00 1549 | 4.324777603149414062e+00 1550 | 4.324593067169189453e+00 1551 | 4.324728012084960938e+00 1552 | 4.325125694274902344e+00 1553 | 4.326378345489501953e+00 1554 | 4.329506874084472656e+00 1555 | 4.335828304290771484e+00 1556 | 4.348924160003662109e+00 1557 | 4.374475002288818359e+00 1558 | 4.427103519439697266e+00 1559 | 4.521428585052490234e+00 1560 | 4.694630146026611328e+00 1561 | 4.903351783752441406e+00 1562 | 5.087869167327880859e+00 1563 | 4.921629428863525391e+00 1564 | 4.534266471862792969e+00 1565 | 4.367339134216308594e+00 1566 | 4.555346965789794922e+00 1567 | 4.653532981872558594e+00 1568 | 4.408173561096191406e+00 1569 | 4.373375892639160156e+00 1570 | 4.533735275268554688e+00 1571 | 4.427394866943359375e+00 1572 | 4.353897571563720703e+00 1573 | 4.459522724151611328e+00 1574 | 4.390659809112548828e+00 1575 | 4.347600460052490234e+00 1576 | 4.423805236816406250e+00 1577 | 4.362884044647216797e+00 1578 | 4.348896026611328125e+00 1579 | 4.395557880401611328e+00 1580 | 4.340564250946044922e+00 1581 | 4.354835987091064453e+00 1582 | 4.371921539306640625e+00 1583 | 4.328012943267822266e+00 1584 | 4.354157447814941406e+00 1585 | 4.350179195404052734e+00 1586 | 4.324922561645507812e+00 1587 | 4.348507404327392578e+00 1588 | 4.332593441009521484e+00 1589 | 4.324607372283935547e+00 1590 | 4.340319633483886719e+00 1591 | 4.321812152862548828e+00 1592 | 4.323613643646240234e+00 1593 | 4.330784797668457031e+00 1594 | 4.315652370452880859e+00 1595 | 4.321439743041992188e+00 1596 | 4.323184013366699219e+00 1597 | 4.311701297760009766e+00 1598 | 4.317746162414550781e+00 1599 | 4.317312717437744141e+00 1600 | 4.309206962585449219e+00 1601 | 4.313843727111816406e+00 1602 | 4.312731742858886719e+00 1603 | 4.306897640228271484e+00 1604 | 4.310261249542236328e+00 1605 | 4.309227943420410156e+00 1606 | 4.304623126983642578e+00 1607 | 4.306826591491699219e+00 1608 | 4.306557178497314453e+00 1609 | 4.302646160125732422e+00 1610 | 4.303552627563476562e+00 1611 | 4.304007530212402344e+00 1612 | 4.301081180572509766e+00 1613 | 4.300791740417480469e+00 1614 | 4.301451683044433594e+00 1615 | 4.299489498138427734e+00 1616 | 4.298495292663574219e+00 1617 | 4.299061775207519531e+00 1618 | 4.298004150390625000e+00 1619 | 4.296631336212158203e+00 1620 | 4.296794414520263672e+00 1621 | 4.296528816223144531e+00 1622 | 4.295193195343017578e+00 1623 | 4.294632434844970703e+00 1624 | 4.294632434844970703e+00 1625 | 4.293838977813720703e+00 1626 | 4.292921543121337891e+00 1627 | 4.292703628540039062e+00 1628 | 4.292396068572998047e+00 1629 | 4.291583538055419922e+00 1630 | 4.291050434112548828e+00 1631 | 4.291016101837158203e+00 1632 | 4.290880680084228516e+00 1633 | 4.290728569030761719e+00 1634 | 4.291344165802001953e+00 1635 | 4.293165683746337891e+00 1636 | 4.296864509582519531e+00 1637 | 4.304372310638427734e+00 1638 | 4.320652961730957031e+00 1639 | 4.352316379547119141e+00 1640 | 4.417976856231689453e+00 1641 | 4.528648853302001953e+00 1642 | 4.729313850402832031e+00 1643 | 4.901586055755615234e+00 1644 | 4.994346618652343750e+00 1645 | 4.699422836303710938e+00 1646 | 4.380630016326904297e+00 1647 | 4.367563724517822266e+00 1648 | 4.560102462768554688e+00 1649 | 4.543184757232666016e+00 1650 | 4.325875759124755859e+00 1651 | 4.365377426147460938e+00 1652 | 4.486151218414306641e+00 1653 | 4.355957984924316406e+00 1654 | 4.325977802276611328e+00 1655 | 4.422474384307861328e+00 1656 | 4.340415000915527344e+00 1657 | 4.311908721923828125e+00 1658 | 4.382230758666992188e+00 1659 | 4.322957038879394531e+00 1660 | 4.309131145477294922e+00 1661 | 4.353134155273437500e+00 1662 | 4.306250095367431641e+00 1663 | 4.309661388397216797e+00 1664 | 4.332764148712158203e+00 1665 | 4.294303417205810547e+00 1666 | 4.308455944061279297e+00 1667 | 4.316349029541015625e+00 1668 | 4.288240909576416016e+00 1669 | 4.304729938507080078e+00 1670 | 4.303359985351562500e+00 1671 | 4.285258769989013672e+00 1672 | 4.299831867218017578e+00 1673 | 4.293895244598388672e+00 1674 | 4.282503128051757812e+00 1675 | 4.294400691986083984e+00 1676 | 4.287653446197509766e+00 1677 | 4.280327796936035156e+00 1678 | 4.288517475128173828e+00 1679 | 4.282761096954345703e+00 1680 | 4.278052330017089844e+00 1681 | 4.284229278564453125e+00 1682 | 4.279457092285156250e+00 1683 | 4.275231361389160156e+00 1684 | 4.279725074768066406e+00 1685 | 4.276843070983886719e+00 1686 | 4.273012161254882812e+00 1687 | 4.276112556457519531e+00 1688 | 4.274893760681152344e+00 1689 | 4.271019458770751953e+00 1690 | 4.272420883178710938e+00 1691 | 4.272550106048583984e+00 1692 | 4.269415855407714844e+00 1693 | 4.269536972045898438e+00 1694 | 4.270399093627929688e+00 1695 | 4.268218040466308594e+00 1696 | 4.267162322998046875e+00 1697 | 4.267971038818359375e+00 1698 | 4.266978740692138672e+00 1699 | 4.265387058258056641e+00 1700 | 4.265574932098388672e+00 1701 | 4.265510559082031250e+00 1702 | 4.264169216156005859e+00 1703 | 4.263567447662353516e+00 1704 | 4.263777256011962891e+00 1705 | 4.263171672821044922e+00 1706 | 4.262231826782226562e+00 1707 | 4.262216091156005859e+00 1708 | 4.262477397918701172e+00 1709 | 4.262331962585449219e+00 1710 | 4.262794971466064453e+00 1711 | 4.264820098876953125e+00 1712 | 4.268819808959960938e+00 1713 | 4.276181697845458984e+00 1714 | 4.292026996612548828e+00 1715 | 4.322390079498291016e+00 1716 | 4.385117530822753906e+00 1717 | 4.489491462707519531e+00 1718 | 4.689200401306152344e+00 1719 | 4.870695590972900391e+00 1720 | 5.052803039550781250e+00 1721 | 4.758959770202636719e+00 1722 | 4.385478496551513672e+00 1723 | 4.293027877807617188e+00 1724 | 4.504136085510253906e+00 1725 | 4.543236255645751953e+00 1726 | 4.305654525756835938e+00 1727 | 4.335693836212158203e+00 1728 | 4.464565277099609375e+00 1729 | 4.321945667266845703e+00 1730 | 4.299439907073974609e+00 1731 | 4.398251533508300781e+00 1732 | 4.303803920745849609e+00 1733 | 4.294664382934570312e+00 1734 | 4.358414173126220703e+00 1735 | 4.283718109130859375e+00 1736 | 4.297547340393066406e+00 1737 | 4.326462745666503906e+00 1738 | 4.270224094390869141e+00 1739 | 4.300360679626464844e+00 1740 | 4.299538612365722656e+00 1741 | 4.264889240264892578e+00 1742 | 4.297169208526611328e+00 1743 | 4.279003620147705078e+00 1744 | 4.264434337615966797e+00 1745 | 4.288721561431884766e+00 1746 | 4.265311241149902344e+00 1747 | 4.264751911163330078e+00 1748 | 4.278138637542724609e+00 1749 | 4.257793903350830078e+00 1750 | 4.263024806976318359e+00 1751 | 4.268379688262939453e+00 1752 | 4.253789424896240234e+00 1753 | 4.260034084320068359e+00 1754 | 4.260919094085693359e+00 1755 | 4.250780582427978516e+00 1756 | 4.256278991699218750e+00 1757 | 4.255755901336669922e+00 1758 | 4.248471736907958984e+00 1759 | 4.252386569976806641e+00 1760 | 4.251786708831787109e+00 1761 | 4.246490955352783203e+00 1762 | 4.248923778533935547e+00 1763 | 4.248689174652099609e+00 1764 | 4.244640350341796875e+00 1765 | 4.245776176452636719e+00 1766 | 4.246156692504882812e+00 1767 | 4.243039131164550781e+00 1768 | 4.243026256561279297e+00 1769 | 4.243731021881103516e+00 1770 | 4.241573810577392578e+00 1771 | 4.240762710571289062e+00 1772 | 4.241474628448486328e+00 1773 | 4.240182399749755859e+00 1774 | 4.238852977752685547e+00 1775 | 4.239253044128417969e+00 1776 | 4.238832950592041016e+00 1777 | 4.237423896789550781e+00 1778 | 4.237128257751464844e+00 1779 | 4.237167358398437500e+00 1780 | 4.236200332641601562e+00 1781 | 4.235399723052978516e+00 1782 | 4.235372543334960938e+00 1783 | 4.234965324401855469e+00 1784 | 4.234076499938964844e+00 1785 | 4.233655929565429688e+00 1786 | 4.233504772186279297e+00 1787 | 4.232930183410644531e+00 1788 | 4.232240676879882812e+00 1789 | 4.231953620910644531e+00 1790 | 4.231746196746826172e+00 1791 | 4.231280326843261719e+00 1792 | 4.230919837951660156e+00 1793 | 4.231033325195312500e+00 1794 | 4.231596946716308594e+00 1795 | 4.232962131500244141e+00 1796 | 4.236515522003173828e+00 1797 | 4.244945049285888672e+00 1798 | 4.263739585876464844e+00 1799 | 4.304735660552978516e+00 1800 | 4.393502712249755859e+00 1801 | 4.570837974548339844e+00 1802 | 4.884840011596679688e+00 1803 | 5.246020317077636719e+00 1804 | 5.271576404571533203e+00 1805 | 4.735482215881347656e+00 1806 | 4.275839328765869141e+00 1807 | 4.540633678436279297e+00 1808 | 4.730338573455810547e+00 1809 | 4.336953639984130859e+00 1810 | 4.368050098419189453e+00 1811 | 4.560486793518066406e+00 1812 | 4.307520389556884766e+00 1813 | 4.381657123565673828e+00 1814 | 4.439887046813964844e+00 1815 | 4.272105216979980469e+00 1816 | 4.404030799865722656e+00 1817 | 4.322122573852539062e+00 1818 | 4.299555301666259766e+00 1819 | 4.353624343872070312e+00 1820 | 4.263605594635009766e+00 1821 | 4.330924034118652344e+00 1822 | 4.273667335510253906e+00 1823 | 4.286627769470214844e+00 1824 | 4.294290065765380859e+00 1825 | 4.255851268768310547e+00 1826 | 4.292375087738037109e+00 1827 | 4.248393535614013672e+00 1828 | 4.274527072906494141e+00 1829 | 4.256336688995361328e+00 1830 | 4.249199390411376953e+00 1831 | 4.264770030975341797e+00 1832 | 4.237439155578613281e+00 1833 | 4.256568908691406250e+00 1834 | 4.240186214447021484e+00 1835 | 4.241662979125976562e+00 1836 | 4.244588375091552734e+00 1837 | 4.231637477874755859e+00 1838 | 4.241390705108642578e+00 1839 | 4.230925083160400391e+00 1840 | 4.232724666595458984e+00 1841 | 4.233439445495605469e+00 1842 | 4.225465297698974609e+00 1843 | 4.231259822845458984e+00 1844 | 4.225269794464111328e+00 1845 | 4.224790096282958984e+00 1846 | 4.226362228393554688e+00 1847 | 4.220660209655761719e+00 1848 | 4.223240375518798828e+00 1849 | 4.221365451812744141e+00 1850 | 4.218672752380371094e+00 1851 | 4.220625877380371094e+00 1852 | 4.217504024505615234e+00 1853 | 4.216983318328857422e+00 1854 | 4.217590332031250000e+00 1855 | 4.214954376220703125e+00 1856 | 4.215353012084960938e+00 1857 | 4.214861392974853516e+00 1858 | 4.212845802307128906e+00 1859 | 4.213447570800781250e+00 1860 | 4.212537288665771484e+00 1861 | 4.211188316345214844e+00 1862 | 4.211575508117675781e+00 1863 | 4.210598945617675781e+00 1864 | 4.209646701812744141e+00 1865 | 4.209766864776611328e+00 1866 | 4.208854675292968750e+00 1867 | 4.208117961883544922e+00 1868 | 4.208101749420166016e+00 1869 | 4.207294464111328125e+00 1870 | 4.206679344177246094e+00 1871 | 4.206592559814453125e+00 1872 | 4.205887317657470703e+00 1873 | 4.205266475677490234e+00 1874 | 4.205123424530029297e+00 1875 | 4.204557418823242188e+00 1876 | 4.203941822052001953e+00 1877 | 4.203709125518798828e+00 1878 | 4.203262805938720703e+00 1879 | 4.202695846557617188e+00 1880 | 4.202424526214599609e+00 1881 | 4.202079772949218750e+00 1882 | 4.201606273651123047e+00 1883 | 4.201375484466552734e+00 1884 | 4.201253414154052734e+00 1885 | 4.201131343841552734e+00 1886 | 4.201426506042480469e+00 1887 | 4.202370166778564453e+00 1888 | 4.204364776611328125e+00 1889 | 4.208428859710693359e+00 1890 | 4.217088222503662109e+00 1891 | 4.233520507812500000e+00 1892 | 4.267737388610839844e+00 1893 | 4.328330516815185547e+00 1894 | 4.453203201293945312e+00 1895 | 4.616654872894287109e+00 1896 | 4.890931129455566406e+00 1897 | 4.902413368225097656e+00 1898 | 4.735986232757568359e+00 1899 | 4.322064399719238281e+00 1900 | 4.249467372894287109e+00 1901 | 4.480819225311279297e+00 1902 | 4.455446720123291016e+00 1903 | 4.238830089569091797e+00 1904 | 4.272242069244384766e+00 1905 | 4.384428977966308594e+00 1906 | 4.270188808441162109e+00 1907 | 4.230468750000000000e+00 1908 | 4.324105262756347656e+00 1909 | 4.257293701171875000e+00 1910 | 4.227354526519775391e+00 1911 | 4.292438030242919922e+00 1912 | 4.232313632965087891e+00 1913 | 4.228256702423095703e+00 1914 | 4.268660545349121094e+00 1915 | 4.214842319488525391e+00 1916 | 4.229753017425537109e+00 1917 | 4.244956970214843750e+00 1918 | 4.205665588378906250e+00 1919 | 4.229008674621582031e+00 1920 | 4.224762916564941406e+00 1921 | 4.202528476715087891e+00 1922 | 4.224028110504150391e+00 1923 | 4.210231781005859375e+00 1924 | 4.202445983886718750e+00 1925 | 4.216421604156494141e+00 1926 | 4.201055049896240234e+00 1927 | 4.202068805694580078e+00 1928 | 4.209125995635986328e+00 1929 | 4.195855140686035156e+00 1930 | 4.199938297271728516e+00 1931 | 4.202595233917236328e+00 1932 | 4.192883014678955078e+00 1933 | 4.197345256805419922e+00 1934 | 4.197458267211914062e+00 1935 | 4.190339565277099609e+00 1936 | 4.194266319274902344e+00 1937 | 4.193928241729736328e+00 1938 | 4.188481330871582031e+00 1939 | 4.191061019897460938e+00 1940 | 4.190867900848388672e+00 1941 | 4.186731815338134766e+00 1942 | 4.188334465026855469e+00 1943 | 4.188589572906494141e+00 1944 | 4.185204982757568359e+00 1945 | 4.185659885406494141e+00 1946 | 4.186430931091308594e+00 1947 | 4.183945178985595703e+00 1948 | 4.183393001556396484e+00 1949 | 4.184308528900146484e+00 1950 | 4.182862281799316406e+00 1951 | 4.181698322296142578e+00 1952 | 4.182200908660888672e+00 1953 | 4.181532859802246094e+00 1954 | 4.180246353149414062e+00 1955 | 4.180301189422607422e+00 1956 | 4.180219173431396484e+00 1957 | 4.179107666015625000e+00 1958 | 4.178606033325195312e+00 1959 | 4.178731441497802734e+00 1960 | 4.178142070770263672e+00 1961 | 4.177324295043945312e+00 1962 | 4.177145957946777344e+00 1963 | 4.176947116851806641e+00 1964 | 4.176257133483886719e+00 1965 | 4.175760746002197266e+00 1966 | 4.175664901733398438e+00 1967 | 4.175405979156494141e+00 1968 | 4.175002574920654297e+00 1969 | 4.175046443939208984e+00 1970 | 4.175745964050292969e+00 1971 | 4.177338600158691406e+00 1972 | 4.181194305419921875e+00 1973 | 4.190629482269287109e+00 1974 | 4.212525844573974609e+00 1975 | 4.261251449584960938e+00 1976 | 4.370359420776367188e+00 1977 | 4.584971427917480469e+00 1978 | 4.953085422515869141e+00 1979 | 5.246942996978759766e+00 1980 | 5.002623081207275391e+00 1981 | 4.409096240997314453e+00 1982 | 4.241328716278076172e+00 1983 | 4.573668003082275391e+00 1984 | 4.531551361083984375e+00 1985 | 4.211771011352539062e+00 1986 | 4.410172462463378906e+00 1987 | 4.433157920837402344e+00 1988 | 4.209123611450195312e+00 1989 | 4.395395755767822266e+00 1990 | 4.311042308807373047e+00 1991 | 4.241856575012207031e+00 1992 | 4.359701633453369141e+00 1993 | 4.216960430145263672e+00 1994 | 4.293504238128662109e+00 1995 | 4.263719081878662109e+00 1996 | 4.218924999237060547e+00 1997 | 4.281784057617187500e+00 1998 | 4.206163883209228516e+00 1999 | 4.253441333770751953e+00 2000 | 4.217480182647705078e+00 2001 | -------------------------------------------------------------------------------- /data/losses_exact_sgd.csv: -------------------------------------------------------------------------------- 1 | 9.754241943359375000e+01 2 | 3.373418426513671875e+01 3 | 3.338163375854492188e+01 4 | 4.785203170776367188e+01 5 | 6.008721160888671875e+01 6 | 5.698876571655273438e+01 7 | 5.661663436889648438e+01 8 | 5.635047531127929688e+01 9 | 5.715383148193359375e+01 10 | 5.607833099365234375e+01 11 | 5.569179916381835938e+01 12 | 5.574220657348632812e+01 13 | 5.581438064575195312e+01 14 | 5.552339935302734375e+01 15 | 5.547266769409179688e+01 16 | 5.628377532958984375e+01 17 | 5.613060379028320312e+01 18 | 5.600774765014648438e+01 19 | 5.608127975463867188e+01 20 | 5.620303726196289062e+01 21 | 5.609058761596679688e+01 22 | 5.609500503540039062e+01 23 | 5.594451141357421875e+01 24 | 5.569974899291992188e+01 25 | 5.575409698486328125e+01 26 | 5.575279235839843750e+01 27 | 5.570509338378906250e+01 28 | 5.566989135742187500e+01 29 | 5.551887893676757812e+01 30 | 5.557960510253906250e+01 31 | 5.562090301513671875e+01 32 | 5.629791259765625000e+01 33 | 5.617068099975585938e+01 34 | 5.598530960083007812e+01 35 | 5.583619689941406250e+01 36 | 5.576497268676757812e+01 37 | 5.585448455810546875e+01 38 | 5.582559204101562500e+01 39 | 5.556398773193359375e+01 40 | 5.537476730346679688e+01 41 | 5.538523101806640625e+01 42 | 5.542067337036132812e+01 43 | 5.541941833496093750e+01 44 | 5.535263061523437500e+01 45 | 5.530610656738281250e+01 46 | 5.584314346313476562e+01 47 | 5.638194274902343750e+01 48 | 5.555432891845703125e+01 49 | 5.522623062133789062e+01 50 | 5.638920974731445312e+01 51 | 5.689710617065429688e+01 52 | 5.605252456665039062e+01 53 | 5.579582977294921875e+01 54 | 5.607145309448242188e+01 55 | 5.583894348144531250e+01 56 | 5.584610366821289062e+01 57 | 5.635843658447265625e+01 58 | 5.599925613403320312e+01 59 | 5.589511871337890625e+01 60 | 5.584754943847656250e+01 61 | 5.607889938354492188e+01 62 | 5.583192443847656250e+01 63 | 5.568931198120117188e+01 64 | 5.582242202758789062e+01 65 | 5.562643051147460938e+01 66 | 5.549168777465820312e+01 67 | 5.543852996826171875e+01 68 | 5.534059143066406250e+01 69 | 5.532194900512695312e+01 70 | 5.521541595458984375e+01 71 | 5.533280563354492188e+01 72 | 5.510844802856445312e+01 73 | 5.504130554199218750e+01 74 | 5.498759078979492188e+01 75 | 5.495623016357421875e+01 76 | 5.477285385131835938e+01 77 | 5.475492477416992188e+01 78 | 5.458019256591796875e+01 79 | 5.457006835937500000e+01 80 | 5.467961883544921875e+01 81 | 5.461753082275390625e+01 82 | 5.450389099121093750e+01 83 | 5.446460342407226562e+01 84 | 5.459606552124023438e+01 85 | 5.466251754760742188e+01 86 | 5.437839126586914062e+01 87 | 5.468621063232421875e+01 88 | 5.445061874389648438e+01 89 | 5.436366653442382812e+01 90 | 5.434947204589843750e+01 91 | 5.442234420776367188e+01 92 | 5.433013153076171875e+01 93 | 5.447443008422851562e+01 94 | 5.471135330200195312e+01 95 | 5.482384872436523438e+01 96 | 5.447554397583007812e+01 97 | 5.439596557617187500e+01 98 | 5.471475601196289062e+01 99 | 5.459124755859375000e+01 100 | 5.431396102905273438e+01 101 | -------------------------------------------------------------------------------- /data/losses_stochastic_sgd.csv: -------------------------------------------------------------------------------- 1 | 9.754241943359375000e+01 2 | 6.233982849121093750e+01 3 | 4.486039352416992188e+01 4 | 7.903101348876953125e+01 5 | 5.605532455444335938e+01 6 | 4.820644760131835938e+01 7 | 4.393406677246093750e+01 8 | 4.030270004272460938e+01 9 | 3.837119674682617188e+01 10 | 3.878179550170898438e+01 11 | 3.613414764404296875e+01 12 | 3.038554382324218750e+01 13 | 2.694277381896972656e+01 14 | 2.406844902038574219e+01 15 | 2.146273803710937500e+01 16 | 1.772295188903808594e+01 17 | 1.575095939636230469e+01 18 | 1.305262660980224609e+01 19 | 1.325567817687988281e+01 20 | 1.078114223480224609e+01 21 | 1.110317230224609375e+01 22 | 8.262114524841308594e+00 23 | 8.280516624450683594e+00 24 | 6.578242778778076172e+00 25 | 6.571306228637695312e+00 26 | 5.154974937438964844e+00 27 | 4.566923618316650391e+00 28 | 3.822735548019409180e+00 29 | 3.431006193161010742e+00 30 | 3.033493757247924805e+00 31 | 2.862568378448486328e+00 32 | 2.665788173675537109e+00 33 | 2.570884704589843750e+00 34 | 2.350254297256469727e+00 35 | 2.148637294769287109e+00 36 | 1.883123397827148438e+00 37 | 1.710891723632812500e+00 38 | 1.551024198532104492e+00 39 | 1.448339462280273438e+00 40 | 1.352167487144470215e+00 41 | 1.270755887031555176e+00 42 | 1.182353138923645020e+00 43 | 1.107083439826965332e+00 44 | 1.041556477546691895e+00 45 | 9.970846176147460938e-01 46 | 9.658975601196289062e-01 47 | 9.502717256546020508e-01 48 | 9.147406220436096191e-01 49 | 8.819558620452880859e-01 50 | 8.420690298080444336e-01 51 | 8.079715371131896973e-01 52 | 7.776800990104675293e-01 53 | 7.503719329833984375e-01 54 | 7.335494160652160645e-01 55 | 7.142430543899536133e-01 56 | 7.057768702507019043e-01 57 | 6.916821599006652832e-01 58 | 6.828399896621704102e-01 59 | 6.619627475738525391e-01 60 | 6.486747860908508301e-01 61 | 6.281179785728454590e-01 62 | 6.086058020591735840e-01 63 | 5.864923000335693359e-01 64 | 5.661154389381408691e-01 65 | 5.490093827247619629e-01 66 | 5.300320386886596680e-01 67 | 5.162346363067626953e-01 68 | 5.030936598777770996e-01 69 | 4.962349832057952881e-01 70 | 4.882148802280426025e-01 71 | 4.898423850536346436e-01 72 | 4.801509678363800049e-01 73 | 4.749349951744079590e-01 74 | 4.647877216339111328e-01 75 | 4.640685021877288818e-01 76 | 4.575448632240295410e-01 77 | 4.573506116867065430e-01 78 | 4.489282071590423584e-01 79 | 4.459122419357299805e-01 80 | 4.367396831512451172e-01 81 | 4.305608868598937988e-01 82 | 4.187212884426116943e-01 83 | 4.123390614986419678e-01 84 | 4.045809209346771240e-01 85 | 4.053118526935577393e-01 86 | 4.048214852809906006e-01 87 | 4.124225974082946777e-01 88 | 4.150598645210266113e-01 89 | 4.189287126064300537e-01 90 | 4.116005301475524902e-01 91 | 4.042877256870269775e-01 92 | 3.918527066707611084e-01 93 | 3.782193362712860107e-01 94 | 3.642461299896240234e-01 95 | 3.523639738559722900e-01 96 | 3.381671607494354248e-01 97 | 3.282057046890258789e-01 98 | 3.184003829956054688e-01 99 | 3.130303621292114258e-01 100 | 3.098244667053222656e-01 101 | -------------------------------------------------------------------------------- /data/vlosses_adam2.csv: -------------------------------------------------------------------------------- 1 | 4.362622451782226562e+01 2 | 4.288741683959960938e+01 3 | 4.188721847534179688e+01 4 | 4.065217208862304688e+01 5 | 3.924925613403320312e+01 6 | 3.780171203613281250e+01 7 | 3.651281738281250000e+01 8 | 3.566295242309570312e+01 9 | 3.518271636962890625e+01 10 | 3.451684951782226562e+01 11 | 3.356711196899414062e+01 12 | 3.260386657714843750e+01 13 | 3.183946228027343750e+01 14 | 3.130952453613281250e+01 15 | 3.091625213623046875e+01 16 | 3.055189895629882812e+01 17 | 3.018450546264648438e+01 18 | 2.983443069458007812e+01 19 | 2.954754638671875000e+01 20 | 2.933464622497558594e+01 21 | 2.914039039611816406e+01 22 | 2.891691398620605469e+01 23 | 2.867391395568847656e+01 24 | 2.845273971557617188e+01 25 | 2.827309036254882812e+01 26 | 2.811838912963867188e+01 27 | 2.796207046508789062e+01 28 | 2.779072380065917969e+01 29 | 2.761067771911621094e+01 30 | 2.744024085998535156e+01 31 | 2.728076553344726562e+01 32 | 2.711328697204589844e+01 33 | 2.692319679260253906e+01 34 | 2.671866226196289062e+01 35 | 2.651561164855957031e+01 36 | 2.631387519836425781e+01 37 | 2.610231590270996094e+01 38 | 2.587748908996582031e+01 39 | 2.564849090576171875e+01 40 | 2.542196273803710938e+01 41 | 2.519144248962402344e+01 42 | 2.495092201232910156e+01 43 | 2.470500564575195312e+01 44 | 2.445885276794433594e+01 45 | 2.420819282531738281e+01 46 | 2.394636154174804688e+01 47 | 2.367654418945312500e+01 48 | 2.340136337280273438e+01 49 | 2.311517524719238281e+01 50 | 2.281396293640136719e+01 51 | 2.250206375122070312e+01 52 | 2.218077087402343750e+01 53 | 2.185028839111328125e+01 54 | 2.152459144592285156e+01 55 | 2.121223640441894531e+01 56 | 2.090874099731445312e+01 57 | 2.060059547424316406e+01 58 | 2.028755760192871094e+01 59 | 1.998360061645507812e+01 60 | 1.968732452392578125e+01 61 | 1.939007186889648438e+01 62 | 1.909013175964355469e+01 63 | 1.879619979858398438e+01 64 | 1.850971221923828125e+01 65 | 1.821783065795898438e+01 66 | 1.791146278381347656e+01 67 | 1.759603881835937500e+01 68 | 1.729288101196289062e+01 69 | 1.702598381042480469e+01 70 | 1.677811431884765625e+01 71 | 1.652683639526367188e+01 72 | 1.628781318664550781e+01 73 | 1.605965995788574219e+01 74 | 1.583410263061523438e+01 75 | 1.561237907409667969e+01 76 | 1.542302322387695312e+01 77 | 1.527816009521484375e+01 78 | 1.504658126831054688e+01 79 | 1.475015735626220703e+01 80 | 1.463831043243408203e+01 81 | 1.448255920410156250e+01 82 | 1.422093105316162109e+01 83 | 1.411827468872070312e+01 84 | 1.399711036682128906e+01 85 | 1.376119518280029297e+01 86 | 1.363776683807373047e+01 87 | 1.351807117462158203e+01 88 | 1.331018543243408203e+01 89 | 1.318353748321533203e+01 90 | 1.308366870880126953e+01 91 | 1.290831279754638672e+01 92 | 1.275647163391113281e+01 93 | 1.266802120208740234e+01 94 | 1.256402111053466797e+01 95 | 1.245241928100585938e+01 96 | 1.239358997344970703e+01 97 | 1.243437480926513672e+01 98 | 1.223922538757324219e+01 99 | 1.201407432556152344e+01 100 | 1.208628177642822266e+01 101 | 1.188914108276367188e+01 102 | 1.173028373718261719e+01 103 | 1.175395584106445312e+01 104 | 1.154404926300048828e+01 105 | 1.153087043762207031e+01 106 | 1.144824218750000000e+01 107 | 1.128351497650146484e+01 108 | 1.129831600189208984e+01 109 | 1.115702819824218750e+01 110 | 1.108696460723876953e+01 111 | 1.104163837432861328e+01 112 | 1.090751838684082031e+01 113 | 1.087923812866210938e+01 114 | 1.079918098449707031e+01 115 | 1.071294403076171875e+01 116 | 1.069341850280761719e+01 117 | 1.063115215301513672e+01 118 | 1.063531780242919922e+01 119 | 1.073174762725830078e+01 120 | 1.074705123901367188e+01 121 | 1.075945854187011719e+01 122 | 1.057664680480957031e+01 123 | 1.046607875823974609e+01 124 | 1.039190483093261719e+01 125 | 1.035132026672363281e+01 126 | 1.027544879913330078e+01 127 | 1.015482234954833984e+01 128 | 1.020259571075439453e+01 129 | 1.001935577392578125e+01 130 | 1.004511356353759766e+01 131 | 9.972114562988281250e+00 132 | 9.892333030700683594e+00 133 | 9.906401634216308594e+00 134 | 9.787626266479492188e+00 135 | 9.788990020751953125e+00 136 | 9.717454910278320312e+00 137 | 9.679726600646972656e+00 138 | 9.629004478454589844e+00 139 | 9.594731330871582031e+00 140 | 9.537680625915527344e+00 141 | 9.496881484985351562e+00 142 | 9.459454536437988281e+00 143 | 9.414511680603027344e+00 144 | 9.370514869689941406e+00 145 | 9.333345413208007812e+00 146 | 9.304446220397949219e+00 147 | 9.251781463623046875e+00 148 | 9.232881546020507812e+00 149 | 9.231882095336914062e+00 150 | 9.254623413085937500e+00 151 | 9.384944915771484375e+00 152 | 9.483821868896484375e+00 153 | 9.382092475891113281e+00 154 | 9.070265769958496094e+00 155 | 9.051414489746093750e+00 156 | 9.184251785278320312e+00 157 | 9.049160003662109375e+00 158 | 8.928410530090332031e+00 159 | 8.991158485412597656e+00 160 | 8.937932968139648438e+00 161 | 8.829620361328125000e+00 162 | 8.865362167358398438e+00 163 | 8.832636833190917969e+00 164 | 8.729436874389648438e+00 165 | 8.750551223754882812e+00 166 | 8.736201286315917969e+00 167 | 8.648792266845703125e+00 168 | 8.639883995056152344e+00 169 | 8.631838798522949219e+00 170 | 8.573830604553222656e+00 171 | 8.548410415649414062e+00 172 | 8.540390014648437500e+00 173 | 8.494914054870605469e+00 174 | 8.452476501464843750e+00 175 | 8.446691513061523438e+00 176 | 8.426490783691406250e+00 177 | 8.383017539978027344e+00 178 | 8.359979629516601562e+00 179 | 8.346921920776367188e+00 180 | 8.334107398986816406e+00 181 | 8.309964179992675781e+00 182 | 8.351840972900390625e+00 183 | 8.514688491821289062e+00 184 | 8.845320701599121094e+00 185 | 8.907490730285644531e+00 186 | 8.476520538330078125e+00 187 | 8.361120223999023438e+00 188 | 8.607028007507324219e+00 189 | 8.357594490051269531e+00 190 | 8.207818984985351562e+00 191 | 8.421295166015625000e+00 192 | 8.178478240966796875e+00 193 | 8.237353324890136719e+00 194 | 8.197097778320312500e+00 195 | 8.072999954223632812e+00 196 | 8.194949150085449219e+00 197 | 8.021104812622070312e+00 198 | 8.095483779907226562e+00 199 | 8.020039558410644531e+00 200 | 8.010582923889160156e+00 201 | 8.002468109130859375e+00 202 | 7.936401367187500000e+00 203 | 7.967220306396484375e+00 204 | 7.897145748138427734e+00 205 | 7.916869163513183594e+00 206 | 7.875406265258789062e+00 207 | 7.852373123168945312e+00 208 | 7.846636295318603516e+00 209 | 7.807757377624511719e+00 210 | 7.800819396972656250e+00 211 | 7.756642818450927734e+00 212 | 7.723410606384277344e+00 213 | 7.716170310974121094e+00 214 | 7.681695938110351562e+00 215 | 7.664362430572509766e+00 216 | 7.636869430541992188e+00 217 | 7.614127159118652344e+00 218 | 7.593970775604248047e+00 219 | 7.573636531829833984e+00 220 | 7.551678657531738281e+00 221 | 7.539579391479492188e+00 222 | 7.525465488433837891e+00 223 | 7.542223453521728516e+00 224 | 7.570854663848876953e+00 225 | 7.677137374877929688e+00 226 | 7.804368495941162109e+00 227 | 7.963166236877441406e+00 228 | 7.760676383972167969e+00 229 | 7.466996669769287109e+00 230 | 7.419018745422363281e+00 231 | 7.586300373077392578e+00 232 | 7.578344345092773438e+00 233 | 7.357042789459228516e+00 234 | 7.389535903930664062e+00 235 | 7.497903823852539062e+00 236 | 7.359483242034912109e+00 237 | 7.298059940338134766e+00 238 | 7.383298397064208984e+00 239 | 7.331093788146972656e+00 240 | 7.249598979949951172e+00 241 | 7.291160106658935547e+00 242 | 7.285202980041503906e+00 243 | 7.216909885406494141e+00 244 | 7.219044208526611328e+00 245 | 7.234040260314941406e+00 246 | 7.190056800842285156e+00 247 | 7.161870956420898438e+00 248 | 7.178641319274902344e+00 249 | 7.162136554718017578e+00 250 | 7.120073318481445312e+00 251 | 7.123180866241455078e+00 252 | 7.130170345306396484e+00 253 | 7.094416141510009766e+00 254 | 7.081117153167724609e+00 255 | 7.100483417510986328e+00 256 | 7.110976219177246094e+00 257 | 7.145317077636718750e+00 258 | 7.251156806945800781e+00 259 | 7.437873840332031250e+00 260 | 7.535180091857910156e+00 261 | 7.312277793884277344e+00 262 | 7.070209980010986328e+00 263 | 7.152225494384765625e+00 264 | 7.294938087463378906e+00 265 | 7.142024993896484375e+00 266 | 7.023557662963867188e+00 267 | 7.108603000640869141e+00 268 | 7.067409038543701172e+00 269 | 6.981475830078125000e+00 270 | 7.025381088256835938e+00 271 | 7.003100395202636719e+00 272 | 6.947898387908935547e+00 273 | 6.973350524902343750e+00 274 | 6.945524215698242188e+00 275 | 6.910274028778076172e+00 276 | 6.930262088775634766e+00 277 | 6.897329330444335938e+00 278 | 6.870611667633056641e+00 279 | 6.892113208770751953e+00 280 | 6.862953662872314453e+00 281 | 6.836262226104736328e+00 282 | 6.854634284973144531e+00 283 | 6.831970214843750000e+00 284 | 6.799245834350585938e+00 285 | 6.811847686767578125e+00 286 | 6.804360866546630859e+00 287 | 6.773519515991210938e+00 288 | 6.776513099670410156e+00 289 | 6.780718803405761719e+00 290 | 6.766175746917724609e+00 291 | 6.776313304901123047e+00 292 | 6.821744918823242188e+00 293 | 6.911623477935791016e+00 294 | 7.123664855957031250e+00 295 | 7.424348354339599609e+00 296 | 7.428917884826660156e+00 297 | 6.987270832061767578e+00 298 | 6.800074100494384766e+00 299 | 7.073730468750000000e+00 300 | 6.998863220214843750e+00 301 | 6.710223197937011719e+00 302 | 6.881329536437988281e+00 303 | 6.899288177490234375e+00 304 | 6.693470001220703125e+00 305 | 6.815494537353515625e+00 306 | 6.777169227600097656e+00 307 | 6.690065383911132812e+00 308 | 6.767279624938964844e+00 309 | 6.676544189453125000e+00 310 | 6.688284397125244141e+00 311 | 6.701010704040527344e+00 312 | 6.626390457153320312e+00 313 | 6.666301250457763672e+00 314 | 6.631647586822509766e+00 315 | 6.612575531005859375e+00 316 | 6.626460075378417969e+00 317 | 6.584414005279541016e+00 318 | 6.598698139190673828e+00 319 | 6.583851337432861328e+00 320 | 6.554516315460205078e+00 321 | 6.572847366333007812e+00 322 | 6.548372745513916016e+00 323 | 6.534819602966308594e+00 324 | 6.542916774749755859e+00 325 | 6.514668464660644531e+00 326 | 6.512663364410400391e+00 327 | 6.516602039337158203e+00 328 | 6.489861011505126953e+00 329 | 6.487677574157714844e+00 330 | 6.486748695373535156e+00 331 | 6.467088699340820312e+00 332 | 6.465498447418212891e+00 333 | 6.463305473327636719e+00 334 | 6.448236465454101562e+00 335 | 6.441931247711181641e+00 336 | 6.441221714019775391e+00 337 | 6.433584690093994141e+00 338 | 6.441327095031738281e+00 339 | 6.483454704284667969e+00 340 | 6.605476379394531250e+00 341 | 6.934345245361328125e+00 342 | 7.591194152832031250e+00 343 | 7.660656929016113281e+00 344 | 6.806327342987060547e+00 345 | 6.552890300750732422e+00 346 | 7.097515583038330078e+00 347 | 6.771188735961914062e+00 348 | 6.494230270385742188e+00 349 | 6.902465343475341797e+00 350 | 6.497015953063964844e+00 351 | 6.636683464050292969e+00 352 | 6.619673252105712891e+00 353 | 6.429080963134765625e+00 354 | 6.635546684265136719e+00 355 | 6.396075725555419922e+00 356 | 6.561185836791992188e+00 357 | 6.394481658935546875e+00 358 | 6.481700897216796875e+00 359 | 6.407533645629882812e+00 360 | 6.408164501190185547e+00 361 | 6.414157867431640625e+00 362 | 6.363628864288330078e+00 363 | 6.404615402221679688e+00 364 | 6.329302310943603516e+00 365 | 6.385368347167968750e+00 366 | 6.314674854278564453e+00 367 | 6.355514526367187500e+00 368 | 6.301551818847656250e+00 369 | 6.329147815704345703e+00 370 | 6.295464038848876953e+00 371 | 6.296343326568603516e+00 372 | 6.290860176086425781e+00 373 | 6.271875381469726562e+00 374 | 6.282343387603759766e+00 375 | 6.252013683319091797e+00 376 | 6.265275001525878906e+00 377 | 6.242502689361572266e+00 378 | 6.244582176208496094e+00 379 | 6.235699653625488281e+00 380 | 6.225198268890380859e+00 381 | 6.225467205047607422e+00 382 | 6.210343360900878906e+00 383 | 6.211651325225830078e+00 384 | 6.201005935668945312e+00 385 | 6.195336818695068359e+00 386 | 6.193076610565185547e+00 387 | 6.181621074676513672e+00 388 | 6.179059028625488281e+00 389 | 6.172305583953857422e+00 390 | 6.165505409240722656e+00 391 | 6.161424160003662109e+00 392 | 6.154163360595703125e+00 393 | 6.149265289306640625e+00 394 | 6.144303321838378906e+00 395 | 6.137583255767822266e+00 396 | 6.133799552917480469e+00 397 | 6.128201484680175781e+00 398 | 6.124061107635498047e+00 399 | 6.123012542724609375e+00 400 | 6.129261493682861328e+00 401 | 6.154609680175781250e+00 402 | 6.246123313903808594e+00 403 | 6.512317180633544922e+00 404 | 7.075206756591796875e+00 405 | 7.779830455780029297e+00 406 | 7.060081958770751953e+00 407 | 6.180658817291259766e+00 408 | 6.812473297119140625e+00 409 | 6.775917053222656250e+00 410 | 6.187295436859130859e+00 411 | 6.662968635559082031e+00 412 | 6.312007904052734375e+00 413 | 6.353498935699462891e+00 414 | 6.396334648132324219e+00 415 | 6.209980010986328125e+00 416 | 6.373075485229492188e+00 417 | 6.158115386962890625e+00 418 | 6.314600944519042969e+00 419 | 6.128636360168457031e+00 420 | 6.257113456726074219e+00 421 | 6.134385108947753906e+00 422 | 6.194214820861816406e+00 423 | 6.121145725250244141e+00 424 | 6.158264636993408203e+00 425 | 6.109912395477294922e+00 426 | 6.125310420989990234e+00 427 | 6.093906879425048828e+00 428 | 6.101958274841308594e+00 429 | 6.077590465545654297e+00 430 | 6.076428413391113281e+00 431 | 6.064935207366943359e+00 432 | 6.059425354003906250e+00 433 | 6.048948764801025391e+00 434 | 6.041346549987792969e+00 435 | 6.034031867980957031e+00 436 | 6.026788711547851562e+00 437 | 6.022289276123046875e+00 438 | 6.010869979858398438e+00 439 | 6.007839202880859375e+00 440 | 6.001550674438476562e+00 441 | 5.993974208831787109e+00 442 | 5.987984657287597656e+00 443 | 5.985039710998535156e+00 444 | 5.975600719451904297e+00 445 | 5.973194599151611328e+00 446 | 5.965203762054443359e+00 447 | 5.962817192077636719e+00 448 | 5.955840110778808594e+00 449 | 5.950276374816894531e+00 450 | 5.947306156158447266e+00 451 | 5.940931320190429688e+00 452 | 5.937312126159667969e+00 453 | 5.931051731109619141e+00 454 | 5.928849220275878906e+00 455 | 5.922531127929687500e+00 456 | 5.917853355407714844e+00 457 | 5.915626525878906250e+00 458 | 5.908987522125244141e+00 459 | 5.906153202056884766e+00 460 | 5.901941299438476562e+00 461 | 5.897389888763427734e+00 462 | 5.893740653991699219e+00 463 | 5.889750003814697266e+00 464 | 5.886269569396972656e+00 465 | 5.882780075073242188e+00 466 | 5.879090309143066406e+00 467 | 5.879105091094970703e+00 468 | 5.878027915954589844e+00 469 | 5.888103485107421875e+00 470 | 5.907783031463623047e+00 471 | 5.972908496856689453e+00 472 | 6.098504543304443359e+00 473 | 6.406790733337402344e+00 474 | 6.649299621582031250e+00 475 | 6.767382621765136719e+00 476 | 6.124888420104980469e+00 477 | 5.914909362792968750e+00 478 | 6.278342247009277344e+00 479 | 6.174111366271972656e+00 480 | 5.870896816253662109e+00 481 | 6.034153461456298828e+00 482 | 6.041476249694824219e+00 483 | 5.860388755798339844e+00 484 | 5.980298042297363281e+00 485 | 5.932982444763183594e+00 486 | 5.845783710479736328e+00 487 | 5.942874908447265625e+00 488 | 5.863406181335449219e+00 489 | 5.858580112457275391e+00 490 | 5.896342754364013672e+00 491 | 5.824413776397705078e+00 492 | 5.867223739624023438e+00 493 | 5.851208686828613281e+00 494 | 5.817677021026611328e+00 495 | 5.852261066436767578e+00 496 | 5.813942909240722656e+00 497 | 5.820713043212890625e+00 498 | 5.825341224670410156e+00 499 | 5.794054508209228516e+00 500 | 5.812411785125732422e+00 501 | 5.797847270965576172e+00 502 | 5.785443782806396484e+00 503 | 5.799408912658691406e+00 504 | 5.778369903564453125e+00 505 | 5.775799274444580078e+00 506 | 5.781771183013916016e+00 507 | 5.764617919921875000e+00 508 | 5.767097473144531250e+00 509 | 5.766956329345703125e+00 510 | 5.753045082092285156e+00 511 | 5.756220817565917969e+00 512 | 5.752806663513183594e+00 513 | 5.742641448974609375e+00 514 | 5.743845462799072266e+00 515 | 5.740878582000732422e+00 516 | 5.732858657836914062e+00 517 | 5.732724666595458984e+00 518 | 5.731357097625732422e+00 519 | 5.723053932189941406e+00 520 | 5.722982406616210938e+00 521 | 5.722155094146728516e+00 522 | 5.717767238616943359e+00 523 | 5.717622280120849609e+00 524 | 5.726884841918945312e+00 525 | 5.735488414764404297e+00 526 | 5.773298740386962891e+00 527 | 5.846784114837646484e+00 528 | 6.024680614471435547e+00 529 | 6.277330875396728516e+00 530 | 6.641216278076171875e+00 531 | 6.472437858581542969e+00 532 | 5.986427307128906250e+00 533 | 5.707217216491699219e+00 534 | 5.979645252227783203e+00 535 | 6.126165390014648438e+00 536 | 5.759104251861572266e+00 537 | 5.791275024414062500e+00 538 | 5.994359970092773438e+00 539 | 5.754518032073974609e+00 540 | 5.755357742309570312e+00 541 | 5.898187160491943359e+00 542 | 5.715858936309814453e+00 543 | 5.756427288055419922e+00 544 | 5.823613643646240234e+00 545 | 5.685918807983398438e+00 546 | 5.764045238494873047e+00 547 | 5.755433559417724609e+00 548 | 5.679285049438476562e+00 549 | 5.754284381866455078e+00 550 | 5.697652339935302734e+00 551 | 5.686382293701171875e+00 552 | 5.722833633422851562e+00 553 | 5.662419319152832031e+00 554 | 5.689674377441406250e+00 555 | 5.683692932128906250e+00 556 | 5.648888111114501953e+00 557 | 5.680248260498046875e+00 558 | 5.651947021484375000e+00 559 | 5.643228530883789062e+00 560 | 5.663238525390625000e+00 561 | 5.633115291595458984e+00 562 | 5.637242317199707031e+00 563 | 5.643445014953613281e+00 564 | 5.621293067932128906e+00 565 | 5.629182815551757812e+00 566 | 5.627232551574707031e+00 567 | 5.611253261566162109e+00 568 | 5.618420124053955078e+00 569 | 5.614394187927246094e+00 570 | 5.602704048156738281e+00 571 | 5.607692718505859375e+00 572 | 5.603302001953125000e+00 573 | 5.593761920928955078e+00 574 | 5.596231937408447266e+00 575 | 5.593771934509277344e+00 576 | 5.585361480712890625e+00 577 | 5.585885047912597656e+00 578 | 5.584845542907714844e+00 579 | 5.577035427093505859e+00 580 | 5.575808048248291016e+00 581 | 5.575828075408935547e+00 582 | 5.569941043853759766e+00 583 | 5.566810131072998047e+00 584 | 5.567694187164306641e+00 585 | 5.564343929290771484e+00 586 | 5.563666820526123047e+00 587 | 5.568611621856689453e+00 588 | 5.584864139556884766e+00 589 | 5.623416900634765625e+00 590 | 5.745083808898925781e+00 591 | 6.016070842742919922e+00 592 | 6.605881214141845703e+00 593 | 6.913734436035156250e+00 594 | 6.574638843536376953e+00 595 | 5.710192203521728516e+00 596 | 6.127451419830322266e+00 597 | 6.385603904724121094e+00 598 | 5.643902778625488281e+00 599 | 6.036314487457275391e+00 600 | 6.044794559478759766e+00 601 | 5.662094593048095703e+00 602 | 6.042337417602539062e+00 603 | 5.653035640716552734e+00 604 | 5.854990482330322266e+00 605 | 5.694759845733642578e+00 606 | 5.707163810729980469e+00 607 | 5.738770961761474609e+00 608 | 5.627290248870849609e+00 609 | 5.729097366333007812e+00 610 | 5.587062835693359375e+00 611 | 5.701817512512207031e+00 612 | 5.575615406036376953e+00 613 | 5.669567584991455078e+00 614 | 5.558978080749511719e+00 615 | 5.642535209655761719e+00 616 | 5.554784297943115234e+00 617 | 5.611257553100585938e+00 618 | 5.553598880767822266e+00 619 | 5.586450099945068359e+00 620 | 5.545970916748046875e+00 621 | 5.563397407531738281e+00 622 | 5.543477058410644531e+00 623 | 5.545058250427246094e+00 624 | 5.537321090698242188e+00 625 | 5.526545524597167969e+00 626 | 5.533581733703613281e+00 627 | 5.513477802276611328e+00 628 | 5.524734973907470703e+00 629 | 5.503151416778564453e+00 630 | 5.516649246215820312e+00 631 | 5.495923995971679688e+00 632 | 5.504014492034912109e+00 633 | 5.493324279785156250e+00 634 | 5.492880344390869141e+00 635 | 5.489078998565673828e+00 636 | 5.480704784393310547e+00 637 | 5.485807895660400391e+00 638 | 5.474427223205566406e+00 639 | 5.476234436035156250e+00 640 | 5.470584869384765625e+00 641 | 5.468974113464355469e+00 642 | 5.466079711914062500e+00 643 | 5.460186481475830078e+00 644 | 5.462256431579589844e+00 645 | 5.455686092376708984e+00 646 | 5.453927993774414062e+00 647 | 5.452708244323730469e+00 648 | 5.448278427124023438e+00 649 | 5.447020530700683594e+00 650 | 5.443565368652343750e+00 651 | 5.441678524017333984e+00 652 | 5.439322948455810547e+00 653 | 5.436123847961425781e+00 654 | 5.434998512268066406e+00 655 | 5.431733608245849609e+00 656 | 5.429090023040771484e+00 657 | 5.427978038787841797e+00 658 | 5.424806594848632812e+00 659 | 5.422541618347167969e+00 660 | 5.421158313751220703e+00 661 | 5.418225765228271484e+00 662 | 5.416270732879638672e+00 663 | 5.414474964141845703e+00 664 | 5.412391185760498047e+00 665 | 5.410523891448974609e+00 666 | 5.409735202789306641e+00 667 | 5.408972740173339844e+00 668 | 5.411149501800537109e+00 669 | 5.417623519897460938e+00 670 | 5.436219215393066406e+00 671 | 5.481471538543701172e+00 672 | 5.583567619323730469e+00 673 | 5.820762157440185547e+00 674 | 6.109329700469970703e+00 675 | 6.489130496978759766e+00 676 | 5.988056659698486328e+00 677 | 5.475376605987548828e+00 678 | 5.559019088745117188e+00 679 | 5.835749149322509766e+00 680 | 5.667306900024414062e+00 681 | 5.425483226776123047e+00 682 | 5.660701751708984375e+00 683 | 5.635597705841064453e+00 684 | 5.419532299041748047e+00 685 | 5.631511211395263672e+00 686 | 5.551924705505371094e+00 687 | 5.427195072174072266e+00 688 | 5.598648071289062500e+00 689 | 5.460762977600097656e+00 690 | 5.461937427520751953e+00 691 | 5.525660037994384766e+00 692 | 5.403007507324218750e+00 693 | 5.483717918395996094e+00 694 | 5.433654308319091797e+00 695 | 5.411223888397216797e+00 696 | 5.455393791198730469e+00 697 | 5.387472152709960938e+00 698 | 5.431951999664306641e+00 699 | 5.404996871948242188e+00 700 | 5.389273643493652344e+00 701 | 5.417010307312011719e+00 702 | 5.372736930847167969e+00 703 | 5.397756099700927734e+00 704 | 5.382994174957275391e+00 705 | 5.368655204772949219e+00 706 | 5.387310981750488281e+00 707 | 5.361037731170654297e+00 708 | 5.371379375457763672e+00 709 | 5.366631507873535156e+00 710 | 5.353297233581542969e+00 711 | 5.364907741546630859e+00 712 | 5.350233078002929688e+00 713 | 5.350633621215820312e+00 714 | 5.353544235229492188e+00 715 | 5.339832305908203125e+00 716 | 5.345212936401367188e+00 717 | 5.341831684112548828e+00 718 | 5.333949565887451172e+00 719 | 5.338410854339599609e+00 720 | 5.332018852233886719e+00 721 | 5.328279018402099609e+00 722 | 5.330114364624023438e+00 723 | 5.324204444885253906e+00 724 | 5.322516441345214844e+00 725 | 5.322834014892578125e+00 726 | 5.317927837371826172e+00 727 | 5.316485404968261719e+00 728 | 5.316488265991210938e+00 729 | 5.311610221862792969e+00 730 | 5.310516834259033203e+00 731 | 5.309710979461669922e+00 732 | 5.306063175201416016e+00 733 | 5.304423332214355469e+00 734 | 5.303830146789550781e+00 735 | 5.300921916961669922e+00 736 | 5.299415111541748047e+00 737 | 5.299057483673095703e+00 738 | 5.298425197601318359e+00 739 | 5.298995971679687500e+00 740 | 5.304882526397705078e+00 741 | 5.318303585052490234e+00 742 | 5.349639892578125000e+00 743 | 5.427845478057861328e+00 744 | 5.575341224670410156e+00 745 | 5.879703044891357422e+00 746 | 6.068168640136718750e+00 747 | 6.048056602478027344e+00 748 | 5.552328586578369141e+00 749 | 5.320807456970214844e+00 750 | 5.591884136199951172e+00 751 | 5.657795906066894531e+00 752 | 5.379011154174804688e+00 753 | 5.353989601135253906e+00 754 | 5.532160758972167969e+00 755 | 5.406654834747314453e+00 756 | 5.320634841918945312e+00 757 | 5.465527534484863281e+00 758 | 5.382233142852783203e+00 759 | 5.318314552307128906e+00 760 | 5.419388294219970703e+00 761 | 5.343470096588134766e+00 762 | 5.325187683105468750e+00 763 | 5.382782936096191406e+00 764 | 5.306429862976074219e+00 765 | 5.330289840698242188e+00 766 | 5.342358589172363281e+00 767 | 5.287569046020507812e+00 768 | 5.325266361236572266e+00 769 | 5.305394172668457031e+00 770 | 5.280109882354736328e+00 771 | 5.310715198516845703e+00 772 | 5.280439853668212891e+00 773 | 5.277963161468505859e+00 774 | 5.291290283203125000e+00 775 | 5.266885757446289062e+00 776 | 5.274319648742675781e+00 777 | 5.273258686065673828e+00 778 | 5.259028911590576172e+00 779 | 5.266624450683593750e+00 780 | 5.259267330169677734e+00 781 | 5.253460884094238281e+00 782 | 5.257719993591308594e+00 783 | 5.248974800109863281e+00 784 | 5.246968269348144531e+00 785 | 5.248549938201904297e+00 786 | 5.241478443145751953e+00 787 | 5.240849018096923828e+00 788 | 5.240497112274169922e+00 789 | 5.235310554504394531e+00 790 | 5.234075069427490234e+00 791 | 5.232932567596435547e+00 792 | 5.229212760925292969e+00 793 | 5.227867603302001953e+00 794 | 5.226433277130126953e+00 795 | 5.223608493804931641e+00 796 | 5.221829414367675781e+00 797 | 5.220657348632812500e+00 798 | 5.218758583068847656e+00 799 | 5.216211318969726562e+00 800 | 5.215425491333007812e+00 801 | 5.213845729827880859e+00 802 | 5.212640762329101562e+00 803 | 5.211980342864990234e+00 804 | 5.213238716125488281e+00 805 | 5.215958595275878906e+00 806 | 5.225562095642089844e+00 807 | 5.247623920440673828e+00 808 | 5.301805496215820312e+00 809 | 5.415420055389404297e+00 810 | 5.653826713562011719e+00 811 | 6.015121936798095703e+00 812 | 6.280319690704345703e+00 813 | 5.969844818115234375e+00 814 | 5.334757328033447266e+00 815 | 5.335915088653564453e+00 816 | 5.711000442504882812e+00 817 | 5.538827419281005859e+00 818 | 5.234858036041259766e+00 819 | 5.495742321014404297e+00 820 | 5.460876941680908203e+00 821 | 5.241770267486572266e+00 822 | 5.448649883270263672e+00 823 | 5.335628986358642578e+00 824 | 5.266571044921875000e+00 825 | 5.397761344909667969e+00 826 | 5.246897697448730469e+00 827 | 5.307940483093261719e+00 828 | 5.302983283996582031e+00 829 | 5.227132320404052734e+00 830 | 5.308912754058837891e+00 831 | 5.222150802612304688e+00 832 | 5.254464149475097656e+00 833 | 5.252618312835693359e+00 834 | 5.210768699645996094e+00 835 | 5.253617286682128906e+00 836 | 5.203589916229248047e+00 837 | 5.226662635803222656e+00 838 | 5.216628074645996094e+00 839 | 5.197330474853515625e+00 840 | 5.218626022338867188e+00 841 | 5.189521312713623047e+00 842 | 5.202488422393798828e+00 843 | 5.195226192474365234e+00 844 | 5.182983875274658203e+00 845 | 5.194581031799316406e+00 846 | 5.178484916687011719e+00 847 | 5.182146072387695312e+00 848 | 5.179948329925537109e+00 849 | 5.170345783233642578e+00 850 | 5.177103996276855469e+00 851 | 5.168478965759277344e+00 852 | 5.166522502899169922e+00 853 | 5.168373107910156250e+00 854 | 5.160217761993408203e+00 855 | 5.161878585815429688e+00 856 | 5.160144805908203125e+00 857 | 5.154767990112304688e+00 858 | 5.156347751617431641e+00 859 | 5.152815341949462891e+00 860 | 5.149863719940185547e+00 861 | 5.150386333465576172e+00 862 | 5.146773815155029297e+00 863 | 5.145030021667480469e+00 864 | 5.144948005676269531e+00 865 | 5.141819953918457031e+00 866 | 5.140087604522705078e+00 867 | 5.139832496643066406e+00 868 | 5.136848926544189453e+00 869 | 5.135580539703369141e+00 870 | 5.134657859802246094e+00 871 | 5.132422447204589844e+00 872 | 5.131079196929931641e+00 873 | 5.130071163177490234e+00 874 | 5.128059864044189453e+00 875 | 5.126660346984863281e+00 876 | 5.125559329986572266e+00 877 | 5.124033927917480469e+00 878 | 5.122329711914062500e+00 879 | 5.121231555938720703e+00 880 | 5.119935035705566406e+00 881 | 5.118402004241943359e+00 882 | 5.116948604583740234e+00 883 | 5.116116046905517578e+00 884 | 5.114536762237548828e+00 885 | 5.113722324371337891e+00 886 | 5.112879276275634766e+00 887 | 5.114387989044189453e+00 888 | 5.116611003875732422e+00 889 | 5.131507873535156250e+00 890 | 5.162660121917724609e+00 891 | 5.268977642059326172e+00 892 | 5.497073650360107422e+00 893 | 6.098115921020507812e+00 894 | 6.591432094573974609e+00 895 | 6.740016460418701172e+00 896 | 5.407641410827636719e+00 897 | 5.476219654083251953e+00 898 | 6.272408962249755859e+00 899 | 5.382391452789306641e+00 900 | 5.606043815612792969e+00 901 | 5.898615837097167969e+00 902 | 5.286243438720703125e+00 903 | 5.810534000396728516e+00 904 | 5.318495273590087891e+00 905 | 5.550365447998046875e+00 906 | 5.313470840454101562e+00 907 | 5.401293277740478516e+00 908 | 5.325156688690185547e+00 909 | 5.339534759521484375e+00 910 | 5.271907329559326172e+00 911 | 5.302194118499755859e+00 912 | 5.239202022552490234e+00 913 | 5.293450832366943359e+00 914 | 5.196889877319335938e+00 915 | 5.263246536254882812e+00 916 | 5.179274559020996094e+00 917 | 5.251442432403564453e+00 918 | 5.159143924713134766e+00 919 | 5.224196910858154297e+00 920 | 5.146760940551757812e+00 921 | 5.210275650024414062e+00 922 | 5.138393402099609375e+00 923 | 5.184453487396240234e+00 924 | 5.129835128784179688e+00 925 | 5.170839786529541016e+00 926 | 5.125970840454101562e+00 927 | 5.149978160858154297e+00 928 | 5.118451595306396484e+00 929 | 5.140971183776855469e+00 930 | 5.113529205322265625e+00 931 | 5.125265598297119141e+00 932 | 5.107148647308349609e+00 933 | 5.118845462799072266e+00 934 | 5.100919246673583984e+00 935 | 5.108714580535888672e+00 936 | 5.094960212707519531e+00 937 | 5.102995395660400391e+00 938 | 5.090015411376953125e+00 939 | 5.094324588775634766e+00 940 | 5.085294246673583984e+00 941 | 5.090317249298095703e+00 942 | 5.080534458160400391e+00 943 | 5.083104610443115234e+00 944 | 5.077380657196044922e+00 945 | 5.078994750976562500e+00 946 | 5.073503017425537109e+00 947 | 5.072685241699218750e+00 948 | 5.071393489837646484e+00 949 | 5.068833827972412109e+00 950 | 5.067492485046386719e+00 951 | 5.064123153686523438e+00 952 | 5.065104484558105469e+00 953 | 5.060974597930908203e+00 954 | 5.060648441314697266e+00 955 | 5.058252334594726562e+00 956 | 5.057617664337158203e+00 957 | 5.055404663085937500e+00 958 | 5.053553104400634766e+00 959 | 5.053390502929687500e+00 960 | 5.050876617431640625e+00 961 | 5.049684524536132812e+00 962 | 5.048246383666992188e+00 963 | 5.047453403472900391e+00 964 | 5.045513153076171875e+00 965 | 5.044075012207031250e+00 966 | 5.043549537658691406e+00 967 | 5.041623115539550781e+00 968 | 5.040425777435302734e+00 969 | 5.039348602294921875e+00 970 | 5.038094997406005859e+00 971 | 5.036738872528076172e+00 972 | 5.035543441772460938e+00 973 | 5.034599304199218750e+00 974 | 5.033106327056884766e+00 975 | 5.032009601593017578e+00 976 | 5.031035900115966797e+00 977 | 5.029621124267578125e+00 978 | 5.028515815734863281e+00 979 | 5.027551651000976562e+00 980 | 5.026253223419189453e+00 981 | 5.025178909301757812e+00 982 | 5.024034023284912109e+00 983 | 5.023100852966308594e+00 984 | 5.021766185760498047e+00 985 | 5.020848751068115234e+00 986 | 5.019835948944091797e+00 987 | 5.018919467926025391e+00 988 | 5.017950534820556641e+00 989 | 5.017757415771484375e+00 990 | 5.017368793487548828e+00 991 | 5.019140720367431641e+00 992 | 5.022178173065185547e+00 993 | 5.032592773437500000e+00 994 | 5.051977157592773438e+00 995 | 5.102300643920898438e+00 996 | 5.196702003479003906e+00 997 | 5.402066230773925781e+00 998 | 5.674953460693359375e+00 999 | 5.951617240905761719e+00 1000 | 5.735409259796142578e+00 1001 | 5.219491481781005859e+00 1002 | 5.064063072204589844e+00 1003 | 5.328687191009521484e+00 1004 | 5.388410568237304688e+00 1005 | 5.078166484832763672e+00 1006 | 5.137835979461669922e+00 1007 | 5.288620948791503906e+00 1008 | 5.078691482543945312e+00 1009 | 5.102169513702392578e+00 1010 | 5.204850673675537109e+00 1011 | 5.051388740539550781e+00 1012 | 5.100458145141601562e+00 1013 | 5.137561321258544922e+00 1014 | 5.033069610595703125e+00 1015 | 5.101635932922363281e+00 1016 | 5.077757358551025391e+00 1017 | 5.036583900451660156e+00 1018 | 5.085474491119384766e+00 1019 | 5.034716129302978516e+00 1020 | 5.048048496246337891e+00 1021 | 5.054547786712646484e+00 1022 | 5.016736507415771484e+00 1023 | 5.049506664276123047e+00 1024 | 5.025746345520019531e+00 1025 | 5.014205455780029297e+00 1026 | 5.036614894866943359e+00 1027 | 5.008316993713378906e+00 1028 | 5.015004634857177734e+00 1029 | 5.019134521484375000e+00 1030 | 5.001647949218750000e+00 1031 | 5.011433601379394531e+00 1032 | 5.003180980682373047e+00 1033 | 4.999117374420166016e+00 1034 | 5.004672527313232422e+00 1035 | 4.993229389190673828e+00 1036 | 4.995713710784912109e+00 1037 | 4.995815277099609375e+00 1038 | 4.987571716308593750e+00 1039 | 4.991898536682128906e+00 1040 | 4.987870216369628906e+00 1041 | 4.982875823974609375e+00 1042 | 4.987260818481445312e+00 1043 | 4.982150077819824219e+00 1044 | 4.978898048400878906e+00 1045 | 4.981702804565429688e+00 1046 | 4.977360248565673828e+00 1047 | 4.975546836853027344e+00 1048 | 4.976697921752929688e+00 1049 | 4.973128795623779297e+00 1050 | 4.971766471862792969e+00 1051 | 4.972346305847167969e+00 1052 | 4.969520568847656250e+00 1053 | 4.967978477478027344e+00 1054 | 4.968201637268066406e+00 1055 | 4.966025829315185547e+00 1056 | 4.964426994323730469e+00 1057 | 4.964266777038574219e+00 1058 | 4.962703227996826172e+00 1059 | 4.961162567138671875e+00 1060 | 4.960812568664550781e+00 1061 | 4.959571361541748047e+00 1062 | 4.958073616027832031e+00 1063 | 4.957290172576904297e+00 1064 | 4.956335544586181641e+00 1065 | 4.955127716064453125e+00 1066 | 4.954166889190673828e+00 1067 | 4.953500747680664062e+00 1068 | 4.952612876892089844e+00 1069 | 4.951636791229248047e+00 1070 | 4.951630115509033203e+00 1071 | 4.951869010925292969e+00 1072 | 4.953342437744140625e+00 1073 | 4.957009315490722656e+00 1074 | 4.966738700866699219e+00 1075 | 4.985781669616699219e+00 1076 | 5.028012275695800781e+00 1077 | 5.113472938537597656e+00 1078 | 5.271671772003173828e+00 1079 | 5.529312133789062500e+00 1080 | 5.745319366455078125e+00 1081 | 5.693117618560791016e+00 1082 | 5.266152858734130859e+00 1083 | 4.986081600189208984e+00 1084 | 5.187667369842529297e+00 1085 | 5.305523872375488281e+00 1086 | 5.058845520019531250e+00 1087 | 4.989647865295410156e+00 1088 | 5.169186592102050781e+00 1089 | 5.078308105468750000e+00 1090 | 4.973736286163330078e+00 1091 | 5.107821941375732422e+00 1092 | 5.034676074981689453e+00 1093 | 4.974092960357666016e+00 1094 | 5.074297428131103516e+00 1095 | 4.992820739746093750e+00 1096 | 4.981681823730468750e+00 1097 | 5.034663200378417969e+00 1098 | 4.963548183441162109e+00 1099 | 4.990610599517822266e+00 1100 | 4.995028495788574219e+00 1101 | 4.951694011688232422e+00 1102 | 4.988568782806396484e+00 1103 | 4.963809013366699219e+00 1104 | 4.954089164733886719e+00 1105 | 4.973791122436523438e+00 1106 | 4.944921493530273438e+00 1107 | 4.956215858459472656e+00 1108 | 4.955852985382080078e+00 1109 | 4.937811374664306641e+00 1110 | 4.952673912048339844e+00 1111 | 4.941129684448242188e+00 1112 | 4.934720039367675781e+00 1113 | 4.945424556732177734e+00 1114 | 4.932143211364746094e+00 1115 | 4.931800842285156250e+00 1116 | 4.936752796173095703e+00 1117 | 4.926283836364746094e+00 1118 | 4.928894519805908203e+00 1119 | 4.929748535156250000e+00 1120 | 4.921313762664794922e+00 1121 | 4.924187183380126953e+00 1122 | 4.924050331115722656e+00 1123 | 4.918031692504882812e+00 1124 | 4.919841289520263672e+00 1125 | 4.918902397155761719e+00 1126 | 4.914425849914550781e+00 1127 | 4.915421485900878906e+00 1128 | 4.914837360382080078e+00 1129 | 4.911162853240966797e+00 1130 | 4.911528110504150391e+00 1131 | 4.911017417907714844e+00 1132 | 4.907846450805664062e+00 1133 | 4.907771110534667969e+00 1134 | 4.907353878021240234e+00 1135 | 4.905115127563476562e+00 1136 | 4.904096126556396484e+00 1137 | 4.903892517089843750e+00 1138 | 4.902174949645996094e+00 1139 | 4.900961399078369141e+00 1140 | 4.900557994842529297e+00 1141 | 4.899525642395019531e+00 1142 | 4.898211002349853516e+00 1143 | 4.897263526916503906e+00 1144 | 4.896924018859863281e+00 1145 | 4.895565509796142578e+00 1146 | 4.894818305969238281e+00 1147 | 4.894060611724853516e+00 1148 | 4.894164562225341797e+00 1149 | 4.892858028411865234e+00 1150 | 4.894168376922607422e+00 1151 | 4.895061016082763672e+00 1152 | 4.900552749633789062e+00 1153 | 4.908415794372558594e+00 1154 | 4.933411121368408203e+00 1155 | 4.973899364471435547e+00 1156 | 5.080464363098144531e+00 1157 | 5.231359004974365234e+00 1158 | 5.559916496276855469e+00 1159 | 5.666441440582275391e+00 1160 | 5.684103012084960938e+00 1161 | 5.142689704895019531e+00 1162 | 4.935481071472167969e+00 1163 | 5.167072772979736328e+00 1164 | 5.200273513793945312e+00 1165 | 4.979498386383056641e+00 1166 | 4.966571331024169922e+00 1167 | 5.097239494323730469e+00 1168 | 4.990492820739746094e+00 1169 | 4.935947895050048828e+00 1170 | 5.039998054504394531e+00 1171 | 4.963368892669677734e+00 1172 | 4.924431324005126953e+00 1173 | 5.003300189971923828e+00 1174 | 4.934683799743652344e+00 1175 | 4.925329208374023438e+00 1176 | 4.970231056213378906e+00 1177 | 4.909868240356445312e+00 1178 | 4.928258419036865234e+00 1179 | 4.937177658081054688e+00 1180 | 4.896284103393554688e+00 1181 | 4.925786018371582031e+00 1182 | 4.909291267395019531e+00 1183 | 4.893118858337402344e+00 1184 | 4.916127204895019531e+00 1185 | 4.892068862915039062e+00 1186 | 4.891975402832031250e+00 1187 | 4.902931213378906250e+00 1188 | 4.883250713348388672e+00 1189 | 4.889759063720703125e+00 1190 | 4.890295982360839844e+00 1191 | 4.878466129302978516e+00 1192 | 4.885734081268310547e+00 1193 | 4.881197452545166016e+00 1194 | 4.874628543853759766e+00 1195 | 4.880147457122802734e+00 1196 | 4.874832153320312500e+00 1197 | 4.870942592620849609e+00 1198 | 4.874166965484619141e+00 1199 | 4.869830608367919922e+00 1200 | 4.867537975311279297e+00 1201 | 4.869402408599853516e+00 1202 | 4.865425586700439453e+00 1203 | 4.863394737243652344e+00 1204 | 4.865044116973876953e+00 1205 | 4.862209796905517578e+00 1206 | 4.859803676605224609e+00 1207 | 4.860684394836425781e+00 1208 | 4.859205722808837891e+00 1209 | 4.856773853302001953e+00 1210 | 4.856757164001464844e+00 1211 | 4.856051445007324219e+00 1212 | 4.853746891021728516e+00 1213 | 4.853226661682128906e+00 1214 | 4.853146553039550781e+00 1215 | 4.851227283477783203e+00 1216 | 4.850076675415039062e+00 1217 | 4.850059986114501953e+00 1218 | 4.848786354064941406e+00 1219 | 4.847412586212158203e+00 1220 | 4.847074031829833984e+00 1221 | 4.846479892730712891e+00 1222 | 4.845191001892089844e+00 1223 | 4.844462394714355469e+00 1224 | 4.844227790832519531e+00 1225 | 4.844464778900146484e+00 1226 | 4.844116687774658203e+00 1227 | 4.846418857574462891e+00 1228 | 4.850960731506347656e+00 1229 | 4.861240386962890625e+00 1230 | 4.882007598876953125e+00 1231 | 4.929082870483398438e+00 1232 | 5.019104003906250000e+00 1233 | 5.189638137817382812e+00 1234 | 5.430735588073730469e+00 1235 | 5.626307487487792969e+00 1236 | 5.496823310852050781e+00 1237 | 5.055807590484619141e+00 1238 | 4.867516517639160156e+00 1239 | 5.083232879638671875e+00 1240 | 5.176282405853271484e+00 1241 | 4.938247680664062500e+00 1242 | 4.897057533264160156e+00 1243 | 5.053777694702148438e+00 1244 | 4.956409454345703125e+00 1245 | 4.872794628143310547e+00 1246 | 4.984990119934082031e+00 1247 | 4.925196170806884766e+00 1248 | 4.865906715393066406e+00 1249 | 4.948370456695556641e+00 1250 | 4.890005111694335938e+00 1251 | 4.869961738586425781e+00 1252 | 4.920280933380126953e+00 1253 | 4.861112117767333984e+00 1254 | 4.872776985168457031e+00 1255 | 4.894012928009033203e+00 1256 | 4.846310615539550781e+00 1257 | 4.872751235961914062e+00 1258 | 4.866878509521484375e+00 1259 | 4.841940402984619141e+00 1260 | 4.866863727569580078e+00 1261 | 4.848299026489257812e+00 1262 | 4.840645790100097656e+00 1263 | 4.856147766113281250e+00 1264 | 4.837981700897216797e+00 1265 | 4.839947223663330078e+00 1266 | 4.844801425933837891e+00 1267 | 4.831037044525146484e+00 1268 | 4.837290287017822266e+00 1269 | 4.835611820220947266e+00 1270 | 4.826858043670654297e+00 1271 | 4.832877635955810547e+00 1272 | 4.828449249267578125e+00 1273 | 4.822745323181152344e+00 1274 | 4.827888488769531250e+00 1275 | 4.823773384094238281e+00 1276 | 4.819595813751220703e+00 1277 | 4.822674751281738281e+00 1278 | 4.819828987121582031e+00 1279 | 4.816200256347656250e+00 1280 | 4.818257808685302734e+00 1281 | 4.816237926483154297e+00 1282 | 4.813117980957031250e+00 1283 | 4.814651489257812500e+00 1284 | 4.812948703765869141e+00 1285 | 4.810372829437255859e+00 1286 | 4.810839653015136719e+00 1287 | 4.810036182403564453e+00 1288 | 4.807796955108642578e+00 1289 | 4.807316303253173828e+00 1290 | 4.807295799255371094e+00 1291 | 4.805694580078125000e+00 1292 | 4.804389476776123047e+00 1293 | 4.804160118103027344e+00 1294 | 4.803748130798339844e+00 1295 | 4.801848411560058594e+00 1296 | 4.801648616790771484e+00 1297 | 4.801115989685058594e+00 1298 | 4.799983024597167969e+00 1299 | 4.799178123474121094e+00 1300 | 4.799338340759277344e+00 1301 | 4.798367023468017578e+00 1302 | 4.798748970031738281e+00 1303 | 4.798830032348632812e+00 1304 | 4.802074909210205078e+00 1305 | 4.805243015289306641e+00 1306 | 4.817478179931640625e+00 1307 | 4.835626602172851562e+00 1308 | 4.884081363677978516e+00 1309 | 4.961005687713623047e+00 1310 | 5.133362293243408203e+00 1311 | 5.337423324584960938e+00 1312 | 5.610584259033203125e+00 1313 | 5.466285228729248047e+00 1314 | 5.074841022491455078e+00 1315 | 4.812684059143066406e+00 1316 | 4.979393482208251953e+00 1317 | 5.163257122039794922e+00 1318 | 4.915400028228759766e+00 1319 | 4.833351612091064453e+00 1320 | 5.013104915618896484e+00 1321 | 4.913367271423339844e+00 1322 | 4.816688060760498047e+00 1323 | 4.931622505187988281e+00 1324 | 4.872955799102783203e+00 1325 | 4.813139915466308594e+00 1326 | 4.892896175384521484e+00 1327 | 4.837963104248046875e+00 1328 | 4.813188552856445312e+00 1329 | 4.867125034332275391e+00 1330 | 4.812202453613281250e+00 1331 | 4.818795680999755859e+00 1332 | 4.844270229339599609e+00 1333 | 4.798913478851318359e+00 1334 | 4.821649551391601562e+00 1335 | 4.823801994323730469e+00 1336 | 4.794254302978515625e+00 1337 | 4.818236351013183594e+00 1338 | 4.806548595428466797e+00 1339 | 4.794344902038574219e+00 1340 | 4.810982227325439453e+00 1341 | 4.794742584228515625e+00 1342 | 4.793148517608642578e+00 1343 | 4.802014350891113281e+00 1344 | 4.787969589233398438e+00 1345 | 4.790615558624267578e+00 1346 | 4.793093204498291016e+00 1347 | 4.783093452453613281e+00 1348 | 4.787098884582519531e+00 1349 | 4.786732196807861328e+00 1350 | 4.778945922851562500e+00 1351 | 4.782271385192871094e+00 1352 | 4.781804561614990234e+00 1353 | 4.775826454162597656e+00 1354 | 4.778048515319824219e+00 1355 | 4.777780532836914062e+00 1356 | 4.773001670837402344e+00 1357 | 4.774064540863037109e+00 1358 | 4.773852348327636719e+00 1359 | 4.770632743835449219e+00 1360 | 4.770736217498779297e+00 1361 | 4.770603656768798828e+00 1362 | 4.768360614776611328e+00 1363 | 4.767361640930175781e+00 1364 | 4.767496585845947266e+00 1365 | 4.766240596771240234e+00 1366 | 4.764482975006103516e+00 1367 | 4.764537811279296875e+00 1368 | 4.764399528503417969e+00 1369 | 4.762185096740722656e+00 1370 | 4.761914253234863281e+00 1371 | 4.761956691741943359e+00 1372 | 4.760394573211669922e+00 1373 | 4.759701728820800781e+00 1374 | 4.759671688079833984e+00 1375 | 4.758872032165527344e+00 1376 | 4.759073734283447266e+00 1377 | 4.759192466735839844e+00 1378 | 4.762099742889404297e+00 1379 | 4.766599655151367188e+00 1380 | 4.779808998107910156e+00 1381 | 4.805636882781982422e+00 1382 | 4.871307373046875000e+00 1383 | 4.995099544525146484e+00 1384 | 5.264702796936035156e+00 1385 | 5.607756614685058594e+00 1386 | 5.929910182952880859e+00 1387 | 5.502411365509033203e+00 1388 | 4.897315979003906250e+00 1389 | 4.908808231353759766e+00 1390 | 5.235307216644287109e+00 1391 | 5.062665939331054688e+00 1392 | 4.801521778106689453e+00 1393 | 5.052458763122558594e+00 1394 | 4.989951610565185547e+00 1395 | 4.816385746002197266e+00 1396 | 5.009685039520263672e+00 1397 | 4.868786811828613281e+00 1398 | 4.858018398284912109e+00 1399 | 4.944483280181884766e+00 1400 | 4.800341129302978516e+00 1401 | 4.894233703613281250e+00 1402 | 4.837351799011230469e+00 1403 | 4.821630001068115234e+00 1404 | 4.860535144805908203e+00 1405 | 4.788465499877929688e+00 1406 | 4.844590663909912109e+00 1407 | 4.797033786773681641e+00 1408 | 4.807269573211669922e+00 1409 | 4.807038784027099609e+00 1410 | 4.782226562500000000e+00 1411 | 4.808128833770751953e+00 1412 | 4.772421360015869141e+00 1413 | 4.790393829345703125e+00 1414 | 4.778810501098632812e+00 1415 | 4.773900508880615234e+00 1416 | 4.779234886169433594e+00 1417 | 4.760923385620117188e+00 1418 | 4.776420116424560547e+00 1419 | 4.760836601257324219e+00 1420 | 4.761035442352294922e+00 1421 | 4.765402793884277344e+00 1422 | 4.751939296722412109e+00 1423 | 4.761429309844970703e+00 1424 | 4.752399444580078125e+00 1425 | 4.749948501586914062e+00 1426 | 4.754920005798339844e+00 1427 | 4.745113849639892578e+00 1428 | 4.748868465423583984e+00 1429 | 4.746984481811523438e+00 1430 | 4.741280555725097656e+00 1431 | 4.745190620422363281e+00 1432 | 4.741297721862792969e+00 1433 | 4.739511013031005859e+00 1434 | 4.740776062011718750e+00 1435 | 4.736665725708007812e+00 1436 | 4.736780643463134766e+00 1437 | 4.736535072326660156e+00 1438 | 4.733848571777343750e+00 1439 | 4.734148502349853516e+00 1440 | 4.732965946197509766e+00 1441 | 4.731197357177734375e+00 1442 | 4.731153488159179688e+00 1443 | 4.729887485504150391e+00 1444 | 4.728586196899414062e+00 1445 | 4.728567600250244141e+00 1446 | 4.727372646331787109e+00 1447 | 4.726333618164062500e+00 1448 | 4.726187229156494141e+00 1449 | 4.724942207336425781e+00 1450 | 4.724176883697509766e+00 1451 | 4.723769187927246094e+00 1452 | 4.722803115844726562e+00 1453 | 4.721954345703125000e+00 1454 | 4.721476554870605469e+00 1455 | 4.720837593078613281e+00 1456 | 4.719880104064941406e+00 1457 | 4.719511985778808594e+00 1458 | 4.718820571899414062e+00 1459 | 4.718129158020019531e+00 1460 | 4.717587947845458984e+00 1461 | 4.717406272888183594e+00 1462 | 4.716806888580322266e+00 1463 | 4.717114448547363281e+00 1464 | 4.717732906341552734e+00 1465 | 4.719917774200439453e+00 1466 | 4.723948001861572266e+00 1467 | 4.734146595001220703e+00 1468 | 4.754000663757324219e+00 1469 | 4.796111583709716797e+00 1470 | 4.881614685058593750e+00 1471 | 5.036080837249755859e+00 1472 | 5.308792114257812500e+00 1473 | 5.564405918121337891e+00 1474 | 5.648788928985595703e+00 1475 | 5.209763526916503906e+00 1476 | 4.773248672485351562e+00 1477 | 4.868886947631835938e+00 1478 | 5.119225025177001953e+00 1479 | 4.936864852905273438e+00 1480 | 4.740203380584716797e+00 1481 | 4.946773052215576172e+00 1482 | 4.929188728332519531e+00 1483 | 4.740137577056884766e+00 1484 | 4.894275188446044922e+00 1485 | 4.863486766815185547e+00 1486 | 4.749235630035400391e+00 1487 | 4.874592781066894531e+00 1488 | 4.792233943939208984e+00 1489 | 4.766139507293701172e+00 1490 | 4.840188026428222656e+00 1491 | 4.743630409240722656e+00 1492 | 4.786509513854980469e+00 1493 | 4.785348892211914062e+00 1494 | 4.735338211059570312e+00 1495 | 4.787172317504882812e+00 1496 | 4.739229679107666016e+00 1497 | 4.749022960662841797e+00 1498 | 4.761902809143066406e+00 1499 | 4.723606109619140625e+00 1500 | 4.753235816955566406e+00 1501 | 4.733036994934082031e+00 1502 | 4.727744579315185547e+00 1503 | 4.741178989410400391e+00 1504 | 4.716994285583496094e+00 1505 | 4.731057167053222656e+00 1506 | 4.725521564483642578e+00 1507 | 4.713009834289550781e+00 1508 | 4.725625038146972656e+00 1509 | 4.713554382324218750e+00 1510 | 4.713357925415039062e+00 1511 | 4.717501163482666016e+00 1512 | 4.706220149993896484e+00 1513 | 4.711745738983154297e+00 1514 | 4.710540294647216797e+00 1515 | 4.702843666076660156e+00 1516 | 4.708183288574218750e+00 1517 | 4.704109668731689453e+00 1518 | 4.700412750244140625e+00 1519 | 4.704440116882324219e+00 1520 | 4.699813842773437500e+00 1521 | 4.698040008544921875e+00 1522 | 4.699964523315429688e+00 1523 | 4.696222782135009766e+00 1524 | 4.695685386657714844e+00 1525 | 4.696848869323730469e+00 1526 | 4.693640708923339844e+00 1527 | 4.692839145660400391e+00 1528 | 4.693606853485107422e+00 1529 | 4.691143512725830078e+00 1530 | 4.690576076507568359e+00 1531 | 4.691030979156494141e+00 1532 | 4.689054012298583984e+00 1533 | 4.688276290893554688e+00 1534 | 4.688294887542724609e+00 1535 | 4.687037944793701172e+00 1536 | 4.685943126678466797e+00 1537 | 4.685893058776855469e+00 1538 | 4.685142040252685547e+00 1539 | 4.684110641479492188e+00 1540 | 4.683745861053466797e+00 1541 | 4.683324337005615234e+00 1542 | 4.682349205017089844e+00 1543 | 4.681669235229492188e+00 1544 | 4.681500434875488281e+00 1545 | 4.680631160736083984e+00 1546 | 4.680048465728759766e+00 1547 | 4.679554939270019531e+00 1548 | 4.679305553436279297e+00 1549 | 4.678524494171142578e+00 1550 | 4.678581714630126953e+00 1551 | 4.678309440612792969e+00 1552 | 4.679179191589355469e+00 1553 | 4.679904460906982422e+00 1554 | 4.683689594268798828e+00 1555 | 4.689205169677734375e+00 1556 | 4.703793048858642578e+00 1557 | 4.728044986724853516e+00 1558 | 4.783923149108886719e+00 1559 | 4.876929759979248047e+00 1560 | 5.057742595672607422e+00 1561 | 5.265183925628662109e+00 1562 | 5.460395336151123047e+00 1563 | 5.284328937530517578e+00 1564 | 4.893163681030273438e+00 1565 | 4.722039699554443359e+00 1566 | 4.912138462066650391e+00 1567 | 5.019078254699707031e+00 1568 | 4.764184951782226562e+00 1569 | 4.729633331298828125e+00 1570 | 4.896871089935302734e+00 1571 | 4.785774707794189453e+00 1572 | 4.711453437805175781e+00 1573 | 4.820876598358154297e+00 1574 | 4.749483585357666016e+00 1575 | 4.704790592193603516e+00 1576 | 4.784140110015869141e+00 1577 | 4.721695423126220703e+00 1578 | 4.706299304962158203e+00 1579 | 4.756062984466552734e+00 1580 | 4.699306964874267578e+00 1581 | 4.712326526641845703e+00 1582 | 4.731743335723876953e+00 1583 | 4.686443805694580078e+00 1584 | 4.711966514587402344e+00 1585 | 4.709019184112548828e+00 1586 | 4.682989120483398438e+00 1587 | 4.706247806549072266e+00 1588 | 4.690056800842285156e+00 1589 | 4.682463645935058594e+00 1590 | 4.698336601257324219e+00 1591 | 4.679073810577392578e+00 1592 | 4.681268692016601562e+00 1593 | 4.688338279724121094e+00 1594 | 4.672910690307617188e+00 1595 | 4.678955078125000000e+00 1596 | 4.680456638336181641e+00 1597 | 4.668753623962402344e+00 1598 | 4.674654006958007812e+00 1599 | 4.674379348754882812e+00 1600 | 4.665898799896240234e+00 1601 | 4.670328140258789062e+00 1602 | 4.669526576995849609e+00 1603 | 4.663254737854003906e+00 1604 | 4.666861534118652344e+00 1605 | 4.665576457977294922e+00 1606 | 4.660839557647705078e+00 1607 | 4.663229465484619141e+00 1608 | 4.662458896636962891e+00 1609 | 4.658815383911132812e+00 1610 | 4.659429550170898438e+00 1611 | 4.659757137298583984e+00 1612 | 4.656898498535156250e+00 1613 | 4.656359672546386719e+00 1614 | 4.657001972198486328e+00 1615 | 4.655103206634521484e+00 1616 | 4.653820037841796875e+00 1617 | 4.654367446899414062e+00 1618 | 4.653562545776367188e+00 1619 | 4.651627063751220703e+00 1620 | 4.652165412902832031e+00 1621 | 4.651643753051757812e+00 1622 | 4.650299549102783203e+00 1623 | 4.649689197540283203e+00 1624 | 4.649731636047363281e+00 1625 | 4.648677825927734375e+00 1626 | 4.648046970367431641e+00 1627 | 4.647531986236572266e+00 1628 | 4.647304534912109375e+00 1629 | 4.646411418914794922e+00 1630 | 4.645955562591552734e+00 1631 | 4.645583152770996094e+00 1632 | 4.645966053009033203e+00 1633 | 4.645044803619384766e+00 1634 | 4.646525382995605469e+00 1635 | 4.647288799285888672e+00 1636 | 4.652328968048095703e+00 1637 | 4.658030509948730469e+00 1638 | 4.677130222320556641e+00 1639 | 4.705276966094970703e+00 1640 | 4.777132987976074219e+00 1641 | 4.881691932678222656e+00 1642 | 5.094754219055175781e+00 1643 | 5.257537841796875000e+00 1644 | 5.365346908569335938e+00 1645 | 5.054018497467041016e+00 1646 | 4.738260269165039062e+00 1647 | 4.725629329681396484e+00 1648 | 4.915697574615478516e+00 1649 | 4.908209800720214844e+00 1650 | 4.682435989379882812e+00 1651 | 4.721078872680664062e+00 1652 | 4.851063728332519531e+00 1653 | 4.713596343994140625e+00 1654 | 4.683168411254882812e+00 1655 | 4.786197662353515625e+00 1656 | 4.698294162750244141e+00 1657 | 4.669334411621093750e+00 1658 | 4.744324207305908203e+00 1659 | 4.681345462799072266e+00 1660 | 4.667260646820068359e+00 1661 | 4.714396953582763672e+00 1662 | 4.664872169494628906e+00 1663 | 4.667840003967285156e+00 1664 | 4.693832874298095703e+00 1665 | 4.653329372406005859e+00 1666 | 4.666254043579101562e+00 1667 | 4.676749229431152344e+00 1668 | 4.647314548492431641e+00 1669 | 4.662495136260986328e+00 1670 | 4.663220405578613281e+00 1671 | 4.644178390502929688e+00 1672 | 4.657663822174072266e+00 1673 | 4.653238296508789062e+00 1674 | 4.641091823577880859e+00 1675 | 4.652130603790283203e+00 1676 | 4.646487236022949219e+00 1677 | 4.638577938079833984e+00 1678 | 4.645944595336914062e+00 1679 | 4.641382694244384766e+00 1680 | 4.636108875274658203e+00 1681 | 4.641146659851074219e+00 1682 | 4.637656688690185547e+00 1683 | 4.632866859436035156e+00 1684 | 4.636504173278808594e+00 1685 | 4.634783744812011719e+00 1686 | 4.630261421203613281e+00 1687 | 4.632904052734375000e+00 1688 | 4.632415294647216797e+00 1689 | 4.627927303314208984e+00 1690 | 4.629091262817382812e+00 1691 | 4.629772663116455078e+00 1692 | 4.626060962677001953e+00 1693 | 4.626055717468261719e+00 1694 | 4.627339363098144531e+00 1695 | 4.624491691589355469e+00 1696 | 4.623656749725341797e+00 1697 | 4.624630928039550781e+00 1698 | 4.623108386993408203e+00 1699 | 4.621748447418212891e+00 1700 | 4.621941089630126953e+00 1701 | 4.621495246887207031e+00 1702 | 4.620402812957763672e+00 1703 | 4.619703769683837891e+00 1704 | 4.619613170623779297e+00 1705 | 4.619330883026123047e+00 1706 | 4.618122577667236328e+00 1707 | 4.618010044097900391e+00 1708 | 4.618517875671386719e+00 1709 | 4.618002891540527344e+00 1710 | 4.618704795837402344e+00 1711 | 4.620636463165283203e+00 1712 | 4.624711513519287109e+00 1713 | 4.632051467895507812e+00 1714 | 4.648537158966064453e+00 1715 | 4.678598403930664062e+00 1716 | 4.743638515472412109e+00 1717 | 4.848263740539550781e+00 1718 | 5.054288864135742188e+00 1719 | 5.235933303833007812e+00 1720 | 5.426553726196289062e+00 1721 | 5.122064113616943359e+00 1722 | 4.744033336639404297e+00 1723 | 4.648808479309082031e+00 1724 | 4.864217281341552734e+00 1725 | 4.907516002655029297e+00 1726 | 4.664645671844482422e+00 1727 | 4.695236206054687500e+00 1728 | 4.827970504760742188e+00 1729 | 4.682969570159912109e+00 1730 | 4.658956050872802734e+00 1731 | 4.759282588958740234e+00 1732 | 4.664829254150390625e+00 1733 | 4.654561519622802734e+00 1734 | 4.718646526336669922e+00 1735 | 4.644605159759521484e+00 1736 | 4.658163070678710938e+00 1737 | 4.686936378479003906e+00 1738 | 4.631144523620605469e+00 1739 | 4.660932064056396484e+00 1740 | 4.659697055816650391e+00 1741 | 4.625702381134033203e+00 1742 | 4.657537937164306641e+00 1743 | 4.638727664947509766e+00 1744 | 4.624695301055908203e+00 1745 | 4.648587226867675781e+00 1746 | 4.624738216400146484e+00 1747 | 4.624532699584960938e+00 1748 | 4.637687683105468750e+00 1749 | 4.617142200469970703e+00 1750 | 4.622700214385986328e+00 1751 | 4.627951145172119141e+00 1752 | 4.612878322601318359e+00 1753 | 4.619236469268798828e+00 1754 | 4.620137691497802734e+00 1755 | 4.609455585479736328e+00 1756 | 4.615118026733398438e+00 1757 | 4.614428997039794922e+00 1758 | 4.606681346893310547e+00 1759 | 4.611055374145507812e+00 1760 | 4.610219478607177734e+00 1761 | 4.604681491851806641e+00 1762 | 4.607277870178222656e+00 1763 | 4.606598377227783203e+00 1764 | 4.602608680725097656e+00 1765 | 4.603777408599853516e+00 1766 | 4.603771209716796875e+00 1767 | 4.600739955902099609e+00 1768 | 4.600655078887939453e+00 1769 | 4.601152420043945312e+00 1770 | 4.599070549011230469e+00 1771 | 4.598116397857666016e+00 1772 | 4.598765373229980469e+00 1773 | 4.597434043884277344e+00 1774 | 4.596021652221679688e+00 1775 | 4.596358299255371094e+00 1776 | 4.595855236053466797e+00 1777 | 4.594439506530761719e+00 1778 | 4.593948364257812500e+00 1779 | 4.594105243682861328e+00 1780 | 4.592991352081298828e+00 1781 | 4.592218399047851562e+00 1782 | 4.592101573944091797e+00 1783 | 4.591703414916992188e+00 1784 | 4.590767383575439453e+00 1785 | 4.590232849121093750e+00 1786 | 4.590122699737548828e+00 1787 | 4.589489459991455078e+00 1788 | 4.588692188262939453e+00 1789 | 4.588514804840087891e+00 1790 | 4.588088989257812500e+00 1791 | 4.587787151336669922e+00 1792 | 4.587185382843017578e+00 1793 | 4.587551116943359375e+00 1794 | 4.587766170501708984e+00 1795 | 4.589567661285400391e+00 1796 | 4.592487812042236328e+00 1797 | 4.602020740509033203e+00 1798 | 4.619616031646728516e+00 1799 | 4.663351535797119141e+00 1800 | 4.750739097595214844e+00 1801 | 4.934890270233154297e+00 1802 | 5.251309871673583984e+00 1803 | 5.620818138122558594e+00 1804 | 5.649824619293212891e+00 1805 | 5.100247383117675781e+00 1806 | 4.630864620208740234e+00 1807 | 4.903137207031250000e+00 1808 | 5.095463752746582031e+00 1809 | 4.698796272277832031e+00 1810 | 4.728740692138671875e+00 1811 | 4.924987316131591797e+00 1812 | 4.671471595764160156e+00 1813 | 4.743747234344482422e+00 1814 | 4.803845405578613281e+00 1815 | 4.635374546051025391e+00 1816 | 4.766129016876220703e+00 1817 | 4.684412002563476562e+00 1818 | 4.662609577178955078e+00 1819 | 4.716537952423095703e+00 1820 | 4.625468254089355469e+00 1821 | 4.694331169128417969e+00 1822 | 4.636487960815429688e+00 1823 | 4.648375511169433594e+00 1824 | 4.657339096069335938e+00 1825 | 4.618768215179443359e+00 1826 | 4.653647899627685547e+00 1827 | 4.609612464904785156e+00 1828 | 4.637603759765625000e+00 1829 | 4.617634773254394531e+00 1830 | 4.609572410583496094e+00 1831 | 4.626757144927978516e+00 1832 | 4.598838329315185547e+00 1833 | 4.616632938385009766e+00 1834 | 4.601172924041748047e+00 1835 | 4.603263378143310547e+00 1836 | 4.604648113250732422e+00 1837 | 4.591687202453613281e+00 1838 | 4.602647304534912109e+00 1839 | 4.591049671173095703e+00 1840 | 4.592139720916748047e+00 1841 | 4.594042301177978516e+00 1842 | 4.585636138916015625e+00 1843 | 4.590279102325439453e+00 1844 | 4.585139751434326172e+00 1845 | 4.584864139556884766e+00 1846 | 4.585180282592773438e+00 1847 | 4.579981327056884766e+00 1848 | 4.582917213439941406e+00 1849 | 4.580014228820800781e+00 1850 | 4.577713489532470703e+00 1851 | 4.579822540283203125e+00 1852 | 4.575978755950927734e+00 1853 | 4.575638294219970703e+00 1854 | 4.576319217681884766e+00 1855 | 4.573340892791748047e+00 1856 | 4.573670387268066406e+00 1857 | 4.573363780975341797e+00 1858 | 4.571088314056396484e+00 1859 | 4.571506023406982422e+00 1860 | 4.570858478546142578e+00 1861 | 4.569283008575439453e+00 1862 | 4.569595336914062500e+00 1863 | 4.568594932556152344e+00 1864 | 4.567648410797119141e+00 1865 | 4.567647933959960938e+00 1866 | 4.566660881042480469e+00 1867 | 4.566163063049316406e+00 1868 | 4.565759181976318359e+00 1869 | 4.565046787261962891e+00 1870 | 4.564529895782470703e+00 1871 | 4.564132690429687500e+00 1872 | 4.563625812530517578e+00 1873 | 4.562973976135253906e+00 1874 | 4.562512397766113281e+00 1875 | 4.562286376953125000e+00 1876 | 4.561444759368896484e+00 1877 | 4.561168670654296875e+00 1878 | 4.560802459716796875e+00 1879 | 4.560178279876708984e+00 1880 | 4.559781074523925781e+00 1881 | 4.559597969055175781e+00 1882 | 4.558965682983398438e+00 1883 | 4.558766841888427734e+00 1884 | 4.558569908142089844e+00 1885 | 4.558598041534423828e+00 1886 | 4.558539390563964844e+00 1887 | 4.559922218322753906e+00 1888 | 4.561473846435546875e+00 1889 | 4.566117763519287109e+00 1890 | 4.574191093444824219e+00 1891 | 4.591935634613037109e+00 1892 | 4.625221729278564453e+00 1893 | 4.689006805419921875e+00 1894 | 4.813622951507568359e+00 1895 | 4.983055114746093750e+00 1896 | 5.260444164276123047e+00 1897 | 5.273389339447021484e+00 1898 | 5.103623867034912109e+00 1899 | 4.680662155151367188e+00 1900 | 4.607698440551757812e+00 1901 | 4.844503879547119141e+00 1902 | 4.819971561431884766e+00 1903 | 4.599523544311523438e+00 1904 | 4.633107662200927734e+00 1905 | 4.748788356781005859e+00 1906 | 4.632512092590332031e+00 1907 | 4.591595649719238281e+00 1908 | 4.687368869781494141e+00 1909 | 4.619547367095947266e+00 1910 | 4.589145183563232422e+00 1911 | 4.655703067779541016e+00 1912 | 4.594529151916503906e+00 1913 | 4.590128421783447266e+00 1914 | 4.631700515747070312e+00 1915 | 4.576885223388671875e+00 1916 | 4.591651439666748047e+00 1917 | 4.607918262481689453e+00 1918 | 4.567653656005859375e+00 1919 | 4.590688228607177734e+00 1920 | 4.587141513824462891e+00 1921 | 4.564057350158691406e+00 1922 | 4.585315704345703125e+00 1923 | 4.572199821472167969e+00 1924 | 4.563907623291015625e+00 1925 | 4.577651977539062500e+00 1926 | 4.562731266021728516e+00 1927 | 4.563444614410400391e+00 1928 | 4.570100307464599609e+00 1929 | 4.556959152221679688e+00 1930 | 4.561111450195312500e+00 1931 | 4.563292026519775391e+00 1932 | 4.553605556488037109e+00 1933 | 4.558277606964111328e+00 1934 | 4.557600975036621094e+00 1935 | 4.550699234008789062e+00 1936 | 4.554858684539794922e+00 1937 | 4.553684234619140625e+00 1938 | 4.548782825469970703e+00 1939 | 4.551262378692626953e+00 1940 | 4.550354480743408203e+00 1941 | 4.546899795532226562e+00 1942 | 4.548100471496582031e+00 1943 | 4.547946929931640625e+00 1944 | 4.545055866241455078e+00 1945 | 4.545107841491699219e+00 1946 | 4.545702457427978516e+00 1947 | 4.543502807617187500e+00 1948 | 4.542637348175048828e+00 1949 | 4.543395996093750000e+00 1950 | 4.542187690734863281e+00 1951 | 4.540689945220947266e+00 1952 | 4.541161060333251953e+00 1953 | 4.540693283081054688e+00 1954 | 4.539006710052490234e+00 1955 | 4.539232730865478516e+00 1956 | 4.539122104644775391e+00 1957 | 4.537764549255371094e+00 1958 | 4.537410259246826172e+00 1959 | 4.537411212921142578e+00 1960 | 4.536724090576171875e+00 1961 | 4.535990715026855469e+00 1962 | 4.535685062408447266e+00 1963 | 4.535413742065429688e+00 1964 | 4.534825801849365234e+00 1965 | 4.534164905548095703e+00 1966 | 4.534020900726318359e+00 1967 | 4.533908367156982422e+00 1968 | 4.533160686492919922e+00 1969 | 4.533544540405273438e+00 1970 | 4.533921241760253906e+00 1971 | 4.535876274108886719e+00 1972 | 4.539309024810791016e+00 1973 | 4.549741744995117188e+00 1974 | 4.570900440216064453e+00 1975 | 4.622570991516113281e+00 1976 | 4.732003688812255859e+00 1977 | 4.955605506896972656e+00 1978 | 5.329213142395019531e+00 1979 | 5.636598110198974609e+00 1980 | 5.378370761871337891e+00 1981 | 4.772851467132568359e+00 1982 | 4.600762367248535156e+00 1983 | 4.940210342407226562e+00 1984 | 4.903820037841796875e+00 1985 | 4.573078632354736328e+00 1986 | 4.773523807525634766e+00 1987 | 4.803425788879394531e+00 1988 | 4.572136402130126953e+00 1989 | 4.759445667266845703e+00 1990 | 4.678470611572265625e+00 1991 | 4.605993270874023438e+00 1992 | 4.723386764526367188e+00 1993 | 4.581454277038574219e+00 1994 | 4.660279750823974609e+00 1995 | 4.627031803131103516e+00 1996 | 4.581632137298583984e+00 1997 | 4.648919582366943359e+00 1998 | 4.570189476013183594e+00 1999 | 4.616522789001464844e+00 2000 | 4.582422256469726562e+00 2001 | -------------------------------------------------------------------------------- /data/vlosses_exact_sgd.csv: -------------------------------------------------------------------------------- 1 | 9.758504486083984375e+01 2 | 3.400662994384765625e+01 3 | 3.367346954345703125e+01 4 | 4.852850723266601562e+01 5 | 6.024071884155273438e+01 6 | 5.707302856445312500e+01 7 | 5.643809127807617188e+01 8 | 5.624697875976562500e+01 9 | 5.701696777343750000e+01 10 | 5.591333007812500000e+01 11 | 5.559650039672851562e+01 12 | 5.568336868286132812e+01 13 | 5.570820999145507812e+01 14 | 5.532520675659179688e+01 15 | 5.534587478637695312e+01 16 | 5.618130493164062500e+01 17 | 5.603034973144531250e+01 18 | 5.590623092651367188e+01 19 | 5.596089172363281250e+01 20 | 5.608708572387695312e+01 21 | 5.598926544189453125e+01 22 | 5.598997497558593750e+01 23 | 5.583658599853515625e+01 24 | 5.558216094970703125e+01 25 | 5.564654922485351562e+01 26 | 5.563145446777343750e+01 27 | 5.560173034667968750e+01 28 | 5.553960418701171875e+01 29 | 5.539214706420898438e+01 30 | 5.546532821655273438e+01 31 | 5.547106170654296875e+01 32 | 5.617616271972656250e+01 33 | 5.602111053466796875e+01 34 | 5.587174224853515625e+01 35 | 5.573401641845703125e+01 36 | 5.562297439575195312e+01 37 | 5.575422286987304688e+01 38 | 5.568199920654296875e+01 39 | 5.544352340698242188e+01 40 | 5.527815628051757812e+01 41 | 5.528773498535156250e+01 42 | 5.532159423828125000e+01 43 | 5.532664108276367188e+01 44 | 5.521370315551757812e+01 45 | 5.521541595458984375e+01 46 | 5.570694732666015625e+01 47 | 5.627413558959960938e+01 48 | 5.550067520141601562e+01 49 | 5.518924713134765625e+01 50 | 5.644663619995117188e+01 51 | 5.691810607910156250e+01 52 | 5.611870956420898438e+01 53 | 5.585436248779296875e+01 54 | 5.613080596923828125e+01 55 | 5.592994689941406250e+01 56 | 5.593198013305664062e+01 57 | 5.630574417114257812e+01 58 | 5.595421218872070312e+01 59 | 5.590480041503906250e+01 60 | 5.577611923217773438e+01 61 | 5.601798629760742188e+01 62 | 5.573671722412109375e+01 63 | 5.567825317382812500e+01 64 | 5.573222351074218750e+01 65 | 5.569287872314453125e+01 66 | 5.559291839599609375e+01 67 | 5.551215362548828125e+01 68 | 5.544824218750000000e+01 69 | 5.546522903442382812e+01 70 | 5.542956161499023438e+01 71 | 5.554744720458984375e+01 72 | 5.517622375488281250e+01 73 | 5.514503097534179688e+01 74 | 5.509885406494140625e+01 75 | 5.515015411376953125e+01 76 | 5.507431030273437500e+01 77 | 5.494500350952148438e+01 78 | 5.481526184082031250e+01 79 | 5.465647888183593750e+01 80 | 5.496338653564453125e+01 81 | 5.477564239501953125e+01 82 | 5.485616683959960938e+01 83 | 5.491510009765625000e+01 84 | 5.496881866455078125e+01 85 | 5.484794235229492188e+01 86 | 5.462095642089843750e+01 87 | 5.464306259155273438e+01 88 | 5.451524734497070312e+01 89 | 5.450964736938476562e+01 90 | 5.443022918701171875e+01 91 | 5.479624938964843750e+01 92 | 5.455032348632812500e+01 93 | 5.477447891235351562e+01 94 | 5.477642440795898438e+01 95 | 5.486117172241210938e+01 96 | 5.474536895751953125e+01 97 | 5.476601028442382812e+01 98 | 5.495127868652343750e+01 99 | 5.495363616943359375e+01 100 | 5.477404403686523438e+01 101 | -------------------------------------------------------------------------------- /data/vlosses_stochastic_sgd.csv: -------------------------------------------------------------------------------- 1 | 9.758504486083984375e+01 2 | 6.243669891357421875e+01 3 | 4.499292373657226562e+01 4 | 7.914521789550781250e+01 5 | 5.643527221679687500e+01 6 | 4.837200546264648438e+01 7 | 4.424446868896484375e+01 8 | 4.064106369018554688e+01 9 | 3.750775146484375000e+01 10 | 3.793084716796875000e+01 11 | 3.486598587036132812e+01 12 | 3.005109596252441406e+01 13 | 2.658629989624023438e+01 14 | 2.584941673278808594e+01 15 | 2.291995429992675781e+01 16 | 2.012074470520019531e+01 17 | 1.905515670776367188e+01 18 | 1.617766761779785156e+01 19 | 1.807382392883300781e+01 20 | 1.461955928802490234e+01 21 | 1.714438247680664062e+01 22 | 1.256489181518554688e+01 23 | 1.494048118591308594e+01 24 | 1.161095046997070312e+01 25 | 1.391752624511718750e+01 26 | 1.082573699951171875e+01 27 | 1.136747074127197266e+01 28 | 1.014708995819091797e+01 29 | 1.059537792205810547e+01 30 | 9.847717285156250000e+00 31 | 1.038200950622558594e+01 32 | 9.895535469055175781e+00 33 | 1.030114078521728516e+01 34 | 9.875215530395507812e+00 35 | 1.003153514862060547e+01 36 | 9.613888740539550781e+00 37 | 9.663681030273437500e+00 38 | 9.467631340026855469e+00 39 | 9.564559936523437500e+00 40 | 9.458808898925781250e+00 41 | 9.545150756835937500e+00 42 | 9.424928665161132812e+00 43 | 9.483310699462890625e+00 44 | 9.428945541381835938e+00 45 | 9.454419136047363281e+00 46 | 9.463463783264160156e+00 47 | 9.504481315612792969e+00 48 | 9.515193939208984375e+00 49 | 9.514415740966796875e+00 50 | 9.511509895324707031e+00 51 | 9.500372886657714844e+00 52 | 9.525661468505859375e+00 53 | 9.480981826782226562e+00 54 | 9.564155578613281250e+00 55 | 9.467595100402832031e+00 56 | 9.629287719726562500e+00 57 | 9.454057693481445312e+00 58 | 9.695550918579101562e+00 59 | 9.452747344970703125e+00 60 | 9.730901718139648438e+00 61 | 9.454604148864746094e+00 62 | 9.731983184814453125e+00 63 | 9.449135780334472656e+00 64 | 9.720386505126953125e+00 65 | 9.469259262084960938e+00 66 | 9.698139190673828125e+00 67 | 9.497591018676757812e+00 68 | 9.670207977294921875e+00 69 | 9.553378105163574219e+00 70 | 9.640308380126953125e+00 71 | 9.619763374328613281e+00 72 | 9.625095367431640625e+00 73 | 9.669338226318359375e+00 74 | 9.606250762939453125e+00 75 | 9.717122077941894531e+00 76 | 9.607691764831542969e+00 77 | 9.759179115295410156e+00 78 | 9.611286163330078125e+00 79 | 9.779539108276367188e+00 80 | 9.622556686401367188e+00 81 | 9.785450935363769531e+00 82 | 9.637899398803710938e+00 83 | 9.779893875122070312e+00 84 | 9.655331611633300781e+00 85 | 9.771986961364746094e+00 86 | 9.699286460876464844e+00 87 | 9.774522781372070312e+00 88 | 9.747869491577148438e+00 89 | 9.783057212829589844e+00 90 | 9.775449752807617188e+00 91 | 9.780313491821289062e+00 92 | 9.778766632080078125e+00 93 | 9.760302543640136719e+00 94 | 9.775089263916015625e+00 95 | 9.755829811096191406e+00 96 | 9.773841857910156250e+00 97 | 9.763814926147460938e+00 98 | 9.772111892700195312e+00 99 | 9.776626586914062500e+00 100 | 9.774176597595214844e+00 101 | -------------------------------------------------------------------------------- /deep_autoencoder.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 40, 6 | "metadata": { 7 | "collapsed": true 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import os\n", 12 | "import sys\n", 13 | "import util as u\n", 14 | "import matplotlib.pyplot as plt\n", 15 | "import torch.nn.functional as F" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "# Deep autoencoder" 23 | ] 24 | }, 25 | { 26 | "cell_type": "markdown", 27 | "metadata": {}, 28 | "source": [ 29 | "Optimize the following sigmoid autoencoder on first 10k MNIST examples, evaluate on second 10k \n", 30 | "\n", 31 | "![deep autoencoder](https://i.stack.imgur.com/f09ot.png)" 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "metadata": {}, 37 | "source": [ 38 | "# Short run" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 105, 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "import kfac_pytorch as kfac_lib" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": 107, 53 | "metadata": {}, 54 | "outputs": [ 55 | { 56 | "name": "stdout", 57 | "output_type": "stream", 58 | "text": [ 59 | "Step 0 loss 97.542419434\n", 60 | "Step 1 loss 32.184158325\n", 61 | "Step 2 loss 31.059148788\n", 62 | "Step 3 loss 30.073652267\n", 63 | "Step 4 loss 28.751443863\n", 64 | "Step 5 loss 28.019514084\n", 65 | "Step 6 loss 27.574556351\n", 66 | "Step 7 loss 27.251935959\n", 67 | "Step 8 loss 27.024274826\n", 68 | "Step 9 loss 26.863992691\n", 69 | "Times: min: 87.59, median: 88.08, mean: 1256.60\n" 70 | ] 71 | } 72 | ], 73 | "source": [ 74 | "losses_adam, vlosses_adam = kfac_lib.train(iters=10, kfac=False, print_interval=1)\n", 75 | "u.summarize_time()" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 106, 81 | "metadata": {}, 82 | "outputs": [ 83 | { 84 | "name": "stdout", 85 | "output_type": "stream", 86 | "text": [ 87 | "Step 0 loss 97.542419434\n", 88 | "Step 1 loss 62.339828491\n", 89 | "Step 2 loss 44.860393524\n", 90 | "Step 3 loss 79.031013489\n", 91 | "Step 4 loss 56.055324554\n", 92 | "Step 5 loss 48.206447601\n", 93 | "Step 6 loss 43.934066772\n", 94 | "Step 7 loss 40.302700043\n", 95 | "Step 8 loss 38.371196747\n", 96 | "Step 9 loss 38.781795502\n", 97 | "Times: min: 384.54, median: 388.57, mean: 1689.78\n" 98 | ] 99 | } 100 | ], 101 | "source": [ 102 | "losses_adam, vlosses_adam = kfac_lib.train(iters=10, kfac=True, print_interval=1)\n", 103 | "u.summarize_time()" 104 | ] 105 | }, 106 | { 107 | "cell_type": "markdown", 108 | "metadata": {}, 109 | "source": [ 110 | "# Long run" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "metadata": {}, 116 | "source": [ 117 | "python run_experiments.py" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": 109, 123 | "metadata": {}, 124 | "outputs": [ 125 | { 126 | "data": { 127 | "text/plain": [ 128 | "" 129 | ] 130 | }, 131 | "metadata": {}, 132 | "output_type": "display_data" 133 | }, 134 | { 135 | "data": { 136 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VdW5//HPc04SQgKEQACZIjgig4CCiENFcUDr1MGp\nVfHW1t4OttXW6V4HvJ2s9/ZafbW1PxzRq61z6zzjUC0qqC0yqMggQcYAgUBChvP8/tg7IcA55AA5\nOUn29/16ndc5e+3pWSGcJ2vttdc2d0dERGR7sWwHICIibZMShIiIJKUEISIiSSlBiIhIUkoQIiKS\nlBKEiIgkpQQhsovM7E9mdt1u7vuamX27pWMSyYScbAcg0prMbDHwbXd/eXeP4e7/3nIRibRdakGI\nNGFm+qNJJKQEIZFhZvcDpcBTZlZpZlea2SAzczO72Mw+B14Nt33EzFaYWYWZvWFmw5oc514z+0X4\neYKZlZnZT81slZktN7N/SzOemJlda2ZLwn3vM7OicF2+mf2fmZWb2Xoze8/M+oTrLjKzhWa20cwW\nmdk3W/hHJQIoQUiEuPsFwOfAae7exd1vbrL6GOAg4KRw+Tlgf6A38D7wwE4OvRdQBPQHLgb+YGbF\naYR0Ufg6FtgH6AL8Plw3OTzmQKAn8O9AlZkVArcBJ7t7V+AI4MM0ziWyy5QgRAJT3H2Tu1cBuPvd\n7r7R3bcAU4CRDX/dJ1EL/Je717r7s0AlcGAa5/wm8L/uvtDdK4FrgHPDbq5agsSwn7vXu/ssd98Q\n7pcAhptZZ3df7u5zdrfSIjujBCESWNrwwcziZnaTmX1mZhuAxeGqkhT7lrt7XZPlzQStgeb0A5Y0\nWV5CMHCkD3A/8ALwFzP7wsxuNrNcd98EnEPQolhuZs+Y2ZA0ziWyy5QgJGpSTV/ctPwbwBnA8QTd\nPIPCcmvhWL4A9m6yXArUASvD1siN7j6UoBvpVOBCAHd/wd1PAPoC84E7WjguEUAJQqJnJUF//850\nBbYA5UAB8KsMxfJn4DIzG2xmXcLzPOTudWZ2rJmNMLM4sIGgyylhZn3M7IzwWsQWgu6sRIbik4hT\ngpCo+TVwbTgy6GcptrmPoLtnGTAXmJGhWO4m6Ep6A1gEVAOXhuv2Ah4lSA7zgNfDbWPA5QStj7UE\nF9e/l6H4JOJMDwwSEZFk1IIQEZGklCBERCQpJQgREUlKCUJERJJq1xOTlZSU+KBBg7IdhohIuzJr\n1qw17t6rue3adYIYNGgQM2fOzHYYIiLtipktaX6rDHYxmdnd4QyVHzUp62FmL5nZp+F7cVhuZnab\nmS0ws3+Z2SGZiktERNKTyWsQ9wKTtiu7GnjF3fcHXgmXAU4mmDlzf+AS4PYMxiUiImnIWIJw9zcI\n7vRs6gxgWvh5GnBmk/L7PDAD6G5mfTMVm4iINK+1r0H0cffl4ecVBLNWQjCP/tIm25WFZcvZjpld\nQtDKoLS0NHORikiHUFtbS1lZGdXV1dkOpdXl5+czYMAAcnNzd2v/rF2kdnc3s12e58PdpwJTAcaM\nGaN5QkRkp8rKyujatSuDBg3CrKUn5G273J3y8nLKysoYPHjwbh2jte+DWNnQdRS+rwrLlxE8OavB\ngLBMRGSPVFdX07Nnz0glBwAzo2fPnnvUcmrtBPEkwaMUCd//1qT8wnA00+FARZOuKBGRPRK15NBg\nT+udyWGufwb+ARwYPtT9YuAm4AQz+5TgYSw3hZs/CywEFhA8/OT7mYoL4Inpf+Kau09n0+aNmTyN\niEi7lslRTOe5e193z3X3Ae5+l7uXu/tEd9/f3Y9397Xhtu7uP3D3fd19hLtn9O63fy19jafji6iu\n2ZTJ04iIALB48WKGDx++Q/l5553HwQcfzC233JKFqJrXru+kFhFpr1asWMF7773HggULsh1KSpqs\nT0SkFS1cuJDRo0czePBgli1bxqhRo3jzzTe54447GDt2LCNHjuRrX/samzdvBmDlypV85StfYeTI\nkYwcOZK333671WKNdAtCD9MTiZYbn5rD3C82tOgxh/brxg2nDUtr248//phzzz2Xe++9l6KiIk49\n9VQ+/PDD4DhDh/Kd73wHgGuvvZa77rqLSy+9lB/96Eccc8wxPPHEE9TX11NZWdmi8e9MpBOEiEhr\nWb16NWeccQaPP/44Q4cOZfHixdus/+ijj7j22mtZv349lZWVnHTSSQC8+uqr3HfffQDE43GKiopa\nLWYlCBGJjHT/0s+EoqIiSktL+fvf/87QoUN3WH/RRRfx17/+lZEjR3Lvvffy2muvtX6Q29E1CBGR\nVpCXl8cTTzzBfffdx4MPPrjD+o0bN9K3b19qa2t54IEHGssnTpzI7bcH85fW19dTUVHRajErQYiI\ntJLCwkKefvppbrnlFp588slt1v385z9n3LhxHHnkkQwZMqSx/NZbb2X69OmMGDGCQw89lLlz57Za\nvObt+ErtmDFjfHceGHTjfefyqM/hldNfoHdxvwxEJiJtxbx58zjooIOyHUbWJKu/mc1y9zHN7RvR\nFkREqy0isgv0TSkiIkkpQYiISFJKECIiklSkE0R7vkAvIpJpkUwQ0ZwZXkRk10QyQYiItAeDBg1i\nzZo1WTt/pBOEephERFKL9FxMugYhIpm2adMmzj77bMrKyqivr+e6666ja9euXH755RQWFnLkkUey\ncOFCnn76acrLyznvvPNYtmwZ48ePz/p3VKQThIhEzHNXw4rZLXvMvUbAyTelXP3888/Tr18/nnnm\nGQAqKioYPnw4b7zxBoMHD+a8885r3PbGG2/kqKOO4vrrr+eZZ57hrrvuatlYd1Gku5hERDJtxIgR\nvPTSS1x11VW8+eabLFq0iH322YfBgwcDbJMg3njjDc4//3wAvvzlL1NcXJyVmBuoBSEi0bGTv/Qz\n5YADDuD999/n2Wef5dprr2XixImtHsPuUgtCRCSDvvjiCwoKCjj//PO54ooreOutt1i4cGHjA4Me\neuihxm2/9KUvNU4F/txzz7Fu3bpshNwo2i0IXaMWkQybPXs2V1xxBbFYjNzcXG6//XaWL1/OpEmT\nKCwsZOzYsY3b3nDDDZx33nkMGzaMI444gtLS0ixGHtUEYabkICKt4qSTTmp8fGiDyspK5s+fj7vz\ngx/8gDFjgpm3e/bsyYsvvpiNMJNSF5OISCu74447GDVqFMOGDaOiooLvfve72Q4pqWi2IEKuZoSI\nZMFll13GZZddlu0wmhXJFoQ1zMak/CAiklIkE4SIiDRPCUJERJJSghARkaSUIEREsuzee+/lhz/8\nYbbD2EGkE4RGMYmIpBbpBCEi0hrOPPNMDj30UIYNG8bUqVMBuOeeezjggAM47LDDeOuttxq3feqp\npxg3bhyjR4/m+OOPZ+XKlQBMmTKFyZMnc/TRR7P33nvz+OOPc+WVVzJixAgmTZpEbW1ti8edlfsg\nzOwy4NsEA01nA/8G9AX+AvQEZgEXuHtNNuITkY7pN+/+hvlr57foMYf0GMJVh121023uvvtuevTo\nQVVVFWPHjuXLX/4yN9xwA7NmzaKoqIhjjz2W0aNHA3DUUUcxY8YMzIw777yTm2++md/+9rcAfPbZ\nZ0yfPp25c+cyfvx4HnvsMW6++Wa+8pWv8Mwzz3DmmWe2aN1aPUGYWX/gR8BQd68ys4eBc4FTgFvc\n/S9m9ifgYuD2TMbiJDJ5eBERAG677TaeeOIJAJYuXcr999/PhAkT6NWrFwDnnHMOn3zyCQBlZWWc\nc845LF++nJqamsZpwQFOPvlkcnNzGTFiBPX19UyaNAkIphRvmPyvJWXrTuocoLOZ1QIFwHLgOOAb\n4fppwBQynSB0CUIkUpr7Sz8TXnvtNV5++WX+8Y9/UFBQwIQJExgyZAhz585Nuv2ll17K5Zdfzumn\nn85rr73GlClTGtd16tQJoHHiPzNrXK6rq2vx2Fv9GoS7LwP+B/icIDFUEHQprXf3hhqWAf2T7W9m\nl5jZTDObuXr16t2KwXZrLxGRXVdRUUFxcTEFBQXMnz+fGTNmUFVVxeuvv055eTm1tbU88sgj22zf\nv3/w9Tdt2rRshQ1kIUGYWTFwBjAY6AcUApPS3d/dp7r7GHcf09A821UbqoM89PnaTbu1v4hIuiZN\nmkRdXR0HHXQQV199NYcffjh9+/ZlypQpjB8/niOPPJKDDjqocfspU6Zw1llnceihh1JSUpLFyMFa\n+6HYZnYWMMndLw6XLwTGA2cBe7l7nZmNB6a4+0k7ORRjxozxmTNn7nIMF/7PmXzQ6zMuKP4DV57+\npV2vhIi0G/PmzdvmCzhqktXfzGa5+5jm9s3GMNfPgcPNrMCCDrSJwFxgOvD1cJvJwN+yEJuIiISy\ncQ3iHeBR4H2CIa4xYCpwFXC5mS0gGOp6V8ZjyfQJRETasayMYnL3G4AbtiteCBzWuoG06tlEJEvc\nvXHET5Ts6SWESN9Jrfwg0vHl5+dTXl6+x1+W7Y27U15eTn5+/m4fI9JPlFOKEOn4BgwYQFlZGbs7\nLL49y8/PZ8CAAbu9f6QThCbrE+n4cnNzt7kbWdIX6S4mERFJTQlCRESSUoIQEZGklCBERCSpaCcI\nXaMWEUkpmgkievfLiIjssmgmiEZqQoiIpBLJBGFhE8KUIEREUopkgmig9CAiklq0E0TE5mYREdkV\nkU4QIiKSmhKEiIgkpQQhIiJJRTpB6AqEiEhqkU4QIiKSWrQThEYxiYikFNEEobk2RESaE8kE0ZAe\n1H4QEUktkgmigRKEiEhqkU4Q6mgSEUkt0glCLQgRkdQinSBERCS1SCcItSBERFKLZoJoHMakFCEi\nkko0E0RIDwwSEUktoglC45dERJoT0QQRUPtBRCS1SCcIERFJLSsJwsy6m9mjZjbfzOaZ2Xgz62Fm\nL5nZp+F7cTZiExGRQLZaELcCz7v7EGAkMA+4GnjF3fcHXgmXM0JXIEREmtfqCcLMioAvAXcBuHuN\nu68HzgCmhZtNA87MdCyuYa4iIillowUxGFgN3GNmH5jZnWZWCPRx9+XhNiuAPsl2NrNLzGymmc1c\nvXp1K4UsIhI92UgQOcAhwO3uPhrYxHbdSR78aZ/0z3t3n+ruY9x9TK9evfYoELUfRERSy0aCKAPK\n3P2dcPlRgoSx0sz6AoTvqzIVgOmJECIizWr1BOHuK4ClZnZgWDQRmAs8CUwOyyYDf8t8LJk+g4hI\n+5WTpfNeCjxgZnnAQuDfCJLVw2Z2MbAEODtLsYmICFlKEO7+ITAmyaqJrRxJ655ORKQd0Z3UIiKS\nVKQThK5BiIikFs0EEQ5iSihBiIikFM0E0UgZQkQklUgmiK0PlFOCEBFJJZoJwoIUoQQhIpJaJBNE\nQxuiPpHlMERE2rBIJoitE22oBSEikkokE0QD9TCJiKQWyQRhjcNclSFERFKJZoII3xO6BiEiklJa\nCcLMfmxm3Sxwl5m9b2YnZjq4TFP7QUQktXRbEN9y9w3AiUAxcAFwU8aiyrSGYa5KESIiKaWbIBp6\nZU4B7nf3OU3K2p3GUUzqYxIRSSndBDHLzF4kSBAvmFlXoN1+uzY8US6hFoSISErpPg/iYmAUsNDd\nN5tZD4KH/LRrupNaRCS1dFsQ44GP3X29mZ0PXAtUZC6szLKw1soPIiKppZsgbgc2m9lI4KfAZ8B9\nGYsqwxqHuSpDiIiklG6CqPOgP+YM4Pfu/gega+bCyrTwGoTyg4hISuleg9hoZtcQDG892sxiQG7m\nwsqsrdN9ZzUMEZE2Ld0WxDnAFoL7IVYAA4D/zlhUrUQJQkQktbQSRJgUHgCKzOxUoNrd2+81CN0o\nJyLSrHSn2jgbeBc4CzgbeMfMvp7JwDJp61xM7fZWDhGRjEv3GsR/AmPdfRWAmfUCXgYezVRgGdU4\nm2t2wxARacvSvQYRa0gOofJd2LcNU4YQEUkl3RbE82b2AvDncPkc4NnMhJR5I/p3Z/p6GDOoR7ZD\nERFps9JKEO5+hZl9DTgyLJrq7k9kLqzM6pwXByAv3gEaQSIiGZJuCwJ3fwx4LIOxtBrdSS0i0ryd\nJggz20jyjnoD3N27ZSSqVqKL1CIiqe00Qbh7O55OI7XGO6l1kVpEJKVodsKHGUI9TCIiqUUyQcTC\nDKEnyomIpJa1BGFmcTP7wMyeDpcHm9k7ZrbAzB4ys7wMnhxox4/EExFpBdlsQfwYmNdk+TfALe6+\nH7CO4Cl2GaFrECIizctKgjCzAcCXgTvDZQOOY+vUHdOAMzN2/vBdo5hERFLLVgvid8CVbO3l6Qms\nd/e6cLkM6J9sRzO7xMxmmtnM1atX71kUukotIpJSqyeIcLrwVe4+a3f2d/ep7j7G3cf06tVrj2Ip\nr6zZo/1FRDqybLQgjgRON7PFwF8IupZuBbqbWcN9GQOAZZkO5MU5KzN9ChGRdqvVE4S7X+PuA9x9\nEHAu8Kq7fxOYDjQ8Y2Iy8LfWjk1ERLZqS/dBXAVcbmYLCK5J3JXpE540vE+mTyEi0m6lPVlfJrj7\na8Br4eeFwGGtcuKGBwbpRjkRkZTaUguiFYU3ymkQk4hIShFNEAGNchURSS3SCUKPHBURSS3SCaJe\n+UFEJKVIJwhdgxARSS3SCUKT9YmIpBbpBJFQE0JEJKVIJoits7nqPggRkVQimSAaUoRrnKuISEoR\nTRAB5QcRkdSUIEREJKlIJwhdoxYRSU0JQkREkop4glCGEBFJJdIJQjfKiYikFskEsfU+CCUIEZFU\nIpkg4hZU2xN1WY5ERKTtimSCyInlAuCuBCEikkokE0Q81vCk1dqsxiEi0pZFMkEU5RYCUEdFliMR\nEWm7cprfpOM5sPv+AFjO47z0ehX5sTxyY3FyYznkWjx4xXLYO7+E3FgOjZe1zYLPDe87lLHt+m3K\nWqt26WhDwVgbikU/l+QsDrEciMWDV9Nli0P3gZDbOdtRSgZEMkGU7H00F766ifuK4PLFj6Xc7ojN\nVfy/latbMTKRdqh7Kfz73yG/KNuRSAuLZIKgax+WdruH4Ws/Z8qZfamu20Jtopa6RB21iTpqvZYf\nz7qZtws6s+k7r1KY0xnwcPImbzKJU5MyaLKeHcvaijY1tLcNxaKfS3Lu4PWQCF9eD4m6cLkONi6H\nF/4DZtwOE67OdrTSwqKZIICaeBfWsjcHDv5S8g1m3QzAqi49GFw0uBUjE2lnFv8d3vkTHHEp5BVm\nOxppQZG8SA0Qi6V3o9yaqjWtEI1IO3bUZVC1Dp67CipXZTsaaUGRTRBmRv1OEsRxA48DYFnlstYK\nSaR9GngYjDgLPrgf/vcgeOh8WPAKJPTExvYusgkibrbTbuffTvgtubFcPl77cesFJdJefe1O+P47\ncPj3YMnb8H9fhdtGwes3w8q5bewaj6QrstcgYrbzLqacWA6H7XUYr5e9zpVjr8Ta0rBDkbao9xA4\n8Rdw3HUw7ymYdS9M/2Xw6twD+o2CvqOg14HQYx8oHgyFJW1rSK9sI8IJwpq9BnHC3icw5R9TeKPs\nDY4ZeEwrRSbSzuV0ghFfD14blsOnL8CyWfDFB/D2bcHopwa5hVA0IHz1h6KBTZYHQLf+wfEkKyKb\nIMys2S7S0/Y9jXvm3MON/7iRF/q9QG48t3WCE+kouvWFQy8KXgB1W2D957B2IaxdBOsWw4YyqCiD\nFbNhU5KL3IW9mySNgU2SSbhc2EutkAyJbIKIpzGKKS+ex08P/Sk/mv4jXljyAqfuc2orRSfSQeV0\ngpL9g1cytdWwYVmQMCrKws9Lg8+r58OCl6F287b7xDsFCaNbkhZIw0vDb3dLqycIMxsI3Af0Ibgj\naKq732pmPYCHgEHAYuBsd1+XqThiZtSn8czRhq6la968hhP3PpG8eF6mQhKR3HzouW/wSsY9GFLb\nkEAqyoIE0pBUFr0e3Lzn23UPdC7e2uLoXgrd94bivbe+d+qa+bq1Q9loQdQBP3X3982sKzDLzF4C\nLgJecfebzOxq4GrgqkwFYWZpPZM6ZjEuHHoh9829j2v/fi2/OvpX5MQi2/ASyS4zKOgRvPoenHyb\n+togSVQs25pAGpLJusWw6A2oqdx2n849gsTRNGl0HxS8x+KQ3z04Z8S0+jeduy8HloefN5rZPKA/\ncAYwIdxsGvAaGUwQ6XQxNbhi7BX07NyTW2bdwqa6Tdwy4Ra1JETaqnhu2EooTb7eHTavhfWLYd0S\nWL8kfP8cVs6Bj5+D+pod97MYDBwHeV2gW7+gNdJ/NOx9ZIedrDCrfwqb2SBgNPAO0CdMHgArCLqg\nku1zCXAJQGlpil+ANMTT7GJq8K3h3yIvlsdv3vsNP3v9Z/zu2N8Rs8jeRiLSfplBYc/g1f/QHdcn\nElC5YmvymPcUzH8aigcFSWRDkptnC3rCXiOg5MBgny59grmpuvZt1xfQs5YgzKwL8BjwE3ff0PQ+\nA3d3M0v67e3uU4GpAGPGjNntu29iseaHuW7v/KHnY2bc9O5N3Dn7Ti45+JLdPb2ItFWxWNBC6NYP\n9h4PI8/ddn19LWxaDas/hvIFsLk86LpaNRc+fGBr99X70yAnP2hd1NdBzcYdzzXkVNhnQjDKqw2O\nksxKgjCzXILk8IC7Px4WrzSzvu6+3Mz6Ahmd1CVmRmIXWhANvjHkG/xr9b/4w4d/YFzfcYzsNTID\n0YlImxXP3ZpA9j1223XuwRDeFbODJLJucXD/x5K3kh9r/tPB69mfBcsDDoOv/r9gRNbmtYAH58mS\nbIxiMuAuYJ67/2+TVU8Ck4Gbwve/ZTKOeGznczGlYmZcP/56nl30LN996bvM+MaMDEQnIu2SWepR\nWO5B99TyfwU3Dn4+A5b8fdttyt6F20ZvWzbpJjjw5CBpxHJatcsqGy2II4ELgNlm9mFY9h8EieFh\nM7sYWAKcnckgYmncKJdKYfjI0k21m7jjX3fwnYO/04KRiUiHZLb1vowhp2wtr60O7u9487fwxfs7\n7vf81cGrwVn3wtAzWyVRtPpVVnf/u7ubux/s7qPC17PuXu7uE919f3c/3t3XZjKOXRnFlMyvjvoV\nALd9cBuuichEZHfl5sNBp8Il02FKBXx/BuxzbOrtH7kIbuweXAPJsMgO6I81M913c44rPa7xc1Vd\nFQW5BS0RlohEXe+D4MK/bl3eUhlcx3jvzqBbqnJFUL7krWDiwwyKdIJwB3ffrZlaG7qZAH48/cfc\nceIdLRmeiEigUxcYfHTwalC9AfK7ZfzUkR3IH48FSWFX7oXY3jNfeQaAGctnqJtJRFpPKyQHUIJI\na7qNVEq7bb1R781lb+5pSCIibUpkE0RDr9KeXKgGeOjUhwD4wSs/YPP2s0yKiLRjkU0QcdvzLiaA\noT2HNn4e9+A4ahO1e3Q8EZG2IroJouEaRAtcO5h5/szGz1PenrLHxxMRaQsimyBe/2Q1AGsrk8za\nuIs6xTs13hfx5GdP8vDHD+/xMUVEsi2yCWLVhi0ArK7c0iLHO23f04hbHICfz/g5KzataJHjiohk\nS2QTxA+P2w+Aos4tN4PiBxd8wAHFBwBwwqMn8O0Xv8366vUtdnwRkdYU2QSRGw+qXlu/mxMyJWFm\nPHb6Y0wsnQjAO8vf4eiHjmZjsml+RUTauAgniJYZxZTM7479Hb8/7veNy0f8+QhGTBuhYbAi0q5E\nNkE0jGKqrc/MHdDHDDyGDy74gPx4fmPZuAfHcfs/b8/I+UREWlpkE0RDF1MmWhANcmI5vHf+e7zw\ntRcay/744R8ZMW0Er3z+SsbOKyLSEiKbIBpaEHW7+1CIXdCvSz9mT57NdYdf11j2k+k/YcS0EdQl\n6jJ+fhGR3RHZBJHTkCAy1MWUzNkHns3M82eSG9s6cmr0/aMZMW2EkoWItDnRTRCt0MWUTKd4J96/\n4H1e+vpLO6wbff9ofvXOrzRdh4i0CZFNEOs3B3dQvzg3Oze07VW4F7Mnz+a9b77HGfue0Vj+5/l/\n5pD7D2HCQxN4ftHzfLru06zEJyIS2QcG9e4ajC7K1CimdOXn5POLo37BL476Be8sf4dvv/htAMqr\ny7nijSuA4GL3V/f7KmfsdwbDS4YTs8jmdRFpRZFNEH2LggQxrF/rPHgjHeP6jmP25NkkPMHrS1/n\nR9N/BEBdoo6HP3mYhz/ZOsfTCXufwC+P+iWdczpnK1wR6eAimyDycoK/wmvqMj+KaVfFLMaxpccy\ne/JsAOasmcPk5yezpX7rvFEvLXmJl5Zsex3jv474L87c78zdeoSqiMj2lCDaYILY3rCSYY1TitfU\n13DLrFv4v3n/t8N21799Pde/fT0A3fK6cdVhV3Hy4JO3GTUlIpIua8/PUh4zZozPnDmz+Q2TcHf2\n+Y9n+eGx+/HTEw9s4chaz+bazUybM40//vOPaW1/3eHXMbr3aPbrvp9aGiIRZWaz3H1Mc9tFtgVh\nZuTFY+2iBbEzBbkFfG/U9/jeqO8BUJ+oZ/66+dz+4e28Xvb6Dtv/fMbPt1ke0GUAlbWVnDL4FH4w\n+gd0zumsFoeIABFOEACd8+JU1dZnO4wWFY/FGdZzGL+f+PttylduWsltH9zG/LXz+WTdJ43lZZVl\nADw4/0EenP/gNvucts9p5MXzOGyvwziu9DhWV61mr8K9lEBEIiLSCaJHQR5rN+35E+Xagz6Fffjl\nUb/cpizhCT5b/xn3z72fJxY8scM+Ty18CoDHPn0s6THP3O9Mvn7A1ynIKSAei1PatZScWKR/pUQ6\nlMhegwA4609vkxOL8edLDm/BqNq/hCdYVrmMWStn8btZv6O8ujztfXNiOY1ThkwZP4WVm1cysXQi\n+xTtg+PkxfMyFbaIpCndaxCRThD/fv8sFq6p5MXLjmnBqDq2+kQ9c8vn8tinj/Hq56+ybsu6PT7m\nt4Z/i+NLj2dg14F0zu1MXiyPTbWb6JLXpQUiFpHtKUGk4T+emM2Lc1Yw89oTWjCqaKupr2FhxULm\nr53PwvULuWfOPS1y3HMOPIfeBb1ZunEp4/qOo2tuV8b1HUduLJequio653QmHou3yLlEOjqNYkpD\nSWFwDaI+4Y3Tf8ueyYvnMaTHEIb0GALA5WMu32GbLfVbWF+9nre/eJvXlr7GlvotVNZWkvAEs9fM\nTnrchz5ZOvsoAAAMZklEQVR+qPHzXxf8Na1YThl8Clvqt7By00qWbFjCCYNOoGtuV0b0GkHvgt58\nsvYTTtv3NDrndCbhCSUYke1EugVx71uLmPLUXGZeezwlXTq1YGTSUjbWbOSz9Z+xtnota6rWcOv7\nt7KhZkNGzlXUqYj8eD4rN69sLMuN5VKbqKV/l/4sq1y2zfbXHHYNA7sOZP2W9RTnFzOgywD27rY3\nlbWV5MfzyYnlYGZU1VVRXVdNcX5xRuIW2VVqQaRh395BH/fssgqOHdI7y9FIMl3zujKq96jG5bMP\nPHun29cl6jCMNVVrWLF5BeVV5bzy+Ss8+dmTzZ6rYksFFVRsU9Yw9fr2yQHg1+/+Op0qpK1fYT9G\n9h7Jc4ueA6B3596sqlrFRcMuakwut8y6hb6FfZk8bDLrt6xnr4K9GNpzKL//8PcM7DqQok5FnFB6\nAvPXzadnfk/6FPRhY+1GeuT3YPrn0ynpXMK+3felV0EveuT3oKa+hnqvZ231Wnrm92TB+gWN+1TX\nVVPatbTxWtDm2s3MXjObmMUYu9dY3J16r9fItQ6sTbUgzGwScCsQB+5095t2tv2etiCqa+sZ/+tX\nOKhvN+6+aCz5uepikEBdoo7Vm1eTF8/j43UfU1tfy4zlM8iJ5fDuindZvXk1vQt6U+/1zF87P+Vx\nSjqXsKZqTStG3vKO7H8kby17K+X6PgV9WLl5JcWdilMOWjiw+EA+XvcxAN07deeo/kfRJbcLjrNg\n/QKq6qpYunEpG2s2ckjvQxjfbzwfrPqAorwiRvUe1ZioN9Rs4PlFz3NA8QG8/PnLjcf/ySE/oXNO\nZ+atnUfc4pR2K+WYAcdQXVfN5xs/p6RzCSs2rWBYyTAenPcgxfnFnL7v6dTW19IppxOLKhYRtzgH\nFB9ATiyHwtxCHCfHclhbvZYe+T1YsH4BhlFeXU7MYgwuGkzD92dJ55Kg5ZiTz6rNq+jfpT8ASzcs\nZUDXAVTVVVGQW9AYb019DbmxXCq2VFCYV9h4b1FlTSV58TziFidBgvpEPZW1lZR0LgGCP1ha4j6k\ndneR2sziwCfACUAZ8B5wnrvPTbXPniYIgGlvL+aGJ+fQq2snjtqvhN7dOtGnaz69unaiS34OBblx\nCvJyyMuJEY8ZOTEL3uPBe9yMnFiMeDz4HItBzCx8oeksJKmEJxq/XGoSNVTXVZMXz6NsYxmOs2rz\nKoo7FfP+qvcZXDSYpRuXErMYLy5+kXqvpyiviHdWvENVXRVnHXAWj3zyCAO7DmTpxqX0LezL8k3L\nWyzWhm42SW37n9HOfmY983vuMHS8uFMxlbWVaf+cx+41lguHXsiEgRN2K9722MV0GLDA3RcCmNlf\ngDOAlAmiJUw+YhD79+nCHW8s5N1Fa1m9cQs19S07/UbMgqTRkCsMg8bP4XvTdeHy1nW2zbZN921c\nt8Pxtttnu/M0Kdkh3u23sSTrrElpqnPvjnR3TXu7JPXb02NuPXamrQFKgXqgX1h2YePaAr5KATB9\ntVHCl6haDSVAbfieCU6Q1AzDSQAe/hzqcUtgHiMRq8KtGnBiia7Ux8uJJbrhVkkitplYohBwErFK\nzAtJxDaQiFUQSxQEZ7AazLsAdeH23YBa6nJWEEsUUJP3KebBV5d5Z+pylhBLFBFP9KQuvpL6+HLc\n6um0ZSSxRDcS8XUkYhXU5iwlr3Z/anMWEEt0J54ooSZ3DrFEEfU5q7BEF3Jr98WIY42x1FGXs5i8\nmuFUFbwS/hBywbb9Io9vPpTa/BlBTIkC4tXDsVglNXlbv746bRnJlrw5bNzQC8vZgscqG9dVbuyF\n+QDoNDs8Rj4eq25cH6vvSSK+Nal8uLScA/LXMGHgHv+T7lRbShD9gaVNlsuAcdtvZGaXAJcAlJaW\ntsiJj9i3hCP2Df5LuTvrN9eyunILG6vrqKqpZ3NNHXUJp7Y+QX3CG191TT43LCfccXcSHjzO1MNj\nNjzaNFgm/Nz4oekb7t5km4aybfdp2vBr+Es01bZsV964nORnsWOD0ndYt825t4un6e672jjdPt6d\nbNiSmwXb7mKwbaPdves/4xY/f1pb9Wt+k905d3OVTzbupLmxKKnWd4amyRm2q3suwPd2fpyGnqFk\n94o212u0/fpcOLJfhrMDbStBpMXdpwJTIehiaunjmxnFhXkUF+qOXxGJtrb07MplQNOUOCAsExGR\nLGhLCeI9YH8zG2xmecC5QPNjE0VEJCPaTBeTu9eZ2Q+BFwiGud7t7nOyHJaISGS1mQQB4O7PAs9m\nOw4REWlbXUwiItKGKEGIiEhSShAiIpKUEoSIiCTVZuZi2h1mthpYspu7lxDMZRAlqnM0qM7RsCd1\n3tvdezW3UbtOEHvCzGamM1lVR6I6R4PqHA2tUWd1MYmISFJKECIiklSUE8TUbAeQBapzNKjO0ZDx\nOkf2GoSIiOxclFsQIiKyE0oQIiKSVCQThJlNMrOPzWyBmV2d7Xj2hJndbWarzOyjJmU9zOwlM/s0\nfC8Oy83Mbgvr/S8zO6TJPpPD7T81s8nZqEs6zGygmU03s7lmNsfMfhyWd+Q655vZu2b2z7DON4bl\ng83snbBuD4XT5GNmncLlBeH6QU2OdU1Y/rGZnZSdGqXPzOJm9oGZPR0ud+g6m9liM5ttZh+a2cyw\nLHu/2x4+IjMqL4KpxD8D9iF4+N8/gaHZjmsP6vMl4BDgoyZlNwNXh5+vBn4Tfj4FeI7gccqHA++E\n5T2AheF7cfi5ONt1S1HfvsAh4eeuwCfA0A5eZwO6hJ9zgXfCujwMnBuW/wn4Xvj5+8Cfws/nAg+F\nn4eGv++dgMHh/4N4tuvXTN0vBx4Eng6XO3SdgcVAyXZlWfvdjmIL4jBggbsvdPca4C/AGVmOabe5\n+xvA2u2KzwCmhZ+nAWc2Kb/PAzOA7mbWFzgJeMnd17r7OuAlYFLmo9917r7c3d8PP28E5hE8z7wj\n19ndveEJ97nhy4HjgEfD8u3r3PCzeBSYaGYWlv/F3be4+yJgAcH/hzbJzAYAXwbuDJeNDl7nFLL2\nux3FBNEfWNpkuSws60j6uPvy8PMKoE/4OVXd2+XPJOxGGE3wF3WHrnPY1fIhsIrgP/xnwHp3rws3\naRp/Y93C9RVAT9pZnYHfAVcCiXC5Jx2/zg68aGazzOySsCxrv9tt6oFB0vLc3c2sw41lNrMuwGPA\nT9x9Q/DHYqAj1tnd64FRZtYdeAIYkuWQMsrMTgVWufssM5uQ7Xha0VHuvszMegMvmdn8pitb+3c7\nii2IZcDAJssDwrKOZGXY1CR8XxWWp6p7u/qZmFkuQXJ4wN0fD4s7dJ0buPt6YDownqBLoeGPvKbx\nN9YtXF8ElNO+6nwkcLqZLSboBj4OuJWOXWfcfVn4vorgD4HDyOLvdhQTxHvA/uFoiDyCC1pPZjmm\nlvYk0DByYTLwtyblF4ajHw4HKsKm6wvAiWZWHI6QODEsa3PCfuW7gHnu/r9NVnXkOvcKWw6YWWfg\nBIJrL9OBr4ebbV/nhp/F14FXPbh6+SRwbjjiZzCwP/Bu69Ri17j7Ne4+wN0HEfwffdXdv0kHrrOZ\nFZpZ14bPBL+TH5HN3+1sX7XPxovg6v8nBP24/5ntePawLn8GlgO1BH2NFxP0vb4CfAq8DPQItzXg\nD2G9ZwNjmhznWwQX8BYA/5bteu2kvkcR9NP+C/gwfJ3Swet8MPBBWOePgOvD8n0IvuwWAI8AncLy\n/HB5Qbh+nybH+s/wZ/ExcHK265Zm/SewdRRTh61zWLd/hq85Dd9N2fzd1lQbIiKSVBS7mEREJA1K\nECIikpQShIiIJKUEISIiSSlBiIhIUkoQImkys+5m9v3wcz8ze7S5fUTaMw1zFUlTOPfT0+4+PMuh\niLQKzcUkkr6bgH3DSfM+BQ5y9+FmdhHBDJuFBHfq/g/BVPIXAFuAU9x9rZntS3BjUy9gM/Add5+/\n42lE2gZ1MYmk72rgM3cfBVyx3brhwFeBscAvgc3uPhr4B3BhuM1U4FJ3PxT4GfDHVolaZDepBSHS\nMqZ78HyKjWZWATwVls8GDg5nnz0CeKTJzLOdWj9MkfQpQYi0jC1NPieaLCcI/p/FCJ5lMKq1AxPZ\nXepiEknfRoLHnO4yd98ALDKzs6DxecIjWzI4kZamBCGSJncvB94ys4+A/96NQ3wTuNjMGmbrbLeP\nupVo0DBXERFJSi0IERFJSglCRESSUoIQEZGklCBERCQpJQgREUlKCUJERJJSghARkaT+P9u1C5rn\nnfyBAAAAAElFTkSuQmCC\n", 137 | "text/plain": [ 138 | "" 139 | ] 140 | }, 141 | "metadata": {}, 142 | "output_type": "display_data" 143 | } 144 | ], 145 | "source": [ 146 | "def plotit(fn, label, subsample=1):\n", 147 | " vals = np.loadtxt('data/'+fn+\".csv\", delimiter=\",\")\n", 148 | " vals = vals[::subsample]\n", 149 | " plt.plot(vals, label=label)\n", 150 | " \n", 151 | "import numpy as np\n", 152 | "import matplotlib.pyplot as plt\n", 153 | "\n", 154 | "plt.figure()\n", 155 | "plotit('losses_sgd_kfac', 'kfac')\n", 156 | "plotit('losses_sgd', 'sgd', 4)\n", 157 | "plotit('losses_adam', 'adam', 4)\n", 158 | "\n", 159 | "plt.xlabel('time')\n", 160 | "plt.ylabel('loss')\n", 161 | "plt.title('train loss')\n", 162 | "\n", 163 | "plt.legend()\n", 164 | "plt.show()" 165 | ] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": 110, 170 | "metadata": {}, 171 | "outputs": [ 172 | { 173 | "data": { 174 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8leWd///X5yxZSQIJEZBFgguyCdS4oLbFreJSl1qr\nVlvtdGpn2rH96oxLZ5xqHzO/Get0aq3fGfvFagVrW+vWqrgraLVqC25sohhQwhIgQAhZz3L9/rjv\ncwh4DgkhJ4dwv5+Px3nknPvcy+cK5LzPdd2bOecQERHZXSjfBYiIyP5JASEiIhkpIEREJCMFhIiI\nZKSAEBGRjBQQIiKSkQJCJMfMbIGZ/W2+6xDZWwoICSQzW21mp/XBeq40s1f7oiaR/Y0CQkREMlJA\nSOCY2f3AGOAJM9thZtf70483sz+b2TYze9fMZnZZ5kozqzOzZjNbZWaXmdkE4BfADH8923qw7ZCZ\n3WRmH5vZRjOba2YV/ntFZvZrM2v0a/irmQ3Ltv0c/GpEdqGAkMBxzn0N+AT4onNukHPuNjMbCcwD\n/h2oBP4JeMTMqs2sFPg5cKZzrgw4AXjHObcc+DvgdX89g3uw+Sv9x8nAOGAQ8H/9964AKoDRQJW/\n7rZs29/HX4NItxQQIp7Lgaecc08555LOueeBhcBZ/vtJYLKZFTvn1jvnlvZyO5cBP3XO1TnndgA/\nAC4xswgQwwuGw5xzCefcIufc9j7evkiPKSBEPIcAF/lDO9v84aKTgBHOuRbgYrxv9OvNbJ6ZHdnL\n7RwMfNzl9cdABBgG3A88C/zOzNaZ2W1mFu3j7Yv0mAJCgmr3yxivAe53zg3u8ih1zt0K4Jx71jl3\nOjACeB+4O8t6urMOL4xSxgBxoME5F3PO/cg5NxFvGOkc4OvdbF8kZxQQElQNePsAUn4NfNHMzjCz\nsL/DeKaZjTKzYWZ2nr8voAPYgTfkk1rPKDMr6OF2fwtcY2Y1ZjYI+A/gQedc3MxONrMpZhYGtuMN\nOSW72b5IziggJKj+E7jJH076J+fcGuA84J+BTXg9iuvw/kZCwLV43/63AJ8H/t5fz0vAUmCDmW3u\nwXbvxRtKegVYBbQDV/vvDQcexguH5cDL/rx72r5IzphuGCQiIpmoByEiIhkpIEREJCMFhIiIZKSA\nEBGRjCL5LmBfDB061I0dOzbfZYiIDCiLFi3a7Jyr7m6+AR0QY8eOZeHChfkuQ0RkQDGzj7ufK4dD\nTGZ2r3+1yiVdplWa2fNm9qH/c4g/3czs52a20szeM7PP5KouERHpmVzug7gPmLXbtBuBF51zhwMv\n+q8BzgQO9x9XAXflsC4REemBnAWEc+4VvLM+uzoPmOM/nwOc32X6XOd5AxhsZiNyVZuIiHSvv/dB\nDHPOrfefb8C7giXASLxLG6TU+9PWsxszuwqvl8GYMWNyV6mIHBBisRj19fW0t7fnu5R+V1RUxKhR\no4hGo71aPm87qZ1zzsz2+jofzrnZwGyA2tpaXSdERPaovr6esrIyxo4di5nlu5x+45yjsbGR+vp6\nampqerWO/j4PoiE1dOT/3OhPX4t3F62UUf40EZF90t7eTlVVVaDCAcDMqKqq2qeeU38HxON4t1XE\n//nHLtO/7h/NdDzQ1GUoSkRknwQtHFL2td25PMz1t8DrwHgzqzezbwK3Aqeb2YfAaf5rgKeAOmAl\n3o1QvpOrugAem/8LfnDvubS0NudyMyIiA1ouj2K61Dk3wjkXdc6Ncs7d45xrdM6d6pw73Dl3mnNu\niz+vc8591zl3qHNuinMup2e/vbdmAU+GV9He2ZLLzYiIALB69WomT578qemXXnopRx11FLfffnse\nquregD6TWkRkoNqwYQN//etfWblyZb5LyUoX6xMR6Ud1dXVMnz6dmpoa1q5dy7Rp0/jTn/7E3Xff\nzTHHHMPUqVO58MILaW1tBaChoYELLriAqVOnMnXqVP785z/3W63qQYhIYPzoiaUsW7e9T9c58eBy\nbv7ipB7Nu2LFCi655BLuu+8+KioqOOecc3jnnXe89UycyLe+9S0AbrrpJu655x6uvvpqvve97/H5\nz3+exx57jEQiwY4dO/q0/j0JdEDobqsi0l82bdrEeeedx6OPPsrEiRNZvXr1Lu8vWbKEm266iW3b\ntrFjxw7OOOMMAF566SXmzp0LQDgcpqKiot9qDnRAiEiw9PSbfi5UVFQwZswYXn31VSZOnPip96+8\n8kr+8Ic/MHXqVO677z4WLFjQ/0XuRvsgRET6QUFBAY899hhz587lN7/5zafeb25uZsSIEcRiMR54\n4IH09FNPPZW77vKuX5pIJGhqauq3mhUQIiL9pLS0lCeffJLbb7+dxx9/fJf3/u3f/o3jjjuOE088\nkSOPPDI9/Y477mD+/PlMmTKFo48+mmXLlvVbveYG8EB8bW2t680Ng3409xIedkt56dznqB6ii8aK\nHMiWL1/OhAkT8l1G3mRqv5ktcs7VdrdsQHsQ3unnjoEbjiIiuRbogBARkewCGhAiItIdBYSIiGSk\ngBARkYwCHRAD+QguEZFcC2RAaBe1iAwEY8eOZfPmzXnbfiADQkREuqdrMYmI5FBLSwtf+cpXqK+v\nJ5FI8K//+q+UlZVx7bXXUlpayoknnkhdXR1PPvkkjY2NXHrppaxdu5YZM2bkfRg80AHhktoHIRIo\nT98IGxb37TqHT4Ezb8369jPPPMPBBx/MvHnzAGhqamLy5Mm88sor1NTUcOmll6bn/dGPfsRJJ53E\nD3/4Q+bNm8c999zTt7XupUAPMSkeRCTXpkyZwvPPP88NN9zAn/70J1atWsW4ceOoqakB2CUgXnnl\nFS6//HIAzj77bIYMGZKXmlMC3YMQkYDZwzf9XDniiCN46623eOqpp7jppps49dRT+72G3gp0D0JE\nJNfWrVtHSUkJl19+Oddddx2vvfYadXV16RsGPfjgg+l5P/e5z6UvBf7000+zdevWfJScph6EiEgO\nLV68mOuuu45QKEQ0GuWuu+5i/fr1zJo1i9LSUo455pj0vDfffDOXXnopkyZN4oQTTmDMmDF5rFwB\nISKSU2eccUb69qEpO3bs4P3338c5x3e/+11qa70rb1dVVfHcc8/lo8yMgj3EpL3UIpIHd999N9Om\nTWPSpEk0NTXx7W9/O98lZRTMHoSZwkFE8uaaa67hmmuuyXcZ3Qp2D0JERLIKdEDojnIiItkFMiB0\nsT4Rke4FMiDSEaEOhIhIVgENCBGR/cd9993HP/zDP+S7jE9RQIiISEYKCBGRHDv//PM5+uijmTRp\nErNnzwbgV7/6FUcccQTHHnssr732WnreJ554guOOO47p06dz2mmn0dDQAMAtt9zCFVdcwWc/+1kO\nOeQQHn30Ua6//nqmTJnCrFmziMVifV53MM+D8OkoJpFg+fFffsz7W97v03UeWXkkNxx7wx7nuffe\ne6msrKStrY1jjjmGs88+m5tvvplFixZRUVHBySefzPTp0wE46aSTeOONNzAzfvnLX3Lbbbfx3//9\n3wB89NFHzJ8/n2XLljFjxgweeeQRbrvtNi644ALmzZvH+eef36dtC3RAiIj0h5///Oc89thjAKxZ\ns4b777+fmTNnUl1dDcDFF1/MBx98AEB9fT0XX3wx69evp7OzM31ZcIAzzzyTaDTKlClTSCQSzJo1\nC/AuKZ66+F9fUkCISGB0900/FxYsWMALL7zA66+/TklJCTNnzuTII49k2bJlGee/+uqrufbaazn3\n3HNZsGABt9xyS/q9wsJCgPSF/8ws/Toej/d57XnZB2Fm15jZUjNbYma/NbMiM6sxszfNbKWZPWhm\nBfmoTUSkLzU1NTFkyBBKSkp4//33eeONN2hra+Pll1+msbGRWCzGQw89tMv8I0eOBGDOnDn5KhvI\nQ0CY2Ujge0Ctc24yEAYuAX4M3O6cOwzYCnwz17U4krnehIgE3KxZs4jH40yYMIEbb7yR448/nhEj\nRnDLLbcwY8YMTjzxRCZMmJCe/5ZbbuGiiy7i6KOPZujQoXmsHKy/b4rtB8QbwFRgO/AH4E7gAWC4\ncy5uZjOAW5xzZ2RfE9TW1rqFCxfudQ3/dv/l/D75Lk/NeoLRw8bu9fIiMnAsX758lw/goMnUfjNb\n5Jyr7W7Zfu9BOOfWAj8BPgHWA03AImCbcy41iFYPjMy0vJldZWYLzWzhpk2belWDLrUhItK9fAwx\nDQHOA2qAg4FSYFZPl3fOzXbO1TrnalNHAOytpN9riic1xCQikk0+dlKfBqxyzm1yzsWAR4ETgcFm\nljqqahSwNlcFrGpsBeCJd9bnahMish/p76H0/cW+tjsfAfEJcLyZlZh3jNapwDJgPvBlf54rgD/m\nqoBEwvultccSudqEiOwnioqKaGxsDFxIOOdobGykqKio1+vo9/MgnHNvmtnDwFtAHHgbmA3MA35n\nZv/uT7sn57XkegMiknejRo2ivr6e3u6zHMiKiooYNWpUr5fPy4lyzrmbgZt3m1wHHNu/hfTr1kQk\nD6LR6C5nI0vP6WJ9IiKSUaADwul4VxGRrAIdEARsp5WIyN4IdEDoct8iItkFOiBERCQ7BYSIiGSk\ngBARkYwUECIiklGwA0L7qEVEsgpmQPjnPygfRESyC2ZAiIhItwIdECHdclREJKuABoQ3xqQhJhGR\n7AIZEKlLMAXt+vAiInsjkAEhIiLdU0CIiEhGCggREclIASEiIhkFOiC0i1pEJLtAB4SIiGQX7IBQ\nF0JEJKtABoTtPBMir3WIiOzPAhkQigURke4FMiDUfxAR6V4gAyJFASEikl2gA8K6n0VEJLACHRDq\nQYiIZBfogBARkewCHRDqQYiIZBfMgNDOBxGRbgUzIHzmdMtREZFsAhoQ6kKIiHQnoAEhIiLdCXRA\naCe1iEh2gQ4IERHJLi8BYWaDzexhM3vfzJab2QwzqzSz583sQ//nkHzUJiIinnz1IO4AnnHOHQlM\nBZYDNwIvOucOB170X+dE+mJ9ToNMIiLZ9HtAmFkF8DngHgDnXKdzbhtwHjDHn20OcH5/1yYiIjvl\nowdRA2wCfmVmb5vZL82sFBjmnFvvz7MBGJaH2kRExJePgIgAnwHucs5NB1rYbTjJeWM/Gcd/zOwq\nM1toZgs3bdq0T4XoNDkRkezyERD1QL1z7k3/9cN4gdFgZiMA/J8bMy3snJvtnKt1ztVWV1f3qoD0\nLUe1D0JEJKt+Dwjn3AZgjZmN9yedCiwDHgeu8KddAfyxv2sTEZGdInna7tXAA2ZWANQB38ALq9+b\n2TeBj4Gv5Kk2EREhTwHhnHsHqM3w1qn9XEn/bk5EZADRmdQiIpKRAkJERDIKZkD4BzElNcIkIpJV\nMAPCp6NcRUSy61FAmNn3zazcPPeY2Vtm9oVcF5cruhaTiEj3etqD+Bvn3HbgC8AQ4GvArTmrKue8\niFBAiIhk19OASH3pPgu43zm3lAF8305Ln0itgBARyaanAbHIzJ7DC4hnzayMAX0pIy8htJNaRCS7\nnp4o901gGlDnnGs1s0q8s58HpPQ+CJ0oJyKSVU97EDOAFc65bWZ2OXAT0JS7svqHRphERLLraUDc\nBbSa2VTgH4GPgLk5qyrX0udBKCFERLLpaUDE/Xs0nAf8X+fc/wBluSsrt9KNVj6IiGTV030QzWb2\nA7zDWz9rZiEgmruy+od2UouIZNfTHsTFQAfe+RAbgFHAf+Wsqn6ifBARya5HAeGHwgNAhZmdA7Q7\n5wbwPgj/RLmBfKSuiEiO9fRSG18B/gJchHcjnzfN7Mu5LCyX0oe5aoxJRCSrnu6D+BfgGOfcRgAz\nqwZewLuf9ICTuid1UoNMIiJZ9XQfRCgVDr7GvVh2v6WjXEVEsutpD+IZM3sW+K3/+mLgqdyUlHuW\nvoqUEkJEJJseBYRz7jozuxA40Z802zn3WO7KyrH0xfryW4aIyP6spz0InHOPAI/ksJZ+k+pA6Exq\nEZHs9hgQZtZM5nEYA5xzrjwnVeWcruYqItKdPQaEc27AXk5jT3beUS6vZYiI7NcG/JFI+0IBISKS\nXTADIn0mtRJCRCSbQAaEzqQWEeleIAMiRUcxiYhkF8iACIe8Zod2njEnIiK7CWRAHDXSOzr3uHGV\nea5ERGT/FciAiEa8Zqv/ICKSXSADIr2TOq9ViIjs34IZEDrMVUSkW4EMiBQd5Soikl0gA0KX2hAR\n6V4wAyI1xOR0T2oRkWwCGRAiItK9vAWEmYXN7G0ze9J/XWNmb5rZSjN70MwKcrdt72dSOyFERLLK\nZw/i+8DyLq9/DNzunDsM2Ap8M3ebTh3FJCIi2eQlIMxsFHA28Ev/tQGnAA/7s8wBzs91HUlFhIhI\nVvnqQfwMuB5I7SWuArY55+L+63pgZKYFzewqM1toZgs3bdrUq43rKCYRke71e0CY2TnARufcot4s\n75yb7Zyrdc7VVldX966Gnevq1fIiIkGwx1uO5siJwLlmdhZQBJQDdwCDzSzi9yJGAWvzUJuIiPj6\nvQfhnPuBc26Uc24scAnwknPuMmA+8GV/tiuAP+a6lngi11sQERm49qfzIG4ArjWzlXj7JO7J9QYf\nWrQm15sQERmw8jHElOacWwAs8J/XAcf2y4bT1/nWPggRkWz2px5EP/ISYuYRvdvJLSISBAENCE80\nrFsGiYhkE+iASGiESUQkq0AHhM6UExHJLtABoWv1iYhkF+iA0BCTiEh2gQ4I3ZNaRCS7QAdEIqk7\nyomIZBPIgEgd3KobBomIZBfIgEhHhI5iEhHJKqAB4VEHQkQku0AHhI5iEhHJLtABoREmEZHsAh0Q\nSSWEiEhWCggREcko0AGhe1KLiGQX6IBI6DAmEZGsAhkQIfPOg0g6nUktIpJNIAMibF6zHYk8VyIi\nsv8KZkCEvFtxOxfLcyUiIvuvgAZE1HuSjOe3EBGR/VggA6IiWgpA3LbnuRIRkf1XJN8F5MOhZYcA\nEI38mc0bl1JWUE40XEAoFAYLgZn/MwSY9/pTdpuWk3m6W96yv5dxWyIiPRfIgBg+6nhOfrWN+YNW\ncfLTl6SnR5wj6hwRByEcl21v5u+3HYi9jByFUk/fy9k2yf5ezrapdjJ8Mpz/C4gU7F6YDHCBDAgq\naxhb+M/MbPwzJ04uZUeinVgyQcz5j2ScB7Yt5n+HDOabtddQYOFdl//UCXYZzqfY13k+NftuE3ZZ\ntrfv7b7Nbmrul23uPuuB2s7dl+2LevZ2m33QzpZNsOQRGDcTPvN15MASzIAAPimdzruNNdz5hZMz\nvv/AnCkAbJp2MSMHjezP0kQGDufgrhPgzdkw/Wsa2jzABHInNUAoZD06k3pz2+Z+qEZkgDKDY6+C\nhsXw4fP5rkb6WGADImzWo2sxbW5VQIjs0VEXQ8UY+M1FMPc8WPY4JHQI+YEgsAERMiOxh4B46oKn\nAKhrquuvkkQGpoISuGoBnHwTbP4Qfv81+NlkmP8f0FSf7+pkHwQ3IEK2x1uOji4fzSHlh7Bo46L+\nK0pkoCqtgs9fB99/Dy75DQybDC/fBj+bAveeCfP/E1Y8A01rdaeuASSwO6lDBslu9kGcMfYMZr83\nm7ptdYwbPK6fKhMZwMIROPJs77H1Y3j717DyeXj5x6SPhiquhKGHw+AxOx8Vo/3HSCgozWsTZKfA\nBkQ4tOchJoDLJlzG3KVzufPtO7n95Nv7qTKRA8SQQ+CUf/EeHc3QsBQ2LIYN78GWVbDmL7DkUXC7\nXTSzeAiUj4KK1GOkFx7lI73XZSO8IJKcC+xvOWTWbQ+isqiSi8dfzJxlc/h4+8ccUn5IP1UncoAp\nLIMxx3uPrhJxaF7n7atoWgtNa7zn2/3nn7wO7dt2XcZCXkhUjNoZGqlHuR8mJZU65LYPBDsgejAU\neuXkK5mzbA7nPHYOi69YnPvCRIIkHNk5zJRNx46dgdG01g+TetheD+vfgffnQaJj12UixX7PY1SX\n3kjX1xrK6onABkQ41LM7yg0tHkp5QTnbO7fzSv0rfG7U5/qhOhFJKxwE1eO9RybOQctmLzAy9UQ+\nehGaN/CpM8KLh+wWILv1RDSU1f8BYWajgbnAMLx/sdnOuTvMrBJ4EBgLrAa+4pzbmqs6vB5Ez46m\nePrCp/naU1/j+/O/z39+9j+ZNXZWrsoSkb1lBoOqvcfB0zPPk4jB9nV+TyQVJGt2hsknf4b2pt3W\nG4KSKu9yIph3OZFDToBwAYw+FqoOg9LqA3ooKx/xGAf+0Tn3lpmVAYvM7HngSuBF59ytZnYjcCNw\nQ66K8A5z7VlAlBeUM/fMuVz90tVc//L1xBIxvnjoF3NVmoj0tXDU22k+ZA/7ETuadw5hpXojjSvh\ng2ch1gp1873H7o6YBUPGwsijvYAqrYaO7XseNhsg+j0gnHPrgfX+82YzWw6MBM4DZvqzzQEWkMOA\nCPdwH0RKRWEF/+/0/8d3X/wu//zqPzO6bDTTDpqWq/JEpL8VlsFBR3qPrpzzHsm419v46CXY9gls\nqfN6HVtWwcoX4c1f7Lrc5Ath6HgoqvB2tI+YBkXlEIp6Q1fhQm/YLLTbxUD3I3kdYDOzscB04E1g\nmB8eABvwhqAyLXMVcBXAmDG9T+iQ9WwfRFfFkWJu+9xtnPz7k/nb5/6WZy58hqHFQ3tdg4gMAObf\nEyZUAFWHeo/dJeKwcZl3CO97v4dVL8Mnb3pXuu2JSV+CmTfC0CP2qyGrvAWEmQ0CHgH+j3Nuu3X5\npTjnnJll/PR2zs0GZgPU1tb2+pTMUMjbXjLp0s97YmjxUB446wEue+oyHv3wUa466qreliAiB4pw\nBEYc5T2mX75zeqwdNq+AhffC+ndh3duZl1/6qPcAOPob3qXTt9R5+0AOzXzF6f6Ql4AwsyheODzg\nnPN/KzSY2Qjn3HozGwFszGUNYT+Qks4R+vQdWPboqOqjALjz7Tv5xuRvEE3d41pEpKtoEYyYCl+8\nY9fpzsHW1d7+jWd2G0lf9CvvkTLmBC8wUkdy9ePhuf1+LSbzugr3AMudcz/t8tbjwBX+8yuAP+ay\njlSvobuzqbMpiZQAcNOrN/VZTSISEGZQWQPH/x3c0gT/2giXPeyFye4++TP84e/g7pPhPw6Gx78H\n29d/er4cyMfF+k4EvgacYmbv+I+zgFuB083sQ+A0/3XObG3pBPZ+P0TK3V+4G4CnVj3Vo8uGi4hk\nFY7A4afDt1/xAuM7b3hHR2Xy1hz46ZFQn/sLiebjKKZXyXBXXd+p/VXHL19dBcCrH27mC5OG7/Xy\nE6ompJ9v7dhKZVFln9UmIgF30AT46oM7XycTULfAu35V3QJY84a3j2LU0TktI9inCQKxRO++/UdD\nUQYXDmZbxzYum3cZT1/4dB9XJiLiC4XhsFO9x8k/gGQSQrkfAArs/SBSenqyXCaPnuvtX6/fUa9h\nJhHpP/0QDhDggBhT6e1kLi/u/RFI1SXV6eevr3t9n2sSEdmfBDYgbjrb24dQUrBvZzE+cNYDAHz7\nhW/THm/f57pERPYXgQ2IT7a0AvDDPy7dp/WkzokA+NLjXyLpkvu0PhGR/UVgA2J7WwyA5eu37/O6\nXrv0NQDWNK/hjrfu6GZuEZGBIbABsTeX1+hOeUE535j8DQDuXXIvT6/SEU0iMvAFNiDCfXxBrGuP\nvjb9/PpXrmdT66Y+Xb+ISH8LbED0ZQ8i5d2vv0vIvF/pKQ+dwmkPncaGlg19vh0Rkf4Q2IDIhZCF\nePfr73LK6FMAaGht4PSHT+eN9W/oPAkRGXACGxCFkdw1/Y5T7uD+M+9Pv/7Wc9/iqLlHsbltc862\nKSLS1wJ7qY0zJg3n3+ctz9n6px00jcVXLGbO0jn8ZOFPADj59yczc/RM7jzlzpxtV2Qgc86RdN4V\nDlyXn45dp6fm6/rT4b2ffu3Pn7ogZzhkhEOWvkEceOtNzdd1/SEDB+nnZtDUFiOWcBRGQoTMiCeT\nJJJQEAmRSHqHtyf9dceTSQzD4cB50xPO0R5L0B5LsKWlk0+2tBJLJKkeVMQhVSXeev2hb68+Ryzh\nSDpH1/vltHXGKYiEOPGwoRxUVpTTf4/ABkQ4B/sgMrli0hV8dcJXufTJS1mxdQUL1ixgypwp3DLj\nFi484sJ+qWF/lkw6lq7bzvb2GBNGlBMOGZ3xZPpnJGwkki79cP4fWup1eyxBPJmkMBImkXQ0tcVY\nt62N1Y2t/OLljwD474umMryiiKRzxBOOeNIRTySJ+cs/t7SBF5Y3cMSwQfzNiTWEzGiLJWjtTAAQ\nDkE07N01xPsQSn0wpT5cdl6yJZn0P6D49AdVarpz3nypD7SuH3hJ/4POOUcyuesHI7usJ7Xsbuvv\nWluXWnfO49IfkDvn8ZalBx/O7NaWZDLTh/NuNSRT69lzDRqF3Ts/OncSV5wwNqfbsIE8Nl5bW+sW\nLlzYq2XbOhNM+OEzAKy+9ey+LCurpY1LueTJS3aZ9uZX36Q4UkzSJQlnuTdtPJEkZEZnIkl7LEEs\n4WjrTLC1tZMbHnmP7W0xph8yhHnvedeIv/fKWtpjSba2dlIYCbOjPUZBJExnPEE4ZGxpibFq8w4O\nqSolHDIatrfTGU/y0KL69DbPnXow7bEEhdEwyaRje3uMDxqaKYqGObiimHDI2NbWSSQUwjlHeyzJ\n9vYY0XCIeCLJuibvrPLBJVHC5n3Yx5POu9Wr/wHf2wslDiRmEDLzvoVi6dddfxreQRMhMwyw1Pzp\nZb0vM6EQ6XnSy/rzpucxIxTythXy3+9RDV3Wk7WGXdaTWja1nk+vK2sNXbfNzunsVkMolPr9WJb1\np+bZdT075/GmJ5yjtSNOUTS8yzpst3+j1HTnf2NPtSEVkB2xBNFwiKJomGjYiIRDxOJJzCASDvm/\nf2+dzkEkZOkeQciMpHPEEknaY0laO+Os2txCPOEYU1nCwYOLKSkMUxDeOfSd+reMJxzRcIhwyLs9\ngQNaOhIcWl3K4JKCXv6/tEXOudru5gtsD6J4Hy+xAaR3PP/i5Tr+d8FKTj3yIA6pKmXooAKWrd/O\n9DFDSCYdHfEkHfEEHbECvj7sIZ7YfANbE3UAHPeb49Lra15xC0dUD2VLS4zNOzr2qpZ17+28gcjf\n3Ne70OzmrQbBAAAN6ElEQVRqwQrvhn7lxVEKIiFaOxI0bO+grCjCoMIIkXAIwyiIhCiOhgmHjCGd\nUcqLorTFEpgZwyuKGD2kmJLCSHqfT8gs3dVPOseStU28trIxvd3rzhhPOGS0dMQpKYhQHA0RjYQI\nm/fHFvb/aFLr6fT/QIujYcyMts4En2xppWF7Ow+8+QkAP7t4GtVlhRRGQkTCISIhIxI2Iv4f7/sb\nmnl+WQOnHHkQk0dWUBAO0RFP0B7zhg2KoiGKCyIU+UML6Q/hTB/OXT/A9qN7C/enRDLBto5tlBeW\n73K3xdTfS8IlMIxwKEwsGaMz0cmOzh0MKRrCmuY1lEZLKY4Us75lPds6tlEULqIj0cHgwsGs3LaS\nhEtQEC7gnY3v0NzZTCQUYVr1NADWtaxjTNkYPtj6AdOqpxFLxFiyeQnDhgzj6Y9fYHjpcKYfNJ3l\nW5Zz+ODD6Uh0EA1F+aT5E5Iuyeiy0SRcIn2YemG4kGdXP0vt8FqGFg8lHo/z+IrHKY2WcszwYwhb\nmIRLkEgmWL5lOeUF5bTGWxldNhqAiEVojbeScAkioQjb2rcRDoUpCBWwoXUDq5pWMWz7MEYOGknC\nJWjubKauqY4xZWMoCBfQEmuhJdZCRWEFW9q30BJrYeSgkURCEb4z9TucNe6snP5bBrYHATD2xnnA\np3sQbZ0JPtq0g3PufDU97aCyQjY2ex/a46pLqdvU0qtthswbs2yPd1I67ieECrZ9ap4a900Wf3gw\nLrHrrQWvO2M8xf63l6JomMaWTm59+n0AqssK2eTX98jfn+D1OkJGeVGUjniC8qIo4ZC3XCyRpKI4\nmv6gjvjfwmT/kXTJ9CHTqW+0AO9uepeDig9iaPFQkiRpjXkfPiu3rSREiIbWBiZWTWRN8xpiyRjb\nO7ZTEi0hnozz0baP+KjpIyIWYWr1VOp31DO+cjwfbPmA3634XXrbBaECqkuqaY+309jeyIWHX0h9\ncz0Ox182/IWRg0YytmIsH275kI1tG5laPZWKwgpaYi0satj1JjZl0TIGFw1mW/s2WuItu1yKpjBc\nSCKZIO7i/fAb3XcRixAOhelIeH9nFYUVRENRwuZ9OdnQsoFoKEosGWPUoFG0J9opjhQTshBJl6Q4\nUkzYwhSEC0i6JFvat9DQ0sChgw+lorACh+Odje8QS8Y44eATCFmIxrZG2uJt1FTU0BZv4431b3D6\nIacTsQgXHH4BMw6e0au29LQHoYDohZPHVzN/ReYT4b40fSQnHT6U5vY4x9ZUUlEcpTASojAapjAS\nIhr+9NFTL3z8AtcsuCbr9v7x6H/kzJozKY2WMqhgUK9qlr3XGmulJFpCR6KDwnAhrTHv+l2vr3ud\ngnAB5YXlNLQ0EEvGKAgXUBopJRqOsmLLClrjrYQsREGogBVbV7CoYRFn1ZzF0salNHc2M7Z8LOta\n1lFeUE7SJVm8eTFb2rcwoXICq7evpi3elh7eKAoXUVVcxdoda/P8G8ls/JDxmBnFkWLe3vj2Lu8d\nNvgwDht8GEmXZNX2VXQmOglbmCFFQ5hYNZGmjia2tm9lVdMqZo6eya+X/5rLJ1xOZVElL9e/TEei\ng6qiKiZWTeTwIYfzZN2TdMQ7mHHwDDoTnbTF2xhWOozhpcNxztHc2UxVcRUAZQVlFEeK2dG5g2g4\nSmpQqSRSQnG02B+6ClEQLiASiuCcIxKKYBjRsNfzSc0DpH8eCBQQPbBgxUau/NVfM773069M5b+e\nXcGMQ6s4ZmwllxwzOuffshPJBP/zzv9w9+K7ezT/zNEzmTJ0Cp856DNsaN3A1KFTGV0+Oqc17i9i\niRiRUIS2eBsbWjawI7aDzkQna5rX0BZvY2jxUDa1bWJT6yYa2xupKqpixdYVfLDlAza2bWRo8VA2\nt20mYhGOP/h4EskE4VCYpo4mVm5bSVu8LS/tGjloZDoIKgorGF4ynObOZiYNnURVUVX6m37Ywowb\nPI7DKg6jZnANv3v/d0yqmsTq7av51pRvkXRJEi7B+pb1DC8ZzohBIyiOFLO1fSvDS4enP/jKC8sJ\nWYiwhSkKF1EQLiBkIQrDhepVHsAUED2U6kVcXDuaH3/5qG7m7l+xZIznVz/PDX+6oU/Wd+HhFzJy\n0Eg2tm7E4SiJlPDVCV9l8ebFdCY6qR1WS1GkiJZYC4MLB1MUKWJHbAdtsTYqiyupb66nubOZQwcf\nmv6ASbgESzYvoa6pjuOGH0c0FKWhtYGX619mfOV4Xvz4RSZUTWB543KmVk+lvLCcp+qeYnDRYN5Y\n/wZNHU0ADC0eyjHDjqGhtYG3Nr5FVVEVje07900MKxlGQ2tDn/weMimOFFMcKaaquIotbVsoihSx\ndsdajhp6FO9tfo+x5WOp31FPPLlzOOT4EcczqmwUZQVllEXLGF46nGElw2jubCYa9u44WF5QTmG4\nkHgyTnlhOWUFZUBqZ7E+gCU/FBAHKOccDa0NPFn3JFvat3D/svu7X+gAkBpuGVcxjrqmuvT0yqJK\ntrRvSb+uqajhnHHnUFZQRiQUoTRSSigUorKwklFloygIF3hDChahJFqSj6aI5J0CIuCcc8SSMRpa\nG1ixZQWNbY3EXZzGtkbqm+tpjbfycv3LXHj4hcyrm0d74tM3O9r9w3h344eMpzBcyJLGJSRdkrPH\nnc3EyolsbN3I6+tfZ/pB03ml/hUuOuIi1u5Yy/jK8dRU1NCZ6GTUoFFUl1QTDUWJhCKEzDtcNmQh\nfbMWyTEFhIiIZNTTgDhwdsuLiEifUkCIiEhGCggREclIASEiIhkpIEREJCMFhIiIZKSAEBGRjBQQ\nIiKS0YA+Uc7MNgEf93LxoUDQbhKtNgeD2hwM+9LmQ5xz1d3NNKADYl+Y2cKenEl4IFGbg0FtDob+\naLOGmEREJCMFhIiIZBTkgJid7wLyQG0OBrU5GHLe5sDugxARkT0Lcg9CRET2QAEhIiIZBTIgzGyW\nma0ws5VmdmO+69kXZnavmW00syVdplWa2fNm9qH/c4g/3czs53673zOzz3RZ5gp//g/N7Ip8tKUn\nzGy0mc03s2VmttTMvu9PP5DbXGRmfzGzd/02/8ifXmNmb/pte9DMCvzphf7rlf77Y7us6wf+9BVm\ndkZ+WtRzZhY2s7fN7En/9QHdZjNbbWaLzewdM1voT8vf/23nXKAeQBj4CBgHFADvAhPzXdc+tOdz\nwGeAJV2m3Qbc6D+/Efix//ws4GnAgOOBN/3plUCd/3OI/3xIvtuWpb0jgM/4z8uAD4CJB3ibDRjk\nP48Cb/pt+T1wiT/9F8Df+8+/A/zCf34J8KD/fKL//70QqPH/DsL5bl83bb8W+A3wpP/6gG4zsBoY\nutu0vP3fDmIP4lhgpXOuzjnXCfwOOC/PNfWac+4VYMtuk88D5vjP5wDnd5k+13neAAab2QjgDOB5\n59wW59xW4HlgVu6r33vOufXOubf8583AcmAkB3abnXNuh/8y6j8ccArwsD999zanfhcPA6ead6Pv\n84DfOec6nHOrgJV4fw/7JTMbBZwN/NJ/bRzgbc4ib/+3gxgQI4E1XV7X+9MOJMOcc+v95xuAYf7z\nbG0fkL8TfxhhOt436gO6zf5QyzvARrw/+I+Abc65uD9L1/rTbfPfbwKqGGBtBn4GXA8k/ddVHPht\ndsBzZrbIzK7yp+Xt/3akNwvJwOGcc2Z2wB3LbGaDgEeA/+Oc2+59WfQciG12ziWAaWY2GHgMODLP\nJeWUmZ0DbHTOLTKzmfmupx+d5Jxba2YHAc+b2ftd3+zv/9tB7EGsBUZ3eT3Kn3YgafC7mvg/N/rT\ns7V9QP1OzCyKFw4POOce9Scf0G1Occ5tA+YDM/CGFFJf8rrWn26b/34F0MjAavOJwLlmthpvGPgU\n4A4O7DbjnFvr/9yI90XgWPL4fzuIAfFX4HD/aIgCvB1aj+e5pr72OJA6cuEK4I9dpn/dP/rheKDJ\n77o+C3zBzIb4R0h8wZ+23/HHle8BljvnftrlrQO5zdV+zwEzKwZOx9v3Mh/4sj/b7m1O/S6+DLzk\nvL2XjwOX+Ef81ACHA3/pn1bsHefcD5xzo5xzY/H+Rl9yzl3GAdxmMys1s7LUc7z/k0vI5//tfO+1\nz8cDb+//B3jjuP+S73r2sS2/BdYDMbyxxm/ijb2+CHwIvABU+vMa8D9+uxcDtV3W8zd4O/BWAt/I\nd7v20N6T8MZp3wPe8R9nHeBtPgp422/zEuCH/vRxeB92K4GHgEJ/epH/eqX//rgu6/oX/3exAjgz\n323rYftnsvMopgO2zX7b3vUfS1OfTfn8v61LbYiISEZBHGISEZEeUECIiEhGCggREclIASEiIhkp\nIEREJCMFhEgPmdlgM/uO//xgM3u4u2VEBjId5irSQ/61n550zk3Ocyki/ULXYhLpuVuBQ/2L5n0I\nTHDOTTazK/GusFmKd6buT/AuJf81oAM4yzm3xcwOxTuxqRpoBb7lnHv/05sR2T9oiEmk524EPnLO\nTQOu2+29ycCXgGOA/w9odc5NB14Hvu7PMxu42jl3NPBPwP/2S9UivaQehEjfmO+8+1M0m1kT8IQ/\nfTFwlH/12ROAh7pcebaw/8sU6TkFhEjf6OjyPNnldRLv7yyEdy+Daf1dmEhvaYhJpOea8W5zutec\nc9uBVWZ2EaTvJzy1L4sT6WsKCJEecs41Aq+Z2RLgv3qxisuAb5pZ6mqdA/ZWtxIMOsxVREQyUg9C\nREQyUkCIiEhGCggREclIASEiIhkpIEREJCMFhIiIZKSAEBGRjP5/zd/y0QoZa+gAAAAASUVORK5C\nYII=\n", 175 | "text/plain": [ 176 | "" 177 | ] 178 | }, 179 | "metadata": {}, 180 | "output_type": "display_data" 181 | } 182 | ], 183 | "source": [ 184 | "plt.figure()\n", 185 | "plotit('vlosses_sgd_kfac', 'kfac')\n", 186 | "plotit('vlosses_sgd', 'sgd', 4)\n", 187 | "plotit('vlosses_adam', 'adam', 4)\n", 188 | "\n", 189 | "plt.xlabel('time')\n", 190 | "plt.ylabel('loss')\n", 191 | "plt.title('test loss')\n", 192 | "\n", 193 | "plt.legend()\n", 194 | "plt.show()" 195 | ] 196 | }, 197 | { 198 | "cell_type": "markdown", 199 | "metadata": {}, 200 | "source": [ 201 | "- final test loss is higher for KFAC (8.48 Adam vs 9.4 KFAC)\n", 202 | "- KFAC takes 100x less iteration\n", 203 | "- KFAC takes 25x less wall-clock time" 204 | ] 205 | } 206 | ], 207 | "metadata": { 208 | "kernelspec": { 209 | "display_name": "Python 3", 210 | "language": "python", 211 | "name": "python3" 212 | }, 213 | "language_info": { 214 | "codemirror_mode": { 215 | "name": "ipython", 216 | "version": 3 217 | }, 218 | "file_extension": ".py", 219 | "mimetype": "text/x-python", 220 | "name": "python", 221 | "nbconvert_exporter": "python", 222 | "pygments_lexer": "ipython3", 223 | "version": "3.5.4" 224 | }, 225 | "toc": { 226 | "nav_menu": {}, 227 | "number_sections": true, 228 | "sideBar": true, 229 | "skip_h1_title": false, 230 | "toc_cell": false, 231 | "toc_position": {}, 232 | "toc_section_display": "block", 233 | "toc_window_display": true 234 | } 235 | }, 236 | "nbformat": 4, 237 | "nbformat_minor": 2 238 | } 239 | -------------------------------------------------------------------------------- /derivation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yaroslavvb/kfac_pytorch/0922cf5c9e35cf2e13905f1606a24ddaf99830ac/derivation.pdf -------------------------------------------------------------------------------- /derivation.tex: -------------------------------------------------------------------------------- 1 | \documentclass{article} 2 | \usepackage{graphicx} 3 | \usepackage{amsmath} 4 | 5 | \begin{document} 6 | 7 | \title{KFAC derivation as Newton step} 8 | \author{Yaroslav Bulatov} 9 | 10 | \maketitle 11 | \section*{Linear Network} 12 | Suppose we have matrix parameter $W$ and are trying to model $Y$ as follows 13 | 14 | $$Y = B'W A$$ 15 | We measure our prediction error as 16 | 17 | $$e = \hat{Y}-B'WA$$ 18 | To minimize prediction error we seek $W$ to minimize the following loss, defined using trace as follows 19 | 20 | $$J = \frac{1}{2}\text{tr}(e'e)$$ 21 | 22 | To find gradient and Hessian, use approach of matrix differentials from Magnus, Nuedecker -- www.janmagnus.nl/misc/mdc2007-3rdedition 23 | 24 | First take differential: 25 | 26 | $$dJ = \text{tr}(e'de) = -\text{tr}(e'B'dWA)$$ 27 | 28 | Rearranging by using properties of trace and then using First Identification table (p.198 of Magnus), the gradient of $W$ is obtained as 29 | 30 | $$G=-BeA'$$ 31 | 32 | Taking differential of this expression we get 33 | 34 | $$dG = BB'dW AA'$$ 35 | 36 | Let lower case versions of variables represent vectorized versions. Vectorizing both sides and applying Kronecker/vec transformation rule, we get 37 | 38 | $$dg = (AA'\otimes BB')dw$$ 39 | 40 | From this we can extract the Hessian by visual inspection as 41 | 42 | $$H = (AA'\otimes BB')$$ 43 | 44 | Hessian is applied to flat (vectorized) gradient, so our vectorized parameter vector after single step of Newton's method is 45 | 46 | $$\text{vec}(W)-H^{-1}\text{vec}(G)$$ 47 | 48 | To invert Hessian, note that inverse distributes over Kronecker product: 49 | $$(A\otimes B)^{-1}=A^{-1}\otimes B^{-1}$$ 50 | 51 | Using this property and the fact that vec distributes over Kronecker product, we get the following equivalent quantity 52 | 53 | $$\text{vec}(W-(BB')^{-1} G (AA')^{-1})$$ 54 | 55 | This gives us Newton update step for original (unvectorized) form. 56 | 57 | 58 | \section*{Piecewise Linear Network} 59 | For a piecewise linear neural network, the loss for each example is locally linear, and we can write our total loss as sum of per-example losses 60 | 61 | $$J = \sum_i J_i$$ 62 | 63 | Where each example is associated with its version of $A_i$, $B_i$ and $e_i$ and $J_i = -\text{tr}(e_i'B_i'dWA_i)$ 64 | 65 | The total Hessian is a sum of per-example Hessians, so we get 66 | 67 | $$H = \sum_i H_i = \sum_i A_iA_i'\otimes B_iB_i'$$ 68 | Now apply Kronecker factorization approximation to get 69 | 70 | $$H \approx \left(\sum_i A_iA_i'\right)\otimes \left(\sum_iB_iB_i'\right)$$ 71 | If our individual examples are vectors, $A_i$ and $B_i$ can be stacked as columns into matrices $A$ and $B$, and the expression above can be written as 72 | $$H \approx \left(AA'\right)\otimes \left(BB'\right)$$ 73 | Since Kronecker product commutes with matrix inverse, preconditioner can be written as 74 | $$H^{-1}\approx \left(AA'\right)^{-1}\otimes \left(BB'\right)^{-1}$$ 75 | 76 | \section*{Convolutional Network} 77 | So far we obtained the Hessian with respect to matrix variable $W$. Suppose our operation is a convolution, in which case we can write it as matrix multiplication where $W$ is a function of another variable, written as $vec(W)=KU$. Here $U$ represents our matrix of tunable parameters and $K$ is the parameter tiling matrix that generates the convolution matmul. The derivative of $W$ with respect to $U$ can be written as (p.205 of Magnus) 78 | 79 | $$\frac{dW}{dU} = (I\otimes K)$$ 80 | 81 | Using this and chain rule for Hessian matrices (p.125 of Magnus), original Hessian of our loss with respect to $U$ can be written 82 | 83 | $$H_u=(I'\otimes K')(AA'\otimes BB')(I\otimes K)$$ 84 | 85 | Note that we have some extra matrix multiplications in our Hessian compared to the case from simple matmul. This means that ``distribute inverse over Kronecker product'' trick no longer works. 86 | 87 | \end{document} 88 | -------------------------------------------------------------------------------- /kfac_pytorch.py: -------------------------------------------------------------------------------- 1 | import util as u 2 | u.check_mkl() 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | import torch.optim as optim 8 | from torch.autograd import Variable 9 | import numpy as np 10 | import scipy 11 | import sys 12 | 13 | from torch.autograd.function import Function 14 | 15 | def regularized_inverse(mat, lambda_=3e-3, inverse_method='numpy', 16 | use_cuda=True): 17 | assert mat.shape[0] == mat.shape[1] 18 | ii = torch.eye(mat.shape[0]) 19 | if use_cuda: 20 | ii = ii.cuda() 21 | regmat = mat + lambda_*ii 22 | 23 | if inverse_method == 'numpy': 24 | import util as u 25 | result = torch.from_numpy(scipy.linalg.inv(regmat.cpu().numpy())) 26 | if use_cuda: 27 | result = result.cuda() 28 | elif inverse_method == 'gpu': 29 | assert use_cuda 30 | result = torch.inverse(regmat).cuda() 31 | else: 32 | assert False, 'unknown inverse_method ' + str(INVERSE_METHOD) 33 | return result 34 | 35 | 36 | def train(optimizer='sgd', nonlin=torch.sigmoid, kfac=True, iters=10, 37 | lr=0.2, newton_matrix='stochastic', eval_every_n_steps=1, 38 | print_interval=200): 39 | """Train on first 10k MNIST examples, evaluate on second 10k.""" 40 | 41 | u.reset_time() 42 | dsize = 10000 43 | 44 | # model options 45 | dtype = np.float32 46 | torch_dtype = 'torch.FloatTensor' 47 | 48 | use_cuda = torch.cuda.is_available() 49 | if use_cuda: 50 | torch_dtype = 'torch.cuda.FloatTensor' 51 | 52 | INVERSE_METHOD = 'numpy' # numpy, gpu 53 | 54 | As = [] 55 | Bs = [] 56 | As_inv = [] 57 | Bs_inv = [] 58 | mode = 'capture' # 'capture', 'kfac', 'standard' 59 | 60 | class KfacAddmm(Function): 61 | @staticmethod 62 | def _get_output(ctx, arg, inplace=False): 63 | if inplace: 64 | ctx.mark_dirty(arg) 65 | return arg 66 | else: 67 | return arg.new().resize_as_(arg) 68 | 69 | @staticmethod 70 | def forward(ctx, add_matrix, matrix1, matrix2, beta=1, alpha=1, inplace=False): 71 | ctx.save_for_backward(matrix1, matrix2) 72 | output = KfacAddmm._get_output(ctx, add_matrix, inplace=inplace) 73 | return torch.addmm(beta, add_matrix, alpha, 74 | matrix1, matrix2, out=output) 75 | 76 | @staticmethod 77 | def backward(ctx, grad_output): 78 | matrix1, matrix2 = ctx.saved_variables 79 | grad_matrix1 = grad_matrix2 = None 80 | 81 | if mode == 'capture': 82 | Bs.insert(0, grad_output.data) 83 | As.insert(0, matrix2.data) 84 | elif mode == 'kfac': 85 | B = grad_output.data 86 | A = matrix2.data 87 | kfac_A = As_inv.pop() @ A 88 | kfac_B = Bs_inv.pop() @ B 89 | grad_matrix1 = Variable(torch.mm(kfac_B, kfac_A.t())) 90 | elif mode == 'standard': 91 | grad_matrix1 = torch.mm(grad_output, matrix2.t()) 92 | 93 | else: 94 | assert False, 'unknown mode '+mode 95 | 96 | if ctx.needs_input_grad[2]: 97 | grad_matrix2 = torch.mm(matrix1.t(), grad_output) 98 | 99 | return None, grad_matrix1, grad_matrix2, None, None, None 100 | 101 | 102 | def kfac_matmul(mat1, mat2): 103 | output = Variable(mat1.data.new(mat1.data.size(0), mat2.data.size(1))) 104 | return KfacAddmm.apply(output, mat1, mat2, 0, 1, True) 105 | 106 | 107 | torch.manual_seed(1) 108 | np.random.seed(1) 109 | if use_cuda: 110 | torch.cuda.manual_seed(1) 111 | 112 | # feature sizes at each layer 113 | fs = [dsize, 28*28, 1024, 1024, 1024, 196, 1024, 1024, 1024, 28*28] 114 | n = len(fs) - 2 # number of matmuls 115 | 116 | class Net(nn.Module): 117 | def __init__(self): 118 | super(Net, self).__init__() 119 | for i in range(1, n+1): 120 | W0 = u.ng_init(fs[i+1], fs[i]) 121 | setattr(self, 'W'+str(i), nn.Parameter(torch.from_numpy(W0))) 122 | 123 | def forward(self, input): 124 | x = input.view(fs[1], -1) 125 | for i in range(1, n+1): 126 | W = getattr(self, 'W'+str(i)) 127 | x = nonlin(kfac_matmul(W, x)) 128 | return x.view_as(input) 129 | 130 | model = Net() 131 | 132 | if use_cuda: 133 | model.cuda() 134 | 135 | images = u.get_mnist_images() 136 | train_data0 = images[:, :dsize].astype(dtype) 137 | train_data = Variable(torch.from_numpy(train_data0)) 138 | test_data0 = images[:, dsize:2*dsize].astype(dtype) 139 | test_data = Variable(torch.from_numpy(test_data0)) 140 | if use_cuda: 141 | train_data = train_data.cuda() 142 | test_data = test_data.cuda() 143 | 144 | model.train() 145 | if optimizer == 'sgd': 146 | optimizer = optim.SGD(model.parameters(), lr=lr) 147 | elif optimizer == 'adam': 148 | optimizer = optim.Adam(model.parameters(), lr=lr) 149 | else: 150 | assert False, 'unknown optimizer '+optimizer 151 | 152 | noise = torch.Tensor(*train_data.data.shape).type(torch_dtype) 153 | assert fs[-1]<=dsize 154 | padding = dsize-fs[-1] 155 | zero_mat = torch.zeros((fs[-1], padding)) 156 | frozen = torch.cat([torch.eye(fs[-1]), zero_mat], 1).type(torch_dtype) 157 | 158 | covA_inv_saved = [None]*n 159 | losses = [] 160 | vlosses = [] 161 | 162 | for step in range(iters): 163 | mode = 'standard' 164 | output = model(train_data) 165 | 166 | if kfac: 167 | mode = 'capture' 168 | optimizer.zero_grad() 169 | del As[:], Bs[:], As_inv[:], Bs_inv[:] 170 | 171 | if newton_matrix == 'stochastic': 172 | noise.normal_() 173 | err_add = noise 174 | elif newton_matrix == 'exact': 175 | err_add = frozen 176 | else: 177 | assert False, 'unknown method for newton matrix '+newton_matrix 178 | 179 | output_hat = Variable(output.data+err_add) 180 | err_hat = output_hat - output 181 | 182 | loss_hat = torch.sum(err_hat*err_hat)/2/dsize 183 | loss_hat.backward(retain_graph=True) 184 | 185 | # compute inverses 186 | for i in range(n): 187 | # first layer activations don't change, only compute once 188 | if i == 0 and covA_inv_saved[i] is not None: 189 | covA_inv = covA_inv_saved[i] 190 | else: 191 | covA_inv = regularized_inverse(As[i] @ As[i].t()/dsize) 192 | covA_inv_saved[i] = covA_inv 193 | As_inv.append(covA_inv) 194 | 195 | covB = (Bs[i]@Bs[i].t())*dsize 196 | # alternative formula: slower but numerically better result 197 | # covB = (Bs[i]*dsize)@(Bs[i].t()*dsize)/dsize 198 | 199 | covB_inv = regularized_inverse(covB) 200 | Bs_inv.append(covB_inv) 201 | mode = 'kfac' 202 | 203 | else: 204 | mode = 'standard' 205 | 206 | if step%eval_every_n_steps==0: 207 | old_mode = mode 208 | mode = 'standard' 209 | test_output = model(test_data) 210 | test_err = test_data - test_output 211 | test_loss = torch.sum(test_err*test_err)/2/dsize 212 | vloss0 = test_loss.data.cpu().numpy()[0] 213 | vlosses.append(vloss0) 214 | mode = old_mode 215 | 216 | optimizer.zero_grad() 217 | err = output - train_data 218 | loss = torch.sum(err*err)/2/dsize 219 | loss.backward() 220 | optimizer.step() 221 | 222 | loss0 = loss.data.cpu().numpy()[0] 223 | losses.append(loss0) 224 | if step%print_interval==0: 225 | print("Step %3d loss %10.9f"%(step, loss0)) 226 | 227 | 228 | u.record_time() 229 | 230 | return losses, vlosses 231 | 232 | 233 | def main(): 234 | losses,vlosses = train(optimizer='sgd', kfac=True, nonlin=F.sigmoid, iters=10, 235 | print_interval=1, lr=0.2) 236 | u.summarize_time() 237 | print(losses) 238 | loss0 = losses[-1] 239 | 240 | use_cuda = torch.cuda.is_available() 241 | if use_cuda: 242 | target = 38.781795502 243 | else: 244 | target = 0 245 | assert abs(loss0-target)<1e-9, abs(loss0-target) 246 | 247 | if __name__=='__main__': 248 | main() 249 | -------------------------------------------------------------------------------- /run_experiments.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | import kfac_pytorch as kfac_lib 3 | import util as u 4 | import torch.nn.functional as F 5 | 6 | nonlin = F.relu 7 | 8 | def run_experiment(name, optimizer, nonlin, kfac, iters, lr): 9 | losses, vlosses = kfac_lib.train(optimizer=optimizer, nonlin=nonlin, 10 | kfac=kfac, iters=iters, lr=lr) 11 | u.dump(losses, 'losses_'+name+'.csv', True) 12 | u.dump(vlosses, 'vlosses_'+name+'.csv', True) 13 | 14 | run_experiment('sgd', 'sgd', F.sigmoid, False, 20000, 0.2) 15 | run_experiment('adam', 'adam', F.sigmoid, False, 20000, 1e-3) 16 | run_experiment('sgd_kfac', 'sgd', F.sigmoid, True, 5000, 0.2) 17 | run_experiment('adam_kfac', 'adam', F.sigmoid, True, 5000, 1e-3) 18 | 19 | # relu explodes, divide learning rates by 10 20 | run_experiment('sgd2', 'sgd', F.relu, False, 20000, 0.02) 21 | run_experiment('adam2', 'adam', F.relu, False, 2000, 1e-4) 22 | run_experiment('sgd_kfac2', 'sgd', F.relu, True, 5000, 0.02) 23 | run_experiment('adam_kfac2', 'adam', F.relu, True, 5000, 1e-4) 24 | -------------------------------------------------------------------------------- /util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import socket 3 | import contextlib 4 | import inspect 5 | import inspect 6 | import networkx as nx 7 | import numpy as np 8 | import os 9 | import sys 10 | import tensorflow as tf 11 | import time 12 | import traceback 13 | from tensorflow.contrib import graph_editor as ge 14 | from collections import OrderedDict 15 | from collections import defaultdict 16 | 17 | # shortcuts to refer to util module, this lets move external code into 18 | # this module unmodified 19 | util = sys.modules[__name__] 20 | u = util 21 | 22 | # for line profiling 23 | try: 24 | profile # throws an exception when profile isn't defined 25 | except NameError: 26 | profile = lambda x: x # if it's not defined simply ignore the decorator. 27 | 28 | 29 | default_tf_dtype = tf.float32 30 | default_np_dtype = np.float32 31 | default_dtype = default_tf_dtype 32 | 33 | USE_MKL_SVD=True # Tensorflow vs MKL SVD 34 | DUMP_BAD_SVD=False # when SVD fails, dump matrix to temp 35 | 36 | if USE_MKL_SVD: 37 | assert np.__config__.get_info("lapack_mkl_info"), "No MKL detected :(" 38 | 39 | 40 | from scipy import linalg 41 | 42 | def check_mkl(): 43 | assert np.__config__.get_info("lapack_mkl_info"), "No MKL detected :(" 44 | print("Using MKL") 45 | 46 | args = None # TODO: replace with object that crashes on access 47 | def set_global_args(local_args): 48 | """Sets args to be reused across several modules. Access as 49 | util.args.somesetting """ 50 | global args 51 | assert args is None 52 | args = local_args 53 | 54 | def concat_blocks(blocks, validate_dims=True): 55 | """Takes 2d grid of blocks representing matrices and concatenates to single 56 | matrix (aka ArrayFlatten)""" 57 | 58 | if validate_dims: 59 | col_dims = np.array([[int(b.shape[1]) for b in row] for row in blocks]) 60 | col_sums = col_dims.sum(1) 61 | assert (col_sums[0] == col_sums).all() 62 | row_dims = np.array([[int(b.shape[0]) for b in row] for row in blocks]) 63 | row_sums = row_dims.sum(0) 64 | assert (row_sums[0] == row_sums).all() 65 | 66 | block_rows = [tf.concat(row, axis=1) for row in blocks] 67 | return tf.concat(block_rows, axis=0) 68 | 69 | def concat_blocks_test(): 70 | blocks = [[tf.constant([[1]]), tf.constant([[1,2]])], 71 | [tf.transpose(tf.constant([[1,2]])), tf.constant([[1,2],[3,4]])]] 72 | result = concat_blocks(blocks) 73 | sess = tf.Session() 74 | result0 = sess.run(result) 75 | check_equal(result0, [[1, 1, 2], [1, 1, 2], [2, 3, 4]]) 76 | 77 | 78 | def partition_matrix_evenly(mat, splits): 79 | """Breaks matrix into 2d grid of equal size.""" 80 | assert int(mat.shape[0])%splits==0 81 | assert int(mat.shape[1])%splits==0 82 | 83 | row_chunks = tf.split(mat, splits, axis=0) 84 | col_chunks = [tf.split(chunk, splits, axis=1) for chunk in row_chunks] 85 | return col_chunks 86 | 87 | def partition_matrix_evenly_test(): 88 | a = tf.reshape([1,2,3,4], (2,2)) 89 | blocks = partition_matrix_evenly(a, 2) 90 | a2 = concat_blocks(blocks) 91 | sess = tf.Session() 92 | check_equal(sess.run(a2), sess.run(a)) 93 | 94 | # inverse of concat blocks 95 | def partition_matrix(mat, sizes): 96 | pass 97 | 98 | def partition_matrix_test(): 99 | pass 100 | 101 | 102 | # TODO: add name property 103 | def pseudo_inverse(mat, eps=1e-10): 104 | """Computes pseudo-inverse of mat, treating eigenvalues below eps as 0.""" 105 | 106 | s, u, v = tf.svd(mat) 107 | eps = 1e-10 # zero threshold for eigenvalues 108 | si = tf.where(tf.less(s, eps), s, 1./s) 109 | return u @ tf.diag(si) @ tf.transpose(v) 110 | 111 | def symsqrt(mat, eps=1e-7): 112 | """Symmetric square root.""" 113 | s, u, v = tf.svd(mat) 114 | # sqrt is unstable around 0, just use 0 in such case 115 | print("Warning, cutting off at eps") 116 | si = tf.where(tf.less(s, eps), s, tf.sqrt(s)) 117 | return u @ tf.diag(si) @ tf.transpose(v) 118 | 119 | def pseudo_inverse_sqrt(mat, eps=1e-7): 120 | """half pseduo-inverse""" 121 | s, u, v = tf.svd(mat) 122 | # zero threshold for eigenvalues 123 | si = tf.where(tf.less(s, eps), s, 1./tf.sqrt(s)) 124 | return u @ tf.diag(si) @ tf.transpose(v) 125 | 126 | def pseudo_inverse_sqrt2(svd, eps=1e-7): 127 | """half pseduo-inverse, accepting existing values""" 128 | # zero threshold for eigenvalues 129 | if svd.__class__.__name__=='SvdTuple': 130 | (s, u, v) = (svd.s, svd.u, svd.v) 131 | elif svd.__class__.__name__=='SvdWrapper': 132 | (s, u, v) = (svd.s, svd.u, svd.v) 133 | else: 134 | assert False, "Unknown type" 135 | si = tf.where(tf.less(s, eps), s, 1./tf.sqrt(s)) 136 | return u @ tf.diag(si) @ tf.transpose(v) 137 | 138 | def pseudo_inverse2(svd, eps=1e-7): 139 | """pseudo-inverse, accepting existing values""" 140 | # use float32 machine precision as cut-off (works for MKL) 141 | # https://www.wolframcloud.com/objects/927b2aa5-de9c-46f5-89fe-c4a58aa4c04b 142 | if svd.__class__.__name__=='SvdTuple': 143 | (s, u, v) = (svd.s, svd.u, svd.v) 144 | elif svd.__class__.__name__=='SvdWrapper': 145 | (s, u, v) = (svd.s, svd.u, svd.v) 146 | else: 147 | assert False, "Unknown type" 148 | max_eigen = tf.reduce_max(s) 149 | si = tf.where(s/max_eigen (2, 10) 452 | def fix_shape(tf_shape): 453 | return tuple(int(dim) for dim in tf_shape) 454 | 455 | def kronecker_cols(a, b): 456 | """Treats rank-1 vectors a, b as columns, returns Kronecker product a x b.""" 457 | 458 | assert len(a.get_shape())==1, "Input a must be rank-1, got shape %s" %(a.get_shape(),) 459 | assert len(b.get_shape())==1, "Input b must be rank-1, got shape %s"%(a.get_shape(),) 460 | segments = [] 461 | for i in range(a.get_shape()[0]): 462 | segments.append(a[i]*b) 463 | result_vec = tf.concat(segments, axis=0) 464 | result_col = tf.expand_dims(result_vec, 1) 465 | return result_col 466 | 467 | def kronecker_cols_test(): 468 | a = tf.constant([1,2]) 469 | b = tf.constant([3,4]) 470 | c = tf.transpose(tf.constant([[3,4,6,8]])) 471 | sess = tf.Session() 472 | assert sess.run(tf.equal(kronecker_cols(a, b), c)).all() 473 | 474 | 475 | def kronecker(A, B, do_shape_inference=True): 476 | """Kronecker product of A,B. 477 | turn_off_shape_inference: if True, makes 10x10 kron go 2.4 sec -> 0.9 sec 478 | """ 479 | 480 | Arows, Acols = fix_shape(A.shape) 481 | Brows, Bcols = fix_shape(B.shape) 482 | Crows, Ccols = Arows*Brows, Acols*Bcols 483 | 484 | temp = tf.reshape(A, [-1, 1, 1])*tf.expand_dims(B, 0) 485 | Bshape = tf.constant((Brows, Bcols)) 486 | 487 | # turn off shape inference 488 | if not do_shape_inference: 489 | disable_shape_inference() 490 | 491 | # [1, n, m] => [n, m] 492 | slices = [tf.reshape(s, Bshape) for s in tf.split(temp, Crows)] 493 | 494 | # import pdb; pdb.set_trace() 495 | grid = list(chunks(slices, Acols)) 496 | assert len(grid) == Arows 497 | result = concat_blocks(grid, validate_dims=do_shape_inference) 498 | 499 | if not do_shape_inference: 500 | enable_shape_inference() 501 | result.set_shape((Arows*Brows, Acols*Bcols)) 502 | 503 | return result 504 | 505 | kr = kronecker 506 | 507 | def kronecker_test(): 508 | A0 = [[1,2],[3,4]] 509 | B0 = [[6,7],[8,9]] 510 | A = tf.constant(A0) 511 | B = tf.constant(B0) 512 | C = kronecker(A, B) 513 | sess = tf.Session() 514 | C0 = sess.run(C) 515 | Ct = [[6, 7, 12, 14], [8, 9, 16, 18], [18, 21, 24, 28], [24, 27, 32, 36]] 516 | Cnp = np.kron(A0, B0) 517 | check_equal(C0, Ct) 518 | check_equal(C0, Cnp) 519 | 520 | 521 | def col(A,i): 522 | """Extracts i'th column of matrix A""" 523 | assert len(A.get_shape())==2 524 | assert i>=0 and i < A.get_shape()[1] 525 | return tf.expand_dims(A[:,i], 1) 526 | 527 | 528 | def khatri_rao(A, B): 529 | Arows, Acols = fix_shape(A.shape) 530 | Brows, Bcols = fix_shape(B.shape) 531 | assert Acols==Bcols 532 | return tf.reshape(tf.einsum("ik,jk->ijk", A, B), (Arows*Brows, Acols)) 533 | 534 | 535 | def khatri_rao_test(): 536 | A = tf.constant([[1, 2], [3, 4]]) 537 | B = tf.constant([[5, 6], [7, 8]]) 538 | C = tf.constant([[5,12], [7,16], [15,24], [21,32]]) 539 | sess = tf.Session() 540 | assert sess.run(tf.equal(khatri_rao(A, B), C)).all() 541 | 542 | 543 | def relu_mask(a, dtype=default_dtype): 544 | """Produces mask of 1s for positive values and 0s for negative values.""" 545 | from tensorflow.python.ops import gen_nn_ops 546 | ones = tf.ones(a.get_shape(), dtype=dtype) 547 | return gen_nn_ops._relu_grad(ones, a) 548 | 549 | def relu_mask_test(): 550 | a = tf.constant([-1,0,1,2], dtype=default_dtype) 551 | sess = tf.Session() 552 | check_equal(sess.run(relu_mask(a)), [0,0,1,1]) 553 | 554 | def assert_rectangular(blocks): 555 | lengths = np.array([len(row) for row in blocks]) 556 | assert (lengths==lengths[0]).all() 557 | 558 | def empty_grid(rows, cols): 559 | """Create empty list of lists of rows-by-cols shape.""" 560 | result = [] 561 | for i in range(rows): 562 | result.append([None]*cols) 563 | return result 564 | 565 | def block_diagonal_inverse(blocks): 566 | """Invert diagonal blocks, leave remaining unchanged.""" 567 | 568 | assert_rectangular(blocks) 569 | num_rows = len(blocks) 570 | num_cols = len(blocks[0]) 571 | 572 | result = empty_grid(num_rows, num_cols) 573 | dtype = blocks[0][0].dtype # TODO: assert same dtype 574 | 575 | for i in range(len(blocks)): 576 | for j in range(len(blocks[0])): 577 | block = blocks[i][j] 578 | if i == j: 579 | result[i][j] = pseudo_inverse(block) 580 | else: 581 | result[i][j] = tf.zeros(shape=block.get_shape(), 582 | dtype=dtype) 583 | return result 584 | 585 | def block_diagonal_inverse_sqrt(blocks): 586 | assert_rectangular(blocks) 587 | num_rows = len(blocks) 588 | num_cols = len(blocks[0]) 589 | 590 | result = empty_grid(num_rows, num_cols) 591 | dtype = blocks[0][0].dtype # TODO: assert same dtype 592 | 593 | for i in range(len(blocks)): 594 | for j in range(len(blocks[0])): 595 | block = blocks[i][j] 596 | if i == j: 597 | result[i][j] = pseudo_inverse_sqrt(block) 598 | else: 599 | result[i][j] = tf.zeros(shape=block.get_shape(), 600 | dtype=dtype) 601 | return result 602 | 603 | 604 | def block_diagonal_inverse_test(): 605 | sess = tf.Session() 606 | blocks = [[2*Identity(3), tf.ones((3, 1))], 607 | [tf.ones((1,3)), 2*Identity(1)]] 608 | new_blocks = block_diagonal_inverse(blocks) 609 | actual = concat_blocks(new_blocks) 610 | expected = 0.5*Identity(4) 611 | check_equal(sess.run(actual), sess.run(expected)) 612 | 613 | 614 | def t(x): 615 | return tf.transpose(x) 616 | 617 | 618 | # Time tracking functions 619 | global_time_list = [] 620 | global_last_time = 0 621 | def reset_time(): 622 | global global_time_list, global_last_time 623 | global_time_list = [] 624 | global_last_time = time.perf_counter() 625 | 626 | def record_time(): 627 | global global_last_time, global_time_list 628 | new_time = time.perf_counter() 629 | global_time_list.append(new_time - global_last_time) 630 | global_last_time = time.perf_counter() 631 | #print("step: %.2f"%(global_time_list[-1]*1000)) 632 | 633 | def last_time(): 634 | """Returns last interval records in millis.""" 635 | global global_last_time, global_time_list 636 | if global_time_list: 637 | return 1000*global_time_list[-1] 638 | else: 639 | return 0 640 | 641 | def summarize_time(time_list=None): 642 | if time_list is None: 643 | time_list = global_time_list 644 | 645 | # delete first large interval if exists 646 | if time_list and time_list[0]>3600*10: 647 | del time_list[0] 648 | 649 | time_list = 1000*np.array(time_list) # get seconds, convert to ms 650 | if len(time_list)>0: 651 | min = np.min(time_list) 652 | median = np.median(time_list) 653 | formatted = ["%.2f"%(d,) for d in time_list[:10]] 654 | print("Times: min: %.2f, median: %.2f, mean: %.2f"%(min, median, 655 | np.mean(time_list))) 656 | else: 657 | print("Times: ") 658 | 659 | def summarize_graph(g=None): 660 | if not g: 661 | g = tf.get_default_graph() 662 | print("Graph: %d ops, %d MBs"%(len(g.get_operations()), 663 | len(str(g.as_graph_def()))/10**6)) 664 | 665 | from tensorflow.python.framework import ops 666 | original_shape_func = ops.set_shapes_for_outputs 667 | def disable_shape_inference(): 668 | ops.set_shapes_for_outputs = lambda _: _ 669 | 670 | def enable_shape_inference(): 671 | ops.set_shapes_for_outputs = original_shape_func 672 | 673 | # work-around for graph_editor.copy_with_input_replacements scaling 674 | # quadratically with size of the graph 675 | from tensorflow.contrib.graph_editor import transform 676 | original_assign_renamed_collections_handler = transform.assign_renamed_collections_handler 677 | def dummy_collections_handler(info, elem, elem_): pass 678 | def disable_collections_handler(): 679 | transform.assign_renamed_collections_handler = dummy_collections_handler 680 | def enable_collections_handler(): 681 | transform.assign_renamed_collections_handler = original_assign_renamed_collections_handler 682 | 683 | 684 | def dump_with_prompt(result, fname, no_prefix=False): 685 | """Helper function to ask for confirmation before overwriting.""" 686 | location = os.getcwd()+"/data/"+fname # TODO: factor out locations logic 687 | if os.path.exists(location): 688 | answer = input("%s exists, overwrite? (Y/n) "%(location,)) 689 | if not answer: 690 | answer = "y" 691 | if answer.lower() != "y": 692 | print("skipping") 693 | else: 694 | u.dump(result, fname, no_prefix) 695 | else: 696 | u.dump(result, fname, no_prefix) 697 | 698 | 699 | def dump(result, fname, no_prefix=False): 700 | """Save result to file.""" 701 | result = result.eval() if hasattr(result, "eval") else result 702 | result = np.asarray(result) 703 | if result.shape == (): # savetxt has problems with scalars 704 | result = np.expand_dims(result, 0) 705 | if no_prefix: 706 | location = os.getcwd()+"/"+fname 707 | else: 708 | location = os.getcwd()+"/data/"+fname 709 | # special handling for integer datatypes 710 | if ( 711 | result.dtype == np.uint8 or result.dtype == np.int8 or 712 | result.dtype == np.uint16 or result.dtype == np.int16 or 713 | result.dtype == np.uint32 or result.dtype == np.int32 or 714 | result.dtype == np.uint64 or result.dtype == np.int64 715 | ): 716 | np.savetxt(location, result, fmt="%d", delimiter=',') 717 | else: 718 | np.savetxt(location, result, delimiter=',') 719 | print(location) 720 | 721 | def dump32(result, fname): 722 | """Efficient dumping of float32 vals""" 723 | result = result.eval() if hasattr(result, "eval") else result 724 | result = np.asarray(result) 725 | location = os.getcwd()+"/data/"+fname 726 | assert is_numeric(result) 727 | # print(location) 728 | return result.astype('float32').tofile(location) 729 | 730 | 731 | def frobenius_np(a): 732 | return np.sqrt(np.sum(np.square(a))) 733 | 734 | def nan_check(result): 735 | result = result.eval() if hasattr(result, "eval") else result 736 | result = np.asarray(result) 737 | print("result any NaNs: %s"% (np.isnan(result).any(),)) 738 | 739 | 740 | def L2(t): 741 | """Squared L2 norm of t.""" 742 | if t.__class__.__name__=='Grads': 743 | t = t.f 744 | else: 745 | assert (t.__class__.__name__.endswith('Tensor') or 746 | t.__class__.__name__.endswith('Variable')) 747 | return tf.reduce_sum(tf.square(t)) 748 | 749 | 750 | global_timeit_dict = OrderedDict() 751 | class timeit: 752 | """Decorator to measure length of time spent in the block in millis and log 753 | it to TensorBoard.""" 754 | 755 | def __init__(self, tag=""): 756 | self.tag = tag 757 | 758 | def __enter__(self): 759 | self.start = time.perf_counter() 760 | return self 761 | 762 | def __exit__(self, *args): 763 | self.end = time.perf_counter() 764 | interval_ms = 1000*(self.end - self.start) 765 | global_timeit_dict.setdefault(self.tag, []).append(interval_ms) 766 | logger = u.get_last_logger(skip_existence_check=True) 767 | if logger: 768 | newtag = 'time/'+self.tag 769 | # since tensorboard doesn't allow hierarchical tags, merge init times 770 | if newtag.startswith('time/init'): 771 | newtag = newtag.replace('time/init', 'timeinit') 772 | logger(newtag, interval_ms) 773 | 774 | 775 | global_record_dict = OrderedDict() 776 | def record(tag, stat): 777 | global global_record_dict 778 | global_record_dict.setdefault(tag, []).append(stat) 779 | 780 | 781 | def timeit_summarize(): 782 | global global_timeit_dict 783 | pass 784 | 785 | # graph traversal 786 | # computation flows from parents to children 787 | # to find path from target to dependency, do 788 | # nx.shortest_path(gg, dependency, target) 789 | def parents(op): return set(input.op for input in op.inputs) 790 | def children(op): return set(op for out in op.outputs for op in out.consumers()) 791 | def dict_graph(): 792 | """Creates dictionary {node: {child1, child2, ..},..} for current 793 | TensorFlow graph. Result is compatible with networkx/toposort""" 794 | 795 | ops = tf.get_default_graph().get_operations() 796 | return {op: children(op) for op in ops} 797 | def nx_graph(): 798 | return nx.DiGraph(dict_graph()) 799 | 800 | def shortest_path(dep, target): 801 | if hasattr(dep, "op"): 802 | dep = dep.op 803 | if hasattr(target, "op"): 804 | target = target.op 805 | return nx.shortest_path(nx_graph(), dep, target) 806 | 807 | def list_or_tuple(k): 808 | return isinstance(k, list) or isinstance(k, tuple) 809 | 810 | def is_numeric(ndarray): 811 | ndarray = np.asarray(ndarray) 812 | return np.issubdtype(ndarray.dtype, np.number) 813 | 814 | class VarInfo: 815 | """Encapsulate variable info.""" 816 | def __init__(self, setter, p): 817 | self.setter = setter 818 | self.p = p 819 | 820 | class SvdTuple: 821 | """Object to store svd tuple. 822 | Create as SvdTuple((s,u,v)) or SvdTuple(s, u, v). 823 | """ 824 | def __init__(self, suvi, *args): 825 | if list_or_tuple(suvi): 826 | if len(suvi) == 3: 827 | s, u, v = suvi 828 | inv = Identity(s.shape[0]) 829 | else: 830 | s, u, v, inv = suvi 831 | else: 832 | s = suvi 833 | u = args[0] 834 | v = args[1] 835 | if len(args)>2: 836 | inv = args[2] 837 | else: 838 | inv = Identity(s.shape[0]) 839 | self.s = s 840 | self.u = u 841 | self.v = v 842 | self.inv = inv 843 | 844 | 845 | class SvdWrapper: 846 | """Encapsulates variables needed to perform SVD of a TensorFlow target. 847 | Initialize: wrapper = SvdWrapper(tensorflow_var) 848 | Trigger SVD: wrapper.update_tf() or wrapper.update_scipy() 849 | Access result as TF vars: wrapper.s, wrapper.u, wrapper.v 850 | """ 851 | 852 | def __init__(self, target, name, do_inverses=False, use_resource=False): 853 | self.name = name 854 | self.target = target 855 | self.do_inverses = do_inverses 856 | self.tf_svd = SvdTuple(tf.svd(target)) 857 | self.update_counter = 0 858 | self.use_resource = use_resource 859 | 860 | self.init = SvdTuple( 861 | ones(target.shape[0], name=name+"_s_init"), 862 | Identity(target.shape[0], name=name+"_u_init"), 863 | Identity(target.shape[0], name=name+"_v_init"), 864 | Identity(target.shape[0], name=name+"_inv_init"), 865 | ) 866 | 867 | assert self.tf_svd.s.shape == self.init.s.shape 868 | assert self.tf_svd.u.shape == self.init.u.shape 869 | assert self.tf_svd.v.shape == self.init.v.shape 870 | # assert self.tf_svd.inv.shape == self.init.inv.shape 871 | 872 | if not self.use_resource: 873 | self.cached = SvdTuple( 874 | tf.Variable(self.init.s, name=name+"_s"), 875 | tf.Variable(self.init.u, name=name+"_u"), 876 | tf.Variable(self.init.v, name=name+"_v"), 877 | tf.Variable(self.init.inv, name=name+"_inv"), 878 | ) 879 | else: 880 | from tensorflow.python.ops import resource_variable_ops as rr 881 | self.cached = SvdTuple( 882 | rr.ResourceVariable(self.init.s, name=name+"_s"), 883 | rr.ResourceVariable(self.init.u, name=name+"_u"), 884 | rr.ResourceVariable(self.init.v, name=name+"_v"), 885 | rr.ResourceVariable(self.init.inv, name=name+"_inv"), 886 | ) 887 | 888 | self.s = self.cached.s 889 | self.u = self.cached.u 890 | self.v = self.cached.v 891 | self.inv = self.cached.inv 892 | 893 | if not use_resource: 894 | self.holder = SvdTuple( 895 | tf.placeholder(default_dtype, shape=self.cached.s.shape, name=name+"_s_holder"), 896 | tf.placeholder(default_dtype, shape=self.cached.u.shape, name=name+"_u_holder"), 897 | tf.placeholder(default_dtype, shape=self.cached.v.shape, name=name+"_v_holder"), 898 | tf.placeholder(default_dtype, shape=self.cached.inv.shape, name=name+"_inv_holder") 899 | ) 900 | else: 901 | self.holder = self.init 902 | 903 | self.update_tf_op = tf.group( 904 | self.cached.s.assign(self.tf_svd.s), 905 | self.cached.u.assign(self.tf_svd.u), 906 | self.cached.v.assign(self.tf_svd.v), 907 | self.cached.inv.assign(self.tf_svd.inv) 908 | ) 909 | 910 | self.update_external_op = tf.group( 911 | self.cached.s.assign(self.holder.s), 912 | self.cached.u.assign(self.holder.u), 913 | self.cached.v.assign(self.holder.v), 914 | ) 915 | 916 | self.update_externalinv_op = tf.group( 917 | self.cached.inv.assign(self.holder.inv), 918 | ) 919 | 920 | 921 | self.init_ops = (self.s.initializer, self.u.initializer, self.v.initializer, 922 | self.inv.initializer) 923 | 924 | 925 | def update(self): 926 | if USE_MKL_SVD: 927 | self.update_scipy() 928 | else: 929 | self.update_tf() 930 | self.update_counter+=1 931 | 932 | def update_tf(self): 933 | sess = u.get_default_session() 934 | sess.run(self.update_tf_op) 935 | 936 | @profile 937 | def update_scipy(self): 938 | if self.do_inverses: 939 | return self.update_scipy_inv() 940 | else: 941 | return self.update_scipy_svd() 942 | 943 | def update_scipy_inv(self): 944 | sess = u.get_default_session() 945 | target0 = sess.run(self.target) 946 | inv0 = linalg.inv(target0) 947 | feed_dict = {self.holder.inv: inv0} 948 | sess.run(self.update_externalinv_op, feed_dict=feed_dict) 949 | 950 | def update_scipy_svd(self): 951 | sess = u.get_default_session() 952 | target0 = sess.run(self.target) 953 | # A=u.diag(s).v', singular vectors are columns 954 | # TODO: catch "ValueError: array must not contain infs or NaNs" 955 | try: 956 | u0, s0, vt0 = linalg.svd(target0) 957 | v0 = vt0.T 958 | except Exception as e: 959 | print("Got error %s"%(repr(e),)) 960 | if DUMP_BAD_SVD: 961 | dump32(target0, "badsvd") 962 | print("gesdd failed, trying gesvd") 963 | u0, s0, vt0 = linalg.svd(target0, lapack_driver="gesvd") 964 | v0 = vt0.T 965 | 966 | feed_dict = {self.holder.u: u0, 967 | self.holder.v: v0, 968 | self.holder.s: s0} 969 | sess.run(self.update_external_op, feed_dict=feed_dict) 970 | 971 | def extract_grad(grads_and_vars, var): 972 | if isinstance(var, str): 973 | varname = var 974 | else: 975 | varname = var.name 976 | vals = [] 977 | for (grad, var) in grads_and_vars: 978 | if var.name == varname: 979 | vals.append(var) 980 | assert length(vals)==1 981 | return vals[0] 982 | 983 | def intersept_op_creation(op_type_name_to_intercept): 984 | """Drops into PDB when particular op type is added to graph.""" 985 | from tensorflow.python.framework import op_def_library 986 | old_apply_op = op_def_library.OpDefLibrary.apply_op 987 | def my_apply_op(obj, op_type_name, name=None, **keywords): 988 | print(op_type_name+"-"+str(name)) 989 | if op_type_name == op_type_name_to_intercept: 990 | import pdb; pdb.set_trace() 991 | return(old_apply_op(obj, op_type_name, name=name, **keywords)) 992 | op_def_library.OpDefLibrary.apply_op=my_apply_op 993 | 994 | 995 | global_variables = {} 996 | def get_variable(name, initializer, reuse=True): 997 | """Lightweight replacement for tf.get_variable() for variables shared within 998 | a single process. Doesn't need variable scopes.""" 999 | 1000 | global global_variables 1001 | if name in global_variables and reuse: 1002 | v = global_variables[name] 1003 | else: 1004 | v = tf.Variable(name=name, initial_value=initializer) 1005 | # print("Creating new variable %s into %s" %(name, v.op.name)) 1006 | global_variables[name] = v 1007 | return v 1008 | 1009 | 1010 | class VarStruct: 1011 | # TODO: refactor to behave more like variable 1012 | """Convenience structure to keep track of variable, its assign op 1013 | and assignment placeholder. 1014 | 1015 | v = Var(6) 1016 | v.set(5) # equivalent to sess.run(v.assign_op, feed_dict={pl: 5}) 1017 | var.var # returns underlying variable 1018 | var.val_ # placeholder to assign op 1019 | var.setter # assign op 1020 | var.set(6) # same as sess.run(var.setter, feed_dict={self.val_: val}) 1021 | var.initialize() # sets variable to initial value 1022 | """ 1023 | 1024 | # TODO: add names to placeholder op 1025 | def __init__(self, initial_value, name, dtype=None): 1026 | 1027 | initial_value = np.array(initial_value) 1028 | assert u.is_numeric(initial_value), "Non-numeric type." 1029 | if not dtype: 1030 | dtype = initial_value.dtype 1031 | else: 1032 | initial_value = initial_value.astype(dtype) 1033 | self.initial_value = initial_value 1034 | self.val_ = tf.placeholder(dtype=initial_value.dtype, 1035 | shape=initial_value.shape, 1036 | name=name+"_holder") 1037 | self.var = tf.Variable(initial_value=self.val_, name=name, dtype=dtype) 1038 | assigned_name = self.var.op.name 1039 | if assigned_name != name: 1040 | print("Warning, conflicting variable %s"%(assigned_name,)) 1041 | self.setter = self.var.assign(self.val_) 1042 | 1043 | def set(self, val): 1044 | sess = u.get_default_session() 1045 | sess.run(self.setter, feed_dict={self.val_: val}) 1046 | 1047 | def initialize(self): 1048 | sess = u.get_default_session() 1049 | sess.run(self.setter, feed_dict={self.val_: self.val}) 1050 | 1051 | 1052 | global_vars = {} 1053 | def get_var(name, initializer, reuse=True): 1054 | """Global get_variable replacement for variables that need to be initialized 1055 | with a large numpy array. 1056 | 1057 | a = tf.get_var([1,2,3]) 1058 | a.var # => gives tf.Variable 1059 | a.val 1060 | """ 1061 | 1062 | global global_vars 1063 | dtype = initializer.dtype 1064 | if name in global_vars and reuse: 1065 | vv = global_vars[name] 1066 | if (np.max(np.abs(vv.initial_value - initializer)))>np.finfo(dtype).eps: 1067 | print("Trying to reinitialize global variable %s with new" 1068 | " value, ignoring new value."%(name,)) 1069 | else: 1070 | vv = VarStruct(initial_value=initializer, name=name) 1071 | global_vars[name] = vv 1072 | return vv 1073 | 1074 | def run_all_tests(module): 1075 | all_functions = inspect.getmembers(module, inspect.isfunction) 1076 | for name,func in all_functions: 1077 | if name.endswith("_test"): 1078 | print("Testing "+name) 1079 | with timeit(): 1080 | func() 1081 | print(module.__name__+" tests passed.") 1082 | 1083 | @contextlib.contextmanager 1084 | def capture_ops(): 1085 | """Decorator to capture ops created in the block. 1086 | with capture_ops() as ops: 1087 | # create some ops 1088 | print(ops) # => prints ops created. 1089 | """ 1090 | 1091 | micros = int(time.perf_counter()*10**6) 1092 | scope_name = str(micros) 1093 | op_list = [] 1094 | with tf.name_scope(scope_name): 1095 | yield op_list 1096 | 1097 | g = tf.get_default_graph() 1098 | op_list.extend(ge.select_ops(scope_name+"/.*", graph=g)) 1099 | 1100 | @contextlib.contextmanager 1101 | def capture_vars(): 1102 | """Decorator to capture global variables created in the block. 1103 | """ 1104 | 1105 | micros = int(time.perf_counter()*10**6) 1106 | scope_name = "capture_vars_"+str(micros) 1107 | op_list = [] 1108 | with tf.variable_scope(scope_name): 1109 | yield op_list 1110 | 1111 | g = tf.get_default_graph() 1112 | for v in tf.global_variables(): 1113 | scope = v.name.split('/', 1)[0] 1114 | if scope == scope_name: 1115 | op_list.append(v) 1116 | 1117 | def Print(op): 1118 | return tf.Print(op, [op], op.name) 1119 | 1120 | 1121 | def get_host_prefix(): 1122 | "ie, returns 10 when on 10.cirrascale..." 1123 | return socket.gethostname().split('.',1)[0] 1124 | 1125 | def summarize_difference(source, target): 1126 | source = np.asarray(source) 1127 | machine_epsilon = np.finfo(source.dtype).eps 1128 | # abs_diff = np.linalg.norm(np.asarray(source)-target, ord=np.inf) 1129 | abs_diff = abs(np.asarray(source)-target) 1130 | rel_diff = abs_diff/abs(source)/machine_epsilon 1131 | print("abs diff: %f, rel diff: %.1f eps " %(np.max(abs_diff), np.max(rel_diff))) 1132 | 1133 | class BufferedWriter: 1134 | """Class that aggregates multiple writes and flushes periodically.""" 1135 | 1136 | def __init__(self, outfn, save_every_secs=60*5): 1137 | self.outfn = outfn 1138 | self.last_save_ts = time.perf_counter() 1139 | self.write_buffer = [] 1140 | self.save_every_secs = save_every_secs 1141 | 1142 | def write(self, line): 1143 | self.write_buffer.append(line) 1144 | if time.perf_counter() - self.last_save_ts > self.save_every_secs: 1145 | self.last_save_ts = time.perf_counter() 1146 | with open(self.outfn, "a") as myfile: 1147 | for line in self.write_buffer: 1148 | myfile.write(line) 1149 | self.write_buffer = [] 1150 | 1151 | def flush(): 1152 | with open(outfn, "a") as myfile: 1153 | for line in self.write_buffer: 1154 | myfile.write(line) 1155 | self.write_buffer = [] 1156 | 1157 | def ossystem(line): 1158 | print(line) 1159 | os.system(line) 1160 | 1161 | def setup_experiment_run_directory(run, safe_mode=True): 1162 | # TODO: factor out to use GLOBAL_RUNS_DIRECTORY 1163 | rundir = "runs/%s"%(run,) 1164 | if os.path.exists(rundir): 1165 | if safe_mode and not run=='default': 1166 | answer = input("%s exists, delete? (Y/n) "%(rundir,)) 1167 | if not answer: 1168 | answer = "y" 1169 | if answer.lower() != "y": 1170 | print("skipping") 1171 | sys.exit() 1172 | print("Removing %s"%(rundir,)) 1173 | ossystem("rm -Rf "+rundir) 1174 | ossystem("mkdir %s"%(rundir,)) 1175 | return rundir 1176 | 1177 | ######################################## 1178 | # Tensorboard logging 1179 | ######################################## 1180 | 1181 | # TODO: have global experiment_base that I can use to move logging to 1182 | # non-current directory 1183 | GLOBAL_RUNS_DIRECTORY='runs' 1184 | global_last_logger = None 1185 | 1186 | def get_last_logger(skip_existence_check=False): 1187 | """Returns last logger, if skip_existence_check is set, doesn't 1188 | throw error if logger doesn't exist.""" 1189 | global global_last_logger 1190 | if not skip_existence_check: 1191 | assert global_last_logger 1192 | return global_last_logger 1193 | 1194 | class TensorboardLogger: 1195 | """Helper class to log to single tensorboard writer from multiple places. 1196 | logger = u.TensorboardLogger("mnist7") 1197 | logger = u.get_last_logger() # gets last logger created 1198 | logger('svd_time', 5) # records "svd_time" stat at 5 1199 | logger.next_step() # advances step counter 1200 | logger.set_step(5) # sets step counter to 5 1201 | """ 1202 | 1203 | def __init__(self, run, step=0): 1204 | # TODO: do nothing for default run 1205 | 1206 | global global_last_logger 1207 | assert global_last_logger is None 1208 | self.run = run 1209 | # sess = tf.get_default_session() 1210 | 1211 | self.summary_writer = tf.summary.FileWriter(GLOBAL_RUNS_DIRECTORY+'/'+run, 1212 | graph=tf.get_default_graph()) 1213 | self.step = step 1214 | self.summary = tf.Summary() 1215 | global_last_logger = self 1216 | self.last_timestamp = time.perf_counter() 1217 | 1218 | def __call__(self, *args): 1219 | assert len(args)%2 == 0 1220 | for (tag, value) in chunks(args, 2): 1221 | self.summary.value.add(tag=tag, simple_value=float(value)) 1222 | 1223 | def next_step(self): 1224 | new_timestamp = time.perf_counter() 1225 | interval_ms = 1000*(new_timestamp - self.last_timestamp) 1226 | self.summary.value.add(tag='time/step', 1227 | simple_value=interval_ms) 1228 | self.last_timestamp = new_timestamp 1229 | self.summary_writer.add_summary(self.summary, self.step) 1230 | self.step+=1 1231 | self.summary = tf.Summary() 1232 | 1233 | 1234 | def as_int32(v): 1235 | """Convert to int32 dtype.""" 1236 | return np.dtype(np.int32).type(v) 1237 | 1238 | def add_dep(from_op, on_op): 1239 | ge.reroute.add_control_inputs(from_op, [on_op]) 1240 | 1241 | # Three functions below are replacements for tf default session/default graph 1242 | # mechanisms that are global (native ones are thread-local because of thread 1243 | # safety issues that have since been fixes (ie, mrry fixed Graph to be thread 1244 | # safe for reading) 1245 | 1246 | sess = None 1247 | def register_default_session(local_sess): 1248 | global sess 1249 | assert sess is None 1250 | sess = local_sess 1251 | 1252 | def get_default_session(): 1253 | # hack, remove 1254 | return tf.get_default_session() 1255 | global sess 1256 | assert sess 1257 | return sess 1258 | 1259 | def get_default_graph(): 1260 | global sess 1261 | assert sess 1262 | return sess.graph 1263 | 1264 | def eval(tensor): 1265 | """tensor.eval() replacement since .eval() is not multi-thread-happy""" 1266 | global sess 1267 | assert sess 1268 | return sess.run(tensor) 1269 | 1270 | def run(fetches): 1271 | return u.eval(fetches) 1272 | 1273 | timeline_counter = 0 1274 | run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) 1275 | def traced_run(fetches): 1276 | """Runs fetches, dumps timeline files in current directory.""" 1277 | global sess 1278 | assert sess 1279 | global timeline_counter 1280 | run_metadata = tf.RunMetadata() 1281 | 1282 | root = os.getcwd()+"/data" 1283 | from tensorflow.python.client import timeline 1284 | 1285 | results = sess.run(fetches, 1286 | options=run_options, 1287 | run_metadata=run_metadata); 1288 | tl = timeline.Timeline(step_stats=run_metadata.step_stats) 1289 | ctf = tl.generate_chrome_trace_format(show_memory=True, 1290 | show_dataflow=False) 1291 | open(root+"/timeline_%d.json"%(timeline_counter,), "w").write(ctf) 1292 | open(root+"/stepstats_%d.pbtxt"%(timeline_counter,), "w").write(str( 1293 | run_metadata.step_stats)) 1294 | timeline_counter+=1 1295 | return results 1296 | 1297 | def get_mnist_images(fold='train'): 1298 | """Returns mnist images, batch dimension last.""" 1299 | 1300 | import gzip 1301 | from tensorflow.contrib.learn.python.learn.datasets import base 1302 | import numpy 1303 | 1304 | def extract_images(f): 1305 | """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. 1306 | Args: 1307 | f: A file object that can be passed into a gzip reader. 1308 | Returns: 1309 | data: A 4D uint8 numpy array [index, y, x, depth]. 1310 | Raises: 1311 | ValueError: If the bytestream does not start with 2051. 1312 | """ 1313 | # print('Extracting', f.name) # todo: remove 1314 | with gzip.GzipFile(fileobj=f) as bytestream: 1315 | magic = _read32(bytestream) 1316 | if magic != 2051: 1317 | raise ValueError('Invalid magic number %d in MNIST image file: %s' % 1318 | (magic, f.name)) 1319 | num_images = _read32(bytestream) 1320 | rows = _read32(bytestream) 1321 | cols = _read32(bytestream) 1322 | buf = bytestream.read(rows * cols * num_images) 1323 | data = numpy.frombuffer(buf, dtype=numpy.uint8) 1324 | data = data.reshape(num_images, rows, cols, 1) 1325 | return data 1326 | 1327 | def _read32(bytestream): 1328 | dt = numpy.dtype(numpy.uint32).newbyteorder('>') 1329 | return numpy.frombuffer(bytestream.read(4), dtype=dt)[0] 1330 | 1331 | if fold == 'train': # todo: rename 1332 | TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' 1333 | elif fold == 'test': 1334 | TRAIN_IMAGES = 't10k-images-idx3-ubyte.gz' 1335 | else: 1336 | assert False, 'unknown fold %s'%(fold) 1337 | 1338 | source_url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' 1339 | local_file = base.maybe_download(TRAIN_IMAGES, '/tmp', 1340 | source_url + TRAIN_IMAGES) 1341 | train_images = extract_images(open(local_file, 'rb')) 1342 | dsize = train_images.shape[0] 1343 | if fold == 'train': 1344 | assert dsize == 60000 1345 | else: 1346 | assert dsize == 10000 1347 | 1348 | train_images = train_images.reshape(dsize, 28**2).T.astype(np.float64)/255 1349 | train_images = np.ascontiguousarray(train_images) 1350 | return train_images.astype(default_np_dtype) 1351 | 1352 | regularizer_cache = {} 1353 | def cachedGpuIdentityRegularizer(n, Lambda): 1354 | global regularizer_cache 1355 | 1356 | n = int(n) 1357 | if (n, Lambda) not in regularizer_cache: 1358 | numpy_diag = Lambda*np.diag(np.ones([n])) 1359 | numpy_diag = numpy_diag.astype(default_np_dtype) 1360 | with tf.device("/gpu:0"): 1361 | regularizer_cache[(n, Lambda)] = tf.constant(numpy_diag) 1362 | 1363 | return regularizer_cache[(n, Lambda)] 1364 | 1365 | # helper utilities 1366 | def ng_init(s1, s2): # uniform weight init from Ng UFLDL 1367 | r = np.sqrt(6) / np.sqrt(s1 + s2 + 1) 1368 | flat = np.random.random(s1*s2)*2*r-r 1369 | return flat.reshape([s1, s2]).astype(default_np_dtype) 1370 | 1371 | 1372 | if __name__=='__main__': 1373 | run_all_tests(sys.modules[__name__]) 1374 | --------------------------------------------------------------------------------