├── .gitignore ├── Codes ├── Jupyter │ ├── Additional examples │ │ ├── Primal_dual_PPA.ipynb │ │ └── Primal_dual_PPA_30.eps │ ├── Exercise01.ipynb │ ├── Exercise02.ipynb │ ├── Exercise03.ipynb │ ├── Exercise04.ipynb │ ├── Exercise05.ipynb │ ├── Exercise06.ipynb │ ├── Exercise07.ipynb │ ├── Exercise08.ipynb │ ├── Exercise09.ipynb │ ├── Exercise10.ipynb │ ├── Exercise11.ipynb │ └── requirements.txt └── Matlab │ ├── Exercise1.m │ ├── Exercise1_dimReduction.m │ ├── Exercise2.m │ ├── Exercise2_SDP_functionvalues.m │ ├── Exercise2_closedform.m │ └── Exercise2_symbolicalLMI.m ├── Course.pdf ├── Course.tex ├── Course_with_corrections.pdf ├── LICENSE ├── README.md └── bib_.bib /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # Distribution / packaging 7 | .Python 8 | build/ 9 | develop-eggs/ 10 | dist/ 11 | downloads/ 12 | eggs/ 13 | .eggs/ 14 | lib/ 15 | lib64/ 16 | parts/ 17 | sdist/ 18 | var/ 19 | wheels/ 20 | pip-wheel-metadata/ 21 | share/python-wheels/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | MANIFEST 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .nox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | *.py,cover 48 | .hypothesis/ 49 | .pytest_cache/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | db.sqlite3 59 | db.sqlite3-journal 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # IPython 78 | profile_default/ 79 | ipython_config.py 80 | 81 | # intellij configs 82 | .idea/ 83 | 84 | # MacOS files 85 | *.DS_Store 86 | 87 | # pyenv 88 | .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 98 | __pypackages__/ 99 | 100 | # Celery stuff 101 | celerybeat-schedule 102 | celerybeat.pid 103 | 104 | # SageMath parsed files 105 | *.sage.py 106 | 107 | # Environments 108 | .env 109 | .venv 110 | env/ 111 | venv/ 112 | ENV/ 113 | env.bak/ 114 | venv.bak/ 115 | 116 | # Spyder project settings 117 | .spyderproject 118 | .spyproject 119 | 120 | # Rope project settings 121 | .ropeproject 122 | 123 | # mkdocs documentation 124 | /site 125 | 126 | # mypy 127 | .mypy_cache/ 128 | .dmypy.json 129 | dmypy.json 130 | 131 | # Pyre type checker 132 | .pyre/ 133 | 134 | # Latex 135 | *.log 136 | *.aux 137 | *.out 138 | *.pdfsync 139 | *.synctex.gz 140 | *.toc 141 | *.bbl 142 | *.blg 143 | *.dvi 144 | 145 | # drafts 146 | untitled.* 147 | 148 | # Matlab temporary files 149 | *.m~ 150 | -------------------------------------------------------------------------------- /Codes/Jupyter/Additional examples/Primal_dual_PPA_30.eps: -------------------------------------------------------------------------------- 1 | %!PS-Adobe-3.0 EPSF-3.0 2 | %%Title: Primal_dual_PPA_30.eps 3 | %%Creator: Matplotlib v3.7.1, https://matplotlib.org/ 4 | %%CreationDate: Thu May 2 02:16:08 2024 5 | %%Orientation: portrait 6 | %%BoundingBox: 140 241 472 551 7 | %%HiResBoundingBox: 140.159191 241.344000 471.840809 550.656000 8 | %%EndComments 9 | %%BeginProlog 10 | /mpldict 12 dict def 11 | mpldict begin 12 | /_d { bind def } bind def 13 | /m { moveto } _d 14 | /l { lineto } _d 15 | /r { rlineto } _d 16 | /c { curveto } _d 17 | /cl { closepath } _d 18 | /ce { closepath eofill } _d 19 | /box { 20 | m 21 | 1 index 0 r 22 | 0 exch r 23 | neg 0 r 24 | cl 25 | } _d 26 | /clipbox { 27 | box 28 | clip 29 | newpath 30 | } _d 31 | /sc { setcachedevice } _d 32 | %!PS-Adobe-3.0 Resource-Font 33 | %%Creator: Converted from TrueType to Type 3 by Matplotlib. 34 | 10 dict begin 35 | /FontName /DejaVuSans-Oblique def 36 | /PaintType 0 def 37 | /FontMatrix [0.00048828125 0 0 0.00048828125 0 0] def 38 | /FontBBox [-2080 -717 3398 2187] def 39 | /FontType 3 def 40 | /Encoding [/x /k /lambda] def 41 | /CharStrings 4 dict dup begin 42 | /.notdef 0 def 43 | /x{1212 0 -53 0 1229 1120 sc 44 | 1229 1120 m 45 | 715 571 l 46 | 1030 0 l 47 | 819 0 l 48 | 582 444 l 49 | 170 0 l 50 | -53 0 l 51 | 498 590 l 52 | 205 1120 l 53 | 416 1120 l 54 | 631 715 l 55 | 1006 1120 l 56 | 1229 1120 l 57 | 58 | ce} _d 59 | /k{1186 0 72 0 1253 1556 sc 60 | 375 1556 m 61 | 559 1556 l 62 | 383 649 l 63 | 1014 1120 l 64 | 1253 1120 l 65 | 549 584 l 66 | 1063 0 l 67 | 840 0 l 68 | 362 547 l 69 | 256 0 l 70 | 72 0 l 71 | 375 1556 l 72 | 73 | ce} _d 74 | /lambda{1212 0 -90 0 1000 1556 sc 75 | 752 1381 m 76 | 1000 0 l 77 | 805 0 l 78 | 652 828 l 79 | 105 0 l 80 | -90 0 l 81 | 609 1074 l 82 | 574 1272 l 83 | 559 1356 516 1398 445 1398 c 84 | 349 1398 l 85 | 379 1556 l 86 | 496 1554 l 87 | 646 1552 731 1494 752 1381 c 88 | 89 | ce} _d 90 | end readonly def 91 | 92 | /BuildGlyph { 93 | exch begin 94 | CharStrings exch 95 | 2 copy known not {pop /.notdef} if 96 | true 3 1 roll get exec 97 | end 98 | } _d 99 | 100 | /BuildChar { 101 | 1 index /Encoding get exch get 102 | 1 index /BuildGlyph get exec 103 | } _d 104 | 105 | FontName currentdict end definefont pop 106 | %!PS-Adobe-3.0 Resource-Font 107 | %%Creator: Converted from TrueType to Type 3 by Matplotlib. 108 | 10 dict begin 109 | /FontName /DejaVuSans def 110 | /PaintType 0 def 111 | /FontMatrix [0.00048828125 0 0 0.00048828125 0 0] def 112 | /FontBBox [-2090 -948 3673 2524] def 113 | /FontType 3 def 114 | /Encoding [/parenleft /parenright /period /zero /one /minus /two /five] def 115 | /CharStrings 9 dict dup begin 116 | /.notdef 0 def 117 | /parenleft{799 0 176 -270 635 1554 sc 118 | 635 1554 m 119 | 546 1401 479 1249 436 1099 c 120 | 393 949 371 797 371 643 c 121 | 371 489 393 336 436 185 c 122 | 480 34 546 -117 635 -270 c 123 | 475 -270 l 124 | 375 -113 300 41 250 192 c 125 | 201 343 176 494 176 643 c 126 | 176 792 201 941 250 1092 c 127 | 299 1243 374 1397 475 1554 c 128 | 635 1554 l 129 | 130 | ce} _d 131 | /parenright{799 0 164 -270 623 1554 sc 132 | 164 1554 m 133 | 324 1554 l 134 | 424 1397 499 1243 548 1092 c 135 | 598 941 623 792 623 643 c 136 | 623 494 598 343 548 192 c 137 | 499 41 424 -113 324 -270 c 138 | 164 -270 l 139 | 253 -117 319 34 362 185 c 140 | 406 336 428 489 428 643 c 141 | 428 797 406 949 362 1099 c 142 | 319 1249 253 1401 164 1554 c 143 | 144 | ce} _d 145 | /period{651 0 219 0 430 254 sc 146 | 219 254 m 147 | 430 254 l 148 | 430 0 l 149 | 219 0 l 150 | 219 254 l 151 | 152 | ce} _d 153 | /zero{1303 0 135 -29 1167 1520 sc 154 | 651 1360 m 155 | 547 1360 469 1309 416 1206 c 156 | 364 1104 338 950 338 745 c 157 | 338 540 364 387 416 284 c 158 | 469 182 547 131 651 131 c 159 | 756 131 834 182 886 284 c 160 | 939 387 965 540 965 745 c 161 | 965 950 939 1104 886 1206 c 162 | 834 1309 756 1360 651 1360 c 163 | 164 | 651 1520 m 165 | 818 1520 946 1454 1034 1321 c 166 | 1123 1189 1167 997 1167 745 c 167 | 1167 494 1123 302 1034 169 c 168 | 946 37 818 -29 651 -29 c 169 | 484 -29 356 37 267 169 c 170 | 179 302 135 494 135 745 c 171 | 135 997 179 1189 267 1321 c 172 | 356 1454 484 1520 651 1520 c 173 | 174 | ce} _d 175 | /one{1303 0 225 0 1114 1493 sc 176 | 254 170 m 177 | 584 170 l 178 | 584 1309 l 179 | 225 1237 l 180 | 225 1421 l 181 | 582 1493 l 182 | 784 1493 l 183 | 784 170 l 184 | 1114 170 l 185 | 1114 0 l 186 | 254 0 l 187 | 254 170 l 188 | 189 | ce} _d 190 | /minus{1716 0 217 557 1499 727 sc 191 | 217 727 m 192 | 1499 727 l 193 | 1499 557 l 194 | 217 557 l 195 | 217 727 l 196 | 197 | ce} _d 198 | /two{1303 0 150 0 1098 1520 sc 199 | 393 170 m 200 | 1098 170 l 201 | 1098 0 l 202 | 150 0 l 203 | 150 170 l 204 | 227 249 331 356 463 489 c 205 | 596 623 679 709 713 748 c 206 | 778 821 823 882 848 932 c 207 | 874 983 887 1032 887 1081 c 208 | 887 1160 859 1225 803 1275 c 209 | 748 1325 675 1350 586 1350 c 210 | 523 1350 456 1339 385 1317 c 211 | 315 1295 240 1262 160 1217 c 212 | 160 1421 l 213 | 241 1454 317 1478 388 1495 c 214 | 459 1512 523 1520 582 1520 c 215 | 737 1520 860 1481 952 1404 c 216 | 1044 1327 1090 1223 1090 1094 c 217 | 1090 1033 1078 974 1055 919 c 218 | 1032 864 991 800 930 725 c 219 | 913 706 860 650 771 557 c 220 | 682 465 556 336 393 170 c 221 | 222 | ce} _d 223 | /five{1303 0 158 -29 1124 1493 sc 224 | 221 1493 m 225 | 1014 1493 l 226 | 1014 1323 l 227 | 406 1323 l 228 | 406 957 l 229 | 435 967 465 974 494 979 c 230 | 523 984 553 987 582 987 c 231 | 749 987 881 941 978 850 c 232 | 1075 759 1124 635 1124 479 c 233 | 1124 318 1074 193 974 104 c 234 | 874 15 733 -29 551 -29 c 235 | 488 -29 424 -24 359 -13 c 236 | 294 -2 227 14 158 35 c 237 | 158 238 l 238 | 218 205 280 181 344 165 c 239 | 408 149 476 141 547 141 c 240 | 662 141 754 171 821 232 c 241 | 888 293 922 375 922 479 c 242 | 922 583 888 665 821 726 c 243 | 754 787 662 817 547 817 c 244 | 493 817 439 811 385 799 c 245 | 332 787 277 768 221 743 c 246 | 221 1493 l 247 | 248 | ce} _d 249 | end readonly def 250 | 251 | /BuildGlyph { 252 | exch begin 253 | CharStrings exch 254 | 2 copy known not {pop /.notdef} if 255 | true 3 1 roll get exec 256 | end 257 | } _d 258 | 259 | /BuildChar { 260 | 1 index /Encoding get exch get 261 | 1 index /BuildGlyph get exec 262 | } _d 263 | 264 | FontName currentdict end definefont pop 265 | end 266 | %%EndProlog 267 | mpldict begin 268 | 140.159 241.344 translate 269 | 331.682 309.312 0 0 clipbox 270 | gsave 271 | 0 0 m 272 | 331.681618 0 l 273 | 331.681618 309.312 l 274 | 0 309.312 l 275 | 0 0 l 276 | cl 277 | grestore 278 | gsave 279 | 21.6 21.6 m 280 | 287.712 21.6 l 281 | 287.712 287.712 l 282 | 21.6 287.712 l 283 | 21.6 21.6 l 284 | cl 285 | grestore 286 | 1.000 setlinewidth 287 | 0 setlinejoin 288 | 0 setlinecap 289 | [] 0 setdash 290 | 0.950 setgray 291 | gsave 292 | 42.365699 106.598326 m 293 | 177.486616 140.360652 l 294 | 178.03937 253.420132 l 295 | 38.821384 225.890798 l 296 | gsave 297 | fill 298 | grestore 299 | stroke 300 | grestore 301 | 0.900 setgray 302 | gsave 303 | 177.486616 140.360652 m 304 | 277.317287 90.820442 l 305 | 281.061852 212.991813 l 306 | 178.03937 253.420132 l 307 | gsave 308 | fill 309 | grestore 310 | stroke 311 | grestore 312 | 0.925 setgray 313 | gsave 314 | 42.365699 106.598326 m 315 | 135.82059 50.366373 l 316 | 277.317287 90.820442 l 317 | 177.486616 140.360652 l 318 | gsave 319 | fill 320 | grestore 321 | stroke 322 | grestore 323 | 0.800 setlinewidth 324 | 1 setlinejoin 325 | 2 setlinecap 326 | 0.000 setgray 327 | gsave 328 | 42.365699 106.598326 m 329 | 135.82059 50.366373 l 330 | stroke 331 | grestore 332 | gsave 333 | 54.4438 51.2982 translate 334 | 328.965 rotate 335 | /DejaVuSans-Oblique 10.0 selectfont 336 | 0 0.53125 moveto 337 | /x glyphshow 338 | /DejaVuSans-Oblique 7.0 selectfont 339 | 5.91797 -1.10938 moveto 340 | /k glyphshow 341 | grestore 342 | 0 setlinecap 343 | 0.690 setgray 344 | gsave 345 | 56.094085 98.337935 m 346 | 192.223322 133.047674 l 347 | 193.213143 247.465605 l 348 | stroke 349 | grestore 350 | gsave 351 | 81.091585 83.2969 m 352 | 218.993182 119.763334 l 353 | 220.807122 236.637112 l 354 | stroke 355 | grestore 356 | gsave 357 | 107.412001 67.459865 m 358 | 247.091295 105.81986 l 359 | 249.812177 225.254882 l 360 | stroke 361 | grestore 362 | 2 setlinecap 363 | 0.000 setgray 364 | gsave 365 | 57.251416 98.633027 m 366 | 53.775904 97.746854 l 367 | stroke 368 | grestore 369 | /DejaVuSans 10.000 selectfont 370 | gsave 371 | 372 | 32.1668 77.9253 translate 373 | 0 rotate 374 | 0 0 m /minus glyphshow 375 | 8.37891 0 m /zero glyphshow 376 | 14.7412 0 m /period glyphshow 377 | 17.9199 0 m /five glyphshow 378 | grestore 379 | gsave 380 | 82.265767 83.607399 m 381 | 78.73956 82.674935 l 382 | stroke 383 | grestore 384 | /DejaVuSans 10.000 selectfont 385 | gsave 386 | 387 | 61.1178 62.5103 translate 388 | 0 rotate 389 | 0 0 m /zero glyphshow 390 | 6.3623 0 m /period glyphshow 391 | 9.54102 0 m /zero glyphshow 392 | grestore 393 | gsave 394 | 108.603221 67.787009 m 395 | 105.025749 66.804531 l 396 | stroke 397 | grestore 398 | /DejaVuSans 10.000 selectfont 399 | gsave 400 | 401 | 87.1957 46.2771 translate 402 | 0 rotate 403 | 0 0 m /zero glyphshow 404 | 6.3623 0 m /period glyphshow 405 | 9.54102 0 m /five glyphshow 406 | grestore 407 | gsave 408 | 277.317287 90.820442 m 409 | 135.82059 50.366373 l 410 | stroke 411 | grestore 412 | gsave 413 | 216.763 32.2748 translate 414 | 15.9553 rotate 415 | /DejaVuSans-Oblique 10.0 selectfont 416 | 0 0.40625 moveto 417 | /lambda glyphshow 418 | /DejaVuSans-Oblique 7.0 selectfont 419 | 5.91797 -1.23438 moveto 420 | /k glyphshow 421 | /DejaVuSans 10.0 selectfont 422 | 10.2451 0.40625 moveto 423 | /parenleft glyphshow 424 | 14.1465 0.40625 moveto 425 | /one glyphshow 426 | 20.5088 0.40625 moveto 427 | /parenright glyphshow 428 | grestore 429 | 0 setlinecap 430 | 0.690 setgray 431 | gsave 432 | 41.780873 226.476015 m 433 | 45.233055 107.314785 l 434 | 138.840106 51.229656 l 435 | stroke 436 | grestore 437 | gsave 438 | 110.597403 240.083979 m 439 | 111.96888 123.989899 l 440 | 208.910247 71.26279 l 441 | stroke 442 | grestore 443 | gsave 444 | 175.419975 252.902166 m 445 | 174.939829 139.724292 l 446 | 274.66526 90.062224 l 447 | stroke 448 | grestore 449 | 2 setlinecap 450 | 0.000 setgray 451 | gsave 452 | 138.018783 51.721756 m 453 | 140.486606 50.243147 l 454 | stroke 455 | grestore 456 | /DejaVuSans 10.000 selectfont 457 | gsave 458 | 459 | 135.324 28.0647 translate 460 | 0 rotate 461 | 0 0 m /minus glyphshow 462 | 8.37891 0 m /zero glyphshow 463 | 14.7412 0 m /period glyphshow 464 | 17.9199 0 m /five glyphshow 465 | grestore 466 | gsave 467 | 208.062051 71.72413 m 468 | 210.610494 70.338014 l 469 | stroke 470 | grestore 471 | /DejaVuSans 10.000 selectfont 472 | gsave 473 | 474 | 209.674 48.7004 translate 475 | 0 rotate 476 | 0 0 m /zero glyphshow 477 | 6.3623 0 m /period glyphshow 478 | 9.54102 0 m /zero glyphshow 479 | grestore 480 | gsave 481 | 273.795004 90.495601 m 482 | 276.409604 89.193562 l 483 | stroke 484 | grestore 485 | /DejaVuSans 10.000 selectfont 486 | gsave 487 | 488 | 275.494 68.0589 translate 489 | 0 rotate 490 | 0 0 m /zero glyphshow 491 | 6.3623 0 m /period glyphshow 492 | 9.54102 0 m /five glyphshow 493 | grestore 494 | gsave 495 | 277.317287 90.820442 m 496 | 281.061852 212.991813 l 497 | stroke 498 | grestore 499 | gsave 500 | 326.147 135.696 translate 501 | 88.2444 rotate 502 | /DejaVuSans-Oblique 10.0 selectfont 503 | 0 0.40625 moveto 504 | /lambda glyphshow 505 | /DejaVuSans-Oblique 7.0 selectfont 506 | 5.91797 -1.23438 moveto 507 | /k glyphshow 508 | /DejaVuSans 10.0 selectfont 509 | 10.2451 0.40625 moveto 510 | /parenleft glyphshow 511 | 14.1465 0.40625 moveto 512 | /two glyphshow 513 | 20.5088 0.40625 moveto 514 | /parenright glyphshow 515 | grestore 516 | 0 setlinecap 517 | 0.690 setgray 518 | gsave 519 | 277.785783 106.105742 m 520 | 177.555932 154.538546 l 521 | 41.921932 121.534399 l 522 | stroke 523 | grestore 524 | gsave 525 | 278.851026 140.860707 m 526 | 177.713371 186.740772 l 527 | 40.913261 155.483634 l 528 | stroke 529 | grestore 530 | gsave 531 | 279.935426 176.24069 m 532 | 177.873398 219.472588 l 533 | 39.886946 190.026749 l 534 | stroke 535 | grestore 536 | 2 setlinecap 537 | 0.000 setgray 538 | gsave 539 | 276.910933 106.528485 m 540 | 279.539347 105.25839 l 541 | stroke 542 | grestore 543 | /DejaVuSans 10.000 selectfont 544 | gsave 545 | 546 | 285.8 97.1936 translate 547 | 0 rotate 548 | 0 0 m /minus glyphshow 549 | 8.37891 0 m /zero glyphshow 550 | 14.7412 0 m /period glyphshow 551 | 17.9199 0 m /five glyphshow 552 | grestore 553 | gsave 554 | 277.967596 141.261466 m 555 | 280.621822 140.057403 l 556 | stroke 557 | grestore 558 | /DejaVuSans 10.000 selectfont 559 | gsave 560 | 561 | 291.236 132.017 translate 562 | 0 rotate 563 | 0 0 m /zero glyphshow 564 | 6.3623 0 m /period glyphshow 565 | 9.54102 0 m /zero glyphshow 566 | grestore 567 | gsave 568 | 279.043248 176.618603 m 569 | 281.723793 175.483165 l 570 | stroke 571 | grestore 572 | /DejaVuSans 10.000 selectfont 573 | gsave 574 | 575 | 292.506 167.468 translate 576 | 0 rotate 577 | 0 0 m /zero glyphshow 578 | 6.3623 0 m /period glyphshow 579 | 9.54102 0 m /five glyphshow 580 | grestore 581 | 1.500 setlinewidth 582 | 1.000 0.000 0.000 setrgbcolor 583 | gsave 584 | 266.112 266.112 21.6 21.6 clipbox 585 | 147.739596 204.887793 m 586 | 159.013096 194.380412 l 587 | 169.682312 182.777209 l 588 | 179.357947 170.561391 l 589 | 187.691365 158.228761 l 590 | 194.405614 146.263718 l 591 | 199.307555 135.113125 l 592 | 202.293176 125.164518 l 593 | 203.346222 116.730421 l 594 | 202.531049 110.039446 l 595 | 199.981261 105.233978 l 596 | 195.88587 102.373431 l 597 | 190.47464 101.441696 l 598 | 184.003955 102.357204 l 599 | 176.744168 104.984202 l 600 | 168.968906 109.14408 l 601 | 160.94651 114.625957 l 602 | 152.933451 121.196099 l 603 | 145.169424 128.605974 l 604 | 137.87375 136.599015 l 605 | 131.242684 144.9163 l 606 | 125.447254 153.30143 l 607 | 120.631373 161.504924 l 608 | 116.90999 169.288445 l 609 | 114.367205 176.429084 l 610 | 113.054357 182.723907 l 611 | 112.988186 187.994793 l 612 | 114.149298 192.093538 l 613 | 116.481231 194.907011 l 614 | 119.890457 196.362052 l 615 | 124.247678 196.429664 l 616 | stroke 617 | grestore 618 | 1.000 setlinewidth 619 | 0 setlinecap 620 | gsave 621 | 266.112 266.112 21.6 21.6 clipbox 622 | /o { 623 | gsave 624 | newpath 625 | translate 626 | 1.0 setlinewidth 627 | 1 setlinejoin 628 | 629 | 0 setlinecap 630 | 631 | 0 -1.5 m 632 | 0.397805 -1.5 0.77937 -1.341951 1.06066 -1.06066 c 633 | 1.341951 -0.77937 1.5 -0.397805 1.5 0 c 634 | 1.5 0.397805 1.341951 0.77937 1.06066 1.06066 c 635 | 0.77937 1.341951 0.397805 1.5 0 1.5 c 636 | -0.397805 1.5 -0.77937 1.341951 -1.06066 1.06066 c 637 | -1.341951 0.77937 -1.5 0.397805 -1.5 0 c 638 | -1.5 -0.397805 -1.341951 -0.77937 -1.06066 -1.06066 c 639 | -0.77937 -1.341951 -0.397805 -1.5 0 -1.5 c 640 | cl 641 | 642 | gsave 643 | 1.000 0.000 0.000 setrgbcolor 644 | fill 645 | grestore 646 | stroke 647 | grestore 648 | } bind def 649 | 147.74 204.888 o 650 | 159.013 194.38 o 651 | 169.682 182.777 o 652 | 179.358 170.561 o 653 | 187.691 158.229 o 654 | 194.406 146.264 o 655 | 199.308 135.113 o 656 | 202.293 125.165 o 657 | 203.346 116.73 o 658 | 202.531 110.039 o 659 | 199.981 105.234 o 660 | 195.886 102.373 o 661 | 190.475 101.442 o 662 | 184.004 102.357 o 663 | 176.744 104.984 o 664 | 168.969 109.144 o 665 | 160.947 114.626 o 666 | 152.933 121.196 o 667 | 145.169 128.606 o 668 | 137.874 136.599 o 669 | 131.243 144.916 o 670 | 125.447 153.301 o 671 | 120.631 161.505 o 672 | 116.91 169.288 o 673 | 114.367 176.429 o 674 | 113.054 182.724 o 675 | 112.988 187.995 o 676 | 114.149 192.094 o 677 | 116.481 194.907 o 678 | 119.89 196.362 o 679 | 124.248 196.43 o 680 | grestore 681 | 0.000 0.000 1.000 setrgbcolor 682 | gsave 683 | 266.112 266.112 21.6 21.6 clipbox 684 | /o { 685 | gsave 686 | newpath 687 | translate 688 | 1.0 setlinewidth 689 | 1 setlinejoin 690 | 691 | 0 setlinecap 692 | 693 | 0 -1.5 m 694 | 0.397805 -1.5 0.77937 -1.341951 1.06066 -1.06066 c 695 | 1.341951 -0.77937 1.5 -0.397805 1.5 0 c 696 | 1.5 0.397805 1.341951 0.77937 1.06066 1.06066 c 697 | 0.77937 1.341951 0.397805 1.5 0 1.5 c 698 | -0.397805 1.5 -0.77937 1.341951 -1.06066 1.06066 c 699 | -1.341951 0.77937 -1.5 0.397805 -1.5 0 c 700 | -1.5 -0.397805 -1.341951 -0.77937 -1.06066 -1.06066 c 701 | -0.77937 -1.341951 -0.397805 -1.5 0 -1.5 c 702 | cl 703 | 704 | gsave 705 | 0.000 0.000 1.000 setrgbcolor 706 | fill 707 | grestore 708 | stroke 709 | grestore 710 | } bind def 711 | 147.74 204.888 o 712 | grestore 713 | 0 setlinejoin 714 | 0.000 setgray 715 | gsave 716 | 266.112 266.112 21.6 21.6 clipbox 717 | /o { 718 | gsave 719 | newpath 720 | translate 721 | 1.0 setlinewidth 722 | 0 setlinejoin 723 | 724 | 0 setlinecap 725 | 726 | -3 -3 m 727 | 3 -3 l 728 | 3 3 l 729 | -3 3 l 730 | cl 731 | 732 | gsave 733 | 0.000 setgray 734 | fill 735 | grestore 736 | stroke 737 | grestore 738 | } bind def 739 | 150.304 155.839 o 740 | grestore 741 | 742 | end 743 | showpage 744 | -------------------------------------------------------------------------------- /Codes/Jupyter/Exercise01.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "3a2daa4d", 6 | "metadata": { 7 | "tags": [] 8 | }, 9 | "source": [ 10 | "# Summary\n", 11 | "This document corresponds to Exercise 1 of [this file](https://github.com/PerformanceEstimation/Learning-Performance-Estimation/blob/main/Course.pdf).\n", 12 | "\n", 13 | "The goal of this exercise is to compute the value of the worst-case ratio $\\frac{\\|x_{k+1}-x_\\star\\|^2}{\\|x_k-x_\\star\\|^2}$ when $x_\\star=\\textrm{argmin}_x f(x)$ for some $f$ that is $L$-smooth and $\\mu$-strongly convex, and where $x_{k+1}=x_k-\\gamma_k \\nabla f(x_k)$ is obtained from a gradient step (with stepsize $\\gamma_k$) from $x_k$.\n", 14 | "\n", 15 | "If [PEPit](https://pypi.org/project/PEPit/) is not already installed, please execute the following cell." 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": null, 21 | "id": "7f2a6e9d", 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "!pip install pepit" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "id": "10f97bd9-4ff5-40e2-9361-ce60d48c8f41", 31 | "metadata": {}, 32 | "source": [ 33 | "Exercises 1.1 to 1.6 are presented in the document, along with their corrections, and do not involve any numerics." 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "id": "12301935", 39 | "metadata": { 40 | "tags": [] 41 | }, 42 | "source": [ 43 | "### Exercise 1.7\n", 44 | "Complete the code for computing the worst-case behavior of the ratio $\\frac{\\|x_{k+1}-x_\\star\\|^2}{\\|x_k-x_\\star\\|^2}$ (we use $k=0$ without loss of generality and for readability below).\n", 45 | "\n", 46 | "As seen in the exercise file, this is done by looking for the worst-case value of $\\|x_{1}-x_\\star\\|^2$ (this is often referred to as the \"performance measure\" in the performance estimation framework, and corresponds to the objective function of the problem of computing the worst-case ratio), when $\\|x_0-x_\\star\\|^2 =1$ (which is often referred to as an \"initial condition\", as it quantifies the quality of the \"initial\" iterate).\n", 47 | "\n", 48 | "To see how to specify such things within the PEPit framework, we refer to [the documentation](https://pepit.readthedocs.io/), which contains numerous examples. In particular, one can check [this example](https://pepit.readthedocs.io/en/0.1.0/_modules/PEPit/examples/composite_convex_minimization/proximal_gradient.html#wc_proximal_gradient), which is a bit more complex than what is asked in the exercise, but very related.\n" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": null, 54 | "id": "6b0d63bc", 55 | "metadata": {}, 56 | "outputs": [], 57 | "source": [ 58 | "from PEPit import PEP\n", 59 | "from PEPit.functions import SmoothStronglyConvexFunction\n", 60 | "\n", 61 | "def wc_gradient(L, mu, gamma, verbose=1):\n", 62 | " # It is intended to compute the worst-case convergence of gradient descent in terms of the distance to \n", 63 | " # an optimal solution: || x_{k+1} - x_\\star ||^2 / || x_k - x_\\star ||^2.\n", 64 | " # Note that we use k = 0 in the code below for readability.\n", 65 | " \n", 66 | " # Instantiate PEP\n", 67 | " problem = PEP()\n", 68 | "\n", 69 | " # Declare a strongly convex smooth function and a closed convex proper function\n", 70 | " f = problem.declare_function(SmoothStronglyConvexFunction, mu=mu, L=L)\n", 71 | "\n", 72 | " # Start by defining its unique optimal point xs = x_\\star\n", 73 | " xs = f.stationary_point()\n", 74 | "\n", 75 | " # Then define the point x0 of the algorithm\n", 76 | " x0 = problem.set_initial_point() \n", 77 | " \n", 78 | " # Perform one iteration of gradient descent\n", 79 | " x = # TODO complete this line. Hint: use f.gradient() to call the gradient of f a a given point.\n", 80 | "\n", 81 | " # Set the \"performance metric\" to the distance between x1 and xs\n", 82 | " # TO COMPLETE (use \"problem.set_performance_metric\" to specify the objective function of the SDP)\n", 83 | " problem.set_performance_metric( ) # TODO complete this line\n", 84 | " \n", 85 | " \n", 86 | " # Set the \"initial condition\" to the distance between x0 and xs\n", 87 | " # TO COMPLETE (use \"problem.set_initial_condition\" or \"problem.add_constraint\" to specify the\n", 88 | " # constraint || x0 - xs ||^2 == 1).\n", 89 | " problem.set_initial_condition( ) # TODO complete this line\n", 90 | "\n", 91 | " # Solve the PEP\n", 92 | " pepit_tau = problem.solve(verbose=verbose)\n", 93 | " \n", 94 | " # Return the worst-case convergence rate output by the SDP solver\n", 95 | " return pepit_tau" 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "id": "59c4a1f8", 101 | "metadata": {}, 102 | "source": [ 103 | "Once the previous code is completed, one can test it for a few values of the problem and algorithmic parameters." 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": null, 109 | "id": "b471f7da", 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "mu = .1\n", 114 | "L = 1\n", 115 | "gamma = 1\n", 116 | "verbose = 1\n", 117 | "\n", 118 | "pepit_tau = wc_gradient(L=L, mu=mu, gamma=gamma[i], verbose=verbose)" 119 | ] 120 | }, 121 | { 122 | "cell_type": "markdown", 123 | "id": "1e9bc404-f223-4c42-bb65-de3b2439e6a3", 124 | "metadata": {}, 125 | "source": [ 126 | "### Exercise 1.8: optimal stepsize and range of acceptable stepsizes?\n", 127 | "\n", 128 | "Compute numerical values of the worst-case ratio for a few values of $\\mu$ and $\\gamma_k$ (with $L=1$), and try to infer rules for the stepsize choices." 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": null, 134 | "id": "389df0e5-68e8-427e-a80c-10c59084763d", 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "import matplotlib.pyplot as plt\n", 139 | "import numpy as np\n", 140 | "import time\n", 141 | "\n", 142 | "nb_test = 20\n", 143 | "\n", 144 | "mu = .1\n", 145 | "L = 1\n", 146 | "gamma = np.linspace(0., 2., num=nb_test)\n", 147 | "verbose = 0\n", 148 | "\n", 149 | "pepit_taus = list()\n", 150 | "\n", 151 | "for i in range(nb_test):\n", 152 | " t0= time.process_time()\n", 153 | " pepit_tau = wc_gradient(L=L, mu=mu, gamma=gamma[i], verbose=verbose)\n", 154 | " pepit_taus.append(pepit_tau)\n", 155 | " t1 = time.process_time() - t0\n", 156 | " print(i+1, '/', nb_test,' done (elapsed time:',\"%.2f\" %t1,'[s])')\n", 157 | " \n", 158 | "plt.plot(gamma, pepit_taus, '-')\n", 159 | "\n", 160 | "plt.xlabel('Step size')\n", 161 | "plt.ylabel('||x_1-x_*||^2 / ||x_0-x_*||^2')\n", 162 | "\n", 163 | "plt.show()" 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "id": "1a6fc73a", 169 | "metadata": {}, 170 | "source": [ 171 | "### Exercise 1.9: variations (performance measures)\n", 172 | "Update the previous code for computing worst-case ratio $\\frac{\\|\\nabla f(x_{k+1})\\|^2}{\\|\\nabla f(x_k)\\|^2}$ and experiment with it.\n", 173 | "\n", 174 | "A good practice in PEPit is to limit the number of calls to evaluate function values and gradients. Indeed, each time a gradient or a function value is evaluated, it corresponds to (i) add points in the discrete representation of the worst-case function, and (ii) thereby, the problem also contain more \"interpolation inequalities\", rendering it numerically more complicated.\n", 175 | "\n", 176 | "\n", 177 | "Compute the worst-case ratios:" 178 | ] 179 | }, 180 | { 181 | "cell_type": "code", 182 | "execution_count": null, 183 | "id": "722e4589", 184 | "metadata": {}, 185 | "outputs": [], 186 | "source": [] 187 | }, 188 | { 189 | "cell_type": "markdown", 190 | "id": "86c8a15b-56e9-44b0-ad32-305d57db88ab", 191 | "metadata": {}, 192 | "source": [ 193 | "Perform numerical experiments for a few values of the parameters:" 194 | ] 195 | }, 196 | { 197 | "cell_type": "code", 198 | "execution_count": null, 199 | "id": "2ff8accc-c1ce-499b-8674-bbc42f3069bc", 200 | "metadata": {}, 201 | "outputs": [], 202 | "source": [] 203 | }, 204 | { 205 | "cell_type": "markdown", 206 | "id": "d8b928df", 207 | "metadata": {}, 208 | "source": [ 209 | "### Exercise 1.10: variations (performance measures)\n", 210 | "Update the previous code for computing worst-case ratio $\\frac{f(x_{k+1})-f_\\star}{f(x_k)-f_\\star}$ and experiment with it.\n", 211 | "\n", 212 | "As before, limit as much as possible the number of gradient/function value evaluations. Note though that for certain classes of functions, PEPit detect when the gradient (or function value) at a point was already evaluated, and does not add the redundant points in the discrete representation." 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": null, 218 | "id": "62106ac3", 219 | "metadata": {}, 220 | "outputs": [], 221 | "source": [] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "id": "b54e4a14", 226 | "metadata": {}, 227 | "source": [ 228 | "### Exercise 1.11: variations (number of iterations)\n", 229 | "Update the previous PEPit code for computing worst-case ratio $\\frac{\\|x_{N}-x_\\star\\|^2}{\\|x_0-x_\\star\\|^2}$ and experiment with it." 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": null, 235 | "id": "f0f73d3f", 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [] 239 | }, 240 | { 241 | "cell_type": "markdown", 242 | "id": "4f14c570-0fb8-48ad-bca3-e495a601e639", 243 | "metadata": {}, 244 | "source": [ 245 | "### Exercise 1.12: dimension of the numerical worst-case\n", 246 | "\n", 247 | "This question does not require numerical experiments." 248 | ] 249 | }, 250 | { 251 | "cell_type": "markdown", 252 | "id": "dc083582", 253 | "metadata": {}, 254 | "source": [ 255 | "### Exercise 1.13: identify low-dimensional counter examples\n", 256 | "The following code is an update of the previous one for computing the worst-case ratio $\\frac{\\|x_{N}-x_\\star\\|^2}{\\|x_0-x_\\star\\|^2}$. It could help identifying low dimensional worst-case examples." 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": null, 262 | "id": "5d6aa27b", 263 | "metadata": {}, 264 | "outputs": [], 265 | "source": [ 266 | "from PEPit import PEP\n", 267 | "from PEPit.functions import SmoothStronglyConvexFunction\n", 268 | "\n", 269 | "def wc_gradient(L, mu, gamma, n, verbose=1):\n", 270 | " # It is intended to compute a worst-case guarantee of gradient descent in terms of the distance to \n", 271 | " # an optimal solution: ||x_{N} - x_\\star ||^2 / || x_0 - x_\\star \\\\^2.\n", 272 | " \n", 273 | " # Instantiate PEP\n", 274 | " problem = PEP()\n", 275 | "\n", 276 | " # Declare a strongly convex smooth function and a closed convex proper function\n", 277 | " f = problem.declare_function(SmoothStronglyConvexFunction, mu=mu, L=L)\n", 278 | "\n", 279 | " # Start by defining its unique optimal point xs = x_\\star\n", 280 | " xs = f.stationary_point()\n", 281 | " fs = f(xs)\n", 282 | "\n", 283 | " # Then define the point x0 of the algorithm\n", 284 | " x0 = problem.set_initial_point()\n", 285 | "\n", 286 | " # Gradient descent\n", 287 | " x = x0\n", 288 | " list_x = list() # store all x's\n", 289 | " list_f = list() # store all f's\n", 290 | " for i in range(n):\n", 291 | " gx,fx = f.oracle(x)\n", 292 | " list_x.append(x)\n", 293 | " list_f.append(fx)\n", 294 | " x = x - gamma * gx\n", 295 | "\n", 296 | " # Set the \"performance metric\" to the distance between xN and xs\n", 297 | " problem.set_performance_metric( (x-xs)**2 )\n", 298 | " \n", 299 | " # Set the \"initial condition\" to the distance between x0 and xs\n", 300 | " problem.set_initial_condition( (x0-xs)**2 == 1) \n", 301 | "\n", 302 | " # Solve the PEP with dimension_reduction_heuristic set to \"trace\" to use the trace heuristic\n", 303 | " pepit_tau = problem.solve(verbose=verbose, dimension_reduction_heuristic=\"trace\")\n", 304 | " \n", 305 | " # INFO: for recovering points in the discrete representation of the function, you can use, e.g.:\n", 306 | " # (x-xs).eval(), (x0-xs).eval()\n", 307 | " # which correspond to the values of x_n-x_* and x_0-x_* obtained in the worst-case scenario.\n", 308 | " \n", 309 | " list_x_solved = list()\n", 310 | " list_f_solved = list()\n", 311 | " for i in range(n):\n", 312 | " list_x_solved.append((list_x[i]-xs).eval())\n", 313 | " list_f_solved.append((list_f[i]-fs).eval())\n", 314 | " \n", 315 | " # Return the output by the SDP solver\n", 316 | " return pepit_tau, list_x_solved, list_f_solved" 317 | ] 318 | }, 319 | { 320 | "cell_type": "code", 321 | "execution_count": null, 322 | "id": "3d5fd133", 323 | "metadata": {}, 324 | "outputs": [], 325 | "source": [ 326 | "L = 1\n", 327 | "mu = .1\n", 328 | "gamma = 1/L \n", 329 | "n = 10\n", 330 | "\n", 331 | "# compute a low-dimensional worst-case example\n", 332 | "pepit_tau, list_x_solved, list_f_solved = wc_gradient(L, mu, gamma, n, verbose=1)" 333 | ] 334 | }, 335 | { 336 | "cell_type": "markdown", 337 | "id": "06de91a8-f183-40dc-8777-cfc8fb085147", 338 | "metadata": {}, 339 | "source": [ 340 | "What is the dimension of the output? (hint: how many nonzero eigenvalue(s) does $G\\succcurlyeq 0$ contain? Check the output message of PEPit)\n", 341 | "\n", 342 | "In PEPit, if $G$ has $r$ nonzero eigenvalue(s), one can use the first $r$ coordinate of the output of the $x_k$'s and $g_k$'s for trying to represent a worst-case function, as follows. What do you observe? (hint: try different values of the stepsize in $(0,2/L)$)." 343 | ] 344 | }, 345 | { 346 | "cell_type": "code", 347 | "execution_count": null, 348 | "id": "6f7efadf-54a7-48df-8742-76e4b877be4e", 349 | "metadata": {}, 350 | "outputs": [], 351 | "source": [ 352 | "import matplotlib.pyplot as plt\n", 353 | "\n", 354 | "# if there is only 1 nonzero eigenvalue, plot the 1-dimensional WC function:\n", 355 | "first_coordinate_x = list()\n", 356 | "for i in range(n):\n", 357 | " first_coordinate_x.append(list_x_solved[i][0])\n", 358 | " \n", 359 | "plt.plot(first_coordinate_x,list_f_solved,'.',label='iterates')\n", 360 | "plt.plot(first_coordinate_x[0],list_f_solved[0],'.',label='x0')\n", 361 | "plt.plot(0,0,'.',label='xs')\n", 362 | "plt.legend()" 363 | ] 364 | }, 365 | { 366 | "cell_type": "markdown", 367 | "id": "e4fe8fd0", 368 | "metadata": {}, 369 | "source": [ 370 | "### Exercise 1.14: Variations (no strong convexity)\n", 371 | "What is the value of the worst-case ratio $\\frac{\\|x_{N}-x_\\star\\|^2}{\\|x_0-x_\\star\\|^2}$ when $\\mu=0$? Can you deduce convergence of gradient descent from it? Can you extract/deduce simple counter-examples from the numerics?" 372 | ] 373 | }, 374 | { 375 | "cell_type": "code", 376 | "execution_count": null, 377 | "id": "13ecdfd6", 378 | "metadata": {}, 379 | "outputs": [], 380 | "source": [] 381 | }, 382 | { 383 | "cell_type": "markdown", 384 | "id": "f9e12fc2-738b-44d7-8f5f-22bcc4b14240", 385 | "metadata": {}, 386 | "source": [ 387 | "Update the previous code for computing worst-case ratios $\\frac{f(x_N)-f_\\star}{\\|x_0-x_\\star\\|^2}$ when $\\mu=0$; can you deduce the apparent dependence on $N$?" 388 | ] 389 | }, 390 | { 391 | "cell_type": "code", 392 | "execution_count": null, 393 | "id": "6d74fa82-7af6-4eb9-b11b-101e94dddcb8", 394 | "metadata": {}, 395 | "outputs": [], 396 | "source": [] 397 | }, 398 | { 399 | "cell_type": "markdown", 400 | "id": "ea1c3022-8963-4213-b356-6bca68e377bf", 401 | "metadata": {}, 402 | "source": [ 403 | "### Exercise 1.15: Variations (no strong convexity & alternate performance measure)\n", 404 | "Update the previous code for computing worst-case ratios $\\frac{\\|\\nabla f(x_N)\\|^2}{\\|x_0-x_\\star\\|^2}$ when $\\mu=0$; can you deduce the apparent dependence on $N$?" 405 | ] 406 | }, 407 | { 408 | "cell_type": "code", 409 | "execution_count": null, 410 | "id": "a39a761d-24d5-4732-916b-538941f98430", 411 | "metadata": {}, 412 | "outputs": [], 413 | "source": [] 414 | }, 415 | { 416 | "cell_type": "markdown", 417 | "id": "5d60135a-aba8-4ae2-b8a7-4028ed1a2dc6", 418 | "metadata": {}, 419 | "source": [ 420 | "### Exercise 1.16: Variations (no strong convexity & alternate performance measure)\n", 421 | "Update the previous code for computing worst-case ratios $\\frac{\\|x_N-x_\\star\\|^2}{\\|\\nabla f(x_0)\\|^2}$ when $\\mu=0$; how does it depend on $N$?" 422 | ] 423 | }, 424 | { 425 | "cell_type": "code", 426 | "execution_count": null, 427 | "id": "f9216088-b964-49e9-b108-4fcfd2318325", 428 | "metadata": {}, 429 | "outputs": [], 430 | "source": [] 431 | }, 432 | { 433 | "cell_type": "markdown", 434 | "id": "ddf2f08a-396c-4499-8291-46c4f1952e78", 435 | "metadata": {}, 436 | "source": [ 437 | "### Exercise 1.17: learning outcomes\n", 438 | "\n", 439 | "This question does not require any numerical experiment." 440 | ] 441 | } 442 | ], 443 | "metadata": { 444 | "kernelspec": { 445 | "display_name": "Python 3 (ipykernel)", 446 | "language": "python", 447 | "name": "python3" 448 | }, 449 | "language_info": { 450 | "codemirror_mode": { 451 | "name": "ipython", 452 | "version": 3 453 | }, 454 | "file_extension": ".py", 455 | "mimetype": "text/x-python", 456 | "name": "python", 457 | "nbconvert_exporter": "python", 458 | "pygments_lexer": "ipython3", 459 | "version": "3.9.7" 460 | } 461 | }, 462 | "nbformat": 4, 463 | "nbformat_minor": 5 464 | } 465 | -------------------------------------------------------------------------------- /Codes/Jupyter/Exercise02.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "4599cf20-337a-455c-b306-54f482068d6a", 6 | "metadata": {}, 7 | "source": [ 8 | "# Summary\n", 9 | "This document corresponds to Exercise 2 of [this file](https://github.com/PerformanceEstimation/Learning-Performance-Estimation/blob/main/Course.pdf).\n", 10 | "\n", 11 | "The goal of this exercise is to compute a closed-form for the worst-case ratio $\\frac{\\|x_{k+1}-x_\\star\\|^2}{\\|x_k-x_\\star\\|^2}$ when $x_\\star=\\textrm{argmin}_x f(x)$ for some $f$ that is $L$-smooth and $\\mu$-strongly convex, and where $x_{k+1}=x_k-\\gamma_k \\nabla f(x_k)$ is obtained from a gradient step (with stepsize $\\gamma_k$) from $x_k$.\n", 12 | "\n", 13 | "\n", 14 | "We first install a few packages easing convex optimization and symbolic computations in Python." 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "id": "54438bf5-ae83-449a-acdf-dbcea276f9c9", 21 | "metadata": {}, 22 | "outputs": [], 23 | "source": [ 24 | "!pip install sympy\n", 25 | "!pip install cvxpy" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "id": "e5121a7e-c27e-45a2-87b0-52e2dd1893c5", 31 | "metadata": {}, 32 | "source": [ 33 | "### Exercises 2.1 & 2.2\n", 34 | "\n", 35 | "In exercises 2.1 and 2.2, there is no numerical experiments." 36 | ] 37 | }, 38 | { 39 | "cell_type": "markdown", 40 | "id": "14f69dcb-525c-40fd-bab8-ec2bda82b340", 41 | "metadata": {}, 42 | "source": [ 43 | "### Exercise 2.3: trick for obtaining the LMI using symbolic computation\n", 44 | "\n", 45 | "For doing this, let us first define a few symbols using SymPyimport sympy as sm.\n" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "id": "58e54232-2275-4a88-af07-b8d55afe0e33", 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "import sympy as sm\n", 56 | "\n", 57 | "# create symbols for the problem parameters:\n", 58 | "L = sm.Symbol('L')\n", 59 | "mu = sm.Symbol('mu')\n", 60 | "gamma = sm.Symbol('gamma')\n", 61 | "\n", 62 | "# create symbols for the \"primal\" variables:\n", 63 | "x0 = sm.Symbol('x0')\n", 64 | "g0 = sm.Symbol('g0')\n", 65 | "f0 = sm.Symbol('f0')\n", 66 | "xs = 0 # wlog, x_* = 0\n", 67 | "gs = 0 # constraint g_* = 0\n", 68 | "fs = 0 # wlog, f_* = 0\n", 69 | "x1 = x0 - gamma * g0 # define x1 using previous symbols:\n", 70 | "\n", 71 | "# create symbols for the \"dual\" variables\n", 72 | "rho = sm.Symbol('rho')\n", 73 | "l1 = sm.Symbol('l1')\n", 74 | "l2 = sm.Symbol('l2')" 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "id": "382f0fe6-72c0-4296-ac9f-4553337bdc43", 80 | "metadata": {}, 81 | "source": [ 82 | "Write all inequalities (in the form \"$...\\leqslant 0$\") and the Lagrangian of the problem:" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": null, 88 | "id": "528dd6c2-d781-4d8e-ae8c-beaff52ed45a", 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "# the two interpolation constraints in the form \"constraint <= 0\"\n", 93 | "constraint1 = f0 - fs + g0*(xs-x0) + 1/2/L * g0**2 + mu/(2*(1-mu/L)) * (x0-xs-1/L*g0)**2\n", 94 | "constraint2 = fs - f0 + 1/2/L * g0**2 + mu/(2*(1-mu/L)) * (x0-xs-1/L*g0)**2\n", 95 | "# the objective and the \"initial condition\" constraint: (also in the form \"constraint <= 0\")\n", 96 | "primal_objective = (x1-xs)**2\n", 97 | "initial_condition = (x0-xs)**2-1\n", 98 | "\n", 99 | "# Lagrangian:\n", 100 | "Lagrangian = - l1*constraint1 - l2*constraint2 - rho * initial_condition + primal_objective" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "id": "7ac1369e-dfcc-4046-87ca-be10978122bf", 106 | "metadata": {}, 107 | "source": [ 108 | "Now, perform the simple trick: obtain the LMI and the linear constraint (i.e., the dual constraints) from differentiating the Lagrangian" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "id": "5ce150ea-ab1b-4dd3-b61a-1f4a99f3302e", 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "# This is the LMI:\n", 119 | "LMI = sm.simplify(sm.hessian( -Lagrangian , (x0,g0))/2) \n", 120 | "# This is the linear constraint ==0 in the LMI\n", 121 | "LinearConst = sm.simplify(sm.diff(-Lagrangian,f0))" 122 | ] 123 | }, 124 | { 125 | "cell_type": "markdown", 126 | "id": "6d0394c1-d914-4fc3-b2f3-16d7a827b8e4", 127 | "metadata": {}, 128 | "source": [ 129 | "Substitute $\\lambda_2$ for getting to the same dual formulation:" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "id": "ff3b81cb-d215-4d5f-a039-4b30f3e20886", 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [ 139 | "# For getting the same LMI as in the document, substitute l2 by l1 and simplify\n", 140 | "LMI_simplified = sm.simplify(LMI.subs(l2,l1))" 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": null, 146 | "id": "15a43c97-0424-4059-a681-45fd7f56eb4a", 147 | "metadata": {}, 148 | "outputs": [], 149 | "source": [ 150 | "LMI_simplified # show the LMI (>=0)" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "id": "495a4458-28b1-4a74-a54a-658f5a841764", 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [ 160 | "LinearConst # show linear constraint (==0)" 161 | ] 162 | }, 163 | { 164 | "cell_type": "markdown", 165 | "id": "8eb10804-8523-4930-8f0c-eaf65e9c9178", 166 | "metadata": {}, 167 | "source": [ 168 | "### Exercise 2.4: numerical solutions to the LMI\n", 169 | "\n", 170 | "For getting a few hints on how to solve this LMI in closed-form, a good practice is to first try it numerically for a few values of $\\mu$, $L$, and $\\gamma_k$.\n", 171 | "\n", 172 | "The following code implements the LMI.\n" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "id": "8a3d9535-e0a8-40d4-b276-64bd262d7662", 179 | "metadata": {}, 180 | "outputs": [], 181 | "source": [ 182 | "import cvxpy as cp\n", 183 | "def lmi_convergence_distance(L, mu, gamma):\n", 184 | "\n", 185 | " # Write the LMI.\n", 186 | " S = cp.Variable((2, 2)) # this is the matrix S\n", 187 | " l1 = cp.Variable() # this is the multiplier (lambda1 == lambda2) which we denote l1\n", 188 | " rho = cp.Variable() # this is the objective\n", 189 | " \n", 190 | " s11 = rho-1+l1*L*mu/(L-mu)\n", 191 | " s12 = gamma-l1*(L+mu)/2/(L-mu)\n", 192 | " s22 = l1/(L-mu)-gamma**2\n", 193 | " \n", 194 | " constraints = [S >> 0,\n", 195 | " S[0,0] == s11,\n", 196 | " S[1,1] == s22,\n", 197 | " S[0,1] == s12,\n", 198 | " S[1,0] == s12,\n", 199 | " l1 >= 0]\n", 200 | " \n", 201 | " prob = cp.Problem(cp.Minimize(rho), constraints)\n", 202 | " prob.solve()\n", 203 | " return rho.value, l1.value, S.value" 204 | ] 205 | }, 206 | { 207 | "cell_type": "markdown", 208 | "id": "dd10fef1-a2c7-4c82-9d32-3a72ff77b5c2", 209 | "metadata": {}, 210 | "source": [ 211 | "The following code solves the LMI for a grid of $\\gamma_k$'s." 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": null, 217 | "id": "a6a6d4ea-7dde-44f2-a5da-1c2b484772cf", 218 | "metadata": {}, 219 | "outputs": [], 220 | "source": [ 221 | "import numpy as np\n", 222 | "import matplotlib.pyplot as plt\n", 223 | "\n", 224 | "nb_test = 50\n", 225 | "mu = .1\n", 226 | "L = 1\n", 227 | "gamma = np.linspace(-1., 3., num=nb_test)\n", 228 | "\n", 229 | "l1_list = list()\n", 230 | "tau_list = list()\n", 231 | "S_list = list()\n", 232 | "\n", 233 | "for i in range(nb_test):\n", 234 | " tau,l1,S = lmi_convergence_distance(L, mu, gamma[i])\n", 235 | " l1_list.append(l1[()])\n", 236 | " tau_list.append(tau[()])\n", 237 | " S_list.append(S)" 238 | ] 239 | }, 240 | { 241 | "cell_type": "markdown", 242 | "id": "7aa2cd5c-7f5a-44e2-b6d9-823a301b89af", 243 | "metadata": {}, 244 | "source": [ 245 | "Plot the output!" 246 | ] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "execution_count": null, 251 | "id": "a2d50935-e28b-4657-b79d-0350e3af9692", 252 | "metadata": {}, 253 | "outputs": [], 254 | "source": [ 255 | "plt.plot(gamma, tau_list, '.-',label='$\\tau_1$')\n", 256 | "plt.plot(gamma, l1_list, '.-',label='$\\lambda_1$')\n", 257 | "\n", 258 | "plt.xlabel('$\\gamma$')\n", 259 | "plt.ylabel('Multipliers')\n", 260 | "plt.legend()\n", 261 | "plt.show()" 262 | ] 263 | }, 264 | { 265 | "cell_type": "markdown", 266 | "id": "52973e72-8727-4ac5-b63c-b037d8ce57e4", 267 | "metadata": {}, 268 | "source": [ 269 | "Recover useful information about $S$: its eigenvalues." 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": null, 275 | "id": "ced278ca-52ef-42e7-86ea-d0a2622f4582", 276 | "metadata": {}, 277 | "outputs": [], 278 | "source": [ 279 | "firsteig_list = list()\n", 280 | "seceig_list = list()\n", 281 | "\n", 282 | "for i in range(nb_test):\n", 283 | " eigsV, _ = np.linalg.eigh(S_list[i])\n", 284 | " firsteig_list.append(np.max(eigsV))\n", 285 | " seceig_list.append(np.min(eigsV))\n", 286 | "\n", 287 | "plt.plot(gamma, firsteig_list, '.-',label='first eigenvalue')\n", 288 | "plt.plot(gamma, seceig_list, '.-',label='second eigenvalue')\n", 289 | "\n", 290 | "plt.xlabel('$\\gamma$')\n", 291 | "plt.ylabel('$\\Lambda(S)$')\n", 292 | "plt.legend()\n", 293 | "plt.show()" 294 | ] 295 | }, 296 | { 297 | "cell_type": "markdown", 298 | "id": "2d4ac8e1-a1b9-4b27-bb6a-6dee602418c1", 299 | "metadata": {}, 300 | "source": [ 301 | "In case you have a candidate expression for $\\tau(\\mu,L,\\gamma_k)$, compare it with the numerics?\n", 302 | "(you can come back to this cell a bit later with a few expressions to test)" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": null, 308 | "id": "0d118c5d-3c5b-449f-bcc4-9208f45365bb", 309 | "metadata": {}, 310 | "outputs": [], 311 | "source": [ 312 | "tau_candidates = [ 1 for i in range(nb_test)] # replace \"1\" by an appropriate candidate.\n", 313 | "plt.plot(gamma, tau_list, '.-',label='LMI')\n", 314 | "plt.plot(gamma, tau_candidates, '--',label='TRIAL')\n", 315 | "\n", 316 | "plt.xlabel('$\\gamma$')\n", 317 | "plt.ylabel('Multipliers')\n", 318 | "plt.legend()\n", 319 | "\n", 320 | "plt.show()" 321 | ] 322 | }, 323 | { 324 | "cell_type": "markdown", 325 | "id": "34dd04dc-880b-4884-9b75-006a2e1edec4", 326 | "metadata": {}, 327 | "source": [ 328 | "### Exercise 2.5: getting closed-form solutions to the LMI\n", 329 | "\n", 330 | "As we saw numerically, the matrix $S$ has rank $1$ for most values of the step-sizes!\n", 331 | "As the problem of finding a bound on the convergence rate $\\frac{\\|x_{k+1}-x_\\star\\|^2}{\\|x_k-x_\\star\\|^2}$ corresponds to a linear problem with an LMI constraint, it is natural that the optimal solution is on the boundary of the feasible set, and we can use that for solving the problem in closed-form.\n", 332 | "\n", 333 | "We use a bit of symbolic computation below for simplicity.\n", 334 | "\n", 335 | "The following cell encodes a symbolic matrix $S$:" 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "execution_count": null, 341 | "id": "d78cb26b-ca29-4fa9-ba7b-dc561cfadacf", 342 | "metadata": {}, 343 | "outputs": [], 344 | "source": [ 345 | "import sympy as sm\n", 346 | "\n", 347 | "rho = sm.Symbol('rho')\n", 348 | "l1 = sm.Symbol('l1')\n", 349 | "gamma = sm.Symbol('gamma')\n", 350 | "L = sm.Symbol('L')\n", 351 | "mu = sm.Symbol('mu')\n", 352 | "\n", 353 | "S = sm.Matrix([[rho-1+l1*L*mu/(L-mu), gamma-l1*(L+mu)/2/(L-mu)], [gamma-l1*(L+mu)/2/(L-mu), l1/(L-mu)-gamma**2]])" 354 | ] 355 | }, 356 | { 357 | "cell_type": "markdown", 358 | "id": "317a7876-12ee-4cb7-8170-6d49280db52b", 359 | "metadata": {}, 360 | "source": [ 361 | "For making $S$ rank defficient, let's choose $\\rho$ for cancelling out the determinant:" 362 | ] 363 | }, 364 | { 365 | "cell_type": "code", 366 | "execution_count": null, 367 | "id": "d9ddb665-a94f-47e1-9a4e-b84fe1bce179", 368 | "metadata": {}, 369 | "outputs": [], 370 | "source": [ 371 | "candidate_rho=sm.solve(S.det(),rho)\n", 372 | "candidate_rho[0]" 373 | ] 374 | }, 375 | { 376 | "cell_type": "markdown", 377 | "id": "918b79f3-3369-4986-9715-08805bc2d8ec", 378 | "metadata": {}, 379 | "source": [ 380 | "There are two possibilities for choosing $\\lambda_1$:\n", 381 | "- on the boundary of the PSD cone ($S=0$),\n", 382 | "- minimize $\\tau$ (and verify that the corresponding $(\\lambda_1,\\rho,S)$ is feasible for the LMI afterwards).\n", 383 | "\n", 384 | "Because we observed numerically that $S$ was rank $1$ for most choices of stepsizes, let's focus on the second possibility." 385 | ] 386 | }, 387 | { 388 | "cell_type": "code", 389 | "execution_count": null, 390 | "id": "9b97743b-753d-46d8-a4ee-b6deeaf33710", 391 | "metadata": {}, 392 | "outputs": [], 393 | "source": [ 394 | "drho=sm.simplify(sm.diff(candidate_rho[0],l1)) #differentiate tau with respect to lambda_1\n", 395 | "drho" 396 | ] 397 | }, 398 | { 399 | "cell_type": "code", 400 | "execution_count": null, 401 | "id": "382c808b-6ee6-48f2-b661-23ab08aa2df0", 402 | "metadata": {}, 403 | "outputs": [], 404 | "source": [ 405 | "l1sol=sm.solve(drho,l1) # solve dtau/dlambda_1 == 0 in lambda1\n", 406 | "l1sol # show the two possibilities!" 407 | ] 408 | }, 409 | { 410 | "cell_type": "markdown", 411 | "id": "73c6ccf5-ce6d-4d33-997e-5932c7321829", 412 | "metadata": {}, 413 | "source": [ 414 | "The corresponding \"final\" expressions for $\\tau$ that must be checked are therefore:" 415 | ] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "execution_count": null, 420 | "id": "29d4f572-a891-4aca-8bc1-fc492c36d4d0", 421 | "metadata": {}, 422 | "outputs": [], 423 | "source": [ 424 | "exprrho1=sm.simplify(candidate_rho[0].subs(l1,l1sol[0]))\n", 425 | "exprrho1.factor()" 426 | ] 427 | }, 428 | { 429 | "cell_type": "code", 430 | "execution_count": null, 431 | "id": "05dd00b2-60da-4b20-bdf3-772ea246258d", 432 | "metadata": {}, 433 | "outputs": [], 434 | "source": [ 435 | "exprrho2=sm.simplify(candidate_rho[0].subs(l1,l1sol[1]))\n", 436 | "exprrho2.factor()" 437 | ] 438 | }, 439 | { 440 | "cell_type": "markdown", 441 | "id": "8bffba1a-8cfd-4c2e-bdff-841c1f1dd168", 442 | "metadata": {}, 443 | "source": [ 444 | "### Exercise 2.6: writing a proof without semidefinite programming / LMI\n", 445 | "\n", 446 | "This exercise requires no coding." 447 | ] 448 | }, 449 | { 450 | "cell_type": "markdown", 451 | "id": "2e836d61-dfe8-4e50-92ee-058c520b0b5c", 452 | "metadata": {}, 453 | "source": [ 454 | "### Exercise 2.7: changing the ratio to $\\frac{\\|\\nabla f(x_{k+1})\\|^2}{\\|\\nabla f(x_k)\\|^2}$\n", 455 | "\n", 456 | "Using similar steps as before, solve the LMI for finding the worst-case value of the ratio $\\frac{\\|\\nabla f(x_{k+1})\\|^2}{\\|\\nabla f(x_k)\\|^2}$: (1) numerically, and (2) in closed-form." 457 | ] 458 | }, 459 | { 460 | "cell_type": "code", 461 | "execution_count": null, 462 | "id": "37ebb26c-ec6a-4459-93d7-881447340caf", 463 | "metadata": {}, 464 | "outputs": [], 465 | "source": [] 466 | }, 467 | { 468 | "cell_type": "markdown", 469 | "id": "d6f0676b-d345-4f7c-b82e-8da8768e14af", 470 | "metadata": {}, 471 | "source": [ 472 | "### Exercise 2.8: changing the ratio to $\\frac{f(x_{k+1})-f_\\star}{f(x_k)-f_\\star}$\n", 473 | "\n", 474 | "Using similar steps as before, solve the LMI for finding the worst-case value of the ratio $\\frac{f(x_{k+1})-f_\\star}{f(x_k)-f_\\star}$: (1) numerically, and (2) in closed-form.\n", 475 | "\n", 476 | "For convenience, a code is provided below." 477 | ] 478 | }, 479 | { 480 | "cell_type": "code", 481 | "execution_count": null, 482 | "id": "59f869eb-5ffa-4239-9379-08d62a448419", 483 | "metadata": {}, 484 | "outputs": [], 485 | "source": [ 486 | "# Import packages.\n", 487 | "import cvxpy as cp\n", 488 | "\n", 489 | "def lmi_functionvalues(L, mu, gamma):\n", 490 | "\n", 491 | " # Write the LMI.\n", 492 | " S = cp.Variable((3, 3))\n", 493 | " lamb = cp.Variable(6)\n", 494 | " rho = cp.Variable()\n", 495 | "\n", 496 | " s11 = mu*L/(L-mu) * (lamb[0]+lamb[1]+lamb[2]+lamb[3])\n", 497 | " s12 = -L/(L-mu) * (lamb[1]+gamma*mu*(lamb[2]+lamb[3])) - mu/(L-mu)*lamb[0]\n", 498 | " s13 = -1/(L-mu) * (L*lamb[3]+mu*lamb[2])\n", 499 | " s22 = 1/(L-mu) * (gamma*mu*(gamma*L*(lamb[2]+lamb[3]+lamb[4]+lamb[5])-2*lamb[4])-2*gamma*L*lamb[5]+lamb[0]+lamb[1]+lamb[4]+lamb[5])\n", 500 | " s23 = 1/(L-mu) * (gamma*L*lamb[3]+lamb[4]*(gamma*L-1)+gamma*mu*(lamb[2]+lamb[5])-lamb[5])\n", 501 | " s33 = 1/(L-mu) * (lamb[2]+lamb[3]+lamb[4]+lamb[5])\n", 502 | " constraints = [0==rho-lamb[0]+lamb[1]-lamb[4]+lamb[5],\n", 503 | " 1==-lamb[2]+lamb[3]+lamb[4]-lamb[5],\n", 504 | " S >> 0,\n", 505 | " S[0,0] == s11,\n", 506 | " S[1,1] == s22,\n", 507 | " S[2,2] == s33,\n", 508 | " S[0,1] == s12,\n", 509 | " S[1,0] == s12,\n", 510 | " S[0,2] == s13,\n", 511 | " S[2,0] == s13,\n", 512 | " S[1,2] == s23,\n", 513 | " S[2,1] == s23,\n", 514 | " lamb >= 0]\n", 515 | " # trick to try later: impose the additional\n", 516 | " # lamb[0]==0,\n", 517 | " # lamb[2]==0\n", 518 | " # lamb[5]==0\n", 519 | " \n", 520 | " prob = cp.Problem(cp.Minimize(rho), constraints)\n", 521 | " prob.solve()\n", 522 | " return rho.value, lamb.value, S.value" 523 | ] 524 | }, 525 | { 526 | "cell_type": "markdown", 527 | "id": "72b016c8-0f13-47a9-893f-2d9e2d7e69e1", 528 | "metadata": {}, 529 | "source": [ 530 | "The following code plots the numerical values of the multipliers:" 531 | ] 532 | }, 533 | { 534 | "cell_type": "code", 535 | "execution_count": null, 536 | "id": "4d03b27a-1f12-4fc8-9ce2-6e8513ade63f", 537 | "metadata": {}, 538 | "outputs": [], 539 | "source": [ 540 | "import matplotlib.pyplot as plt\n", 541 | "import numpy as np\n", 542 | "\n", 543 | "nb_test = 50\n", 544 | "\n", 545 | "mu = .1\n", 546 | "L = 1\n", 547 | "gamma = np.linspace(0, 2., num=nb_test)\n", 548 | "verbose = 0\n", 549 | "taus = np.empty([nb_test])\n", 550 | "lambdas = np.empty([6, nb_test])\n", 551 | "S_list = list()\n", 552 | "\n", 553 | "for i in range(nb_test):\n", 554 | " tau, lamb,S = lmi_functionvalues(L=L, mu=mu, gamma=gamma[i])\n", 555 | " taus[i]=tau[()]\n", 556 | " lambdas[:,i]=lamb\n", 557 | " S_list.append(S)\n", 558 | "\n", 559 | "\n", 560 | "plt.plot(gamma, lambdas[0,:], '-',label='$\\lambda_1$')\n", 561 | "plt.plot(gamma, lambdas[1,:], '-',label='$\\lambda_2$')\n", 562 | "plt.plot(gamma, lambdas[2,:], '-',label='$\\lambda_3$')\n", 563 | "plt.plot(gamma, lambdas[3,:], '-',label='$\\lambda_4$')\n", 564 | "plt.plot(gamma, lambdas[4,:], '-',label='$\\lambda_5$')\n", 565 | "plt.plot(gamma, lambdas[5,:], '-',label='$\\lambda_6$')\n", 566 | "plt.legend()\n", 567 | "plt.xlabel('$\\gamma$')\n", 568 | "plt.ylabel('Multipliers')\n", 569 | "\n", 570 | "plt.show()" 571 | ] 572 | }, 573 | { 574 | "cell_type": "code", 575 | "execution_count": null, 576 | "id": "b2a86252-ad61-4fa3-a802-637629b9a765", 577 | "metadata": {}, 578 | "outputs": [], 579 | "source": [ 580 | "eigsV = np.empty([3, nb_test])\n", 581 | "\n", 582 | "for i in range(nb_test):\n", 583 | " eigsV[:,i], _ = np.linalg.eigh(S_list[i])\n", 584 | "\n", 585 | "plt.plot(gamma, eigsV[0,:], '.-')\n", 586 | "plt.plot(gamma, eigsV[1,:], '.-')\n", 587 | "plt.plot(gamma, eigsV[2,:], '.-')\n", 588 | "\n", 589 | "plt.xlabel('$\\gamma$')\n", 590 | "plt.ylabel('$\\Lambda(S)$')\n", 591 | "plt.show()" 592 | ] 593 | }, 594 | { 595 | "cell_type": "markdown", 596 | "id": "7c9d367b-4c33-4767-b87b-b44803ebb69a", 597 | "metadata": {}, 598 | "source": [ 599 | "### Exercise 2.9: obtain the LMI formulations from exercises 2.7 and 2.8 using symbolic computation.\n", 600 | "\n", 601 | "By adapting the idea from exercise 2.3, obtain the two LMI formulations for the different ratios." 602 | ] 603 | }, 604 | { 605 | "cell_type": "code", 606 | "execution_count": null, 607 | "id": "a1f2fe7c-071d-42a4-80ff-41038ee778b8", 608 | "metadata": {}, 609 | "outputs": [], 610 | "source": [] 611 | }, 612 | { 613 | "cell_type": "code", 614 | "execution_count": null, 615 | "id": "a2b0b41d-e4ac-414a-a449-3b69b34c71d4", 616 | "metadata": {}, 617 | "outputs": [], 618 | "source": [] 619 | } 620 | ], 621 | "metadata": { 622 | "kernelspec": { 623 | "display_name": "Python 3 (ipykernel)", 624 | "language": "python", 625 | "name": "python3" 626 | }, 627 | "language_info": { 628 | "codemirror_mode": { 629 | "name": "ipython", 630 | "version": 3 631 | }, 632 | "file_extension": ".py", 633 | "mimetype": "text/x-python", 634 | "name": "python", 635 | "nbconvert_exporter": "python", 636 | "pygments_lexer": "ipython3", 637 | "version": "3.9.7" 638 | } 639 | }, 640 | "nbformat": 4, 641 | "nbformat_minor": 5 642 | } 643 | -------------------------------------------------------------------------------- /Codes/Jupyter/Exercise03.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "68c42883-09d9-48cc-9931-30148162af85", 6 | "metadata": {}, 7 | "source": [ 8 | "# Summary\n", 9 | "This document corresponds to Exercise 3 of [this file](https://github.com/PerformanceEstimation/Learning-Performance-Estimation/blob/main/Course.pdf).\n", 10 | "\n", 11 | "If [PEPit](https://pypi.org/project/PEPit/) is not already installed, please execute the following cell." 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "id": "9b22c3c7-10c3-4408-b77d-b884ac12ca57", 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "!pip install pepit" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "id": "7350a5f9-41a3-48b5-bedb-e4e7bf2b705e", 27 | "metadata": {}, 28 | "source": [ 29 | "### Exercise 3.1\n", 30 | "\n", 31 | "This exercise requires no code.\n", 32 | "\n", 33 | "### Exercise 3.2\n", 34 | "\n", 35 | "Codes for computing the worst-case values of $\\frac{f(x_N)-f_\\star}{\\|x_0-x_\\star\\|^2}$ and $\\frac{\\min_{0\\leqslant i\\leqslant N}\\{f(x_i)-f_\\star\\}}{\\|x_0-x_\\star\\|^2}$ for the accelerated gradient method are provided below.\n", 36 | "\n", 37 | "We let the reader adapt the codes for the gradient method and for the heavy-ball method." 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "id": "77390d9f-ace3-490d-b340-d0e502f98d28", 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "from PEPit import PEP\n", 48 | "from PEPit.functions import SmoothConvexFunction\n", 49 | "\n", 50 | "def wc_accelerated_method_lastfunctionvalue(L, n, verbose=1):\n", 51 | " \n", 52 | " # Instantiate PEP\n", 53 | " problem = PEP()\n", 54 | "\n", 55 | " # Declare a strongly convex smooth function\n", 56 | " func = problem.declare_function(SmoothConvexFunction, L=L)\n", 57 | "\n", 58 | " # Start by defining its unique optimal point xs = x_* and corresponding function value fs = f_*\n", 59 | " xs = func.stationary_point()\n", 60 | " fs = func(xs)\n", 61 | "\n", 62 | " # Then define the starting point x0 of the algorithm\n", 63 | " x0 = problem.set_initial_point()\n", 64 | "\n", 65 | " # Set the initial constraint that is the distance between x0 and x^*\n", 66 | " problem.set_initial_condition((x0 - xs) ** 2 <= 1)\n", 67 | "\n", 68 | " # Run n steps of the fast gradient method\n", 69 | " x_new = x0\n", 70 | " y = x0\n", 71 | " for i in range(n):\n", 72 | " x_old = x_new\n", 73 | " x_new = y - 1 / L * func.gradient(y)\n", 74 | " y = x_new + i / (i + 3) * (x_new - x_old)\n", 75 | "\n", 76 | " # Set the performance metric to the function value accuracy\n", 77 | " problem.set_performance_metric(func.value(x_new)-fs)\n", 78 | "\n", 79 | " # Solve the PEP\n", 80 | " pepit_tau = problem.solve(verbose=verbose)\n", 81 | "\n", 82 | "\n", 83 | " # Return the worst-case guarantee of the evaluated method\n", 84 | " return pepit_tau\n", 85 | "\n", 86 | "def wc_accelerated_method_bestfunctionvalue(L, n, verbose=1):\n", 87 | " \n", 88 | " # Instantiate PEP\n", 89 | " problem = PEP()\n", 90 | "\n", 91 | " # Declare a strongly convex smooth function\n", 92 | " func = problem.declare_function(SmoothConvexFunction, L=L)\n", 93 | "\n", 94 | " # Start by defining its unique optimal point xs = x_* and corresponding function value fs = f_*\n", 95 | " xs = func.stationary_point()\n", 96 | " fs = func(xs)\n", 97 | "\n", 98 | " # Then define the starting point x0 of the algorithm\n", 99 | " x0 = problem.set_initial_point()\n", 100 | "\n", 101 | " # Set the initial constraint that is the distance between x0 and x^*\n", 102 | " problem.set_initial_condition((x0 - xs) ** 2 <= 1)\n", 103 | "\n", 104 | " # Run n steps of the fast gradient method\n", 105 | " x_new = x0\n", 106 | " y = x0\n", 107 | " for i in range(n):\n", 108 | " x_old = x_new\n", 109 | " # the performance metric corresponds to the min over all specified metrics!\n", 110 | " # here: \\min_{i=0,...,N}{ f(x_i)-f_*}\n", 111 | " problem.set_performance_metric(func.value(x_old)-fs) \n", 112 | " x_new = y - 1 / L * func.gradient(y)\n", 113 | " y = x_new + i / (i + 3) * (x_new - x_old)\n", 114 | "\n", 115 | " problem.set_performance_metric(func.value(x_new)-fs)\n", 116 | "\n", 117 | " # Solve the PEP\n", 118 | " pepit_tau = problem.solve(verbose=verbose)\n", 119 | "\n", 120 | "\n", 121 | " # Return the worst-case guarantee of the evaluated method\n", 122 | " return pepit_tau" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": null, 128 | "id": "95715b5c-3249-42ce-bf8c-bf02c7c0c0ff", 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "import numpy as np\n", 133 | "import time\n", 134 | "import matplotlib.pyplot as plt\n", 135 | "\n", 136 | "n_max = 30\n", 137 | "n_list = list(range(n_max))\n", 138 | "L = 1\n", 139 | "verbose = 0\n", 140 | "\n", 141 | "pepits_taus_accelerated_last = list()\n", 142 | "pepits_taus_accelerated_best = list()\n", 143 | "\n", 144 | "for i in range(n_max):\n", 145 | " t0= time.process_time()\n", 146 | " pepit_tau = wc_accelerated_method_lastfunctionvalue(L, i, verbose)\n", 147 | " pepits_taus_accelerated_last.append(pepit_tau)\n", 148 | " pepit_tau = wc_accelerated_method_bestfunctionvalue(L, i, verbose)\n", 149 | " pepits_taus_accelerated_best.append(pepit_tau)\n", 150 | " t1 = time.process_time() - t0\n", 151 | " print(i+1, '/', n_max,' done (elapsed time:',\"%.2f\" %t1,'[s])')" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": null, 157 | "id": "5fca134e-e981-4a88-8bef-ed2b83812ca1", 158 | "metadata": {}, 159 | "outputs": [], 160 | "source": [ 161 | "plt.plot(n_list, pepits_taus_accelerated_last, '-', label='[AGM] last f(x_i)-f_\\star')\n", 162 | "plt.plot(n_list, pepits_taus_accelerated_best, 'o--', label='[AGM] best f(x_i)-f_\\star')\n", 163 | "\n", 164 | "plt.legend()\n", 165 | "plt.xlabel('N')\n", 166 | "plt.ylabel('f(x_i)-f_\\star')\n", 167 | "plt.loglog()\n", 168 | "plt.show()" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": null, 174 | "id": "633d034a-f92e-4f2f-96e9-126c758da967", 175 | "metadata": {}, 176 | "outputs": [], 177 | "source": [] 178 | } 179 | ], 180 | "metadata": { 181 | "kernelspec": { 182 | "display_name": "Python 3 (ipykernel)", 183 | "language": "python", 184 | "name": "python3" 185 | }, 186 | "language_info": { 187 | "codemirror_mode": { 188 | "name": "ipython", 189 | "version": 3 190 | }, 191 | "file_extension": ".py", 192 | "mimetype": "text/x-python", 193 | "name": "python", 194 | "nbconvert_exporter": "python", 195 | "pygments_lexer": "ipython3", 196 | "version": "3.9.7" 197 | } 198 | }, 199 | "nbformat": 4, 200 | "nbformat_minor": 5 201 | } 202 | -------------------------------------------------------------------------------- /Codes/Jupyter/Exercise04.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "8c46ab64-4c9e-4697-bdd4-dac4f18c849e", 6 | "metadata": { 7 | "tags": [] 8 | }, 9 | "source": [ 10 | "# Summary\n", 11 | "This document corresponds to Exercise 4 of [this file](https://github.com/PerformanceEstimation/Learning-Performance-Estimation/blob/main/Course.pdf).\n", 12 | "\n", 13 | "If [PEPit](https://pypi.org/project/PEPit/) is not already installed, please execute the following cell." 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "id": "2e9edc82-9fcd-428c-b3b7-8154cc5f7e93", 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "!pip install pepit" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "id": "eb01f937-5012-4774-b656-a27e0fe32082", 29 | "metadata": {}, 30 | "source": [ 31 | "### Exercise 4.1\n", 32 | "\n", 33 | "This exercise requires no coding.\n", 34 | "\n", 35 | "### Exercise 4.2\n", 36 | "\n", 37 | "The following lines allow computing the worst-case value of the ratio $\\frac{f(x_N)-f_\\star}{\\|x_0-x_\\star\\|^2}$ for the proximal point algorithm when $f$ is a (closed, proper) convex function." 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "id": "de1b6b5f-f5be-4cd0-a1a8-2eb40ebda9f5", 44 | "metadata": {}, 45 | "outputs": [], 46 | "source": [ 47 | "from PEPit import PEP\n", 48 | "from PEPit.functions import ConvexFunction\n", 49 | "from PEPit.primitive_steps import proximal_step\n", 50 | "\n", 51 | "def wc_proximal_point(gamma, n, verbose=1):\n", 52 | " \n", 53 | " # Instantiate PEP\n", 54 | " problem = PEP()\n", 55 | "\n", 56 | " # Declare a convex function\n", 57 | " func = problem.declare_function(ConvexFunction)\n", 58 | "\n", 59 | " # Start by defining its unique optimal point xs = x_* and corresponding function value fs = f_*\n", 60 | " xs = func.stationary_point()\n", 61 | " fs = func(xs)\n", 62 | "\n", 63 | " # Then define the starting point x0 of the algorithm\n", 64 | " x0 = problem.set_initial_point()\n", 65 | "\n", 66 | " # Set the initial constraint that is the distance between x0 and x^*\n", 67 | " problem.set_initial_condition((x0 - xs) ** 2 <= 1)\n", 68 | "\n", 69 | " # Run n steps of the proximal point method\n", 70 | " x = x0\n", 71 | " for _ in range(n):\n", 72 | " x, _, fx = proximal_step(x, func, gamma)\n", 73 | "\n", 74 | " # Set the performance metric to the final distance to optimum in function values\n", 75 | " problem.set_performance_metric(fx - fs)\n", 76 | "\n", 77 | " # Solve the PEP\n", 78 | " pepit_tau = problem.solve(verbose)\n", 79 | "\n", 80 | " # Return the worst-case guarantee of the evaluated method \n", 81 | " return pepit_tau" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": null, 87 | "id": "c834a646-4864-42a3-bf1d-2c3afb41140e", 88 | "metadata": {}, 89 | "outputs": [], 90 | "source": [ 91 | "import numpy as np\n", 92 | "import time\n", 93 | "import matplotlib.pyplot as plt\n", 94 | "\n", 95 | "n_max = 30\n", 96 | "n_list = list(range(1,n_max))\n", 97 | "gamma = 1\n", 98 | "verbose = 0\n", 99 | "\n", 100 | "pepits_taus_ppm = list()\n", 101 | "\n", 102 | "for i in range(n_max-1):\n", 103 | " t0= time.process_time()\n", 104 | " pepit_tau,_ = wc_proximal_point(gamma, i+1, verbose)\n", 105 | " pepits_taus_ppm.append(pepit_tau)\n", 106 | " t1 = time.process_time() - t0\n", 107 | " print(i+1, '/', n_max-1,' done (elapsed time:',\"%.2f\" %t1,'[s])')" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": null, 113 | "id": "2bc5fa5e-c91f-45a3-b939-fc40bd64ee5c", 114 | "metadata": {}, 115 | "outputs": [], 116 | "source": [ 117 | "plt.plot(n_list, pepits_taus_ppm, '-')\n", 118 | "\n", 119 | "plt.xlabel('N')\n", 120 | "plt.ylabel('f(x_i)-f_\\star')\n", 121 | "plt.loglog()\n", 122 | "plt.show()" 123 | ] 124 | }, 125 | { 126 | "cell_type": "markdown", 127 | "id": "877a29ef-b4eb-42e7-9fc5-38f67496b6b8", 128 | "metadata": {}, 129 | "source": [ 130 | "### Exercise 4.3\n", 131 | "\n", 132 | "Using the previous code, can you guess the dependence (of the value of the worst-case ratio) on $N$ and $\\gamma$?" 133 | ] 134 | }, 135 | { 136 | "cell_type": "markdown", 137 | "id": "e14550d9-3781-4e55-bac7-f5f090d10754", 138 | "metadata": {}, 139 | "source": [ 140 | "### Exercise 4.4\n", 141 | "\n", 142 | "Can you find a low-dimensional worst-case example?\n", 143 | "\n", 144 | "The following cells use the trace heuristic for trying to identify a low-dimensional worst-case example." 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": null, 150 | "id": "5296937e-ac53-4128-a045-4923d8d1ac68", 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "from PEPit import PEP\n", 155 | "from PEPit.functions import ConvexFunction\n", 156 | "from PEPit.primitive_steps import proximal_step\n", 157 | "\n", 158 | "def wc_proximal_point(gamma, n, verbose=1):\n", 159 | " \n", 160 | " # Instantiate PEP\n", 161 | " problem = PEP()\n", 162 | "\n", 163 | " # Declare a convex function\n", 164 | " func = problem.declare_function(ConvexFunction)\n", 165 | "\n", 166 | " # Start by defining its unique optimal point xs = x_* and corresponding function value fs = f_*\n", 167 | " xs = func.stationary_point()\n", 168 | " fs = func(xs)\n", 169 | "\n", 170 | " # Then define the starting point x0 of the algorithm\n", 171 | " x0 = problem.set_initial_point()\n", 172 | "\n", 173 | " # Set the initial constraint that is the distance between x0 and x^*\n", 174 | " problem.set_initial_condition((x0 - xs) ** 2 <= 1)\n", 175 | "\n", 176 | " # Run n steps of the proximal point method\n", 177 | " x = x0\n", 178 | " list_x = list() # store all x's\n", 179 | " list_f = list() # store all f's\n", 180 | " for _ in range(n):\n", 181 | " x, _, fx = proximal_step(x, func, gamma)\n", 182 | " list_x.append(x)\n", 183 | " list_f.append(fx)\n", 184 | "\n", 185 | " # Set the performance metric to the final distance to optimum in function values\n", 186 | " problem.set_performance_metric(fx - fs)\n", 187 | "\n", 188 | " # Solve the PEP\n", 189 | " pepit_tau = problem.solve(verbose,dimension_reduction_heuristic=\"trace\")\n", 190 | " \n", 191 | " # Store all the points\n", 192 | " list_x_solved = list()\n", 193 | " list_f_solved = list()\n", 194 | " for i in range(n):\n", 195 | " list_x_solved.append((list_x[i]-xs).eval())\n", 196 | " list_f_solved.append((list_f[i]-fs).eval())\n", 197 | "\n", 198 | " # Return the worst-case guarantee of the evaluated method \n", 199 | " return pepit_tau, list_x_solved, list_f_solved" 200 | ] 201 | }, 202 | { 203 | "cell_type": "markdown", 204 | "id": "0766778b-3878-44ca-9e36-dc5f964faa3d", 205 | "metadata": {}, 206 | "source": [ 207 | "Let's test this code: what's the output for certain values of $\\gamma$ and $N$? What's the dimension of the worst-case found?" 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": null, 213 | "id": "365c8d11-5e61-46d3-a156-114a4fb9e877", 214 | "metadata": {}, 215 | "outputs": [], 216 | "source": [ 217 | "gamma = 1\n", 218 | "n = 20\n", 219 | "\n", 220 | "# compute a low-dimensional worst-case example\n", 221 | "pepit_tau, list_x_solved, list_f_solved = wc_proximal_point(gamma, n, verbose=1)" 222 | ] 223 | }, 224 | { 225 | "cell_type": "markdown", 226 | "id": "7aa29032-3f40-495b-80d4-226792d416d7", 227 | "metadata": {}, 228 | "source": [ 229 | "Plot the one-dimensional worst-case example!" 230 | ] 231 | }, 232 | { 233 | "cell_type": "code", 234 | "execution_count": null, 235 | "id": "f1e83e0e-f255-43b4-8694-8aa429dc5167", 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [ 239 | "import matplotlib.pyplot as plt\n", 240 | "\n", 241 | "# if there is only 1 nonzero eigenvalue, plot the 1-dimensional WC function:\n", 242 | "first_coordinate_x = list()\n", 243 | "for i in range(n):\n", 244 | " first_coordinate_x.append(list_x_solved[i][0])\n", 245 | " \n", 246 | "plt.plot(first_coordinate_x,list_f_solved,'.',label='iterates')\n", 247 | "plt.plot(first_coordinate_x[0],list_f_solved[0],'.',label='x0')\n", 248 | "plt.plot(0,0,'.',label='xs')\n", 249 | "plt.legend()" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": null, 255 | "id": "5d76ebf4-c062-42dc-a086-6f9145743e46", 256 | "metadata": {}, 257 | "outputs": [], 258 | "source": [] 259 | } 260 | ], 261 | "metadata": { 262 | "kernelspec": { 263 | "display_name": "Python 3 (ipykernel)", 264 | "language": "python", 265 | "name": "python3" 266 | }, 267 | "language_info": { 268 | "codemirror_mode": { 269 | "name": "ipython", 270 | "version": 3 271 | }, 272 | "file_extension": ".py", 273 | "mimetype": "text/x-python", 274 | "name": "python", 275 | "nbconvert_exporter": "python", 276 | "pygments_lexer": "ipython3", 277 | "version": "3.9.7" 278 | } 279 | }, 280 | "nbformat": 4, 281 | "nbformat_minor": 5 282 | } 283 | -------------------------------------------------------------------------------- /Codes/Jupyter/Exercise05.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "ee74a11b-4b7f-4a01-a883-7e44d38eef0a", 6 | "metadata": {}, 7 | "source": [ 8 | "# Summary\n", 9 | "This document corresponds to Exercise 5 of [this file](https://github.com/PerformanceEstimation/Learning-Performance-Estimation/blob/main/Course.pdf).\n", 10 | "\n", 11 | "If [PEPit](https://pypi.org/project/PEPit/) is not already installed, please execute the following cell" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": null, 17 | "id": "b18f7f31-caa9-4b09-999a-15951ad2e49f", 18 | "metadata": {}, 19 | "outputs": [], 20 | "source": [ 21 | "!pip install pepit" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "id": "a60d934d-472f-48a9-9ac0-be83701dd409", 27 | "metadata": {}, 28 | "source": [ 29 | "### Exercise 5.1\n", 30 | "\n", 31 | "This exercise requires no coding.\n", 32 | "\n", 33 | "### Exercise 5.2\n", 34 | "\n", 35 | "This exercise requires no coding.\n", 36 | "\n", 37 | "### Exercise 5.3\n", 38 | "\n", 39 | "The following cell provides codes for computing the worst-case value of $f(x_N)-f_\\star$ for both projected gradient and Frank-Wolfe when $f$ is an $L$-smooth function which is minimized on a bounded set $Q$ of diameter bounded by $0=0); 21 | constraints = constraints + (l1>=0); 22 | 23 | 24 | options = sdpsettings('verbose',0); 25 | status = optimize(constraints,tau,options); 26 | 27 | double(tau) % this is the worst-case value computed by the SDP solver 28 | double(l1) 29 | eig(double(S)) 30 | -------------------------------------------------------------------------------- /Codes/Matlab/Exercise2_SDP_functionvalues.m: -------------------------------------------------------------------------------- 1 | clear all; clc; 2 | % This code corresponds to exercise 2.6 of the document 3 | % https://github.com/PerformanceEstimation/Learning-Performance-Estimation/blob/main/Exercises/Course.pdf 4 | 5 | % parameters to be tested: 6 | L = 1; 7 | mu = .1; 8 | gamma = .9/L; 9 | 10 | % SDP: 11 | lamb = sdpvar(6,1); 12 | tau = sdpvar(1); 13 | 14 | s11 = mu*L/(L-mu) * (lamb(1)+lamb(2)+lamb(3)+lamb(4)); 15 | s12 = -L/(L-mu) * (lamb(2)+gamma*mu*(lamb(3)+lamb(4))) - mu/(L-mu)*lamb(1); 16 | s13 = -1/(L-mu) * (L*lamb(4)+mu*lamb(3)); 17 | s22 = 1/(L-mu) * (gamma*mu*(gamma*L*(lamb(3)+lamb(4)+lamb(5)+lamb(6))-... 18 | 2*lamb(5))-2*gamma*L*lamb(6)+lamb(1)+lamb(2)+lamb(5)+lamb(6)); 19 | s23 = 1/(L-mu) * (gamma*L*lamb(4)+lamb(5)*(gamma*L-1)+... 20 | gamma*mu*(lamb(3)+lamb(6))-lamb(6)); 21 | s33 = 1/(L-mu) * (lamb(3)+lamb(4)+lamb(5)+lamb(6)); 22 | 23 | S = [s11 s12 s13; s12 s22 s23; s13 s23 s33]; 24 | 25 | constraints = (S>=0); 26 | constraints = constraints + (lamb>=0); 27 | constraints = constraints + (tau-lamb(1)+lamb(2)-lamb(5)+lamb(6)==0); 28 | constraints = constraints + (-lamb(3)+lamb(4)+lamb(5)-lamb(6)==1); 29 | 30 | 31 | options = sdpsettings('verbose',0); 32 | status = optimize(constraints,tau,options); 33 | 34 | double(tau) % this is the worst-case value computed by the SDP solver 35 | double(lamb) -------------------------------------------------------------------------------- /Codes/Matlab/Exercise2_closedform.m: -------------------------------------------------------------------------------- 1 | clear all; clc; 2 | % This code corresponds to exercise 2.3 of the document 3 | % https://github.com/PerformanceEstimation/Learning-Performance-Estimation/blob/main/Exercises/Course.pdf 4 | 5 | % symbolic parameters 6 | syms gamma L mu; 7 | % symbolic variables 8 | syms l1 tau; % lambda_1 and tau 9 | 10 | % LMI: 11 | 12 | s11 = tau-1+l1*L*mu/(L-mu); 13 | s12 = gamma-l1*(L+mu)/2/(L-mu); 14 | s22 = l1/(L-mu)-gamma^2; 15 | 16 | S = [s11 s12; s12 s22]; 17 | 18 | % find the value of tau that cancels out the determinant of S 19 | tau_candidate=solve(simplify(det(S))==0,tau,'ReturnConditions',true); 20 | 21 | % differentiate the expression of tau wrt l1 22 | dtau = simplify(diff(tau_candidate.tau,l1)); 23 | l1_candidates=solve(dtau==0,l1,'ReturnConditions',true); 24 | 25 | 26 | %first candidate expression (to be verified afterwards): 27 | simplify(subs(tau_candidate.tau,l1,l1_candidates.l1(1))) 28 | 29 | %second candidate expression (to be verified afterwards): 30 | simplify(subs(tau_candidate.tau,l1,l1_candidates.l1(2))) -------------------------------------------------------------------------------- /Codes/Matlab/Exercise2_symbolicalLMI.m: -------------------------------------------------------------------------------- 1 | clear all; clc; 2 | % This code corresponds to exercise 2.2 of the document 3 | % https://github.com/PerformanceEstimation/Learning-Performance-Estimation/blob/main/Exercises/Course.pdf 4 | 5 | % parameters to be tested: 6 | L = 1; 7 | mu = .1; 8 | gamma = 1/L; 9 | 10 | % SDP: 11 | l1 = sdpvar(1,1); % lambda_1 12 | tau = sdpvar(1); 13 | 14 | s11 = tau-1+l1*L*mu/(L-mu); 15 | s12 = gamma-l1*(L+mu)/2/(L-mu); 16 | s22 = l1/(L-mu)-gamma^2; 17 | 18 | S = [s11 s12; s12 s22]; 19 | 20 | constraints = (S>=0); 21 | constraints = constraints + (l1>=0); 22 | 23 | 24 | options = sdpsettings('verbose',0); 25 | status = optimize(constraints,tau,options); 26 | 27 | double(tau) % this is the worst-case value computed by the SDP solver 28 | double(l1) 29 | eig(double(S)) 30 | -------------------------------------------------------------------------------- /Course.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PerformanceEstimation/Learning-Performance-Estimation/2b057f2fa16e8503f6fad847383bb4dcedcac121/Course.pdf -------------------------------------------------------------------------------- /Course_with_corrections.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PerformanceEstimation/Learning-Performance-Estimation/2b057f2fa16e8503f6fad847383bb4dcedcac121/Course_with_corrections.pdf -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 PerformanceEstimation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Learning Performance Estimation 2 | 3 | This repository contains teaching ressources about performance estimation problems. In short, if you are interested in using worst-case analyses in the context of first-order optimization methods, those ressources might be of interest to you. 4 | 5 | A friendly informal introduction to this formalism is available in this [blog post](https://francisbach.com/computer-aided-analyses/). 6 | 7 | ## Content 8 | 9 | - The document [Course.pdf](Course.pdf) contains a set of exercises (latex source code [here](Course.tex)) for getting familiar with performance estimation problems. 10 | - The folder [Codes](/Codes/) contains the corresponding codes (with a few missing parts to be completed while doing the assignements for the main exercises). 11 | 12 | ## Contributing 13 | If you want to correct some typos, or contribute to new exercises that you find relevant, please don't hesitate to do a pull request with your suggestions. We'll add you to the list of contributors. 14 | 15 | ## Contributors 16 | - [**Adrien Taylor**](https://adrientaylor.github.io/) 17 | - [**Baptiste Goujaud**](https://scholar.google.com/citations?user=93PAG2AAAAAJ&hl=en) 18 | -------------------------------------------------------------------------------- /bib_.bib: -------------------------------------------------------------------------------- 1 | %%% USED 2 | @article{drori2014performance, 3 | year={2014}, 4 | journal={Mathematical Programming}, 5 | volume={145}, 6 | number={1-2}, 7 | title={Performance of first-order methods for smooth convex minimization: a novel approach}, 8 | publisher={Springer}, 9 | author={Drori, Y. and Teboulle, M.}, 10 | pages={451-482} 11 | } 12 | @article{taylor2015smooth, 13 | title={Smooth strongly convex interpolation and exact worst-case performance of first-order methods}, 14 | author={Taylor, A. B. and Hendrickx, J. M. and Glineur, F.}, 15 | journal={Mathematical Programming}, 16 | volume={161}, 17 | number={1-2}, 18 | pages={307--345}, 19 | year={2017}, 20 | publisher={Springer} 21 | } 22 | @article{taylor2015exact, 23 | title={Exact worst-case performance of first-order methods for composite convex optimization}, 24 | author={Taylor, A. B. and Hendrickx, J. M. and Glineur, F.}, 25 | journal={SIAM Journal on Optimization}, 26 | volume={27}, 27 | number={3}, 28 | pages={1283--1313}, 29 | year={2017}, 30 | publisher={SIAM} 31 | } 32 | @article{boyd2003subgradient, 33 | title={Subgradient methods}, 34 | author={Boyd, Stephen and Xiao, Lin and Mutapcic, Almir}, 35 | journal={lecture notes of EE392o, Stanford University}, 36 | year={2003} 37 | } 38 | 39 | % Solvers & packages 40 | @INPROCEEDINGS {Yalmip2004, 41 | AUTHOR = { J. L\"{o}fberg }, 42 | TITLE = { {YALMIP} : A Toolbox for Modeling and Optimization in {MATLAB} }, 43 | BOOKTITLE = { Proceedings of the CACSD Conference }, 44 | YEAR = { 2004 } 45 | } 46 | @article{diamond2016cvxpy, 47 | author = {S. Diamond and S. Boyd}, 48 | title = {{CVXPY}: {A} {P}ython-embedded modeling language for convex optimization}, 49 | journal = {Journal of Machine Learning Research}, 50 | year = {2016}, 51 | volume = {17}, 52 | number = {83}, 53 | pages = {1--5}, 54 | } 55 | @article{sedumi1999, 56 | author = {Sturm, J. F.}, 57 | journal = {Optimization Methods and Software}, 58 | pages = {625--653}, 59 | title = {Using {S}e{D}u{M}i 1.02, a {MATLAB} toolbox for optimization over symmetric cones}, 60 | volume = {11--12}, 61 | year = {1999} 62 | } 63 | 64 | @article{mosek2010, 65 | title={The {MOSEK} optimization software}, 66 | author={Mosek, APS}, 67 | journal={Online at http://www.mosek.com}, 68 | volume={54}, 69 | year={2010} 70 | } 71 | @article{goujaud2022pepit, 72 | title={{PEPit}: computer-assisted worst-case analyses of first-order optimization methods in {P}ython}, 73 | author={Goujaud, B. and Moucer, C. and Glineur, F. and Hendrickx, J. and Taylor, A. and Dieuleveut, A.}, 74 | journal={preprint arXiv:2201.04040}, 75 | year={2022} 76 | } 77 | 78 | @inproceedings{pesto2017, 79 | title={Performance {E}stimation {T}oolbox ({PESTO}): automated worst-case analysis of first-order optimization methods}, 80 | author={Taylor, A. and Hendrickx, J. and Glineur, F.}, 81 | booktitle={Proceedings of the 56th IEEE Conference on Decision and Control (CDC 2017)}, 82 | year={2017} 83 | } 84 | @article{meurer2017sympy, 85 | title={SymPy: symbolic computing in Python}, 86 | author={Meurer, A. and Smith, C. P. and Paprocki, M. and {\v{C}}ert{\'\i}k, O. and Kirpichev, S. B. and Rocklin, M. and Kumar, A. and Ivanov, S. and Moore, J. K. and Singh, S. and others}, 87 | journal={PeerJ Computer Science}, 88 | volume={3}, 89 | pages={e103}, 90 | year={2017}, 91 | publisher={PeerJ Inc.} 92 | } 93 | 94 | 95 | %% REFS for methods 96 | 97 | 98 | @article{guler1991convergence, 99 | title={On the convergence of the proximal point algorithm for convex minimization}, 100 | author={G{\"u}ler, O.}, 101 | journal={SIAM Journal on Control and Optimization}, 102 | volume={29}, 103 | number={2}, 104 | pages={403--419}, 105 | year={1991}, 106 | publisher={SIAM} 107 | } 108 | 109 | @inproceedings{jaggi2013revisiting, 110 | title={Revisiting {F}rank-{W}olfe: Projection-Free Sparse Convex Optimization}, 111 | author={Jaggi, M.}, 112 | booktitle={International Conference on Machine Learning (ICML)}, 113 | pages={427--435}, 114 | year={2013} 115 | } 116 | @book{Book:Nesterov2, 117 | title = "Lectures on Convex Optimization", 118 | author = "Nesterov, Y.", 119 | series = "Springer Optimization and Its Applications", 120 | publisher = "Springer International Publishing", 121 | year = 2018 122 | } 123 | @article{Nesterov:1983wy, 124 | author = {Nesterov, Y.}, 125 | journal = {Soviet Mathematics Doklady}, 126 | pages = {372--376}, 127 | priority = {2}, 128 | title = {{A method of solving a convex programming problem with convergence rate O($1/k^2$))}}, 129 | volume = {27}, 130 | year = {1983} 131 | } 132 | @inproceedings{ghadimi2015global, 133 | title={Global convergence of the heavy-ball method for convex optimization}, 134 | author={Ghadimi, Euhanna and Feyzmahdavian, Hamid Reza and Johansson, Mikael}, 135 | booktitle={2015 European control conference (ECC)}, 136 | pages={310--315}, 137 | year={2015}, 138 | organization={IEEE} 139 | } 140 | @book{Book:polyak1987, 141 | title={Introduction to Optimization}, 142 | author={Polyak, B. T.}, 143 | year={1987}, 144 | publisher={Optimization Software New York} 145 | } 146 | @article{polyak1964some, 147 | title={Some methods of speeding up the convergence of iteration methods}, 148 | author={Polyak, B. T.}, 149 | journal={USSR Computational Mathematics and Mathematical Physics}, 150 | volume={4}, 151 | number={5}, 152 | pages={1--17}, 153 | year={1964}, 154 | publisher={Elsevier} 155 | } 156 | @article{frank1956algorithm, 157 | title={An algorithm for quadratic programming}, 158 | author={Frank, M. and Wolfe, P.}, 159 | journal={Naval research logistics quarterly}, 160 | volume={3}, 161 | number={1-2}, 162 | pages={95--110}, 163 | year={1956}, 164 | publisher={Wiley Online Library} 165 | }@article{nesterov2013gradient, 166 | title={Gradient methods for minimizing composite functions}, 167 | author={Nesterov, Y.}, 168 | journal={Mathematical Programming}, 169 | volume={140}, 170 | number={1}, 171 | pages={125--161}, 172 | year={2013}, 173 | publisher={Springer} 174 | } 175 | @article{taylor2018exact, 176 | title={Exact worst-case convergence rates of the proximal gradient method for composite convex minimization}, 177 | author={Taylor, A. B. and Hendrickx, J. M. and Glineur, F.}, 178 | journal={Journal of Optimization Theory and Applications}, 179 | volume={178}, 180 | number={2}, 181 | pages={455--476}, 182 | year={2018}, 183 | publisher={Springer} 184 | } 185 | @article{gu2020tight, 186 | title={Tight sublinear convergence rate of the proximal point algorithm for maximal monotone inclusion problems}, 187 | author={Gu, G. and Yang, J.}, 188 | journal={SIAM Journal on Optimization}, 189 | volume={30}, 190 | number={3}, 191 | pages={1905--1921}, 192 | year={2020}, 193 | publisher={SIAM} 194 | } 195 | @inproceedings{fazel2003log, 196 | title={Log-det heuristic for matrix rank minimization with applications to Hankel and Euclidean distance matrices}, 197 | author={Fazel, M. and Hindi, H. and Boyd, S.}, 198 | booktitle={Proceedings of the 2003 American Control Conference, 2003.}, 199 | volume={3}, 200 | pages={2156--2162}, 201 | year={2003}, 202 | organization={IEEE} 203 | } 204 | 205 | 206 | %%% NOT USED 207 | 208 | @book{Shor:Subgradient, 209 | added-at = {2011-08-17T16:08:47.000+0200}, 210 | address = {New York, NY, USA}, 211 | author = {Shor, N. Z. and Kiwiel, Krzysztof C. and Ruszcay\`{n}ski, Andrzej}, 212 | biburl = {https://www.bibsonomy.org/bibtex/27642f439d63a8db8b2a3cdf784747a8f/pcbouman}, 213 | interhash = {00e79165be422ac4d57b64cb9457d283}, 214 | intrahash = {7642f439d63a8db8b2a3cdf784747a8f}, 215 | isbn = {0-387-12763-1}, 216 | keywords = {book gradient methods minimization}, 217 | publisher = {Springer-Verlag New York, Inc.}, 218 | timestamp = {2011-08-18T09:10:27.000+0200}, 219 | title = {Minimization methods for non-differentiable functions}, 220 | year = 1985 221 | } 222 | 223 | 224 | 225 | 226 | @inproceedings{moulines2011non, 227 | title={Non-asymptotic analysis of stochastic approximation algorithms for machine learning}, 228 | author={Bach, F. R. and Moulines, E.}, 229 | booktitle={Advances in Neural Information Processing Systems (NIPS)}, 230 | pages={451--459}, 231 | year={2011} 232 | } 233 | 234 | 235 | 236 | @article{ryu2016primer, 237 | title={Primer on monotone operator methods}, 238 | author={Ryu, E. K. and Boyd, S.}, 239 | journal={Appl. Comput. Math}, 240 | volume={15}, 241 | number={1}, 242 | pages={3--43}, 243 | year={2016} 244 | } 245 | 246 | @book{bauschke2011convex, 247 | title={Convex analysis and monotone operator theory in Hilbert spaces}, 248 | author={Bauschke, H. H. and Combettes, P. L.}, 249 | volume={408}, 250 | year={2011}, 251 | publisher={Springer} 252 | } 253 | 254 | 255 | 256 | @InProceedings{pmlr-v99-taylor19a, 257 | title = {Stochastic first-order methods: non-asymptotic and computer-aided analyses via potential functions}, 258 | author = {Taylor, A. and Bach, F.}, 259 | year = {2019}, 260 | booktitle={Conference on Learning Theory (COLT)} 261 | } 262 | 263 | @article{deKlerkELS2016, 264 | title={On the worst-case complexity of the gradient method with exact line search for smooth strongly convex functions}, 265 | author={de Klerk, E. and Glineur, F. and Taylor, A. B.}, 266 | journal={Optimization Letters}, 267 | volume={11}, 268 | number={7}, 269 | pages={1185--1199}, 270 | year={2017}, 271 | publisher={Springer} 272 | } 273 | @article{de2017worst, 274 | title={Worst-case convergence analysis of inexact gradient and Newton methods through semidefinite programming performance estimation}, 275 | author={De Klerk, E. and Glineur, F. and Taylor, A.B.}, 276 | journal={SIAM Journal on Optimization}, 277 | volume={30}, 278 | number={3}, 279 | pages={2053--2082}, 280 | year={2020}, 281 | publisher={SIAM} 282 | } 283 | 284 | 285 | @article{guler1992new, 286 | title={New proximal point algorithms for convex minimization}, 287 | author={G{\"u}ler, O.}, 288 | journal={SIAM Journal on Optimization}, 289 | volume={2}, 290 | number={4}, 291 | pages={649--664}, 292 | year={1992}, 293 | publisher={SIAM} 294 | } 295 | @article{mann1953mean, 296 | title={Mean value methods in iteration}, 297 | author={Mann, W. R.}, 298 | journal={Proceedings of the American Mathematical Society}, 299 | volume={4}, 300 | number={3}, 301 | pages={506--510}, 302 | year={1953}, 303 | publisher={JSTOR} 304 | } 305 | @article{halpern1967fixed, 306 | title={Fixed points of nonexpanding maps}, 307 | author={Halpern, B.}, 308 | journal={Bulletin of the American Mathematical Society}, 309 | volume={73}, 310 | number={6}, 311 | pages={957--961}, 312 | year={1967} 313 | } 314 | 315 | @Article{drori2014optimal, 316 | author="Drori, Y. 317 | and Teboulle, M.", 318 | title="An optimal variant of {K}elley's cutting-plane method", 319 | journal="Mathematical Programming", 320 | year="2016", 321 | volume="160", 322 | number="1", 323 | pages="321--351" 324 | } 325 | @phdthesis{drori2014contributions, 326 | title={Contributions to the Complexity Analysis of Optimization Algorithms}, 327 | author={Drori, Y.}, 328 | year={2014}, 329 | school={Tel-Aviv University} 330 | } 331 | @phdthesis{Taylor2017PEPs, 332 | title={Convex Interpolation and Performance Estimation of First-order Methods for Convex Optimization}, 333 | author={Taylor, A. B.}, 334 | year={2017}, 335 | school={Universit\'e catholique de Louvain} 336 | } 337 | @inproceedings{defazio2014saga, 338 | title={{SAGA}: A fast incremental gradient method with support for non-strongly convex composite objectives}, 339 | author={Defazio, A. and Bach, F. and Lacoste-Julien, S.}, 340 | booktitle={Advances in Neural Information Processing Systems (NIPS)}, 341 | pages={1646--1654}, 342 | year={2014} 343 | } 344 | 345 | @InProceedings{pmlr-v89-zhou19c, 346 | title = {Direct Acceleration of SAGA using Sampled Negative Momentum}, 347 | author = {Zhou, K. and Ding, Q. and Shang, F. and Cheng, J. and Li, D. and Luo, Z.-Q.}, 348 | booktitle = {Proceedings of Machine Learning Research}, 349 | pages = {1602--1610}, 350 | year = {2019}, 351 | volume = {89} 352 | } 353 | 354 | @inproceedings{defazio2016simple, 355 | title={A simple practical accelerated method for finite sums}, 356 | author={Defazio, A.}, 357 | booktitle={Advances in Neural Information Processing Systems (NIPS)}, 358 | pages={676--684}, 359 | year={2016} 360 | } 361 | 362 | @article{kim2014optimized, 363 | title={Optimized first-order methods for smooth convex minimization}, 364 | author={Kim, D. and Fessler, J. A.}, 365 | journal={Mathematical Programming}, 366 | volume={159}, 367 | number={1-2}, 368 | pages={81--107}, 369 | year={2016}, 370 | publisher={Springer} 371 | } 372 | 373 | @article{kim2015convergence, 374 | title={On the convergence analysis of the optimized gradient method}, 375 | author={Kim, D. and Fessler, J. A.}, 376 | journal={Journal of optimization theory and applications}, 377 | volume={172}, 378 | number={1}, 379 | pages={187--205}, 380 | year={2017}, 381 | publisher={Springer} 382 | } 383 | @article{kim2019accelerated, 384 | title={Accelerated proximal point method for maximally monotone operators}, 385 | author={Kim, D.}, 386 | journal={Mathematical Programming}, 387 | pages={1--31}, 388 | year={2021}, 389 | publisher={Springer} 390 | } 391 | 392 | 393 | 394 | @article{lessard2014analysis, 395 | title={Analysis and design of optimization algorithms via integral quadratic constraints}, 396 | author={Lessard, L. and Recht, B. and Packard, A.}, 397 | journal={SIAM Journal on Optimization}, 398 | volume={26}, 399 | number={1}, 400 | pages={57--95}, 401 | year={2016}, 402 | publisher={SIAM} 403 | } 404 | @inproceedings{taylor2018lyapunov, 405 | author = {Taylor, A. and Van Scoy, B. and Lessard, L.}, 406 | title = {Lyapunov functions for first-order methods: {T}ight automated convergence guarantees}, 407 | booktitle = {International Conference on Machine Learning (ICML)}, 408 | year = {2018} 409 | } 410 | @article{nesterov2015quasi, 411 | title={Quasi-monotone subgradient methods for nonsmooth convex minimization}, 412 | author={Nesterov, Y. and Shikhman, V.}, 413 | journal={Journal of Optimization Theory and Applications}, 414 | volume={165}, 415 | number={3}, 416 | pages={917--940}, 417 | year={2015}, 418 | publisher={Springer} 419 | } 420 | 421 | 422 | @inproceedings{cyrus2018robust, 423 | title={A robust accelerated optimization algorithm for strongly convex functions}, 424 | author={Cyrus, S. and Hu, B. and Van Scoy, B. and Lessard, L.}, 425 | booktitle={2018 Annual American Control Conference (ACC)}, 426 | pages={1376--1381}, 427 | year={2018}, 428 | organization={IEEE} 429 | } 430 | @article{van2018fastest, 431 | title={The fastest known globally convergent first-order method for minimizing strongly convex functions}, 432 | author={Van Scoy, B. and Freeman, R. A. and Lynch, K. M.}, 433 | journal={IEEE Control Systems Letters}, 434 | volume={2}, 435 | number={1}, 436 | pages={49--54}, 437 | year={2018} 438 | } 439 | @article{drori2018efficient, 440 | title={Efficient first-order methods for convex minimization: a constructive approach}, 441 | author={Drori, Y. and Taylor, A.B.}, 442 | journal={Mathematical Programming}, 443 | volume={184}, 444 | number={1}, 445 | pages={183--220}, 446 | year={2020}, 447 | publisher={Springer} 448 | } 449 | 450 | @article{kim2018optimizing, 451 | title={Optimizing the efficiency of first-order methods for decreasing the gradient of smooth convex functions}, 452 | author={Kim, D. and Fessler, J.A.}, 453 | journal={Journal of Optimization Theory and Applications}, 454 | volume={188}, 455 | number={1}, 456 | pages={192--219}, 457 | year={2021}, 458 | publisher={Springer} 459 | } 460 | 461 | @book{Book:Nesterov2, 462 | title = "Lectures on Convex Optimization", 463 | author = "Nesterov, Y.", 464 | series = "Springer Optimization and Its Applications", 465 | publisher = "Springer International Publishing", 466 | year = 2018 467 | } 468 | @article{Nesterov:1983wy, 469 | author = {Nesterov, Y.}, 470 | journal = {Soviet Mathematics Doklady}, 471 | pages = {372--376}, 472 | priority = {2}, 473 | title = {{A method of solving a convex programming problem with convergence rate O($1/k^2$))}}, 474 | volume = {27}, 475 | year = {1983} 476 | } 477 | @article{ryu2018operator, 478 | title={Operator splitting performance estimation: Tight contraction factors and optimal parameter selection}, 479 | author={Ryu, E.K. and Taylor, A.B. and Bergeling, C. and Giselsson, P.}, 480 | journal={SIAM Journal on Optimization}, 481 | volume={30}, 482 | number={3}, 483 | pages={2251--2271}, 484 | year={2020}, 485 | publisher={SIAM} 486 | } 487 | @article{Dragomir2019optimal, 488 | title={Optimal complexity and certification of Bregman first-order methods}, 489 | author={Dragomir, R.-A. and Taylor, A.B. and d’Aspremont, A. and Bolte, J.}, 490 | journal={Mathematical Programming}, 491 | pages={1--43}, 492 | year={2021}, 493 | publisher={Springer} 494 | } 495 | 496 | @inproceedings{Barre2020Polyak, 497 | title={Complexity guarantees for {P}olyak steps with momentum}, 498 | author={Barr{\'e}, M. and Taylor, A. and d’Aspremont, A.}, 499 | booktitle={Conference on Learning Theory (COLT)}, 500 | year={2020} 501 | } 502 | 503 | @article{Barre2020inexact, 504 | title={Principled analyses and design of first-order methods with inexact proximal operators}, 505 | author={Barr\'e, M. and Taylor, A. and Bach, F.}, 506 | journal={preprint arXiv:2006.06041}, 507 | year={2020} 508 | } 509 | 510 | @article{nesterov2013gradient, 511 | title={Gradient methods for minimizing composite functions}, 512 | author={Nesterov, Y.}, 513 | journal={Mathematical Programming}, 514 | volume={140}, 515 | number={1}, 516 | pages={125--161}, 517 | year={2013}, 518 | publisher={Springer} 519 | } 520 | @article{douglas1956, 521 | author = {J. Douglas and H. H. Rachford}, 522 | title = {On the numerical solution of heat conduction problems in two and three space variables}, 523 | journal = {Transactions of the American Mathematical Society}, 524 | ajournal={Trans. Amer. Math. Soc.}, 525 | volume = {82}, 526 | pages = {421--439}, 527 | year = {1956} 528 | } 529 | @article{moursi2019douglas, 530 | title={Douglas--Rachford Splitting for the Sum of a Lipschitz Continuous and a Strongly Monotone Operator}, 531 | author={Moursi, W.M. and Vandenberghe, L.}, 532 | journal={Journal of Optimization Theory and Applications}, 533 | volume={183}, 534 | number={1}, 535 | pages={179--198}, 536 | year={2019}, 537 | publisher={Springer} 538 | } 539 | 540 | @article{bauschke2017douglas, 541 | title={On the {D}ouglas--{R}achford algorithm}, 542 | author={Bauschke, H. H. and Moursi, W. M.}, 543 | journal={Mathematical Programming}, 544 | volume={164}, 545 | number={1-2}, 546 | pages={263--284}, 547 | year={2017}, 548 | publisher={Springer} 549 | } 550 | @article{bansal2017potential, 551 | title={Potential-Function Proofs for Gradient Methods}, 552 | author={Bansal, N. and Gupta, A.}, 553 | journal={Theory of Computing}, 554 | volume={15}, 555 | number={1}, 556 | pages={1--32}, 557 | year={2019}, 558 | publisher={Theory of Computing Exchange} 559 | } 560 | 561 | @inproceedings{patrinos2014douglas, 562 | title={{D}ouglas--{R}achford splitting: Complexity estimates and accelerated variants}, 563 | author={Patrinos, P. and Stella, L. and Bemporad, A.}, 564 | booktitle={53rd IEEE Conference on Decision and Control}, 565 | pages={4234--4239}, 566 | year={2014}, 567 | organization={IEEE} 568 | } 569 | @article{gu2019optimal, 570 | title={Optimal Nonergodic Sublinear Convergence Rate of Proximal Point Algorithm for Maximal Monotone Inclusion Problems}, 571 | author={Gu, G. and Yang, J.}, 572 | journal={preprint arXiv:1904.05495}, 573 | year={2019} 574 | } 575 | @article{eckstein1993nonlinear, 576 | title={Nonlinear proximal point algorithms using Bregman functions, with applications to convex programming}, 577 | author={Eckstein, J.}, 578 | journal={Mathematics of Operations Research}, 579 | volume={18}, 580 | number={1}, 581 | pages={202--226}, 582 | year={1993}, 583 | publisher={INFORMS} 584 | } 585 | @article{auslender2006interior, 586 | title={Interior gradient and proximal methods for convex and conic optimization}, 587 | author={Auslender, A. and Teboulle, M.}, 588 | journal={SIAM Journal on Optimization}, 589 | volume={16}, 590 | number={3}, 591 | pages={697--725}, 592 | year={2006}, 593 | publisher={SIAM} 594 | } 595 | @article{bauschke2016descent, 596 | title={A descent lemma beyond {L}ipschitz gradient continuity: first-order methods revisited and applications}, 597 | author={Bauschke, H. H. and Bolte, J. and Teboulle, M.}, 598 | journal={Mathematics of Operations Research}, 599 | volume={42}, 600 | number={2}, 601 | pages={330--348}, 602 | year={2016}, 603 | publisher={INFORMS} 604 | } 605 | @article{bolte2018first, 606 | title={First order methods beyond convexity and {L}ipschitz gradient continuity with applications to quadratic inverse problems}, 607 | author={Bolte, J. and Sabach, S. and Teboulle, M. and Vaisbourd, Y.}, 608 | journal={SIAM Journal on Optimization}, 609 | volume={28}, 610 | number={3}, 611 | pages={2131--2151}, 612 | year={2018}, 613 | publisher={SIAM} 614 | } 615 | 616 | @article{lu2018relatively, 617 | title={Relatively smooth convex optimization by first-order methods, and applications}, 618 | author={Lu, H. and Freund, R. M. and Nesterov, Y.}, 619 | journal={SIAM Journal on Optimization}, 620 | volume={28}, 621 | number={1}, 622 | pages={333--354}, 623 | year={2018}, 624 | publisher={SIAM} 625 | } 626 | 627 | @article{beck2009fast, 628 | title={A fast iterative shrinkage-thresholding algorithm for linear inverse problems}, 629 | author={Beck, A. and Teboulle, M.}, 630 | journal={SIAM journal on imaging sciences}, 631 | volume={2}, 632 | number={1}, 633 | pages={183--202}, 634 | year={2009}, 635 | publisher={SIAM} 636 | } 637 | @article{lieder2017convergence, 638 | title={On the convergence rate of the {H}alpern-iteration}, 639 | author={Lieder, F.}, 640 | journal={Optimization Letters}, 641 | volume={15}, 642 | number={2}, 643 | pages={405--418}, 644 | year={2021}, 645 | publisher={Springer} 646 | } 647 | 648 | @phdthesis{lieder2018, 649 | title={Projection Based Methods for Conic Linear Programming Optimal First Order Complexities and Norm Constrained Quasi Newton Methods}, 650 | author={Lieder, F.}, 651 | year={2018}, 652 | school={Universitats-und Landesbibliothek der Heinrich-Heine-Universitat Dusseldorf} 653 | } 654 | 655 | @article{drori2017exact, 656 | title={The exact information-based complexity of smooth convex minimization}, 657 | author={Drori, Y.}, 658 | journal={Journal of Complexity}, 659 | volume={39}, 660 | pages={1--16}, 661 | year={2017}, 662 | publisher={Elsevier} 663 | } 664 | @article{davis2017three, 665 | title={A three-operator splitting scheme and its optimization applications}, 666 | author={Davis, D. and Yin, W.}, 667 | journal={Set-valued and variational analysis}, 668 | volume={25}, 669 | number={4}, 670 | pages={829--858}, 671 | year={2017}, 672 | publisher={Springer} 673 | } 674 | 675 | @article{colla2021decentralized, 676 | title={Automated Worst-Case Performance Analysis of Decentralized Gradient Descent}, 677 | author={S. Colla and J. M. Hendrickx}, 678 | year={2021}, 679 | journal={preprint arXiv:2103.14396} 680 | } 681 | 682 | @article{Nedic2009Distrib, 683 | author = {Nedic, A. and Ozdaglar, A.}, 684 | year = {2009}, 685 | month = {02}, 686 | pages = {48 - 61}, 687 | title = {Distributed Subgradient Methods for Multi-Agent Optimization}, 688 | volume = {54}, 689 | journal = {Automatic Control, IEEE Transactions on}, 690 | doi = {10.1109/TAC.2008.2009515} 691 | } 692 | @article{giselsson2016linear, 693 | title={Linear convergence and metric selection for {D}ouglas-{R}achford splitting and {ADMM}}, 694 | author={Giselsson, P. and Boyd, S.}, 695 | journal={IEEE Transactions on Automatic Control}, 696 | volume={62}, 697 | number={2}, 698 | pages={532--544}, 699 | year={2016} 700 | } 701 | @article{goujaud2022optimal, 702 | title={Optimal first-order methods for convex functions with a quadratic upper bound}, 703 | author={Goujaud, B. and Taylor, A. and Dieuleveut, A.}, 704 | journal={preprint arXiv:2205.15033}, 705 | year={2022} 706 | } 707 | @article{taylor2022optimal, 708 | title={An optimal gradient method for smooth strongly convex minimization}, 709 | author={Taylor, A. and Drori, Y.}, 710 | journal={Mathematical Programming}, 711 | pages={1--38}, 712 | year={2022}, 713 | publisher={Springer} 714 | } 715 | @article{gupta2022branch, 716 | title={Branch-and-Bound Performance Estimation Programming: A Unified Methodology for Constructing Optimal Optimization Methods}, 717 | author={Das Gupta, S. and Van Parys, B.P.G. and Ryu, E. K.}, 718 | journal={preprint arXiv:2203.07305}, 719 | year={2022} 720 | } 721 | @article{gorbunov2022last, 722 | title={Last-Iterate Convergence of Optimistic Gradient Method for Monotone Variational Inequalities}, 723 | author={Gorbunov, E. and Taylor, A. and Gidel, G.}, 724 | journal={preprint arXiv:2205.08446}, 725 | year={2022} 726 | } 727 | @inproceedings{colla2021automated, 728 | title={Automated worst-case performance analysis of decentralized gradient descent}, 729 | author={Colla, S. and Hendrickx, J. M.}, 730 | booktitle={2021 60th IEEE Conference on Decision and Control (CDC)}, 731 | pages={2627--2633}, 732 | year={2021} 733 | } 734 | @article{colla2022automatic, 735 | title={Automatic Performance Estimation for Decentralized Optimization}, 736 | author={Colla, S. and Hendrickx, J. M.}, 737 | journal={preprint arXiv:2203.05963}, 738 | year={2022} 739 | } 740 | @article{sundararajan2020analysis, 741 | title={Analysis and design of first-order distributed optimization algorithms over time-varying graphs}, 742 | author={Sundararajan, A. and Van Scoy, B. and Lessard, L.}, 743 | journal={IEEE Transactions on Control of Network Systems}, 744 | volume={7}, 745 | number={4}, 746 | pages={1597--1608}, 747 | year={2020} 748 | } 749 | @article{moucer2022systematic, 750 | title={A systematic approach to Lyapunov analyses of continuous-time models in convex optimization}, 751 | author={Moucer, C. and Taylor, A. and Bach, F.}, 752 | journal={preprint arXiv:2205.12772}, 753 | year={2022} 754 | } 755 | @article{fazlyab2018analysis, 756 | title={Analysis of optimization algorithms via integral quadratic constraints: Nonstrongly convex problems}, 757 | author={Fazlyab, M. and Ribeiro, A. and Morari, M. and Preciado, V.M.}, 758 | journal={SIAM Journal on Optimization}, 759 | volume={28}, 760 | number={3}, 761 | pages={2654--2689}, 762 | year={2018} 763 | } 764 | @article{hu2021analysis, 765 | title={Analysis of biased stochastic gradient descent using sequential semidefinite programs}, 766 | author={Hu, B. and Seiler, P. and Lessard, L.}, 767 | journal={Mathematical Programming}, 768 | volume={187}, 769 | number={1}, 770 | pages={383--408}, 771 | year={2021}, 772 | publisher={Springer} 773 | } 774 | @inproceedings{sundararajan2017robust, 775 | title={Robust convergence analysis of distributed optimization algorithms}, 776 | author={Sundararajan, A. and Hu, B. and Lessard, L.}, 777 | booktitle={2017 55th Annual Allerton Conference on Communication, Control, and Computing (Allerton)}, 778 | pages={1206--1212}, 779 | year={2017} 780 | } 781 | @inproceedings{hu2017unified, 782 | title={A unified analysis of stochastic optimization methods using jump system theory and quadratic constraints}, 783 | author={Hu, B. and Seiler, P. and Rantzer, A.}, 784 | booktitle={Conference on Learning Theory (COLT)}, 785 | year={2017} 786 | } 787 | 788 | 789 | --------------------------------------------------------------------------------