├── Active Learning for Cost-Sensitive Classification.pdf ├── Lab1_unconstrainedOptimization ├── .idea │ ├── Lab1_unconstrainedOptimization.iml │ ├── misc.xml │ ├── modules.xml │ └── workspace.xml ├── __pycache__ │ ├── derivativeFreeMethod.cpython-36.pyc │ ├── firstDerivativeMethod.cpython-36.pyc │ ├── hyperGradientDescent.cpython-36.pyc │ ├── lassoRegression.cpython-36.pyc │ ├── myClass.cpython-36.pyc │ ├── oneDimensionalSearch.cpython-36.pyc │ └── secondDerivativeMethod.cpython-36.pyc ├── derivativeFreeMethod.py ├── firstDerivativeMethod.py ├── hyperGradientDescent.py ├── lassoRegression.py ├── main.py ├── myClass.py ├── oneDimensionalSearch.py ├── secondDerivativeMethod.py └── sonar.all-data ├── Lab2_constrainedOptimization ├── .idea │ ├── Lab2_constrainedOptimization.iml │ ├── misc.xml │ ├── modules.xml │ └── workspace.xml ├── PCA.py ├── ShoppingMall │ ├── IALM_background_subtraction.mat │ ├── data │ │ ├── ShoppingMall1481.bmp │ │ ├── ShoppingMall1482.bmp │ │ ├── ShoppingMall1483.bmp │ │ ├── ShoppingMall1484.bmp │ │ ├── ShoppingMall1485.bmp │ │ ├── ShoppingMall1486.bmp │ │ ├── ShoppingMall1487.bmp │ │ ├── ShoppingMall1488.bmp │ │ ├── ShoppingMall1489.bmp │ │ ├── ShoppingMall1490.bmp │ │ ├── ShoppingMall1491.bmp │ │ ├── ShoppingMall1492.bmp │ │ ├── ShoppingMall1493.bmp │ │ ├── ShoppingMall1494.bmp │ │ ├── ShoppingMall1495.bmp │ │ ├── ShoppingMall1496.bmp │ │ ├── ShoppingMall1497.bmp │ │ ├── ShoppingMall1498.bmp │ │ ├── ShoppingMall1499.bmp │ │ ├── ShoppingMall1500.bmp │ │ ├── ShoppingMall1501.bmp │ │ ├── ShoppingMall1502.bmp │ │ ├── ShoppingMall1503.bmp │ │ ├── ShoppingMall1504.bmp │ │ ├── ShoppingMall1505.bmp │ │ ├── ShoppingMall1506.bmp │ │ ├── ShoppingMall1507.bmp │ │ ├── ShoppingMall1508.bmp │ │ ├── ShoppingMall1509.bmp │ │ ├── ShoppingMall1510.bmp │ │ ├── ShoppingMall1511.bmp │ │ ├── ShoppingMall1512.bmp │ │ ├── ShoppingMall1513.bmp │ │ ├── ShoppingMall1514.bmp │ │ ├── ShoppingMall1515.bmp │ │ ├── ShoppingMall1516.bmp │ │ ├── ShoppingMall1517.bmp │ │ ├── ShoppingMall1518.bmp │ │ ├── ShoppingMall1519.bmp │ │ ├── ShoppingMall1520.bmp │ │ ├── ShoppingMall1521.bmp │ │ ├── ShoppingMall1522.bmp │ │ ├── ShoppingMall1523.bmp │ │ ├── ShoppingMall1524.bmp │ │ ├── ShoppingMall1525.bmp │ │ ├── ShoppingMall1526.bmp │ │ ├── ShoppingMall1527.bmp │ │ ├── ShoppingMall1528.bmp │ │ ├── ShoppingMall1529.bmp │ │ ├── ShoppingMall1530.bmp │ │ ├── ShoppingMall1531.bmp │ │ ├── ShoppingMall1532.bmp │ │ ├── ShoppingMall1533.bmp │ │ ├── ShoppingMall1534.bmp │ │ ├── ShoppingMall1535.bmp │ │ ├── ShoppingMall1536.bmp │ │ ├── ShoppingMall1537.bmp │ │ ├── ShoppingMall1538.bmp │ │ ├── ShoppingMall1539.bmp │ │ ├── ShoppingMall1540.bmp │ │ ├── ShoppingMall1541.bmp │ │ ├── ShoppingMall1542.bmp │ │ ├── ShoppingMall1543.bmp │ │ ├── ShoppingMall1544.bmp │ │ ├── ShoppingMall1545.bmp │ │ ├── ShoppingMall1546.bmp │ │ ├── ShoppingMall1547.bmp │ │ ├── ShoppingMall1548.bmp │ │ ├── ShoppingMall1549.bmp │ │ ├── ShoppingMall1550.bmp │ │ ├── ShoppingMall1551.bmp │ │ ├── ShoppingMall1552.bmp │ │ ├── ShoppingMall1553.bmp │ │ ├── ShoppingMall1554.bmp │ │ ├── ShoppingMall1555.bmp │ │ ├── ShoppingMall1556.bmp │ │ ├── ShoppingMall1557.bmp │ │ ├── ShoppingMall1558.bmp │ │ ├── ShoppingMall1559.bmp │ │ ├── ShoppingMall1560.bmp │ │ ├── ShoppingMall1561.bmp │ │ ├── ShoppingMall1562.bmp │ │ ├── ShoppingMall1563.bmp │ │ ├── ShoppingMall1564.bmp │ │ ├── ShoppingMall1565.bmp │ │ ├── ShoppingMall1566.bmp │ │ ├── ShoppingMall1567.bmp │ │ ├── ShoppingMall1568.bmp │ │ ├── ShoppingMall1569.bmp │ │ ├── ShoppingMall1570.bmp │ │ ├── ShoppingMall1571.bmp │ │ ├── ShoppingMall1572.bmp │ │ ├── ShoppingMall1573.bmp │ │ ├── ShoppingMall1574.bmp │ │ ├── ShoppingMall1575.bmp │ │ ├── ShoppingMall1576.bmp │ │ ├── ShoppingMall1577.bmp │ │ ├── ShoppingMall1578.bmp │ │ ├── ShoppingMall1579.bmp │ │ ├── ShoppingMall1580.bmp │ │ ├── ShoppingMall1581.bmp │ │ ├── ShoppingMall1582.bmp │ │ ├── ShoppingMall1583.bmp │ │ ├── ShoppingMall1584.bmp │ │ ├── ShoppingMall1585.bmp │ │ ├── ShoppingMall1586.bmp │ │ ├── ShoppingMall1587.bmp │ │ ├── ShoppingMall1588.bmp │ │ ├── ShoppingMall1589.bmp │ │ ├── ShoppingMall1590.bmp │ │ ├── ShoppingMall1591.bmp │ │ ├── ShoppingMall1592.bmp │ │ ├── ShoppingMall1593.bmp │ │ ├── ShoppingMall1594.bmp │ │ ├── ShoppingMall1595.bmp │ │ ├── ShoppingMall1596.bmp │ │ ├── ShoppingMall1597.bmp │ │ ├── ShoppingMall1598.bmp │ │ ├── ShoppingMall1599.bmp │ │ ├── ShoppingMall1600.bmp │ │ ├── ShoppingMall1601.bmp │ │ ├── ShoppingMall1602.bmp │ │ ├── ShoppingMall1603.bmp │ │ ├── ShoppingMall1604.bmp │ │ ├── ShoppingMall1605.bmp │ │ ├── ShoppingMall1606.bmp │ │ ├── ShoppingMall1607.bmp │ │ └── ShoppingMall1608.bmp │ └── result │ │ ├── IALM_result000.png │ │ ├── IALM_result001.png │ │ ├── IALM_result002.png │ │ ├── IALM_result003.png │ │ ├── IALM_result004.png │ │ ├── IALM_result005.png │ │ ├── IALM_result006.png │ │ ├── IALM_result007.png │ │ ├── IALM_result008.png │ │ ├── IALM_result009.png │ │ ├── IALM_result010.png │ │ ├── IALM_result011.png │ │ ├── IALM_result012.png │ │ ├── IALM_result013.png │ │ ├── IALM_result014.png │ │ ├── IALM_result015.png │ │ ├── IALM_result016.png │ │ ├── IALM_result017.png │ │ ├── IALM_result018.png │ │ ├── IALM_result019.png │ │ ├── IALM_result020.png │ │ ├── IALM_result021.png │ │ ├── IALM_result022.png │ │ ├── IALM_result023.png │ │ ├── IALM_result024.png │ │ ├── IALM_result025.png │ │ ├── IALM_result026.png │ │ ├── IALM_result027.png │ │ ├── IALM_result028.png │ │ ├── IALM_result029.png │ │ ├── IALM_result030.png │ │ ├── IALM_result031.png │ │ ├── IALM_result032.png │ │ ├── IALM_result033.png │ │ ├── IALM_result034.png │ │ ├── IALM_result035.png │ │ ├── IALM_result036.png │ │ ├── IALM_result037.png │ │ ├── IALM_result038.png │ │ ├── IALM_result039.png │ │ ├── IALM_result040.png │ │ ├── IALM_result041.png │ │ ├── IALM_result042.png │ │ ├── IALM_result043.png │ │ ├── IALM_result044.png │ │ ├── IALM_result045.png │ │ ├── IALM_result046.png │ │ ├── IALM_result047.png │ │ ├── IALM_result048.png │ │ ├── IALM_result049.png │ │ ├── IALM_result050.png │ │ ├── IALM_result051.png │ │ ├── IALM_result052.png │ │ ├── IALM_result053.png │ │ ├── IALM_result054.png │ │ ├── IALM_result055.png │ │ ├── IALM_result056.png │ │ ├── IALM_result057.png │ │ ├── IALM_result058.png │ │ ├── IALM_result059.png │ │ ├── IALM_result060.png │ │ ├── IALM_result061.png │ │ ├── IALM_result062.png │ │ ├── IALM_result063.png │ │ ├── IALM_result064.png │ │ ├── IALM_result065.png │ │ ├── IALM_result066.png │ │ ├── IALM_result067.png │ │ ├── IALM_result068.png │ │ ├── IALM_result069.png │ │ ├── IALM_result070.png │ │ ├── IALM_result071.png │ │ ├── IALM_result072.png │ │ ├── IALM_result073.png │ │ ├── IALM_result074.png │ │ ├── IALM_result075.png │ │ ├── IALM_result076.png │ │ ├── IALM_result077.png │ │ ├── IALM_result078.png │ │ ├── IALM_result079.png │ │ ├── IALM_result080.png │ │ ├── IALM_result081.png │ │ ├── IALM_result082.png │ │ ├── IALM_result083.png │ │ ├── IALM_result084.png │ │ ├── IALM_result085.png │ │ ├── IALM_result086.png │ │ ├── IALM_result087.png │ │ ├── IALM_result088.png │ │ ├── IALM_result089.png │ │ ├── IALM_result090.png │ │ ├── IALM_result091.png │ │ ├── IALM_result092.png │ │ ├── IALM_result093.png │ │ ├── IALM_result094.png │ │ ├── IALM_result095.png │ │ ├── IALM_result096.png │ │ ├── IALM_result097.png │ │ ├── IALM_result098.png │ │ └── IALM_result099.png ├── __pycache__ │ ├── PCA.cpython-37.pyc │ ├── constrainedOptimization.cpython-36.pyc │ ├── constrainedOptimization.cpython-37.pyc │ ├── myClass.cpython-36.pyc │ └── myClass.cpython-37.pyc ├── constrainedOptimization.py ├── main.py └── myClass.py ├── 实验一、阅读并实现比较Online learning rate adaptation with hypergradient descent 1703.04782.pdf ├── 实验报告(一)-19S003037-于晟健.pdf ├── 实验报告(二)-19S003037-于晟健.pdf ├── 组合优化与凸优化-实验要求及课程报告要求.pdf └── 阅读报告-19S003037-于晟健.pdf /Active Learning for Cost-Sensitive Classification.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NeilYSJ/Combinatorial-Optimization-and-Convex-Optimization/ecc3ae911a10373c9357ca694b884c40bf050796/Active Learning for Cost-Sensitive Classification.pdf -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/.idea/Lab1_unconstrainedOptimization.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 12 | 13 | 23 | 24 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 1589617500055 176 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 221 | 222 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | 339 | 340 | 341 | 342 | 343 | 344 | 345 | 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | 373 | 374 | 375 | 376 | 377 | 378 | 379 | 380 | 381 | 382 | 383 | 384 | 385 | 386 | 387 | 388 | 389 | 390 | 391 | 392 | 393 | 394 | 395 | 396 | 397 | 398 | 399 | 400 | 401 | 402 | 403 | 404 | 405 | 406 | 407 | 408 | 409 | 410 | 411 | 412 | 413 | 414 | 415 | 416 | 417 | 418 | 419 | 420 | 421 | 422 | 423 | 424 | 425 | 426 | 427 | 428 | 429 | 430 | 431 | 432 | 433 | 434 | 435 | 436 | 437 | 438 | 439 | 440 | 441 | 442 | 443 | 444 | 445 | 446 | 447 | 448 | 449 | 450 | 451 | 452 | 453 | 454 | 455 | 456 | 457 | 458 | 459 | 460 | 461 | 462 | 463 | 464 | 465 | 466 | 467 | 468 | 469 | 470 | 471 | 472 | 473 | 474 | 475 | 476 | 477 | 478 | 479 | 480 | 481 | 482 | 483 | 484 | 485 | 486 | 487 | 488 | 489 | 490 | 491 | 492 | 493 | 494 | 495 | 496 | 497 | 498 | 499 | 500 | 501 | 502 | 503 | 504 | 505 | 506 | 507 | 508 | 509 | 510 | 511 | 512 | 513 | 514 | 515 | 516 | 517 | 518 | 519 | 520 | 521 | 522 | 523 | 524 | 525 | 526 | 527 | 528 | 529 | 530 | 531 | 532 | 533 | 534 | 535 | 536 | 537 | 538 | 539 | 540 | 541 | 542 | 543 | 544 | 545 | 546 | 547 | 548 | 549 | 550 | 551 | 552 | 553 | 554 | 555 | 556 | 557 | 558 | 559 | 560 | 561 | 562 | 563 | 564 | 565 | 566 | 567 | 568 | 569 | 570 | 571 | 572 | 573 | 574 | 575 | 576 | 577 | 578 | 579 | 580 | 581 | 582 | 583 | 584 | 585 | 586 | 587 | 588 | 589 | 590 | 591 | 592 | 593 | 594 | 595 | 596 | 597 | 598 | 599 | 600 | 601 | 602 | 603 | 604 | 605 | 606 | 607 | 608 | 609 | 610 | 611 | 612 | 613 | 614 | 615 | 616 | 617 | 618 | 619 | 620 | 621 | 622 | 623 | 624 | 625 | 626 | 627 | 628 | 629 | 630 | 631 | 632 | 633 | 634 | 635 | 636 | 637 | 638 | 639 | 640 | 641 | 642 | 643 | 644 | 645 | 646 | 647 | 648 | 649 | 650 | 651 | 652 | 653 | 654 | 655 | 656 | 657 | 658 | 659 | 660 | 661 | 662 | 663 | 664 | 665 | 666 | -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/__pycache__/derivativeFreeMethod.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NeilYSJ/Combinatorial-Optimization-and-Convex-Optimization/ecc3ae911a10373c9357ca694b884c40bf050796/Lab1_unconstrainedOptimization/__pycache__/derivativeFreeMethod.cpython-36.pyc -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/__pycache__/firstDerivativeMethod.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NeilYSJ/Combinatorial-Optimization-and-Convex-Optimization/ecc3ae911a10373c9357ca694b884c40bf050796/Lab1_unconstrainedOptimization/__pycache__/firstDerivativeMethod.cpython-36.pyc -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/__pycache__/hyperGradientDescent.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NeilYSJ/Combinatorial-Optimization-and-Convex-Optimization/ecc3ae911a10373c9357ca694b884c40bf050796/Lab1_unconstrainedOptimization/__pycache__/hyperGradientDescent.cpython-36.pyc -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/__pycache__/lassoRegression.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NeilYSJ/Combinatorial-Optimization-and-Convex-Optimization/ecc3ae911a10373c9357ca694b884c40bf050796/Lab1_unconstrainedOptimization/__pycache__/lassoRegression.cpython-36.pyc -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/__pycache__/myClass.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NeilYSJ/Combinatorial-Optimization-and-Convex-Optimization/ecc3ae911a10373c9357ca694b884c40bf050796/Lab1_unconstrainedOptimization/__pycache__/myClass.cpython-36.pyc -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/__pycache__/oneDimensionalSearch.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NeilYSJ/Combinatorial-Optimization-and-Convex-Optimization/ecc3ae911a10373c9357ca694b884c40bf050796/Lab1_unconstrainedOptimization/__pycache__/oneDimensionalSearch.cpython-36.pyc -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/__pycache__/secondDerivativeMethod.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NeilYSJ/Combinatorial-Optimization-and-Convex-Optimization/ecc3ae911a10373c9357ca694b884c40bf050796/Lab1_unconstrainedOptimization/__pycache__/secondDerivativeMethod.cpython-36.pyc -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/derivativeFreeMethod.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | @author: Neil.YU 4 | @license: (C) Copyright 2013-2018, Node Supply Chain Manager Corporation Limited. 5 | @contact: neil_yushengjian@foxmail.com 6 | @software: PyCharm 2018.1.2 7 | @file: derivativeFreeMethod.py 8 | @time: 2020/5/16 17:47 9 | @desc: 10 | ''' 11 | 12 | import math 13 | import numpy as np 14 | from myClass import point, rosenbrock 15 | from oneDimensionalSearch import golden_search, fibonacci_search, dichotomous_search 16 | from oneDimensionalSearch import armijo_goldstein_search,wolfe_powell_search 17 | 18 | 19 | def cyclic_coordinate_method(loss_function: rosenbrock, start: point, method='golden_search', epsilon=10e-1, 20 | k_max=10000) -> list: 21 | """ 22 | 23 | :param loss_function: 24 | :param start: 25 | :param method: 26 | :param epsilon: 27 | :param k_max: 28 | :return: 29 | """ 30 | x, M, k = [start], len(start), 0 31 | 32 | while True: 33 | # if meet the termination conditions then break 34 | gradient = loss_function.g(x[k]) 35 | if k > k_max or np.linalg.norm(gradient) < epsilon: break 36 | # find the new x 37 | direction = [0] * M 38 | direction[np.mod(k, M)] = 1 39 | if method == 'golden_search': 40 | step = golden_search(loss_function, x[k], direction) 41 | elif method == 'fibonacci_search': 42 | step = fibonacci_search(loss_function, x[k], direction) 43 | elif method == 'dichotomous_search': 44 | step = dichotomous_search(loss_function, x[k], direction) 45 | else: 46 | return x 47 | x.append(x[k] + point(direction[0] * step, direction[1] * step)) 48 | k += 1 49 | 50 | return x 51 | 52 | -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/firstDerivativeMethod.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | @author: Neil.YU 4 | @license: (C) Copyright 2013-2018, Node Supply Chain Manager Corporation Limited. 5 | @contact: neil_yushengjian@foxmail.com 6 | @software: PyCharm 2018.1.2 7 | @file: firstDerivativeMethod.py 8 | @time: 2020/5/16 17:45 9 | @desc: 10 | ''' 11 | 12 | import numpy as np 13 | from myClass import point, rosenbrock 14 | from oneDimensionalSearch import golden_search, fibonacci_search, dichotomous_search 15 | from oneDimensionalSearch import armijo_goldstein_search, wolfe_powell_search 16 | 17 | 18 | def steepest_descent(loss_function: rosenbrock, start: point, method='golden_search', epsilon=10e-2, 19 | k_max=10000) -> list: 20 | """ 21 | 22 | :param loss_function: 23 | :param start: 24 | :param method: 25 | :param epsilon: 26 | :param k_max: 27 | :return: 28 | """ 29 | x, k = [start], 0 30 | 31 | while True: 32 | # if meet the termination conditions then break 33 | gradient = loss_function.g(x[k]) 34 | if k > k_max or np.linalg.norm(gradient) < epsilon: break 35 | 36 | # find the new x 37 | direction = -gradient / np.linalg.norm(gradient) 38 | if method == 'golden_search': 39 | step = golden_search(loss_function, x[k], direction) 40 | elif method == 'fibonacci_search': 41 | step = fibonacci_search(loss_function, x[k], direction) 42 | elif method == 'dichotomous_search': 43 | step = dichotomous_search(loss_function, x[k], direction) 44 | else: 45 | return x 46 | p = step * direction 47 | x.append(x[k] + point(p[0], p[1])) 48 | k += 1 49 | 50 | return x 51 | 52 | 53 | def conjugate_gradient(loss_function: rosenbrock, start: point, method='golden_search', epsilon=10e-2, 54 | k_max=10000) -> list: 55 | """ 56 | 57 | :param loss_function: 58 | :param start: 59 | :param method: 60 | :param epsilon: 61 | :param k_max: 62 | :return: 63 | """ 64 | x, direction, k = [start], -1 * loss_function.g(start) / np.linalg.norm(loss_function.g(start)), 0 65 | 66 | while True: 67 | # if meet the termination conditions then break 68 | gradient_old = loss_function.g(x[k]) / np.linalg.norm(loss_function.g(x[k])) 69 | if np.linalg.norm(loss_function.g(x[k])) < epsilon or k > k_max: break 70 | 71 | # find the new x 72 | if method == 'golden_search': 73 | step = golden_search(loss_function, x[k], direction) 74 | elif method == 'fibonacci_search': 75 | step = fibonacci_search(loss_function, x[k], direction) 76 | elif method == 'dichotomous_search': 77 | step = dichotomous_search(loss_function, x[k], direction) 78 | else: 79 | return x 80 | x.append(x[k] + point(direction[0] * step, direction[1] * step)) 81 | 82 | # update the direction 83 | gradient_new = loss_function.g(x[k + 1]) / np.linalg.norm(loss_function.g(x[k + 1])) 84 | alpha = np.dot(gradient_new, gradient_new) / np.dot(gradient_old, gradient_old) 85 | direction = -gradient_new + alpha * direction 86 | k += 1 87 | 88 | return x 89 | 90 | 91 | def plain_gradient_descent(loss_function: rosenbrock, start: point, step=0.1, epsilon=10e-2, k_max=10000) -> list: 92 | """ 93 | 94 | :param loss_function: 95 | :param start: 96 | :param step: 97 | :param epsilon: 98 | :param k_max: 99 | :return: 100 | """ 101 | x, k = [start], 0 102 | 103 | while True: 104 | # if meet the termination conditions then break 105 | gradient = loss_function.g(x[k]) 106 | if k > k_max or np.linalg.norm(gradient) < epsilon: break 107 | 108 | # find the new x 109 | direction = -gradient / np.linalg.norm(gradient) 110 | p = step * direction 111 | x.append(x[k] + point(p[0], p[1])) 112 | k += 1 113 | 114 | return x 115 | 116 | 117 | def Momentum(loss_function: rosenbrock, start: point, step=0.1, rho=0.7, epsilon=10e-2, k_max=10000) -> list: 118 | """ 119 | 120 | :param loss_function: 121 | :param start: 122 | :param step: 123 | :param rho: the influence of historical gradients 124 | :param epsilon: 125 | :param k_max: 126 | :return: 127 | """ 128 | x, k = [start], 0 129 | direction = -loss_function.g(start) / np.linalg.norm(loss_function.g(start)) 130 | p = step * direction 131 | x.append(x[k] + point(p[0], p[1])) 132 | 133 | while True: 134 | k += 1 135 | # if meet the termination conditions then break 136 | gradient = loss_function.g(x[k]) 137 | if k > k_max or np.linalg.norm(gradient) < epsilon: break 138 | 139 | # find the new x 140 | direction = -gradient / np.linalg.norm(gradient) 141 | # add the historical p 142 | p = rho * p + step * direction 143 | x.append(x[k] + point(p[0], p[1])) 144 | 145 | return x 146 | 147 | 148 | def Nesterov_momentum(loss_function: rosenbrock, start: point, step=0.1, rho=0.7, mu=0.2, epsilon=10e-2, 149 | k_max=10000) -> list: 150 | """ 151 | 152 | :param loss_function: 153 | :param start: 154 | :param step: 155 | :param rho: the influence of historical gradients 156 | :param beta: ahead rate 157 | :param epsilon: 158 | :param k_max: 159 | :return: 160 | """ 161 | x, k = [start], 0 162 | direction = -loss_function.g(start) / np.linalg.norm(loss_function.g(start)) 163 | p = step * direction 164 | x.append(x[k] + point(p[0], p[1])) 165 | 166 | while True: 167 | k += 1 168 | # if meet the termination conditions then break 169 | if k > k_max or np.linalg.norm(loss_function.g(x[k])) < epsilon: break 170 | 171 | # find the new x 172 | # ahead p * mu 173 | gradient = loss_function.g(x[k] + point(p[0] * mu, p[1] * mu)) 174 | direction = -gradient / np.linalg.norm(gradient) 175 | # add the historical p 176 | p = rho * p + step * direction 177 | x.append(x[k] + point(p[0], p[1])) 178 | 179 | return x 180 | 181 | 182 | def Adagrad(loss_function: rosenbrock, start: point, initial_step=0.1, epsilon=10e-2, k_max=10000) -> list: 183 | """ 184 | Adaptive Gradient 185 | :param loss_function: 186 | :param start: 187 | :param initial_step: 188 | :param epsilon: 189 | :param k_max: 190 | :return: 191 | """ 192 | x, k = [start], 0 193 | delte = np.array([10e-7] * len(start)) 194 | 195 | r = np.zeros(len(start)) 196 | while True: 197 | gradient = loss_function.g(x[k]) 198 | # if meet the termination conditions then break 199 | if k > k_max or np.linalg.norm(gradient) < epsilon: break 200 | gradient = -gradient / np.linalg.norm(gradient) 201 | # find the new x 202 | r = r + gradient ** 2 203 | p = initial_step * gradient / (r + delte) ** 0.5 204 | x.append(x[k] + point(p[0], p[1])) 205 | k += 1 206 | 207 | return x 208 | 209 | 210 | def RMSprop(loss_function: rosenbrock, start: point, initial_step=0.1, rho=0.99, epsilon=10e-2, k_max=10000) -> list: 211 | """ 212 | root mean squared 213 | :param loss_function: 214 | :param start: 215 | :param initial_step: 216 | :param rho: 217 | :param epsilon: 218 | :param k_max: 219 | :return: 220 | """ 221 | x, k = [start], 0 222 | delta = np.array([10e-7] * len(start)) 223 | 224 | r = np.zeros(len(start)) 225 | while True: 226 | gradient = loss_function.g(x[k]) 227 | # if meet the termination conditions then break 228 | if k > k_max or np.linalg.norm(gradient) < epsilon: break 229 | gradient = -gradient / np.linalg.norm(gradient) 230 | # find the new x 231 | # add the influence rate of historical to r 232 | r = rho * r + (1 - rho) * gradient ** 2 233 | p = initial_step * gradient / (r + delta) ** 0.5 234 | x.append(x[k] + point(p[0], p[1])) 235 | k += 1 236 | 237 | return x 238 | 239 | 240 | def Adadelta(loss_function: rosenbrock, start: point, rho=0.99, epsilon=10e-2, k_max=10000) -> list: 241 | """ 242 | 243 | :param loss_function: 244 | :param start: 245 | :param rho: 246 | :param epsilon: 247 | :param k_max: 248 | :return: 249 | """ 250 | x, k = [start], 0 251 | delta = np.array([10e-7] * len(start)) 252 | 253 | step = np.zeros(len(start)) 254 | r = np.zeros(len(start)) 255 | while True: 256 | gradient = loss_function.g(x[k]) 257 | # if meet the termination conditions then break 258 | if k > k_max or np.linalg.norm(gradient) < epsilon: break 259 | gradient = -gradient / np.linalg.norm(gradient) 260 | # find the new x 261 | # add the influence rate of historical to r 262 | r = rho * r + (1 - rho) * gradient ** 2 263 | p = gradient * ((step + delta) / (r + delta)) ** 0.5 264 | x.append(x[k] + point(p[0], p[1])) 265 | step = rho * step + (1 - rho) * p ** 2 266 | k += 1 267 | 268 | return x 269 | 270 | 271 | def Adam(loss_function: rosenbrock, start: point, initial_step=0.1, rho0=0.9, rho1=0.99, epsilon=10e-2, 272 | k_max=10000) -> list: 273 | """ 274 | Adaptive momentum 275 | :param loss_function: 276 | :param start: 277 | :param initial_step: 278 | :param rho0: 279 | :param rho1: 280 | :param epsilon: 281 | :param k_max: 282 | :return: 283 | """ 284 | x, k = [start], 0 285 | delta = np.array([10e-7] * len(start)) 286 | 287 | r = np.zeros(len(start)) 288 | direction = -loss_function.g(start) / np.linalg.norm(loss_function.g(start)) 289 | while True: 290 | gradient = loss_function.g(x[k]) 291 | # if meet the termination conditions then break 292 | if k > k_max or np.linalg.norm(gradient) < epsilon: break 293 | gradient = -gradient / np.linalg.norm(gradient) 294 | # add the influence rate of historical to direction 295 | direction = rho0 * direction + (1 - rho0) * gradient 296 | # find the new x 297 | # add the influence rate of historical to r 298 | r = rho1 * r + (1 - rho1) * gradient ** 2 299 | p = initial_step * direction / (r + delta) ** 0.5 300 | x.append(x[k] + point(p[0], p[1])) 301 | k += 1 302 | 303 | return x 304 | -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/hyperGradientDescent.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | @author: Neil.YU 4 | @license: (C) Copyright 2013-2018, Node Supply Chain Manager Corporation Limited. 5 | @contact: neil_yushengjian@foxmail.com 6 | @software: PyCharm 2018.1.2 7 | @file: hyperGradientDescent.py 8 | @time: 2020/5/22 15:48 9 | @desc: 10 | ''' 11 | 12 | import numpy as np 13 | from myClass import point, rosenbrock 14 | 15 | 16 | def plain_gradient_descent_HD(loss_function: rosenbrock, start: point, initial_step=0.1, beta=0.001, epsilon=10e-2, 17 | k_max=10000) -> list: 18 | """ 19 | 20 | :param loss_function: 21 | :param start: 22 | :param step: 23 | :param epsilon: 24 | :param k_max: 25 | :return: 26 | """ 27 | x, k, step = [start], 0, initial_step 28 | direction_old = -loss_function.g(start) / np.linalg.norm(loss_function.g(start)) 29 | p = step * direction_old 30 | x.append(x[k] + point(p[0], p[1])) 31 | 32 | while True: 33 | k += 1 34 | # if meet the termination conditions then break 35 | gradient = loss_function.g(x[k]) 36 | if k > k_max or np.linalg.norm(gradient) < epsilon: break 37 | 38 | # find the new x 39 | direction_new = -gradient / np.linalg.norm(gradient) 40 | # update the step 41 | step = step + beta * np.dot(direction_new, direction_old) 42 | p = step * direction_new 43 | x.append(x[k] + point(p[0], p[1])) 44 | 45 | direction_old = direction_new 46 | 47 | return x 48 | 49 | 50 | def Nesterov_momentum_HD(loss_function: rosenbrock, start: point, initial_step=0.1, rho=0.7, mu=0.2, beta=0.001, 51 | epsilon=10e-2, k_max=10000) -> list: 52 | """ 53 | 54 | :param loss_function: 55 | :param start: 56 | :param step: 57 | :param rho: the influence of historical gradients 58 | :param beta: ahead rate 59 | :param epsilon: 60 | :param k_max: 61 | :return: 62 | """ 63 | x, k, step = [start], 0, initial_step 64 | direction_old = -loss_function.g(start) / np.linalg.norm(loss_function.g(start)) 65 | p = step * direction_old 66 | x.append(x[k] + point(p[0], p[1])) 67 | 68 | while True: 69 | k += 1 70 | # if meet the termination conditions then break 71 | if k > k_max or np.linalg.norm(loss_function.g(x[k])) < epsilon: break 72 | 73 | # find the new x 74 | # ahead p * beta 75 | gradient = loss_function.g(x[k] + point(p[0] * mu, p[1] * mu)) 76 | direction_new = -gradient / np.linalg.norm(gradient) 77 | # update the step 78 | step = step + beta * np.dot(direction_new, direction_old) 79 | # add the historical p 80 | p = rho * p + step * direction_new 81 | x.append(x[k] + point(p[0], p[1])) 82 | 83 | direction_old = direction_new 84 | 85 | return x 86 | 87 | 88 | def Adam_HD(loss_function: rosenbrock, start: point, initial_step=0.1, rho0=0.9, rho1=0.99, beta=10e-7, epsilon=10e-2, 89 | k_max=10000) -> list: 90 | """ 91 | Adaptive momentum 92 | :param loss_function: 93 | :param start: 94 | :param initial_step: 95 | :param rho0: 96 | :param rho1: 97 | :param epsilon: 98 | :param k_max: 99 | :return: 100 | """ 101 | x, k, step = [start], 0, initial_step 102 | delta = np.array([10e-7] * len(start)) 103 | 104 | r = np.zeros(len(start)) 105 | direction = -loss_function.g(start) / np.linalg.norm(loss_function.g(start)) 106 | uk_old = direction 107 | r = rho1 * r + (1 - rho1) * direction ** 2 108 | p = step * uk_old 109 | x.append(x[k] + point(p[0], p[1])) 110 | 111 | while True: 112 | k += 1 113 | gradient = loss_function.g(x[k]) 114 | # if meet the termination conditions then break 115 | if k > k_max or np.linalg.norm(gradient) < epsilon: break 116 | gradient = -gradient / np.linalg.norm(gradient) 117 | # add the influence rate of historical to direction 118 | direction = rho0 * direction + (1 - rho0) * gradient 119 | uk_new = direction / (r + delta) ** 0.5 120 | # find the new x 121 | # add the influence rate of historical to r 122 | r = rho1 * r + (1 - rho1) * gradient ** 2 123 | # update the step 124 | step = step + beta * np.dot(uk_new, uk_old) 125 | p = step * uk_new 126 | x.append(x[k] + point(p[0], p[1])) 127 | 128 | uk_old = uk_new 129 | 130 | return x 131 | -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/lassoRegression.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | @author: Neil.YU 4 | @license: (C) Copyright 2013-2018, Node Supply Chain Manager Corporation Limited. 5 | @contact: neil_yushengjian@foxmail.com 6 | @software: PyCharm 2018.1.2 7 | @file: lassoRegression.py 8 | @time: 2020/5/23 14:42 9 | @desc: 10 | ''' 11 | 12 | # LASSO Regression: Least Absolute Shrinkage and Selection Operator Regression whose loss function is not differentiable 13 | # CD/LARS: coordinate descent/least Angle Regression 14 | 15 | import math 16 | import pandas as pd 17 | import numpy as np 18 | import numpy.matlib 19 | from myClass import lassoregression 20 | 21 | 22 | def advance_retreat_method(loss_function: lassoregression, start: np.mat, direction: np.mat, step=0, 23 | delta=0.1) -> tuple: 24 | """ 25 | find the initial section of step 26 | :param loss_function: 27 | :param start: 28 | :param direction: 29 | :param step: 30 | :param delta: 31 | :return: 32 | """ 33 | alpha0, point0 = step, start 34 | 35 | alpha1 = alpha0 + delta 36 | point1 = point0 + direction * delta 37 | if loss_function.f(point0) < loss_function.f(point1): 38 | while True: 39 | delta *= 2 40 | alpha2 = alpha0 - delta 41 | point2 = point0 - direction * delta 42 | if loss_function.f(point2) < loss_function.f(point0): 43 | alpha1, alpha0 = alpha0, alpha2 44 | point1, point0 = point0, point2 45 | else: 46 | return alpha2, alpha1 47 | else: 48 | while True: 49 | delta *= 2 50 | alpha2 = alpha1 + delta 51 | point2 = point1 + direction * delta 52 | if loss_function.f(point2) < loss_function.f(point1): 53 | alpha0, alpha1 = alpha1, alpha2 54 | point0, point1 = point1, point2 55 | else: 56 | return alpha0, alpha2 57 | 58 | 59 | def golden_search(loss_function: lassoregression, start: np.mat, direction: np.mat, epsilon=0.1) -> float: 60 | """ 61 | derivative-free to search the longest step 62 | :param loss_function: 63 | :param start: 64 | :param direction: 65 | :param epsilon: 66 | :return: 67 | """ 68 | a, b = advance_retreat_method(loss_function, start, direction) 69 | 70 | # find the minimum 71 | golden_num = (math.sqrt(5) - 1) / 2 72 | p, q = a + (1 - golden_num) * (b - a), a + golden_num * (b - a) 73 | while abs(a - b) > epsilon: 74 | f_p = loss_function.f(start + direction * p) 75 | f_q = loss_function.f(start + direction * q) 76 | if f_p < f_q: 77 | b, q = q, p 78 | p = a + (1 - golden_num) * (b - a) 79 | else: 80 | a, p = p, q 81 | q = a + golden_num * (b - a) 82 | 83 | return (a + b) / 2 84 | 85 | 86 | def coordinate_descent(loss_function: lassoregression, epsilon=10e-2, k_max=350) -> list: 87 | """ 88 | 89 | :param loss_function: 90 | :param epsilon: 91 | :param k_max: 92 | :return: 93 | """ 94 | theta = np.matlib.zeros((loss_function.X.shape[1], 1)) 95 | M, k = theta.shape[0], 0 96 | 97 | temp = np.around(theta.T.tolist()[0], 3) 98 | theta_list = [temp] 99 | direction = np.matlib.zeros((M, 1)) 100 | while True: 101 | # update the theta on each dimension 102 | for i in range(M): 103 | direction[np.mod(i - 1, M), 0] = 0 104 | direction[np.mod(i, M), 0] = 1 105 | step = golden_search(loss_function, theta, direction) 106 | theta = theta + direction * step 107 | k += 1 108 | temp = np.around(theta.T.tolist()[0], 3) 109 | theta_list.append(temp) 110 | # if meet the termination conditions then break 111 | if k > k_max or np.max(abs(theta_list[k] - theta_list[k - 1])) < epsilon: 112 | break 113 | 114 | return theta_list 115 | 116 | 117 | def least_angle_regression(X: pd.DataFrame, Y: pd.DataFrame, k_max=350) -> list: 118 | """ 119 | 120 | :param X: 121 | :param Y: 122 | :param k_max: 123 | :return: 124 | """ 125 | # set the length of step 126 | step = 0.004 127 | 128 | theta = np.array([0.0] * X.shape[1]) 129 | theta_list = [list(theta)] 130 | for k in range(k_max): 131 | # count the residual 132 | residual = Y - X.dot(theta) 133 | # count the correlation between features and residual 134 | corr = residual.dot(X) 135 | # find the feature which has the max correlation, update its theta 136 | idx = np.abs(corr).argmax() 137 | corrMax = corr[idx] 138 | theta[idx] += step * corrMax / abs(corrMax) 139 | # save the result 140 | theta_list.append(list(theta)) 141 | 142 | return theta_list 143 | -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/main.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | @author: Neil.YU 4 | @license: (C) Copyright 2013-2018, Node Supply Chain Manager Corporation Limited. 5 | @contact: neil_yushengjian@foxmail.com 6 | @software: PyCharm 2018.1.2 7 | @file: main.py 8 | @time: 2020/5/16 16:25 9 | @desc: 10 | ''' 11 | 12 | import time 13 | import pandas as pd 14 | import numpy as np 15 | 16 | from myClass import point, rosenbrock, drawResult, drawResult2, drawGifResult 17 | from derivativeFreeMethod import cyclic_coordinate_method 18 | from firstDerivativeMethod import steepest_descent, conjugate_gradient 19 | from firstDerivativeMethod import plain_gradient_descent, Momentum, Nesterov_momentum 20 | from firstDerivativeMethod import Adagrad, RMSprop, Adadelta, Adam 21 | from secondDerivativeMethod import Newton_method, DFP, BFGS 22 | from hyperGradientDescent import plain_gradient_descent_HD, Nesterov_momentum_HD, Adam_HD 23 | 24 | from myClass import lassoregression, drawTheta 25 | from lassoRegression import coordinate_descent, least_angle_regression 26 | 27 | 28 | def mission_1(): 29 | a, b, step = 1, 10, 0.1 30 | loss_function, start = rosenbrock(a, b), point(5, -10) 31 | 32 | ###### derivative-free method 33 | # step search method 34 | epsilon = 10e-1 35 | method = ['golden_search', 'fibonacci_search', 'dichotomous_search'] 36 | # """ 37 | for m in method: 38 | points = cyclic_coordinate_method(loss_function, start, method=m, epsilon=epsilon) 39 | #drawResult(loss_function, points, start, 'cyclic_coordinate_method by %s' % m, epsilon) 40 | drawResult2(loss_function, points, start, 'cyclic_coordinate_method by %s' % m, epsilon) 41 | # """ 42 | 43 | ###### first derivative method 44 | # rho denotes the influence rate of historical p on the new p, mu denotes the ahead rate in Momentum 45 | # rho0 denotes the influence rate of historical direction on the new direction in Adam 46 | # rho1 denotes the influence rate of historical r on the new r in Adam 47 | epsilon = 10e-1 48 | rho, mu = 0.8, 0.2 49 | rho0, rho1 = 0.9, 0.99 50 | # """ 51 | points = steepest_descent(loss_function, start, method=method[0], epsilon=epsilon) 52 | # drawResult(loss_function, points, start, 'steepest_descent by %s' % method[0], epsilon) 53 | drawResult2(loss_function, points, start, 'steepest_descent by %s' % method[0], epsilon) 54 | 55 | points = conjugate_gradient(loss_function, start, method=method[0], epsilon=epsilon) 56 | # drawResult(loss_function, points, start, 'conjugate_gradient by %s' % method[0], epsilon) 57 | drawResult2(loss_function, points, start, 'conjugate_gradient by %s' % method[0], epsilon) 58 | # """ 59 | 60 | # """ 61 | points = plain_gradient_descent(loss_function, start, step=step, epsilon=epsilon) 62 | # drawResult(loss_function, points, start, 'plain_gradient_descent', epsilon, otherlabel=',step=%.2g' % step) 63 | drawResult2(loss_function, points, start, 'plain_gradient_descent', epsilon, otherlabel=',step=%.2g' % step) 64 | 65 | points = Momentum(loss_function, start, step=step, rho=rho, epsilon=epsilon) 66 | # drawResult(loss_function, points, start, 'Momentum', epsilon, otherlabel=',step=%.2g,rho=%.2g' % (step, rho)) 67 | drawResult2(loss_function, points, start, 'Momentum', epsilon, otherlabel=',step=%.2g,rho=%.2g' % (step, rho)) 68 | 69 | points = Nesterov_momentum(loss_function, start, step=step, rho=rho, mu=mu, epsilon=epsilon) 70 | # drawResult(loss_function, points, start, 'Nesterov_momentum', epsilon, 71 | # otherlabel=',step=%.2g,rho=%.2g,mu=%.2g' % (step, rho, mu)) 72 | drawResult2(loss_function, points, start, 'Nesterov_momentum', epsilon, 73 | otherlabel=',step=%.2g,rho=%.2g,mu=%.2g' % (step, rho, mu)) 74 | # """ 75 | 76 | # """ 77 | points = Adagrad(loss_function, start, initial_step=step, epsilon=epsilon) 78 | # drawResult(loss_function, points, start, 'Adagrad', epsilon, otherlabel=',initial_step=%.2g' % step) 79 | drawResult2(loss_function, points, start, 'Adagrad', epsilon, otherlabel=',initial_step=%.2g' % step) 80 | 81 | points = RMSprop(loss_function, start, initial_step=step, rho=rho1, epsilon=epsilon) 82 | # drawResult(loss_function, points, start, 'RMSProp', epsilon, 83 | # otherlabel=',initial_step=%.2g,rho=%.2g' % (step, rho1)) 84 | drawResult2(loss_function, points, start, 'RMSProp', epsilon, 85 | otherlabel=',initial_step=%.2g,rho=%.2g' % (step, rho1)) 86 | 87 | points = Adadelta(loss_function, start, rho=rho1, epsilon=epsilon) 88 | # drawResult(loss_function, points, start, 'Adadelta', epsilon, otherlabel=',rho=%.2g' % rho1) 89 | drawResult2(loss_function, points, start, 'Adadelta', epsilon, otherlabel=',rho=%.2g' % rho1) 90 | 91 | points = Adam(loss_function, start, initial_step=step, rho0=rho0, rho1=rho1, epsilon=epsilon) 92 | # drawResult(loss_function, points, start, 'Adam', epsilon, 93 | # otherlabel=',initial_step=%.2g,rho0=%.2g,rho1=%.2g' % (step, rho0, rho1)) 94 | drawResult2(loss_function, points, start, 'Adam', epsilon, 95 | otherlabel=',initial_step=%.2g,rho0=%.2g,rho1=%.2g' % (step, rho0, rho1)) 96 | # """ 97 | 98 | ###### hyper gradient descent 99 | # beta denotes the influence rate of historical direction on the new step in hyperGradientDescent 100 | # """ 101 | beta0, beta1 = 0.01, 10e-7 102 | 103 | points = plain_gradient_descent_HD(loss_function, start, initial_step=step, beta=beta0, epsilon=epsilon) 104 | # drawResult(loss_function, points, start, 'plain_gradient_descent_HD', epsilon, 105 | # otherlabel=',initial_step=%.2g,beta=%.2g' % (step, beta0)) 106 | drawResult2(loss_function, points, start, 'plain_gradient_descent_HD', epsilon, 107 | otherlabel=',initial_step=%.2g,beta=%.2g' % (step, beta0)) 108 | 109 | points = Nesterov_momentum_HD(loss_function, start, initial_step=step, rho=rho, mu=mu, beta=beta0, epsilon=epsilon) 110 | # drawResult(loss_function, points, start, 'Nesterov_momentum_HD', epsilon, 111 | # otherlabel=',initial_step=%.2g,rho=%.2g,mu=%.2g,beta=%.2g' % (step, rho, mu, beta0)) 112 | drawResult2(loss_function, points, start, 'Nesterov_momentum_HD', epsilon, 113 | otherlabel=',initial_step=%.2g,rho=%.2g,mu=%.2g,beta=%.2g' % (step, rho, mu, beta0)) 114 | 115 | points = Adam_HD(loss_function, start, initial_step=step, rho0=rho0, rho1=rho1, beta=beta1, epsilon=epsilon) 116 | # drawResult(loss_function, points, start, 'Adam_HD', epsilon, 117 | # otherlabel=',initial_step=%.2g,rho0=%.2g,rho1=%.2g,beta=%.2g' % (step, rho0, rho1, beta1)) 118 | drawResult2(loss_function, points, start, 'Adam_HD', epsilon, 119 | otherlabel=',initial_step=%.2g,rho0=%.2g,rho1=%.2g,beta=%.2g' % (step, rho0, rho1, beta1)) 120 | # """ 121 | 122 | ###### second derivative method 123 | # """ 124 | epsilon = 10e-2 125 | 126 | points = Newton_method(loss_function, start, method=method[0], epsilon=epsilon) 127 | # drawResult(loss_function, points, start, 'Newton_method by %s' % method[0], epsilon) 128 | drawResult2(loss_function, points, start, 'Newton_method by %s' % method[0], epsilon) 129 | 130 | points = DFP(loss_function, start, method=method[0], epsilon=epsilon) 131 | # drawResult(loss_function, points, start, 'DFP by %s' % method[0], epsilon) 132 | drawResult2(loss_function, points, start, 'DFP by %s' % method[0], epsilon) 133 | 134 | points = BFGS(loss_function, start, method=method[0], epsilon=epsilon) 135 | # drawResult(loss_function, points, start, 'BFGS by %s' % method[0], epsilon) 136 | drawResult2(loss_function, points, start, 'BFGS by %s' % method[0], epsilon) 137 | # """ 138 | 139 | 140 | def mission_2(): 141 | # data.columns denotes the number of features, data.index denotes the number of data 142 | data = pd.read_csv('sonar.all-data', header=None, prefix='V') 143 | data['V60'] = data.iloc[:, -1].apply(lambda v: 1.0 if v == 'M' else 0.0) 144 | # print(len(data.index),len(data.columns)) 145 | # normalizing 146 | norm_data = (data - data.mean()) / data.std() 147 | # set the test data 148 | N = len(data.index) // 8 149 | X = data.iloc[N:, : -1].append(data.iloc[len(data.index) - N:, : -1]) 150 | Y = data.iloc[N:, -1].append(data.iloc[len(data.index) - N:, -1]) 151 | 152 | ###### coordinate_descent 153 | epsilon = 0.025 154 | loss_function = lassoregression(X=norm_data.values[N:len(data.index) - N, : -1], 155 | Y=norm_data.values[N:len(data.index) - N, -1], 156 | N=len(norm_data.index) - 2 * N) 157 | start = time.time() 158 | thetas = coordinate_descent(loss_function, epsilon=epsilon) 159 | end = time.time() 160 | # count the error of coordinate_descent 161 | theta = np.array(thetas[-1]) 162 | error = np.linalg.norm(Y - X.dot(theta)) / len(Y.index) 163 | print("coordinate_descent\ttime:%f\terror:%f" % (end - start, error)) 164 | # print(pd.value_counts(thetas[-1])) 165 | drawTheta('coordinate_descent (epsilon=%.3g)' % epsilon, thetas) 166 | 167 | ###### least_angle_regression 168 | start = time.time() 169 | thetas = least_angle_regression(X=norm_data.values[N:len(data.index) - N, : -1], 170 | Y=norm_data.values[N:len(data.index) - N, -1]) 171 | end = time.time() 172 | # count the error of least_angle_regression 173 | theta = np.array(thetas[-1]) 174 | error = np.linalg.norm(Y - X.dot(theta)) / len(Y.index) 175 | print("least_angle_regression\ttime:%f\terror:%f" % (end - start, error)) 176 | # print(pd.value_counts(thetas[-1])) 177 | drawTheta('least_angle_regression', thetas) 178 | 179 | 180 | if __name__ == '__main__': 181 | mission_1() 182 | # mission_2() 183 | -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/myClass.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | @author: Neil.YU 4 | @license: (C) Copyright 2013-2018, Node Supply Chain Manager Corporation Limited. 5 | @contact: neil_yushengjian@foxmail.com 6 | @software: PyCharm 2018.1.2 7 | @file: myClass.py 8 | @time: 2020/5/16 18:09 9 | @desc: 10 | ''' 11 | 12 | import numpy as np 13 | import matplotlib.pyplot as plt 14 | from mpl_toolkits.mplot3d import Axes3D 15 | 16 | 17 | class point: 18 | x, y = 0, 0 19 | 20 | def __init__(self, x, y): 21 | self.x, self.y = x, y 22 | 23 | def __add__(self, other): 24 | return point(self.x + other.x, self.y + other.y) 25 | 26 | def __sub__(self, other): 27 | return point(self.x - other.x, self.y - other.y) 28 | 29 | def __len__(self): 30 | return 2 31 | 32 | def __str__(self): 33 | return str(self.x) + "," + str(self.y) 34 | 35 | 36 | class rosenbrock: 37 | a, b = 1.0, 1.0 38 | 39 | def __init__(self, a, b): 40 | self.a, self.b = a * 1.0, b * 1.0 41 | 42 | def f(self, p: point): 43 | return (self.a - p.x) ** 2 + self.b * (p.y - p.x ** 2) ** 2 44 | 45 | def g(self, p: point): 46 | return np.array([-4 * self.b * p.x * (p.y - p.x ** 2) - 2 * (self.a - p.x), 2 * self.b * (p.y - p.x ** 2)]) 47 | 48 | def H(self, p: point): 49 | return np.array( 50 | [[12 * self.b * p.x * p.x - 4 * self.b * p.y + 2, -4 * self.b * p.x], [-4 * self.b * p.x, 2 * self.b]]) 51 | 52 | 53 | class lassoregression: 54 | beta, N = 0.002, 0 55 | X, Y = 0, 0 56 | 57 | def __init__(self, X, Y, N): 58 | self.X, self.Y, self.N = np.mat(X), np.mat(Y).T, N 59 | 60 | def f(self, theta: np.array): 61 | return (((self.X * theta - self.Y).T * (self.X * theta - self.Y)) / (2 * self.N) 62 | + self.beta * np.linalg.norm(theta, ord=1))[0, 0] 63 | 64 | def drawTheta(label: str, result: list): 65 | plt.plot(result) 66 | plt.grid(True) 67 | 68 | plt.title(label) 69 | plt.xlabel("step") 70 | plt.ylabel("theta") 71 | 72 | plt.show() 73 | 74 | 75 | def drawResult(loss_function: rosenbrock, points: list, start: point, label: str, epsilon: float, otherlabel=''): 76 | fig = plt.figure() 77 | ax = Axes3D(fig) 78 | plt.title( 79 | label + ' (a=' + str(loss_function.a) + ',b=' + str( 80 | loss_function.b) + otherlabel + ',epsilon=' + '%.e' % epsilon + ',iteration=' + str(len(points)) + ')') 81 | # draw the function 82 | X = np.arange(-5.5, 5.5 + 0.05, 0.05) 83 | Y = np.arange(-30, 30 + 0.05, 0.05) 84 | X, Y = np.meshgrid(X, Y) 85 | Z = loss_function.f(point(X, Y)) 86 | ax.plot_surface(Y, X, Z, cmap='rainbow', alpha=0.55) 87 | plt.contour(Y, X, Z, 60, colors='black', alpha=0.55) 88 | # draw the line of points 89 | x, y, z = [], [], [] 90 | for p in points: 91 | x.append(p.x) 92 | y.append(p.y) 93 | z.append(loss_function.f(p)) 94 | ax.plot3D(y, x, z, 'r*-') 95 | # draw the start point 96 | ax.scatter(start.y, start.x, loss_function.f(start), color='black') 97 | ax.text(start.y, start.x, loss_function.f(start), 'start(' + str(start.x) + ',' + str(start.y) + ')') 98 | # draw the end point 99 | end = points[len(points) - 1] 100 | ax.scatter(end.y, end.x, loss_function.f(end), color='black') 101 | ax.text(end.y, end.x, loss_function.f(end), 102 | 'end(' + '%.2f' % end.x + ',' + '%.2f' % end.y + ',' + '%.2f' % loss_function.f(end) + ')', 103 | verticalalignment='top') 104 | # draw the target point 105 | target = point(loss_function.a, loss_function.a ** 2) 106 | ax.scatter(target.y, target.x, loss_function.f(target), color='black') 107 | ax.text(target.y, target.x, loss_function.f(target), 108 | 'target(' + str(target.x) + ',' + str(target.y) + ',' + str(loss_function.f(target)) + ')', 109 | verticalalignment='bottom') 110 | 111 | plt.show() 112 | 113 | 114 | def drawGifResult(loss_function: rosenbrock, points: list, start: point, label: str, epsilon: float, otherlabel=''): 115 | plt.ion() 116 | fig = plt.figure() 117 | ax = Axes3D(fig) 118 | # draw the function 119 | plt.title( 120 | label + ' (a=' + str(loss_function.a) + ',b=' + str( 121 | loss_function.b) + otherlabel + ',epsilon=' + '%.e' % epsilon + ',iteration=' + str(len(points)) + ')') 122 | X = np.arange(-5.5, 5.5 + 0.05, 0.05) 123 | Y = np.arange(-30, 30 + 0.05, 0.05) 124 | X, Y = np.meshgrid(X, Y) 125 | Z = loss_function.f(point(X, Y)) 126 | ax.plot_surface(Y, X, Z, cmap='rainbow', alpha=0.55) 127 | plt.contour(Y, X, Z, 60, colors='black', alpha=0.55) 128 | # draw the start point 129 | ax.scatter(start.y, start.x, loss_function.f(start), color='black') 130 | ax.text(start.y, start.x, loss_function.f(start), 'start(' + str(start.x) + ',' + str(start.y) + ')') 131 | # draw the target point 132 | target = point(loss_function.a, loss_function.a ** 2) 133 | ax.scatter(target.y, target.x, loss_function.f(target), color='black') 134 | ax.text(target.y, target.x, loss_function.f(target), 135 | 'target(' + str(target.x) + ',' + str(target.y) + ',' + str(loss_function.f(target)) + ')', 136 | verticalalignment='bottom') 137 | # draw the line of points 138 | k = 0 139 | end = points[len(points) - 1] 140 | x, y, z = [], [], [] 141 | while True: 142 | if k == len(points): 143 | ax.plot3D(y, x, z, 'r*-') 144 | ax.scatter(end.y, end.x, loss_function.f(end), color='black') 145 | ax.text(end.y, end.x, loss_function.f(end), 146 | 'end(' + '%.2f' % end.x + ',' + '%.2f' % end.y + ',' + '%.2f' % loss_function.f(end) + ')', 147 | verticalalignment='top') 148 | break 149 | x.append(points[k].x) 150 | y.append(points[k].y) 151 | z.append(loss_function.f(points[k])) 152 | ax.plot3D(y, x, z, 'r*-') 153 | k += 1 154 | plt.pause(0.1) 155 | 156 | plt.pause(1000) 157 | 158 | def drawResult2(loss_function: rosenbrock, points: list, start: point, label: str, epsilon: float, otherlabel=''): 159 | plt.figure() 160 | plt.title( 161 | label + ' (a=' + str(loss_function.a) + ',b=' + str( 162 | loss_function.b) +',iteration=' + str(len(points)) + ')') 163 | # draw the function 164 | X = np.arange(-5.5, 5.5 + 0.05, 0.05) 165 | Y = np.arange(-30, 30 + 0.05, 0.05) 166 | X, Y = np.meshgrid(X, Y) 167 | Z = loss_function.f(point(X, Y)) 168 | contour = plt.contour(Y, X, Z, colors='k') 169 | plt.clabel(contour, fontsize=8, colors='k') 170 | 171 | # draw the line of points 172 | x, y= [], [] 173 | for p in points: 174 | x.append(p.x) 175 | y.append(p.y) 176 | plt.plot(y, x, 'r*-') 177 | # draw the start point 178 | plt.scatter(start.y, start.x, color='black') 179 | plt.text(start.y, start.x, 'start(' + str(start.x) + ',' + str(start.y) + ')') 180 | # draw the end point 181 | end = points[len(points) - 1] 182 | plt.scatter(end.y, end.x, color='black') 183 | plt.text(end.y, end.x, 184 | 'end(' + '%.2f' % end.x + ',' + '%.2f' % end.y + ',' + '%.2f' % loss_function.f(end) + ')', 185 | verticalalignment='top') 186 | # draw the target point 187 | target = point(loss_function.a, loss_function.a ** 2) 188 | plt.scatter(target.y, target.x, color='black') 189 | plt.text(target.y, target.x, 190 | 'target(' + str(target.x) + ',' + str(target.y) + ',' + str(loss_function.f(target)) + ')', 191 | verticalalignment='bottom') 192 | 193 | plt.show() -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/oneDimensionalSearch.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | @author: Neil.YU 4 | @license: (C) Copyright 2013-2018, Node Supply Chain Manager Corporation Limited. 5 | @contact: neil_yushengjian@foxmail.com 6 | @software: PyCharm 2018.1.2 7 | @file: oneDimensionalSearch.py 8 | @time: 2020/5/19 14:05 9 | @desc: 10 | ''' 11 | 12 | import math 13 | import random 14 | import numpy as np 15 | from myClass import point, rosenbrock 16 | 17 | 18 | # Accurate One-dimensional Search 19 | 20 | def advance_retreat_method(loss_function: rosenbrock, start: point, direction: list, step=0, delta=0.1) -> tuple: 21 | """ 22 | find the initial section of step 23 | :param loss_function: 24 | :param start: 25 | :param direction: 26 | :param step: 27 | :param delta: 28 | :return: 29 | """ 30 | alpha0, point0 = step, start 31 | 32 | alpha1 = alpha0 + delta 33 | point1 = point0 + point(direction[0] * delta, direction[1] * delta) 34 | if loss_function.f(point0) < loss_function.f(point1): 35 | while True: 36 | delta *= 2 37 | alpha2 = alpha0 - delta 38 | point2 = point0 - point(direction[0] * delta, direction[1] * delta) 39 | if loss_function.f(point2) < loss_function.f(point0): 40 | alpha1, alpha0 = alpha0, alpha2 41 | point1, point0 = point0, point2 42 | else: 43 | return alpha2, alpha1 44 | else: 45 | while True: 46 | delta *= 2 47 | alpha2 = alpha1 + delta 48 | point2 = point1 + point(direction[0] * delta, direction[1] * delta) 49 | if loss_function.f(point2) < loss_function.f(point1): 50 | alpha0, alpha1 = alpha1, alpha2 51 | point0, point1 = point1, point2 52 | else: 53 | return alpha0, alpha2 54 | 55 | 56 | def golden_search(loss_function: rosenbrock, start: point, direction: list, epsilon=0.1) -> float: 57 | """ 58 | derivative-free to search the longest step 59 | :param loss_function: 60 | :param start: 61 | :param direction: 62 | :param epsilon: 63 | :return: 64 | """ 65 | a, b = advance_retreat_method(loss_function, start, direction) 66 | 67 | # find the minimum 68 | golden_num = (math.sqrt(5) - 1) / 2 69 | p, q = a + (1 - golden_num) * (b - a), a + golden_num * (b - a) 70 | while abs(a - b) > epsilon: 71 | f_p = loss_function.f(start + point(direction[0] * p, direction[1] * p)) 72 | f_q = loss_function.f(start + point(direction[0] * q, direction[1] * q)) 73 | if f_p < f_q: 74 | b, q = q, p 75 | p = a + (1 - golden_num) * (b - a) 76 | else: 77 | a, p = p, q 78 | q = a + golden_num * (b - a) 79 | 80 | return (a + b) / 2 81 | 82 | 83 | def fibonacci_search(loss_function: rosenbrock, start: point, direction: list, epsilon=1) -> float: 84 | """ 85 | derivative-free to search the longest step 86 | :param loss_function: 87 | :param start: 88 | :param direction: 89 | :param epsilon: 90 | :return: 91 | """ 92 | a, b = advance_retreat_method(loss_function, start, direction) 93 | 94 | # build the Fibonacci series 95 | F, d = [1.0, 2.0], (b - a) / epsilon 96 | while F[-1] < d: F.append(F[-1] + F[-2]) 97 | 98 | # find the minimum 99 | N = len(F) - 1 100 | p, q = a + (1 - F[N - 1] / F[N]) * (b - a), a + F[N - 1] / F[N] * (b - a) 101 | while abs(a - b) > epsilon and N > 0: 102 | N = N - 1 103 | f_p = loss_function.f(start + point(direction[0] * p, direction[1] * p)) 104 | f_q = loss_function.f(start + point(direction[0] * q, direction[1] * q)) 105 | if f_p < f_q: 106 | b, q = q, p 107 | p = a + (1 - F[N - 1] / F[N]) * (b - a) 108 | else: 109 | a, p = p, q 110 | q = a + F[N - 1] / F[N] * (b - a) 111 | 112 | return (a + b) / 2 113 | 114 | 115 | def dichotomous_search(loss_function: rosenbrock, start: point, direction: list, epsilon=0.1) -> float: 116 | """ 117 | derivative-free to search the longest step 118 | :param loss_function: 119 | :param start: 120 | :param direction: 121 | :param epsilon: 122 | :return: 123 | """ 124 | a, b = advance_retreat_method(loss_function, start, direction) 125 | 126 | # find the minimum 127 | e = epsilon / 3 128 | p, q = (a + b) / 2 - e, (a + b) / 2 + e 129 | while abs(a - b) > epsilon: 130 | f_p = loss_function.f(start + point(direction[0] * p, direction[1] * p)) 131 | f_q = loss_function.f(start + point(direction[0] * q, direction[1] * q)) 132 | if f_p < f_q: 133 | b = q 134 | else: 135 | a = p 136 | p, q = (a + b) / 2 - e, (a + b) / 2 + e 137 | 138 | return (a + b) / 2 139 | 140 | 141 | # Inaccurate One-dimensional Search 142 | 143 | def armijo_goldstein_search(loss_function: rosenbrock, start: point, direction: list, rho=0.1) -> float: 144 | """ 145 | 146 | :param loss_function: 147 | :param start: 148 | :param direction: 149 | :param rho: meet condition 0= (1 - rho) * alpha * gradient_f1: 167 | print(alpha) 168 | return alpha 169 | else: 170 | a, b = alpha, b 171 | if b < alpha: 172 | alpha = (a + b) / 2 173 | else: 174 | alpha = 2 * alpha 175 | else: 176 | a, b = a, alpha 177 | alpha = (a + b) / 2 178 | 179 | 180 | def wolfe_powell_search(loss_function: rosenbrock, start: point, direction: list, rho=0.1) -> float: 181 | pass 182 | -------------------------------------------------------------------------------- /Lab1_unconstrainedOptimization/secondDerivativeMethod.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | ''' 3 | @author: Neil.YU 4 | @license: (C) Copyright 2013-2018, Node Supply Chain Manager Corporation Limited. 5 | @contact: neil_yushengjian@foxmail.com 6 | @software: PyCharm 2018.1.2 7 | @file: secondDerivativeMethod.py 8 | @time: 2020/5/16 17:45 9 | @desc: 10 | ''' 11 | 12 | import numpy as np 13 | from myClass import point, rosenbrock 14 | from oneDimensionalSearch import golden_search, fibonacci_search, dichotomous_search 15 | 16 | 17 | def Newton_method(loss_function: rosenbrock, start: point, method='golden_search', epsilon=10e-2, k_max=10000) -> list: 18 | """ 19 | 20 | :param loss_function: 21 | :param start: 22 | :param step: 23 | :param epsilon: 24 | :param k_max: 25 | :return: 26 | """ 27 | x, k = [start], 0 28 | 29 | while True: 30 | # if meet the termination conditions then break 31 | gradient = loss_function.g(x[k]) 32 | if k > k_max or np.linalg.norm(gradient) < epsilon: break 33 | 34 | # find the new x 35 | inverse = np.linalg.inv(loss_function.H(x[k])) 36 | direction = -np.matmul(inverse, gradient) 37 | if method == 'golden_search': 38 | step = golden_search(loss_function, x[k], direction) 39 | elif method == 'fibonacci_search': 40 | step = fibonacci_search(loss_function, x[k], direction) 41 | elif method == 'dichotomous_search': 42 | step = dichotomous_search(loss_function, x[k], direction) 43 | else: 44 | return x 45 | p = step * direction 46 | x.append(x[k] + point(p[0], p[1])) 47 | k += 1 48 | 49 | return x 50 | 51 | 52 | def DFP(loss_function: rosenbrock, start: point, method='golden_search', epsilon=10e-2, k_max=10000) -> list: 53 | """ 54 | Davidon Fletcher Powell, quasi_newton_method 55 | :param loss_function: 56 | :param start: 57 | :param step: 58 | :param epsilon: 59 | :param k_max: 60 | :return: 61 | """ 62 | x, k = [start], 0 63 | D = np.identity(len(start)) # Identity matrix 64 | 65 | while True: 66 | # if meet the termination conditions then break 67 | gradient = loss_function.g(x[k]) 68 | if k > k_max or np.linalg.norm(gradient) < epsilon: break 69 | 70 | # find the new x 71 | gradient = gradient / np.linalg.norm(gradient) 72 | direction = -np.matmul(D, gradient) 73 | if method == 'golden_search': 74 | step = golden_search(loss_function, x[k], direction) 75 | elif method == 'fibonacci_search': 76 | step = fibonacci_search(loss_function, x[k], direction) 77 | elif method == 'dichotomous_search': 78 | step = dichotomous_search(loss_function, x[k], direction) 79 | else: 80 | return x 81 | p = step * direction 82 | x.append(x[k] + point(p[0], p[1])) 83 | # update the D 84 | yk = np.mat(loss_function.g(x[k + 1]) / np.linalg.norm(loss_function.g(x[k + 1])) - gradient).T 85 | pk = np.mat(p).T 86 | Dk = np.mat(D) 87 | D = D + np.array((pk * pk.T) / (pk.T * yk) - (Dk * yk * yk.T * Dk) / (yk.T * Dk * yk)) 88 | k += 1 89 | 90 | return x 91 | 92 | 93 | def BFGS(loss_function: rosenbrock, start: point, method='golden_search', epsilon=10e-2, k_max=10000) -> list: 94 | """ 95 | Broyden Fletcher Goldfarb Shanno 96 | :param loss_function: 97 | :param start: 98 | :param method: 99 | :param epsilon: 100 | :param k_max: 101 | :return: 102 | """ 103 | x, k = [start], 0 104 | B = np.identity(len(start)) # Identity matrix 105 | 106 | while True: 107 | # if meet the termination conditions then break 108 | gradient = loss_function.g(x[k]) 109 | if k > k_max or np.linalg.norm(gradient) < epsilon: break 110 | 111 | # find the new x 112 | gradient = gradient / np.linalg.norm(gradient) 113 | direction = -np.matmul(np.linalg.inv(B), gradient) 114 | if method == 'golden_search': 115 | step = golden_search(loss_function, x[k], direction) 116 | elif method == 'fibonacci_search': 117 | step = fibonacci_search(loss_function, x[k], direction) 118 | elif method == 'dichotomous_search': 119 | step = dichotomous_search(loss_function, x[k], direction) 120 | else: 121 | return x 122 | p = step * direction 123 | x.append(x[k] + point(p[0], p[1])) 124 | # update the B 125 | yk = np.mat(loss_function.g(x[k + 1]) / np.linalg.norm(loss_function.g(x[k + 1])) - gradient).T 126 | pk = np.mat(p).T 127 | Bk = np.mat(B) 128 | B = B + np.array((yk * yk.T) / (yk.T * pk) - (Bk * pk * pk.T * Bk) / (pk.T * Bk * pk)) 129 | k += 1 130 | 131 | return x 132 | -------------------------------------------------------------------------------- /Lab2_constrainedOptimization/.idea/Lab2_constrainedOptimization.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | -------------------------------------------------------------------------------- /Lab2_constrainedOptimization/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | -------------------------------------------------------------------------------- /Lab2_constrainedOptimization/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /Lab2_constrainedOptimization/.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 70 | 71 | 72 | 80 | 81 | 82 | 83 | 84 | true 85 | DEFINITION_ORDER 86 | 87 | 88 | 89 | 90 | 91 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 |