├── .github └── workflows │ └── wheels.yml ├── .gitignore ├── .ipynb_checkpoints ├── FairSVM-checkpoint.ipynb ├── QR-checkpoint.ipynb └── SVM-checkpoint.ipynb ├── .readthedocs.yaml ├── .vscode └── settings.json ├── CMakeLists.txt ├── LICENSE ├── README.md ├── doc ├── Makefile ├── make.bat └── source │ ├── _static │ ├── benchmark │ │ ├── benchmark_FairSVM.html │ │ ├── benchmark_FairSVM_benchopt_run_2023-10-12_22h11m38.html │ │ ├── benchmark_Huber.html │ │ ├── benchmark_Huber_benchopt_run_2023-10-10_22h40m00.html │ │ ├── benchmark_QR.html │ │ ├── benchmark_QR_benchopt_run_2023-10-14_22h48m25.html │ │ ├── benchmark_SVM.html │ │ ├── benchmark_SVM_benchopt_run_2023-10-10_20h12m15.html │ │ ├── benchmark_sSVM.html │ │ └── benchmark_sSVM_benchopt_run_2023-10-10_20h52m22.html │ └── css │ │ └── label.css │ ├── _templates │ └── autoapi │ │ ├── index.rst │ │ ├── macros.rst │ │ └── python │ │ ├── class.rst │ │ └── module.rst │ ├── autoapi │ ├── index.rst │ └── rehline │ │ └── index.rst │ ├── benchmark.rst │ ├── clean_notebooks.py │ ├── conf.py │ ├── example.rst │ ├── examples │ ├── .ipynb_checkpoints │ │ ├── FairSVM-checkpoint.ipynb │ │ └── RankRegression-checkpoint.ipynb │ ├── CQR.ipynb │ ├── FairSVM.ipynb │ ├── Path_solution.ipynb │ ├── QR.ipynb │ ├── RankRegression.ipynb │ ├── SVM.ipynb │ ├── Sklearn_Mixin.ipynb │ └── Warm_start.ipynb │ ├── figs │ ├── logo.png │ ├── res.png │ └── tab.png │ ├── getting_started.rst │ ├── index.rst │ ├── tutorials.rst │ └── tutorials │ ├── ReHLine_ERM.rst │ ├── ReHLine_MF.rst │ ├── ReHLine_manual.rst │ ├── ReHLine_sklearn.rst │ ├── constraint.rst │ ├── loss.rst │ └── warmstart.rst ├── figs ├── res.png └── tab.png ├── pyproject.toml ├── rehline ├── __init__.py ├── _base.py ├── _class.py ├── _data.py ├── _loss.py ├── _path_sol.py └── _sklearn_mixin.py ├── requirements.txt ├── setup.py ├── src ├── rehline.cpp └── rehline.h ├── tests ├── _test_cqr.py ├── _test_fairsvm.py ├── _test_path_sol.py ├── _test_sklearn_mixin.py ├── _test_svm.py ├── _test_svr.py └── _test_warmstart.py └── to-do.md /.github/workflows/wheels.yml: -------------------------------------------------------------------------------- 1 | name: Build and upload to PyPI 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - .gitignore 7 | - README.md 8 | - LICENSE 9 | - doc/ 10 | release: 11 | types: 12 | - published 13 | 14 | jobs: 15 | build_wheels: 16 | name: Build wheels on ${{ matrix.os }} 17 | runs-on: ${{ matrix.os }} 18 | strategy: 19 | # macos-13 is an intel runner, macos-14 is apple silicon 20 | matrix: 21 | os: [ubuntu-latest, windows-latest, macos-13, macos-latest] 22 | 23 | steps: 24 | - uses: actions/checkout@v4 25 | 26 | - name: Build wheels 27 | uses: pypa/cibuildwheel@v2.21.3 28 | env: 29 | CIBW_BUILD_VERBOSITY: 1 30 | 31 | - uses: actions/upload-artifact@v4 32 | with: 33 | name: rehline-wheels-${{ matrix.os }}-${{ strategy.job-index }} 34 | path: ./wheelhouse/*.whl 35 | 36 | build_sdist: 37 | name: Build source distribution 38 | runs-on: ubuntu-latest 39 | steps: 40 | - uses: actions/checkout@v4 41 | 42 | - name: Build sdist 43 | run: pipx run build --sdist 44 | 45 | - uses: actions/upload-artifact@v4 46 | with: 47 | name: rehline-sdist 48 | path: dist/*.tar.gz 49 | 50 | upload_pypi: 51 | needs: [build_wheels, build_sdist] 52 | runs-on: ubuntu-latest 53 | environment: pypi 54 | permissions: 55 | id-token: write 56 | if: github.event_name == 'release' && github.event.action == 'published' 57 | # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) 58 | # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') 59 | steps: 60 | - uses: actions/download-artifact@v4 61 | with: 62 | # unpacks all regot artifacts into dist/ 63 | pattern: rehline-* 64 | path: dist 65 | merge-multiple: true 66 | 67 | - uses: pypa/gh-action-pypi-publish@release/v1 68 | # with: 69 | # To test: repository-url: https://test.pypi.org/legacy/ 70 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Config files 2 | 3 | # Binary files 4 | *.o 5 | *.so 6 | *.dll 7 | .DS_Store 8 | *.csv 9 | 10 | # Python cache and build files 11 | **/__pycache__/ 12 | build/ 13 | rehline.egg-info/ 14 | *.log 15 | env/ 16 | 17 | # Files during package building 18 | eigen-3.4.0/ 19 | eigen3.zip 20 | 21 | # Package sdist 22 | *.tar.gz 23 | *.whl 24 | 25 | .vscode/* 26 | !.vscode/settings.json 27 | !.vscode/tasks.json 28 | !.vscode/launch.json 29 | !.vscode/extensions.json 30 | !.vscode/*.code-snippets 31 | 32 | # Local History for Visual Studio Code 33 | .history/ 34 | 35 | # Built Visual Studio Code Extensions 36 | *.vsix 37 | 38 | # python checkpoint 39 | *-checkpoint.ipynb 40 | *-checkpoint.py 41 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/SVM-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "fbcb401d-6ca6-4933-abd5-f8f504282416", 6 | "metadata": {}, 7 | "source": [ 8 | "# **SVM**\n", 9 | "\n", 10 | "[![Slides](https://img.shields.io/badge/🦌-ReHLine-blueviolet)](https://rehline-python.readthedocs.io/en/latest/)\n", 11 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1f_7t1t6FNxAooQOmpyhHCOVq0IKgMxe-?usp=sharing)\n", 12 | "\n", 13 | "SVMs solve the following optimization problem:\n", 14 | "$$\n", 15 | " \\min_{\\mathbf{\\beta} \\in \\mathbb{R}^d} \\ C \\sum_{i=1}^n ( 1 - y_i \\mathbf{\\beta}^\\intercal \\mathbf{x}_i )_+ + \\frac{1}{2} \\| \\mathbf{\\beta} \\|_2^2\n", 16 | "$$\n", 17 | "where $\\mathbf{x}_i \\in \\mathbb{R}^d$ is a feature vector, and $y_i \\in \\{-1, 1\\}$ is a binary label.\n", 18 | "\n", 19 | "> **Note.** Since the hinge loss is a plq function, thus we can solve it by `rehline.plqERM_Ridge`." 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 1, 25 | "id": "2dd1c096-e0df-492f-be63-8ac272007237", 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "## simulate data\n", 30 | "from sklearn.datasets import make_classification\n", 31 | "from sklearn.preprocessing import StandardScaler\n", 32 | "import numpy as np\n", 33 | "\n", 34 | "scaler = StandardScaler()\n", 35 | "\n", 36 | "n, d = 10000, 5\n", 37 | "X, y = make_classification(n_samples=n, n_features=d)\n", 38 | "## convert y to +1/-1\n", 39 | "y = 2*y - 1\n", 40 | "X = scaler.fit_transform(X)" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 2, 46 | "id": "aece9fbe-f9be-40ae-8179-b44849fb0fd3", 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "## solve SVM via `plqERM_Ridge`\n", 51 | "from rehline import plqERM_Ridge\n", 52 | "\n", 53 | "clf = plqERM_Ridge(loss={'name': 'svm'}, C=1.0)\n", 54 | "clf.fit(X=X, y=y)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 3, 60 | "id": "93719987-c6b3-4a9b-9b40-c35e5bf90ef0", 61 | "metadata": {}, 62 | "outputs": [ 63 | { 64 | "data": { 65 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2jUlEQVR4nO3de3xU9Z3/8ffMJIRwScItCZFcwFK5iIKAkGJbkJhI2a4s1EulFNFCoYEiab1QqVC84GIrURpBrQW7laWr1q0FqoZYcFeClyiWm2wRMQMhiTSQAEouc87vD36ZNUsyM0kmc2ZOXs/H4zzqzPnOmc+ZJuQ93/M936/DNE1TAAAANuW0ugAAAICORNgBAAC2RtgBAAC2RtgBAAC2RtgBAAC2RtgBAAC2RtgBAAC2FmV1AeHAMAyVlZWpZ8+ecjgcVpcDAAACYJqmzpw5o5SUFDmdLfffEHYklZWVKTU11eoyAABAG7jdbg0YMKDF/YQdST179pR04cOKi4uzuBoAABCImpoapaamev+Ot4SwI3kvXcXFxRF2AACIMP6GoDBAGQAA2BphBwAA2BphBwAA2BpjdgAAiBAej0f19fVWlxEy0dHRcrlc7T4OYQcAgDBnmqbKy8t1+vRpq0sJuYSEBCUnJ7drHjzCDgAAYa4x6CQmJqpbt26dYgJc0zT1+eefq7KyUpLUv3//Nh+LsAMAQBjzeDzeoNOnTx+rywmp2NhYSVJlZaUSExPbfEmLAcoAAISxxjE63bp1s7gSazSed3vGKhF2AACIAJ3h0lVzgnHehB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AFzEMQx6PJ6DNMAyrywUQZL/73e/Up08f1dbWNnl+2rRpmjVrlkVVtR1hB0AThmEoIyNdUVFRAW0ZGekEHsBmbrzxRnk8Hr3yyive5yorK7V161bdfvvtFlbWNsygDKAJ0zTldh/TqU+P+J2t1OPxqFf6IJmmGaLqAIRCbGysbr31Vm3YsEE33nijJOn3v/+90tLSNHHiRGuLawPCDoBmuVyuoKw2DCAyzZ07V2PHjtXx48d1ySWXaOPGjbrtttsicnJDwg4AALjIqFGjdOWVV+p3v/udsrOztX//fm3dutXqstqEsAMAAJr1gx/8QPn5+Tp+/LiysrKUmppqdUltwgBlAADQrFtvvVXHjh3TM888E5EDkxsRdgAAQLPi4+M1Y8YM9ejRQ9OmTbO6nDYj7AAAgBYdP35cM2fOVExMjNWltBljdgAAwEVOnTqlHTt2aMeOHXryySetLqddCDsA2s3j8fht43A45HTSmQxEilGjRunUqVP613/9V1122WVWl9MuhB0AbWYYhlwuZ0Dd26mpA3T06KcEHiBCHD161OoSgoawA6DNTNOUx2Oo6ujHiopq+Z8TZloGYCXCDoB2Y7ZlAOGM/mQAAGBrhB0AAGBrhB0AAGBrjNkBACBClZaW6uTJkyF5r759+yotLS0k7xVshB0AACJQaWmphg4dqs8//zwk79etWzcdPHiwXYHnj3/8o9avX6+SkhJVVVXpgw8+0MiRI4NXZAsIOwAQQQzDCOgWfiZxtL+TJ0/q888/17O/fkKXDR7coe916O9/1x0Lf6yTJ0+2K+ycO3dO11xzjW666SbNnTs3iBX6RtgBgAhhGIZSU1NVVlbmt21KSorcbjeBpxO4bPBgjbpihNVlBGTWrFmSQj9hIWEHACKEaZoqKyvT46uf9hliDMPQ4rvnMYkj8P8RdgAgwjidTnpsgFaw9LdlxYoVcjgcTbYhQ4Z4958/f165ubnq06ePevTooRkzZqiioqLJMUpLSzV16lR169ZNiYmJuuuuu9TQ0BDqUwEAAF/y/PPPq0ePHt7tv/7rvyyrxfKeneHDh2v79u3ex19eX2fJkiXaunWrXnjhBcXHx2vhwoWaPn263nrrLUkX1tuZOnWqkpOTtWvXLp04cULf//73FR0drYcffjjk5wKEu0AGtwaygjkA+PPP//zPGjdunPfxJZdcYlktloedqKgoJScnX/R8dXW1nn32WW3atEnXXnutJGnDhg0aOnSodu/erfHjx+v111/XgQMHtH37diUlJWnkyJF64IEHdM8992jFihXq0qVLs+9ZW1ur2tpa7+OampqOOTkgjBiGoYyMdLndxwJqz3gPAO3Rs2dP9ezZ0+oyJIXBDMp///vflZKSokGDBmnmzJkqLS2VJJWUlKi+vl5ZWVnetkOGDFFaWpqKi4slScXFxRoxYoSSkpK8bXJyclRTU6P9+/e3+J6rVq1SfHy8d0tNTe2gswPCh2macruP6dSnR1Rz7NMWt5NH/m51qQBsqqqqSnv27NGBAwckSYcOHdKePXtUXl7eoe9rac/OuHHjtHHjRl122WU6ceKEfvGLX+jrX/+69u3bp/LycnXp0kUJCQlNXpOUlOT9UMrLy5sEncb9jftasnTpUuXl5Xkf19TUEHjQafhboZzVy4HIcujvHf8FJVjv8corr2jOnDnex7fccoskafny5VqxYkVQ3qM5loadKVOmeP/7iiuu0Lhx45Senq7/+I//UGxsbIe9b0xMjGJiYjrs+AAAdLS+ffuqW7duumPhj0Pyft26dVPfvn3bdYzbbrtNt912W3AKagXLx+x8WUJCgr761a/q8OHDuu6661RXV6fTp0836d2pqKjwjvFJTk7WO++80+QYjXdrNTcOCAAAu0hLS9PBgwdZGysAYRV2zp49q48//lizZs3S6NGjFR0draKiIs2YMUPShWt7paWlyszMlCRlZmbqoYceUmVlpRITEyVJhYWFiouL07Bhwyw7DwAAQiEtLS1iA0goWRp2fvrTn+rb3/620tPTVVZWpuXLl8vlcum73/2u4uPjdccddygvL0+9e/dWXFycFi1apMzMTI0fP16SlJ2drWHDhmnWrFlavXq1ysvLtWzZMuXm5nKZCgAASLI47Bw7dkzf/e539Y9//EP9+vXTNddco927d6tfv36SpDVr1sjpdGrGjBmqra1VTk6OnnzySe/rXS6XtmzZogULFigzM1Pdu3fX7NmztXLlSqtOCQAAhBlLw87mzZt97u/atasKCgpUUFDQYpv09HRt27Yt2KUBAACbCKsxOwCA0ApkVu1GDoeDNbkQkfipBYBOyjAMpaamKioqKqAtNTVVhmFYXTbQavTsAEAnZZqmysrK9Pjqp/322BiGocV3z2MZEUQkwg4AdHJOp5PLU7A1wg4AABGqtLSUSQUDQNgBACAClZaWasiQIfriiy9C8n6xsbH66KOPWhV43nzzTT366KMqKSnRiRMn9PLLL2vatGkdV2QLCDsAAESgkydP6osvvtDsmXOVnJTSoe9VXlGm555/RidPnmxV2Dl37pyuvPJK3X777Zo+fXoHVugbYQcAgAiWnJSitAHpVpfRrClTpjRZ9NsqjEgDAAC2RtgBAAC2RtgBAAC2xpgdACHj8Xj8tmFJAgDBRtgB0OEMw5DL5VRMTIzftqmpA3T06KcEHgBBQ9gB0OFM05THY6jq6MeKimr5nx2Px6Ne6YNYkgCwibNnz+rw4cPex5988on27Nmj3r17h3SCQsIOgJBxuVxyuVxWlwHYSnlFWdi+x3vvvadJkyZ5H+fl5UmSZs+erY0bNwajtIAQdgAAiEB9+/ZVbGysnnv+mZC8X2xsrPr27duq10ycODEsemoJOwAARKC0tDR99NFHrI0VAMIOAAARKi0tLWIDSChxuwMAALA1wg4AALA1LmMBNmAYht9BgIFM6AcgfIXDQF8rBOO8CTtAhDMMQxkZ6XK7jwXUvrP+gwlEqujoaEnS559/rtjYWIurCb3PP/9c0v9+Dm1B2AEinGmacruP6dSnR3zOYVNXV6e+gwaHsDJYzV9vHr19kcHlcikhIUGVlZWSpG7dusnhcFhcVcczTVOff/65KisrlZCQ0K45ugg7gE34m7CPyfw6D8MwJCmg5TkkevsiQXJysiR5A09nkpCQ4D3/tiLsAIBN5T+yXi4fy3M0NDRoyb3zJcJO2HM4HOrfv78SExNVX19vdTkhEx0dHZQvaoQdALApp9Ppc0FVFluNPCy50jb8pAMAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFuLsroAAEDk8Hg8fts4HA45nXyXRvgg7AAA/DIMQ5IUExPjt21KSorcbjeBB2GDsAMACFj+I+vlimr5T4dhGFp89zyZphnCqgDfCDsAgIA5nU56bBBx+IkFAAC2RtgBAAC2RtgBAAC2RtgBAAC2RtgBAAC2xt1YABAGDMPwe7t2IBP6AbgYYQcALGYYhlJTU1VWVhZQe+awAVonbMLOI488oqVLl2rx4sXKz8+XJJ0/f14/+clPtHnzZtXW1ionJ0dPPvmkkpKSvK8rLS3VggUL9Ne//lU9evTQ7NmztWrVKkX5mPQKAMKJaZoqKyvT46uf9jmHTUNDg5bcO18i7ACtEhZjdt5991099dRTuuKKK5o8v2TJEv35z3/WCy+8oJ07d6qsrEzTp0/37vd4PJo6darq6uq0a9cuPffcc9q4caPuv//+UJ8CALRb44R9vjYArWf5b87Zs2c1c+ZMPfPMM+rVq5f3+erqaj377LN67LHHdO2112r06NHasGGDdu3apd27d0uSXn/9dR04cEC///3vNXLkSE2ZMkUPPPCACgoKVFdX1+J71tbWqqampskGAADsyfKwk5ubq6lTpyorK6vJ8yUlJaqvr2/y/JAhQ5SWlqbi4mJJUnFxsUaMGNHkslZOTo5qamq0f//+Ft9z1apVio+P926pqalBPisAABAuLA07mzdv1vvvv69Vq1ZdtK+8vFxdunRRQkJCk+eTkpJUXl7ubfPloNO4v3FfS5YuXarq6mrv5na723kmAAAgXFk2itftdmvx4sUqLCxU165dQ/reMTExiomJCel7AgAAa1jWs1NSUqLKykpdddVVioqKUlRUlHbu3KknnnhCUVFRSkpKUl1dnU6fPt3kdRUVFUpOTpYkJScnq6Ki4qL9jfsAAAAsCzuTJ0/W3r17tWfPHu82ZswYzZw50/vf0dHRKioq8r7m0KFDKi0tVWZmpiQpMzNTe/fuVWVlpbdNYWGh4uLiNGzYsJCfEwAACD+WXcbq2bOnLr/88ibPde/eXX369PE+f8cddygvL0+9e/dWXFycFi1apMzMTI0fP16SlJ2drWHDhmnWrFlavXq1ysvLtWzZMuXm5nKZCgAASAqjSQWbs2bNGjmdTs2YMaPJpIKNXC6XtmzZogULFigzM1Pdu3fX7NmztXLlSgurBgAA4SSsws6OHTuaPO7atasKCgpUUFDQ4mvS09O1bdu2Dq4MAABEKsvn2QEAAOhIhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrYTWpIADYiWEYMk3TbzuPxxOCaoDOi7ADAB3AMAylpqaqrKws4NcEEowAtB5hBwA6gGmaKisr0+Orn5bT6XvEQENDg5bcO18i7AAdgrADAB3I6XT6DTv+9gNoH37DAACArdGzAwAIukAGXTscDnq1EBKEHQBA0BiGIUmKiYnx2zYlJUVut5vAgw5H2AEABF3+I+vlimr5T4xhGFp89zzuQENIEHYAAEEXyMBsIFT4SQQAALZG2AEAALZG2AEAALZG2AEAALZG2AEAALZG2AEAALZG2AEAALZG2AEAALZG2AEAALZG2AEAALbGchFAGDMMw+/aQYGsLg0AnRlhBwhThmEoIyNdbvexgNqzoCIANI+wA4Qp0zTldh/TqU+PyOVytdiurq5OfQcNDmFlABBZCDtAmHO5XD7Djq99AAAGKAMAAJsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFuLsroAAPi/PB6P3zYOh0NOJ9/XAPhH2AEQNgzDkMvlVExMjN+2qakDdPTopwQeAH4RdgCEDdM05fEYqjr6saKiWv7nyePxqFf6IJmmGcLqAEQqwg6AsONyueRyuawuA4BN0P8LAABsjbADAABsrU1hZ9CgQfrHP/5x0fOnT5/WoEGD2l0UAABAsLQp7Bw9erTZW0Nra2t1/PjxdhcFAAAQLK0aoPzKK694//u1115TfHy897HH41FRUZEyMjKCVhwAAEB7tapnZ9q0aZo2bZocDodmz57tfTxt2jTdcsstKiws1K9+9auAj7du3TpdccUViouLU1xcnDIzM/WXv/zFu//8+fPKzc1Vnz591KNHD82YMUMVFRVNjlFaWqqpU6eqW7duSkxM1F133aWGhobWnBYAALCxVvXsGIYhSRo4cKDeffdd9e3bt11vPmDAAD3yyCMaPHiwTNPUc889pxtuuEEffPCBhg8friVLlmjr1q164YUXFB8fr4ULF2r69Ol66623JF3oTZo6daqSk5O1a9cunThxQt///vcVHR2thx9+uF21AQAAe2jTPDuffPJJUN7829/+dpPHDz30kNatW6fdu3drwIABevbZZ7Vp0yZde+21kqQNGzZo6NCh2r17t8aPH6/XX39dBw4c0Pbt25WUlKSRI0fqgQce0D333KMVK1aoS5cuQakTAABErjZPKlhUVKSioiJVVlZ6e3wa/fa3v2318Twej1544QWdO3dOmZmZKikpUX19vbKysrxthgwZorS0NBUXF2v8+PEqLi7WiBEjlJSU5G2Tk5OjBQsWaP/+/Ro1alSz71VbW6va2lrv45qamlbXCwAAIkOb7sb6xS9+oezsbBUVFenkyZM6depUk6019u7dqx49eigmJkbz58/Xyy+/rGHDhqm8vFxdunRRQkJCk/ZJSUkqLy+XJJWXlzcJOo37G/e1ZNWqVYqPj/duqampraoZAABEjjb17Kxfv14bN27UrFmz2l3AZZddpj179qi6ulovvviiZs+erZ07d7b7uL4sXbpUeXl53sc1NTUEHgAAbKpNYaeurk5f+9rXglJAly5d9JWvfEWSNHr0aL377rt6/PHHdfPNN6uurk6nT59u0rtTUVGh5ORkSVJycrLeeeedJsdrvFursU1zYmJiAlpVGQAARL42Xcb6wQ9+oE2bNgW7FkkX7viqra3V6NGjFR0draKiIu++Q4cOqbS0VJmZmZKkzMxM7d27V5WVld42hYWFiouL07BhwzqkPgBA8Hg8Hr/b/x0XCrRWm3p2zp8/r6efflrbt2/XFVdcoejo6Cb7H3vssYCOs3TpUk2ZMkVpaWk6c+aMNm3apB07dngnLLzjjjuUl5en3r17Ky4uTosWLVJmZqbGjx8vScrOztawYcM0a9YsrV69WuXl5Vq2bJlyc3PpuQGAMNYYYAL5tzolJUVut1tOJ8s5om3aFHb+9re/aeTIkZKkffv2NdnncDgCPk5lZaW+//3v68SJE4qPj9cVV1yh1157Tdddd50kac2aNXI6nZoxY4Zqa2uVk5OjJ5980vt6l8ulLVu2aMGCBcrMzFT37t01e/ZsrVy5si2nBQAIsfxH1ssV1fKfIsMwtPjueTJNM4RVwW7aFHb++te/BuXNn332WZ/7u3btqoKCAhUUFLTYJj09Xdu2bQtKPQCA0HI6nfTYoMPxEwYAAGytTT07kyZN8nm56o033mhzQQAAAMHUprDTOF6nUX19vfbs2aN9+/Zp9uzZwagLAAAgKNoUdtasWdPs8ytWrNDZs2fbVRAAAEAwBXXMzve+9702rYsFAADQUYIadoqLi9W1a9dgHhIAAKBd2nQZa/r06U0em6apEydO6L333tPPf/7zoBQGAP54PJ6A2jkcDm5vBjqxNoWd+Pj4Jo+dTqcuu+wyrVy5UtnZ2UEpDABaYhiGXC5nwDOlp6YO0NGjnxJ4gE6qTWFnw4YNwa4DAAJmmqY8HkNVRz9WlI/Zd6ULvT+90gcxAy/QibUp7DQqKSnRwYMHJUnDhw/XqFGjglIUAATC5XLJ5XJZXQaAMNemsFNZWalbbrlFO3bsUEJCgiTp9OnTmjRpkjZv3qx+/foFs0YAAIA2a9MF7EWLFunMmTPav3+/qqqqVFVVpX379qmmpkY//vGPg10jAABAm7WpZ+fVV1/V9u3bNXToUO9zw4YNU0FBAQOUAQBAWGlTz45hGIqOjr7o+ejoaBmG0e6iAAAAgqVNYefaa6/V4sWLVVZW5n3u+PHjWrJkiSZPnhy04gAAANqrTWHn17/+tWpqapSRkaFLL71Ul156qQYOHKiamhqtXbs22DUCQFgxDEMej8fvBiA8tGnMTmpqqt5//31t375dH330kSRp6NChysrKCmpxABBuDMNQampqk55tX5jfB7Beq8LOG2+8oYULF2r37t2Ki4vTddddp+uuu06SVF1dreHDh2v9+vX6+te/3iHFAoDVTNNUWVmZHl/9tM8ZmRsaGrTk3vkSYQewXKsuY+Xn52vu3LmKi4u7aF98fLx++MMf6rHHHgtacQAQrpxOp98NQHho1W/jhx9+qOuvv77F/dnZ2SopKWl3UQAAAMHSqrBTUVHR7C3njaKiovTZZ5+1uygAAIBgaVXYueSSS7Rv374W9//tb39T//79210UAABAsLQq7HzrW9/Sz3/+c50/f/6ifV988YWWL1+uf/qnfwpacQAAAO3Vqruxli1bpj/+8Y/66le/qoULF+qyyy6TJH300UcqKCiQx+PRfffd1yGFAgAAtEWrwk5SUpJ27dqlBQsWaOnSpd75IxwOh3JyclRQUKCkpKQOKRQAAKAtWj2pYHp6urZt26ZTp07p8OHDMk1TgwcPVq9evTqiPgAAgHZp0wzKktSrVy+NHTs2mLUAAAAEHbNeAQAAWyPsAAAAWyPsAAAAWyPsAAAAWyPsAAAAWyPsAAAAWyPsAAAAWyPsAAAAWyPsAAAAW2vzDMoAAISKx+MJqJ3D4ZDTyfd4NEXYAQCELcMwJEkxMTEBtU9JSZHb7SbwoAnCDgAg7OU/sl6uKN9/sgzD0OK758k0zRBVhUhB2AEAhD2n00lvDdqMnxwAAGBrhB0AAGBrhB0AAGBrhB0AAGBrDFAGQswwjIDuFgl0XhEAgG+EHSCEDMNQRka63O5jAb+G22gBoH0IO0AImaYpt/uYTn16RC6Xy2fburo69R00OESVAYB9EXYAC7hcLr9hx99+AEBgGKAMAABsjZ4dAJ1CIAO+TdOUw+Fo93EAhBfCDgBbMwxDLpczoIUku0RHq66+PqDjMnAciByEHQC2ZpqmPB5DVUc/VpSPhSQbGhrUO+NSrXlkvd92S+6dLxF2gIhB2AHQKfgbFN7YU+NvwUkWowQiD7+1AADA1iwNO6tWrdLYsWPVs2dPJSYmatq0aTp06FCTNufPn1dubq769OmjHj16aMaMGaqoqGjSprS0VFOnTlW3bt2UmJiou+66Sw0NDaE8FQAAEKYsDTs7d+5Ubm6udu/ercLCQtXX1ys7O1vnzp3ztlmyZIn+/Oc/64UXXtDOnTtVVlam6dOne/d7PB5NnTpVdXV12rVrl5577jlt3LhR999/vxWnBAAAwoylY3ZeffXVJo83btyoxMRElZSU6Bvf+Iaqq6v17LPPatOmTbr22mslSRs2bNDQoUO1e/dujR8/Xq+//roOHDig7du3KykpSSNHjtQDDzyge+65RytWrFCXLl2sODUAABAmwmrMTnV1tSSpd+/ekqSSkhLV19crKyvL22bIkCFKS0tTcXGxJKm4uFgjRoxQUlKSt01OTo5qamq0f//+Zt+ntrZWNTU1TTYAAGBPYRN2DMPQnXfeqQkTJujyyy+XJJWXl6tLly5KSEho0jYpKUnl5eXeNl8OOo37G/c1Z9WqVYqPj/duqampQT4bAAAQLsIm7OTm5mrfvn3avHlzh7/X0qVLVV1d7d3cbneHvycAALBGWMyzs3DhQm3ZskVvvvmmBgwY4H0+OTlZdXV1On36dJPenYqKCiUnJ3vbvPPOO02O13i3VmOb/ysmJiag2VQBAEDks7RnxzRNLVy4UC+//LLeeOMNDRw4sMn+0aNHKzo6WkVFRd7nDh06pNLSUmVmZkqSMjMztXfvXlVWVnrbFBYWKi4uTsOGDQvNiQAAgLBlac9Obm6uNm3apD/96U/q2bOnd4xNfHy8YmNjFR8frzvuuEN5eXnq3bu34uLitGjRImVmZmr8+PGSpOzsbA0bNkyzZs3S6tWrVV5ermXLlik3N5feGwAAYG3YWbdunSRp4sSJTZ7fsGGDbrvtNknSmjVr5HQ6NWPGDNXW1ionJ0dPPvmkt63L5dKWLVu0YMECZWZmqnv37po9e7ZWrlwZqtMAAABhzNKwE8iqwV27dlVBQYEKCgpabJOenq5t27YFszQAAGATYXM3FgAAQEcg7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsLi1XPAQAIFo/H47eNw+GQ08n3/c6CsAMAsAXDMCQpoEWgU1JS5Ha7CTydBGEHAGAr+Y+slyuq5T9vhmFo8d3zAlqfEfZA2AEA2IrT6aTHBk0QdgB0CoZh+Pwm33gJBID9EHYA2FpjwHnooYd8fttvDDtc2gDsh7ADoFPImTxV0dHRLe6vr6/XL59+ViLsALZD2AHQKTgcDjkcDp/7AdgTI7gAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtces5ECT+ZuiVAluNGQAQXIQdIAgMw1BGRrrc7mMBtWeWXgAIHcIOEASmacrtPqZTnx6Ry+VqsV1dXZ36DhocwsoAAIQdIIhcLpfPsONrHwCgYzBAGQAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2Bq3ngPAlxiGIcMwfO4HEFkIOwCgCyHG5XIq72c/8tvW5XJeaB+CugC0H2EHACSZMuXxGHrt319UdHR0i+0aGhqUfcuMEFYGoL0IOwDwJU6n0+dM11zGAiIPA5QBAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtMakg4IdhGDJN02cbj8cTomoAAK1F2AF8MAxDGRnpcruPBdTeXyiCfbBgKBA5CDuAD6Zpyu0+plOfHvG5hEBdXZ36DhocwspgFRYMBSIPYQcIgMvl8hl2fO2DvbBgKBB5CDsA0AYsGApEDu7GAgAAtkbYAQAAtkbYAQAAtsaYHQBApxTI/FgOh0NOJ/0CkY6wAwDoVBoHj8fExPhtm5KSIrfbTeCJcJaGnTfffFOPPvqoSkpKdOLECb388suaNm2ad79pmlq+fLmeeeYZnT59WhMmTNC6des0ePD/zmdSVVWlRYsW6c9//rOcTqdmzJihxx9/XD169LDgjACEUiCzW3NXFFqS/8h6uaJa/jNoGIYW3z2PyUJtwNKwc+7cOV155ZW6/fbbNX369Iv2r169Wk888YSee+45DRw4UD//+c+Vk5OjAwcOqGvXrpKkmTNn6sSJEyosLFR9fb3mzJmjefPmadOmTaE+HQAh0vjH56GHHvL7jbuhoeHCazq8KkQap9NJj00nYWnYmTJliqZMmdLsPtM0lZ+fr2XLlumGG26QJP3ud79TUlKS/vM//1O33HKLDh48qFdffVXvvvuuxowZI0lau3atvvWtb+mXv/ylUlJSmj12bW2tamtrvY9ramqCfGYAQiFn8lSfE/tJ0vna83rsNxskvp0DnVbYRtpPPvlE5eXlysrK8j4XHx+vcePGqbi4WJJUXFyshIQEb9CRpKysLDmdTr399tstHnvVqlWKj4/3bqmpqR13IgA6jMPhCGgD0LmFbdgpLy+XJCUlJTV5PikpybuvvLxciYmJTfZHRUWpd+/e3jbNWbp0qaqrq72b2+0OcvUAACBcdMq7sWJiYgIahQ8AACJf2PbsJCcnS5IqKiqaPF9RUeHdl5ycrMrKyib7GxoaVFVV5W0DAAA6t7ANOwMHDlRycrKKioq8z9XU1Ojtt99WZmamJCkzM1OnT59WSUmJt80bb7whwzA0bty4kNcMAADCj6WXsc6ePavDhw97H3/yySfas2ePevfurbS0NN1555168MEHNXjwYO+t5ykpKd65eIYOHarrr79ec+fO1fr161VfX6+FCxfqlltuafFOLAAA0LlYGnbee+89TZo0yfs4Ly9PkjR79mxt3LhRd999t86dO6d58+bp9OnTuuaaa/Tqq69659iRpOeff14LFy7U5MmTvZMKPvHEEyE/FwAAEJ4sDTsTJ070OTOlw+HQypUrtXLlyhbb9O7dmwkEAQBAi8J2zA4AAEAwEHYAAICtEXYAAICtEXYAAICtdcoZlAGEN8MwfN68YBhGCKtpH8Mw/NYbSecDRCLCDoCw0RhwHnroITmdLXc8NzQ0XGgfkqraxjAMuVxO5f3sRwG1dzodamhokMPHeROKgLYh7AAIOzmTpyo6OrrF/edrz+ux32yQfPT+WM2UKY/H0Gv//qLPc5Gkuro6TZ11c0DByOVyXghSwSoU6AQIOwDCjsPhkMPh8Lk/UjidTrlcvqOJw+kIKBg1NDQo+5YZwS4RsD3CDjotf+NCJMnj8YSoms7BTmNxOoK/YNTZPx+grQg76JQMw1BGRrrc7mMBtfcXiuCbncbiAIg8hB10SqZpyu0+plOfHvH5Tbqurk59Bw0OYWX2ZoexOAAiD2EHnZrL5fIZdvyNtUDr2GksDjqPQC5nOxwOn72WsBZhBwCAZjSOkYqJifHbNiUlRW63m8ATpgg7AAD4kP/IermiWv5zaRiGFt89j7F9YYywAwCAD06nkx6bCMf/ewAAwNbo2QEAmzIMQw4fc/Mwbw86C8IOAEQYf4uLNjQ0yOVy6s575/s9FstPoDMg7ABAhGjt4qIsPwFcQNgBgAgR6OKitXW1mnLrTSw/Afx/hB0AiDD+Qgx3DgFN8RsBAABsjbADAABsjbADAABsjbADAABsjbADAABsjbADAABsjVvPYSuGYQS08rDH4wlBNZ2Hv8+d+VwAWImwA9swDEMZGelyu48F/JpAghFa1vj5PfTQQz7ndmloaLjQPiRVAUBThB3YhmmacruP6dSnR3xOuCZJdXV16jtocIgqs7+cyVN9zuh7vva8HvvNBolwCcAChB3Yjsvl8ht2/O1H6zgcDjkcDp/7AcAqDFAGAAC2RtgBAAC2xmUsAOjkDMPwe8ccd9T5F+hdng6Hg8VaQ4ywAwCdlGEYcrmcyvvZjwJq73I5L7ymg+uKNI1BMCYmJqD2KSkpcrvdBJ4QIuwAQCdlypTHY+i1f3/R59100oXpA7JvmeG3F6gz9wDlP7Jerijff1YNw9Diu+cx7UWIEXYAoJNzOp1+71BsaGgIuBeos/YAOZ1OemvCFGEHQLMCmY26M3+L72wC7QVq7AECwglhB0ATgc6KLDEzcmfkrxeIAIxwRNgB0Cx/syJLzIwMIDIQdgA0y9+syI1tACDcMZIKAADYGmEHAADYGpexYKlA7viRLgya9XfJJNDZSwEAnQthB5YxDEMZGelyu4/5bdslOlp19fUBHZfJugDYQaBfBll+wj/CDixjmqbc7mM69ekRn7ey1tXVqe+gwao6+rGifMxO2tgOAMKdv57oC18GM1RWVub3WCw/4R9hB5ZzuVw+w07jvkDbwTd/3xaZJwXoOK1dRyv/X5/yO68Ry0/4R9gBOolAJwtkokAEA2to+eZvHa2GhgYtuXe+nFyiCgrCDmATgfbY+JsskIkC0R6tWUnd6XSooaFBDh9/zO0aivyto0XACS7CDhDhWttjIz+TBTJRINoj0DW06urqNHXWzSwsipAg7KBDBHIXAbeKBxc9Nggn/tbQcjgdLCyKkCHsIKhz3TQe79JBg+Q+5v+W8sbjomWBXp7yt7wDPTYIRywsilAg7HRyHTXXjSRuFW8nBhQDQHAQdjq5YM918+W2nfFWcX+9MI37AulNY0Ax8L/83d3V2Kbxfx1BvBMs2MfrCIEMC+jMkw/aJuwUFBTo0UcfVXl5ua688kqtXbtWV199tdVlRYxgzXXz5badSWt7Yfy1+3JbBhSjM2vN3V3ShX9/7rx3fgDt/A96bnzvYB2vI7Rm3p7OPPmgLcLOH/7wB+Xl5Wn9+vUaN26c8vPzlZOTo0OHDikxMdHq8izDIOHQC7QXJvvab6lLly4+j0WPDRD43V2SVFtXqym33hTwoOdA5gLyeAy9vvklnz3a4TCI2t+8PR01+WCkLGlhi7Dz2GOPae7cuZozZ44kaf369dq6dat++9vf6t5777W0to74QQj0EgiDhIMn2IOE/bX7clsA/gcyN7YJpG1DQ0OreoscDkdAg6itvNQWqLq6uoB73/21i6QlLSI+7NTV1amkpERLly71Pud0OpWVlaXi4uJmX1NbW6va2lrv4+rqaklSTU1NUGszDEOXDx+u44H8IPTvrw/27PH7g2AYhkaNGhXQD5ckffzB+3JFtXzM+vp6Db5qrKqqqhTl45tQfd2Fgcn+2rWmrVXtWtO2rrZOkvTz++/3GT4ae8iqz5xRtM9B2Rd+7mrOnPH7LTXQtla1o0ZqDKd2rT2mx2Poxac3+P79r6/TTfNuD+B4dXI6HQGHJ6czsEtjDodUU1OtaB+9wI3vHcjxnE6nunXrFlCNDocj4C/BD97/S7mcvsPgfSt/ourq6qAPc2j8u+23VjPCHT9+3JRk7tq1q8nzd911l3n11Vc3+5rly5ebunDzChsbGxsbG1uEb26322dWiPienbZYunSp8vLyvI8Nw1BVVZX69Olji0sHNTU1Sk1NldvtVlxcnNXlhA0+l4vxmTSPz6V5fC4X4zNpXqg+F9M0debMGaWkpPhsF/Fhp2/fvnK5XKqoqGjyfEVFhZKTk5t9TUxMzEUj1xMSEjqqRMvExcXxy9cMPpeL8Zk0j8+leXwuF+MzaV4oPpf4+Hi/bSL+/rMuXbpo9OjRKioq8j5nGIaKioqUmZlpYWUAACAcRHzPjiTl5eVp9uzZGjNmjK6++mrl5+fr3Llz3ruzAABA52WLsHPzzTfrs88+0/3336/y8nKNHDlSr776qpKSkqwuzRIxMTFavnx5QJNMdSZ8LhfjM2ken0vz+FwuxmfSvHD7XBymyQQrAADAviJ+zA4AAIAvhB0AAGBrhB0AAGBrhB0AAGBrhJ1OYOvWrRo3bpxiY2PVq1cvTZs2zeqSwkZtba1Gjhwph8OhPXv2WF2OpY4ePao77rhDAwcOVGxsrC699FItX75cdXV1VpcWcgUFBcrIyFDXrl01btw4vfPOO1aXZJlVq1Zp7Nix6tmzpxITEzVt2jQdOnTI6rLCziOPPCKHw6E777zT6lIsd/z4cX3ve99Tnz59FBsbqxEjRui9996ztCbCjs299NJLmjVrlubMmaMPP/xQb731lm699Varywobd999t99pxjuLjz76SIZh6KmnntL+/fu1Zs0arV+/Xj/72c+sLi2k/vCHPygvL0/Lly/X+++/ryuvvFI5OTmqrKy0ujRL7Ny5U7m5udq9e7cKCwtVX1+v7OxsnTt3zurSwsa7776rp556SldccYXVpVju1KlTmjBhgqKjo/WXv/xFBw4c0K9+9Sv16tXL2sKCsxwnwlF9fb15ySWXmL/5zW+sLiUsbdu2zRwyZIi5f/9+U5L5wQcfWF1S2Fm9erU5cOBAq8sIqauvvtrMzc31PvZ4PGZKSoq5atUqC6sKH5WVlaYkc+fOnVaXEhbOnDljDh482CwsLDS/+c1vmosXL7a6JEvdc8895jXXXGN1GRehZ8fG3n//fR0/flxOp1OjRo1S//79NWXKFO3bt8/q0ixXUVGhuXPn6t/+7d/UrVs3q8sJW9XV1erdu7fVZYRMXV2dSkpKlJWV5X3O6XQqKytLxcXFFlYWPqqrqyWpU/1c+JKbm6upU6c2+ZnpzF555RWNGTNGN954oxITEzVq1Cg988wzVpfFZSw7O3LkiCRpxYoVWrZsmbZs2aJevXpp4sSJqqqqsrg665imqdtuu03z58/XmDFjrC4nbB0+fFhr167VD3/4Q6tLCZmTJ0/K4/FcNPt6UlKSysvLLaoqfBiGoTvvvFMTJkzQ5ZdfbnU5ltu8ebPef/99rVq1yupSwsaRI0e0bt06DR48WK+99poWLFigH//4x3ruuecsrYuwE4HuvfdeORwOn1vj+AtJuu+++zRjxgyNHj1aGzZskMPh0AsvvGDxWQRfoJ/L2rVrdebMGS1dutTqkkMi0M/ly44fP67rr79eN954o+bOnWtR5Qg3ubm52rdvnzZv3mx1KZZzu91avHixnn/+eXXt2tXqcsKGYRi66qqr9PDDD2vUqFGaN2+e5s6dq/Xr11taly3WxupsfvKTn+i2227z2WbQoEE6ceKEJGnYsGHe52NiYjRo0CCVlpZ2ZImWCPRzeeONN1RcXHzRmi1jxozRzJkzLf8GEmyBfi6NysrKNGnSJH3ta1/T008/3cHVhZe+ffvK5XKpoqKiyfMVFRVKTk62qKrwsHDhQm3ZskVvvvmmBgwYYHU5lispKVFlZaWuuuoq73Mej0dvvvmmfv3rX6u2tlYul8vCCq3Rv3//Jn9zJGno0KF66aWXLKroAsJOBOrXr5/69evnt93o0aMVExOjQ4cO6ZprrpEk1dfX6+jRo0pPT+/oMkMu0M/liSee0IMPPuh9XFZWppycHP3hD3/QuHHjOrJESwT6uUgXenQmTZrk7QV0OjtX52+XLl00evRoFRUVeadoMAxDRUVFWrhwobXFWcQ0TS1atEgvv/yyduzYoYEDB1pdUliYPHmy9u7d2+S5OXPmaMiQIbrnnns6ZdCRpAkTJlw0NcH//M//WP43h7BjY3FxcZo/f76WL1+u1NRUpaen69FHH5Uk3XjjjRZXZ520tLQmj3v06CFJuvTSSzv1N9bjx49r4sSJSk9P1y9/+Ut99tln3n2dqVcjLy9Ps2fP1pgxY3T11VcrPz9f586d05w5c6wuzRK5ubnatGmT/vSnP6lnz57esUvx8fGKjY21uDrr9OzZ86JxS927d1efPn069XimJUuW6Gtf+5oefvhh3XTTTXrnnXf09NNPW95LTNixuUcffVRRUVGaNWuWvvjiC40bN05vvPGG9XMeIOwUFhbq8OHDOnz48EWhzzRNi6oKvZtvvlmfffaZ7r//fpWXl2vkyJF69dVXLxq03FmsW7dOkjRx4sQmz2/YsMHv5VF0PmPHjtXLL7+spUuXauXKlRo4cKDy8/M1c+ZMS+tymJ3pXzEAANDpdK4L8gAAoNMh7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7ACApLq6OqtLANBBCDsAwtqLL76oESNGKDY2Vn369FFWVpbOnTsnSfrtb3+r4cOHKyYmRv3799fChQu9rystLdUNN9ygHj16KC4uTjfddJMqKiq8+1esWKGRI0fqN7/5jQYOHKiuXbtKkk6fPq0f/OAH6tevn+Li4nTttdfqww8/DO1JAwgqwg6AsHXixAl997vf1e23366DBw9qx44dmj59ukzT1Lp165Sbm6t58+Zp7969euWVV/SVr3xFkmQYhm644QZVVVVp586dKiws1JEjR3TzzTc3Of7hw4f10ksv6Y9//KP27NkjSbrxxhtVWVmpv/zlLyopKdFVV12lyZMnq6qqKtSnDyBIWPUcQNh6//33NXr0aB09elTp6elN9l1yySWaM2eOHnzwwYteV1hYqClTpuiTTz5RamqqJOnAgQMaPny43nnnHY0dO1YrVqzQww8/rOPHj6tfv36SpP/+7//W1KlTVVlZqZiYGO/xvvKVr+juu+/WvHnzOvBsAXSUKKsLAICWXHnllZo8ebJGjBihnJwcZWdn6zvf+Y7q6+tVVlamyZMnN/u6gwcPKjU11Rt0JGnYsGFKSEjQwYMHNXbsWElSenq6N+hI0ocffqizZ8+qT58+TY73xRdf6OOPP+6AMwQQCoQdAGHL5XKpsLBQu3bt0uuvv661a9fqvvvuU1FRUVCO37179yaPz549q/79+2vHjh0XtU1ISAjKewIIPcIOgLDmcDg0YcIETZgwQffff7/S09NVWFiojIwMFRUVadKkSRe9ZujQoXK73XK73U0uY50+fVrDhg1r8b2uuuoqlZeXKyoqShkZGR11SgBCjLADIGy9/fbbKioqUnZ2thITE/X222/rs88+09ChQ7VixQrNnz9fiYmJmjJlis6cOaO33npLixYtUlZWlkaMGKGZM2cqPz9fDQ0N+tGPfqRvfvObGjNmTIvvl5WVpczMTE2bNk2rV6/WV7/6VZWVlWnr1q36l3/5F5+vBRC+CDsAwlZcXJzefPNN5efnq6amRunp6frVr36lKVOmSJLOnz+vNWvW6Kc//an69u2r73znO5Iu9Ab96U9/0qJFi/SNb3xDTqdT119/vdauXevz/RwOh7Zt26b77rtPc+bM0Weffabk5GR94xvfUFJSUoefL4COwd1YAADA1phnBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2Nr/AwKdXcya05rFAAAAAElFTkSuQmCC", 66 | "text/plain": [ 67 | "
" 68 | ] 69 | }, 70 | "metadata": {}, 71 | "output_type": "display_data" 72 | } 73 | ], 74 | "source": [ 75 | "import seaborn as sns\n", 76 | "import pandas as pd\n", 77 | "import warnings\n", 78 | "import matplotlib.pyplot as plt\n", 79 | "warnings.filterwarnings(\"ignore\", \"is_categorical_dtype\")\n", 80 | "warnings.filterwarnings(\"ignore\", \"use_inf_as_na\")\n", 81 | "\n", 82 | "score = clf.decision_function(X)\n", 83 | "df = pd.DataFrame({'score': score, 'y': y})\n", 84 | "sns.histplot(df, x=\"score\", hue=\"y\")\n", 85 | "plt.show()" 86 | ] 87 | } 88 | ], 89 | "metadata": { 90 | "kernelspec": { 91 | "display_name": "Python 3 (ipykernel)", 92 | "language": "python", 93 | "name": "python3" 94 | }, 95 | "language_info": { 96 | "codemirror_mode": { 97 | "name": "ipython", 98 | "version": 3 99 | }, 100 | "file_extension": ".py", 101 | "mimetype": "text/x-python", 102 | "name": "python", 103 | "nbconvert_exporter": "python", 104 | "pygments_lexer": "ipython3", 105 | "version": "3.10.12" 106 | } 107 | }, 108 | "nbformat": 4, 109 | "nbformat_minor": 5 110 | } 111 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the OS, Python version and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.10" 13 | # You can also specify other tool versions: 14 | # nodejs: "19" 15 | # rust: "1.64" 16 | # golang: "1.19" 17 | 18 | # Build documentation in the "docs/" directory with Sphinx 19 | sphinx: 20 | configuration: doc/source/conf.py 21 | 22 | # Optionally build your docs in additional formats such as PDF and ePub 23 | # formats: 24 | # - pdf 25 | # - epub 26 | 27 | # Optional but recommended, declare the Python requirements required 28 | # to build your documentation 29 | # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 30 | python: 31 | install: 32 | - requirements: ./requirements.txt -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "esbonio.sphinx.confDir": "", 3 | "files.associations": { 4 | "random": "cpp", 5 | "type_traits": "cpp", 6 | "tuple": "cpp", 7 | "utility": "cpp" 8 | } 9 | } -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0) 2 | project(rehline LANGUAGES CXX) 3 | 4 | set(PYBIND11_FINDPYTHON ON) 5 | find_package(pybind11 CONFIG REQUIRED) 6 | 7 | pybind11_add_module(rehline MODULE src/rehline.cpp) 8 | 9 | install(TARGETS rehline DESTINATION .) 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022-2024 SoftMin 4 | Copyright (c) 2022-2024 Ben Dai 5 | Copyright (c) 2022-2024 Yixuan Qiu 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in all 15 | copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ReHLine 2 | 3 | **ReHLine** is designed to be a computationally efficient and practically useful software package for large-scale empirical risk minimization (ERM) problems. 4 | 5 | - Documentation: [https://rehline-python.readthedocs.io](https://rehline-python.readthedocs.io) 6 | - Project homepage: [https://rehline.github.io](https://rehline.github.io) 7 | - GitHub repo: [https://github.com/softmin/ReHLine-python](https://github.com/softmin/ReHLine-python) 8 | - PyPi: [https://pypi.org/project/rehline](https://pypi.org/project/rehline) 9 | - Paper: [NeurIPS | 2023](https://openreview.net/pdf?id=3pEBW2UPAD) 10 | 11 | 12 | The **ReHLine** solver has four appealing 13 | "linear properties": 14 | 15 | - It applies to any convex piecewise linear-quadratic loss function, including the hinge loss, the check loss, the Huber loss, etc. 16 | - In addition, it supports linear equality and inequality constraints on the parameter vector. 17 | - The optimization algorithm has a provable linear convergence rate. 18 | - The per-iteration computational complexity is linear in the sample size. 19 | 20 | 21 | ## ✨ New Features: Scikit-Learn Compatible Estimators 22 | 23 | We are excited to introduce full scikit-learn compatibility! `ReHLine` now provides `plq_Ridge_Classifier` and `plq_Ridge_Regressor` estimators that integrate seamlessly with the entire scikit-learn ecosystem. 24 | 25 | This means you can: 26 | - Drop `ReHLine` estimators directly into your existing scikit-learn `Pipeline`. 27 | - Perform robust hyperparameter tuning using `GridSearchCV`. 28 | - Use standard scikit-learn evaluation metrics and cross-validation tools. 29 | 30 | 54 | 55 | ## ⌛ Benchmark (powered by benchopt) 56 | 57 | Some existing problems of recent interest in statistics and machine 58 | learning can be solved by **ReHLine**, and we provide reproducible 59 | benchmark code and results at the 60 | [ReHLine-benchmark](https://github.com/softmin/ReHLine-benchmark) repository. 61 | 62 | | Problem | Results | 63 | |---------- |:-----------------:| 64 | |[FairSVM](https://github.com/softmin/ReHLine-benchmark/tree/main/benchmark_FairSVM) | [Result](https://rehline-python.readthedocs.io/en/latest/_static/benchmark/benchmark_FairSVM.html)| 65 | |[ElasticQR](https://github.com/softmin/ReHLine-benchmark/tree/main/benchmark_QR) | [Result](https://rehline-python.readthedocs.io/en/latest/_static/benchmark/benchmark_QR.html)| 66 | |[RidgeHuber](https://github.com/softmin/ReHLine-benchmark/tree/main/benchmark_Huber) | [Result](https://rehline-python.readthedocs.io/en/latest/_static/benchmark/benchmark_Huber.html)| 67 | |[SVM](https://github.com/softmin/ReHLine-benchmark/tree/main/benchmark_SVM) | [Result](https://rehline-python.readthedocs.io/en/latest/_static/benchmark/benchmark_SVM.html)| 68 | |[Smoothed SVM](https://github.com/softmin/ReHLine-benchmark/tree/main/benchmark_sSVM) | [Result](https://rehline-python.readthedocs.io/en/latest/_static/benchmark/benchmark_sSVM.html)| 69 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /doc/source/_static/css/label.css: -------------------------------------------------------------------------------- 1 | .summarylabel { 2 | background-color: var(--color-foreground-secondary); 3 | color: var(--color-background-secondary); 4 | font-size: 70%; 5 | padding-left: 2px; 6 | padding-right: 2px; 7 | border-radius: 3px; 8 | vertical-align: 15%; 9 | padding-bottom: 2px; 10 | filter: opacity(40%); 11 | } 12 | 13 | 14 | table.summarytable { 15 | width: 100%; 16 | } -------------------------------------------------------------------------------- /doc/source/_templates/autoapi/index.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | ============= 3 | 4 | This page contains auto-generated API reference documentation. 5 | 6 | .. toctree:: 7 | :titlesonly: 8 | 9 | {% for page in pages %} 10 | {{ page.include_path }} 11 | {% endfor %} 12 | 13 | -------------------------------------------------------------------------------- /doc/source/_templates/autoapi/macros.rst: -------------------------------------------------------------------------------- 1 | {# AutoApiSummary replacement macro #} 2 | {# 3 | The intent of this macro is to replace the autoapisummary directive with the following 4 | improvements: 5 | 1) Method signature is generated without typing annotation regardless of the value of 6 | `autodoc_typehints` 7 | 2) Properties are treated like attribute (but labelled as properties). 8 | 3) Label are added as summary prefix to indicate if the member is a property, class 9 | method, or static method. 10 | 11 | Copyright: Antoine Beyeler, 2022 12 | License: MIT 13 | #} 14 | 15 | {# Renders an object's name with a proper reference and optional signature. 16 | 17 | The signature is re-generated from obj.obj.args, which is undocumented but is the 18 | only way to have a type-less signature if `autodoc_typehints` is `signature` or 19 | `both`. #} 20 | {% macro _render_item_name(obj, sig=False) -%} 21 | :py:obj:`{{ obj.name }} <{{ obj.id }}>` 22 | {%- if sig -%} 23 | \ ( 24 | {%- for arg in obj.obj.args -%} 25 | {%- if arg[0] %}{{ arg[0]|replace('*', '\*') }}{% endif -%}{{ arg[1] -}} 26 | {%- if not loop.last %}, {% endif -%} 27 | {%- endfor -%} 28 | ){%- endif -%} 29 | {%- endmacro %} 30 | 31 | 32 | {# Generates a single object optionally with a signature and a labe. #} 33 | {% macro _item(obj, sig=False, label='') %} 34 | * - {{ _render_item_name(obj, sig) }} 35 | - {% if label %}:summarylabel:`{{ label }}` {% endif %}{% if obj.summary %}{{ obj.summary }}{% else %}\-{% endif +%} 36 | {% endmacro %} 37 | 38 | 39 | 40 | {# Generate an autosummary-like table with the provided members. #} 41 | {% macro auto_summary(objs, title='') -%} 42 | 43 | .. list-table:: {{ title }} 44 | :header-rows: 0 45 | :widths: auto 46 | :class: summarytable {#- apply CSS class to customize styling +#} 47 | 48 | {% for obj in objs -%} 49 | {#- should the full signature be used? -#} 50 | {%- set sig = (obj.type in ['method', 'function'] and not 'property' in obj.properties) -%} 51 | 52 | {#- compute label -#} 53 | {%- if 'property' in obj.properties -%} 54 | {%- set label = 'prop' -%} 55 | {%- elif 'classmethod' in obj.properties -%} 56 | {%- set label = 'class' -%} 57 | {%- elif 'abstractmethod' in obj.properties -%} 58 | {%- set label = 'abc' -%} 59 | {%- elif 'staticmethod' in obj.properties -%} 60 | {%- set label = 'static' -%} 61 | {%- else -%} 62 | {%- set label = '' -%} 63 | {%- endif -%} 64 | 65 | {{- _item(obj, sig=sig, label=label) -}} 66 | {%- endfor -%} 67 | 68 | {% endmacro %} 69 | -------------------------------------------------------------------------------- /doc/source/_templates/autoapi/python/class.rst: -------------------------------------------------------------------------------- 1 | {% import 'macros.rst' as macros %} 2 | 3 | {% if obj.display %} 4 | .. py:{{ obj.type }}:: {{ obj.short_name }}{% if obj.args %}({{ obj.args }}){% endif %} 5 | {% for (args, return_annotation) in obj.overloads %} 6 | {{ " " * (obj.type | length) }} {{ obj.short_name }}{% if args %}({{ args }}){% endif %} 7 | {% endfor %} 8 | 9 | 10 | {% if obj.bases %} 11 | {% if "show-inheritance" in autoapi_options %} 12 | Bases: {% for base in obj.bases %}{{ base|link_objs }}{% if not loop.last %}, {% endif %}{% endfor %} 13 | {% endif %} 14 | 15 | 16 | {% if "show-inheritance-diagram" in autoapi_options and obj.bases != ["object"] %} 17 | .. autoapi-inheritance-diagram:: {{ obj.obj["full_name"] }} 18 | :parts: 1 19 | {% if "private-members" in autoapi_options %} 20 | :private-bases: 21 | {% endif %} 22 | 23 | {% endif %} 24 | {% endif %} 25 | {% if obj.docstring %} 26 | {{ obj.docstring|indent(3) }} 27 | {% endif %} 28 | {% if "inherited-members" in autoapi_options %} 29 | {% set visible_classes = obj.classes|selectattr("display")|list %} 30 | {% else %} 31 | {% set visible_classes = obj.classes|rejectattr("inherited")|selectattr("display")|list %} 32 | {% endif %} 33 | {% for klass in visible_classes %} 34 | {{ klass.render()|indent(3) }} 35 | {% endfor %} 36 | {% if "inherited-members" in autoapi_options %} 37 | {% set visible_attributes = obj.attributes|selectattr("display")|list %} 38 | {% else %} 39 | {% set visible_attributes = obj.attributes|rejectattr("inherited")|selectattr("display")|list %} 40 | {% endif %} 41 | {% if "inherited-members" in autoapi_options %} 42 | {% set visible_methods = obj.methods|selectattr("display")|list %} 43 | {% else %} 44 | {% set visible_methods = obj.methods|rejectattr("inherited")|selectattr("display")|list %} 45 | {% endif %} 46 | 47 | {% if visible_methods or visible_attributes %} 48 | Overview 49 | ======== 50 | 51 | {% set summary_methods = visible_methods|rejectattr("properties", "contains", "property")|list %} 52 | {% set summary_attributes = visible_attributes + visible_methods|selectattr("properties", "contains", "property")|list %} 53 | {% if summary_attributes %} 54 | {{ macros.auto_summary(summary_attributes, title="Attributes")|indent(3) }} 55 | {% endif %} 56 | 57 | {% if summary_methods %} 58 | {{ macros.auto_summary(summary_methods, title="Methods")|indent(3) }} 59 | {% endif %} 60 | 61 | Members 62 | ======= 63 | 64 | {% for attribute in visible_attributes %} 65 | {{ attribute.render()|indent(3) }} 66 | {% endfor %} 67 | {% for method in visible_methods %} 68 | {{ method.render()|indent(3) }} 69 | {% endfor %} 70 | {% endif %} 71 | {% endif %} 72 | -------------------------------------------------------------------------------- /doc/source/_templates/autoapi/python/module.rst: -------------------------------------------------------------------------------- 1 | {% import 'macros.rst' as macros %} 2 | 3 | {% if not obj.display %} 4 | :orphan: 5 | 6 | {% endif %} 7 | {{ obj.name }} 8 | {{ "=" * obj.name|length }} 9 | 10 | .. py:module:: {{ obj.name }} 11 | 12 | {% if obj.docstring %} 13 | .. autoapi-nested-parse:: 14 | 15 | {{ obj.docstring|indent(3) }} 16 | 17 | {% endif %} 18 | 19 | {% block subpackages %} 20 | {% set visible_subpackages = obj.subpackages|selectattr("display")|list %} 21 | {% if visible_subpackages %} 22 | Subpackages 23 | ----------- 24 | .. toctree:: 25 | :titlesonly: 26 | :maxdepth: 3 27 | 28 | {% for subpackage in visible_subpackages %} 29 | {{ subpackage.short_name }}/index.rst 30 | {% endfor %} 31 | 32 | 33 | {% endif %} 34 | {% endblock %} 35 | {% block submodules %} 36 | {% set visible_submodules = obj.submodules|selectattr("display")|list %} 37 | {% if visible_submodules %} 38 | Submodules 39 | ---------- 40 | .. toctree:: 41 | :titlesonly: 42 | :maxdepth: 1 43 | 44 | {% for submodule in visible_submodules %} 45 | {{ submodule.short_name }}/index.rst 46 | {% endfor %} 47 | 48 | 49 | {% endif %} 50 | {% endblock %} 51 | {% block content %} 52 | {% if obj.all is not none %} 53 | {% set visible_children = obj.children|selectattr("display")|selectattr("short_name", "in", obj.all)|list %} 54 | {% elif obj.type is equalto("package") %} 55 | {% set visible_children = obj.children|selectattr("display")|list %} 56 | {% else %} 57 | {% set visible_children = obj.children|selectattr("display")|rejectattr("imported")|list %} 58 | {% endif %} 59 | {% if visible_children %} 60 | Overview 61 | -------- 62 | 63 | {% set visible_classes = visible_children|selectattr("type", "equalto", "class")|list %} 64 | {% set visible_functions = visible_children|selectattr("type", "equalto", "function")|list %} 65 | {% set visible_attributes = visible_children|selectattr("type", "equalto", "data")|list %} 66 | {% if "show-module-summary" in autoapi_options and (visible_classes or visible_functions) %} 67 | {% block classes scoped %} 68 | {% if visible_classes %} 69 | {{ macros.auto_summary(visible_classes, title="Classes") }} 70 | {% endif %} 71 | {% endblock %} 72 | 73 | {% block functions scoped %} 74 | {% if visible_functions %} 75 | {{ macros.auto_summary(visible_functions, title="Function") }} 76 | {% endif %} 77 | {% endblock %} 78 | 79 | {% block attributes scoped %} 80 | {% if visible_attributes %} 81 | {{ macros.auto_summary(visible_attributes, title="Attributes") }} 82 | {% endif %} 83 | {% endblock %} 84 | {% endif %} 85 | 86 | {% if visible_classes %} 87 | Classes 88 | ------- 89 | {% for obj_item in visible_classes %} 90 | {{ obj_item.render()|indent(0) }} 91 | {% endfor %} 92 | {% endif %} 93 | 94 | {% if visible_functions %} 95 | Functions 96 | --------- 97 | {% for obj_item in visible_functions %} 98 | {{ obj_item.render()|indent(0) }} 99 | {% endfor %} 100 | {% endif %} 101 | 102 | {% if visible_attributes %} 103 | Attributes 104 | ---------- 105 | {% for obj_item in visible_attributes %} 106 | {{ obj_item.render()|indent(0) }} 107 | {% endfor %} 108 | {% endif %} 109 | 110 | 111 | {% endif %} 112 | {% endblock %} 113 | -------------------------------------------------------------------------------- /doc/source/autoapi/index.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | ============= 3 | 4 | This page contains auto-generated API reference documentation. 5 | 6 | .. toctree:: 7 | :titlesonly: 8 | 9 | /autoapi/rehline/index 10 | -------------------------------------------------------------------------------- /doc/source/benchmark.rst: -------------------------------------------------------------------------------- 1 | Benchmark 2 | ========= 3 | 4 | We have developed a benchmark for evaluating the performance of optimization methods, which is built on top of the `benchopt `_. For those interested in reproducing the benchmark results presented in our paper, we provide a dedicated repository, `ReHLine-benchmark `_, which contains all the necessary resources and instructions. 5 | 6 | .. table:: **Benchmark Results** 7 | :align: left 8 | 9 | +-------------+--------------------------------------------------------+ 10 | | Problem | Results | 11 | +=============+========================================================+ 12 | | FairSVM_ | `Result <./_static/benchmark/benchmark_FairSVM.html>`__| 13 | +-------------+--------------------------------------------------------+ 14 | | ElasticQR_ | `Result <./_static/benchmark/benchmark_QR.html>`__ | 15 | +-------------+--------------------------------------------------------+ 16 | | RidgeHuber_ | `Result <./_static/benchmark/benchmark_Huber.html>`__ | 17 | +-------------+--------------------------------------------------------+ 18 | | SVM_ | `Result <./_static/benchmark/benchmark_SVM.html>`__ | 19 | +-------------+--------------------------------------------------------+ 20 | | sSVM_ | `Result <./_static/benchmark/benchmark_sSVM.html>`__ | 21 | +-------------+--------------------------------------------------------+ 22 | 23 | .. _FairSVM: https://github.com/softmin/ReHLine-benchmark/tree/main/benchmark_FairSVM 24 | .. _ElasticQR: https://github.com/softmin/ReHLine-benchmark/tree/main/benchmark_QR 25 | .. _RidgeHuber: https://github.com/softmin/ReHLine-benchmark/tree/main/benchmark_Huber 26 | .. _SVM: https://github.com/softmin/ReHLine-benchmark/tree/main/benchmark_SVM 27 | .. _sSVM: https://github.com/softmin/ReHLine-benchmark/tree/main/benchmark_sSVM 28 | 29 | 30 | .. admonition:: Note 31 | :class: tip 32 | 33 | You may select the `log-log scale` option in the left sidebar, as this will significantly improve the readability of the results. 34 | 35 | Results Overview 36 | ---------------- 37 | 38 | .. image:: ./figs/res.png 39 | :width: 90% 40 | :align: center 41 | 42 | -------------------------------------------------------------------------------- /doc/source/clean_notebooks.py: -------------------------------------------------------------------------------- 1 | import json 2 | import sys 3 | from pathlib import Path 4 | 5 | def clean_notebook(file_path): 6 | """Removes the 'id' field from all cells in a Jupyter notebook.""" 7 | try: 8 | with open(file_path, 'r', encoding='utf-8') as f: 9 | notebook = json.load(f) 10 | 11 | changes_made = False 12 | if 'cells' in notebook and isinstance(notebook['cells'], list): 13 | for cell in notebook['cells']: 14 | if isinstance(cell, dict) and 'id' in cell: 15 | del cell['id'] 16 | changes_made = True 17 | 18 | if changes_made: 19 | with open(file_path, 'w', encoding='utf-8') as f: 20 | json.dump(notebook, f, indent=1, ensure_ascii=False) 21 | f.write('\n') # Add a newline at the end of the file 22 | print(f"Cleaned: {file_path}") 23 | else: 24 | print(f"No changes needed: {file_path}") 25 | 26 | except Exception as e: 27 | print(f"Error processing {file_path}: {e}") 28 | 29 | def main(): 30 | if len(sys.argv) != 2: 31 | print("Usage: python clean_notebooks.py ") 32 | sys.exit(1) 33 | 34 | target_dir = Path(sys.argv[1]) 35 | if not target_dir.is_dir(): 36 | print(f"Error: {target_dir} is not a valid directory.") 37 | sys.exit(1) 38 | 39 | print(f"Searching for notebooks in {target_dir}...") 40 | notebook_files = list(target_dir.rglob('*.ipynb')) 41 | 42 | if not notebook_files: 43 | print("No notebook files found.") 44 | return 45 | 46 | for notebook_file in notebook_files: 47 | clean_notebook(notebook_file) 48 | 49 | if __name__ == "__main__": 50 | main() 51 | -------------------------------------------------------------------------------- /doc/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | 17 | import os.path as osp 18 | 19 | import furo 20 | import nbsphinx 21 | import renku_sphinx_theme 22 | 23 | # import sphinx.apidoc 24 | # -- Project information ----------------------------------------------------- 25 | 26 | project = 'ReHLine' 27 | copyright = '2023, Ben Dai and Yixuan Qiu' 28 | author = 'Ben Dai, Yixuan Qiu' 29 | # The full version, including alpha/beta/rc tags 30 | # release = '0.10' 31 | 32 | import os 33 | import sys 34 | 35 | sys.path.append('.') 36 | sys.path.insert(0, os.path.abspath('../..')) 37 | sys.path.insert(0, os.path.abspath('../rehline')) 38 | # sys.path.append('../..') 39 | # sys.path.insert(0, os.path.abspath('.')) 40 | # sys.path.insert(0, os.path.abspath('../')) 41 | # sys.path.insert(0, os.path.abspath('../..')) 42 | # sys.path.insert(1, os.path.dirname(os.path.abspath("../")) + os.sep + "feature_engine") 43 | # -- General configuration --------------------------------------------------- 44 | 45 | # Add any Sphinx extension module names here, as strings. They can be 46 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 47 | # ones. 48 | master_doc = 'index' 49 | extensions = [ 50 | # 'sphinx.ext.autodoc', 51 | 'autoapi.extension', 52 | # "sphinx.ext.linkcode", 53 | "sphinx.ext.intersphinx", 54 | "sphinx_autodoc_typehints", 55 | # 'sphinx.ext.autosummary', 56 | # 'sphinx_gallery.gen_gallery', 57 | # 'numpydoc', 58 | 'nbsphinx', 59 | ] 60 | 61 | # -- Plausible support 62 | ENABLE_PLAUSIBLE = os.environ.get("READTHEDOCS_VERSION_TYPE", "") in ["branch", "tag"] 63 | html_context = {"enable_plausible": ENABLE_PLAUSIBLE} 64 | 65 | # -- autoapi configuration --------------------------------------------------- 66 | autodoc_typehints = "signature" # autoapi respects this 67 | 68 | autoapi_type = "python" 69 | autoapi_dirs = ['../../rehline/'] 70 | autoapi_template_dir = "_templates/autoapi" 71 | autoapi_options = [ 72 | "members", 73 | "undoc-members", 74 | "show-inheritance", 75 | "show-module-summary", 76 | "imported-members", 77 | ] 78 | autoapi_keep_files = True 79 | 80 | 81 | # -- custom auto_summary() macro --------------------------------------------- 82 | def contains(seq, item): 83 | """Jinja2 custom test to check existence in a container. 84 | 85 | Example of use: 86 | {% set class_methods = methods|selectattr("properties", "contains", "classmethod") %} 87 | 88 | Related doc: https://jinja.palletsprojects.com/en/3.1.x/api/#custom-tests 89 | """ 90 | return item in seq 91 | 92 | 93 | def prepare_jinja_env(jinja_env) -> None: 94 | """Add `contains` custom test to Jinja environment.""" 95 | jinja_env.tests["contains"] = contains 96 | 97 | 98 | autoapi_prepare_jinja_env = prepare_jinja_env 99 | 100 | # Custom role for labels used in auto_summary() tables. 101 | rst_prolog = """ 102 | .. role:: summarylabel 103 | """ 104 | 105 | # Related custom CSS 106 | html_css_files = [ 107 | "css/label.css", 108 | ] 109 | 110 | # autosummary_generate = True 111 | # numpydoc_show_class_members = False 112 | nbsphinx_execute = 'never' 113 | nbsphinx_allow_errors = True 114 | # autodoc_mock_imports = ['numpy'] 115 | # Add any paths that contain templates here, relative to this directory. 116 | templates_path = ['_templates'] 117 | 118 | # List of patterns, relative to source directory, that match files and 119 | # directories to ignore when looking for source files. 120 | # This pattern also affects html_static_path and html_extra_path. 121 | exclude_patterns = [] 122 | 123 | 124 | # -- Options for HTML output ------------------------------------------------- 125 | 126 | # The theme to use for HTML and HTML Help pages. See the documentation for 127 | # a list of builtin themes. 128 | 129 | # html_theme_path = [hachibee_sphinx_theme.get_html_themes_path()] 130 | 131 | 132 | html_theme = 'furo' 133 | 134 | # html_permalinks_icon = '§' 135 | 136 | # html_permalinks_icon = 'alpha' 137 | # html_theme = 'sphinxawesome_theme' 138 | 139 | # import sphinx_theme_pd 140 | # html_theme = 'sphinx_theme_pd' 141 | # html_theme_path = [sphinx_theme_pd.get_html_theme_path()] 142 | 143 | # import solar_theme 144 | # html_theme = 'solar_theme' 145 | # html_theme_path = [solar_theme.theme_path] 146 | 147 | # Add any paths that contain custom static files (such as style sheets) here, 148 | # relative to this directory. They are copied after the builtin static files, 149 | # so a file named "default.css" will overwrite the builtin "default.css". 150 | html_static_path = ['_static'] 151 | 152 | # html_css_files = [ 153 | # 'css/custom.css', 154 | # ] 155 | 156 | def autoapi_skip_members(app, what, name, obj, skip, options): 157 | if what == "attribute": 158 | skip = True 159 | return skip 160 | 161 | def setup(sphinx): 162 | sphinx.connect("autoapi-skip-member", autoapi_skip_members) 163 | 164 | -------------------------------------------------------------------------------- /doc/source/example.rst: -------------------------------------------------------------------------------- 1 | Example 2 | ======= 3 | 4 | Example Gallery 5 | --------------- 6 | 7 | .. nblinkgallery:: 8 | :caption: Emprical Risk Minimization 9 | :name: rst-link-gallery 10 | 11 | examples/QR.ipynb 12 | examples/CQR.ipynb 13 | examples/SVM.ipynb 14 | examples/FairSVM.ipynb 15 | examples/RankRegression.ipynb 16 | examples/Path_solution.ipynb 17 | examples/Warm_start.ipynb 18 | examples/Sklearn_Mixin.ipynb 19 | 20 | List of Examples 21 | ---------------- 22 | 23 | .. toctree:: 24 | :maxdepth: 1 25 | 26 | examples/QR.ipynb 27 | examples/CQR.ipynb 28 | examples/SVM.ipynb 29 | examples/FairSVM.ipynb 30 | examples/RankRegression.ipynb 31 | examples/Path_solution.ipynb 32 | examples/Warm_start.ipynb 33 | examples/Sklearn_Mixin.ipynb -------------------------------------------------------------------------------- /doc/source/examples/SVM.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# SVM\n", 8 | "\n", 9 | "[![Slides](https://img.shields.io/badge/🦌-ReHLine-blueviolet)](https://rehline-python.readthedocs.io/en/latest/)\n", 10 | "\n", 11 | "SVMs solve the following optimization problem:\n", 12 | "$$\n", 13 | " \\min_{\\mathbf{\\beta} \\in \\mathbb{R}^d} \\ C \\sum_{i=1}^n ( 1 - y_i \\mathbf{\\beta}^\\intercal \\mathbf{x}_i )_+ + \\frac{1}{2} \\| \\mathbf{\\beta} \\|_2^2\n", 14 | "$$\n", 15 | "where $\\mathbf{x}_i \\in \\mathbb{R}^d$ is a feature vector, and $y_i \\in \\{-1, 1\\}$ is a binary label.\n", 16 | "\n", 17 | "> **Note.** Since the hinge loss is a plq function, thus we can solve it by `rehline.plqERM_Ridge`." 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 1, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "## simulate data\n", 27 | "from sklearn.datasets import make_classification\n", 28 | "from sklearn.preprocessing import StandardScaler\n", 29 | "import numpy as np\n", 30 | "\n", 31 | "scaler = StandardScaler()\n", 32 | "\n", 33 | "n, d = 10000, 5\n", 34 | "X, y = make_classification(n_samples=n, n_features=d)\n", 35 | "## convert y to +1/-1\n", 36 | "y = 2*y - 1\n", 37 | "X = scaler.fit_transform(X)" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": 2, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "## solve SVM via `plqERM_Ridge`\n", 47 | "from rehline import plqERM_Ridge\n", 48 | "\n", 49 | "clf = plqERM_Ridge(loss={'name': 'svm'}, C=1.0)\n", 50 | "clf.fit(X=X, y=y)" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 3, 56 | "metadata": {}, 57 | "outputs": [ 58 | { 59 | "data": { 60 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2jUlEQVR4nO3de3xU9Z3/8ffMJIRwScItCZFcwFK5iIKAkGJbkJhI2a4s1EulFNFCoYEiab1QqVC84GIrURpBrQW7laWr1q0FqoZYcFeClyiWm2wRMQMhiTSQAEouc87vD36ZNUsyM0kmc2ZOXs/H4zzqzPnOmc+ZJuQ93/M936/DNE1TAAAANuW0ugAAAICORNgBAAC2RtgBAAC2RtgBAAC2RtgBAAC2RtgBAAC2RtgBAAC2FmV1AeHAMAyVlZWpZ8+ecjgcVpcDAAACYJqmzpw5o5SUFDmdLfffEHYklZWVKTU11eoyAABAG7jdbg0YMKDF/YQdST179pR04cOKi4uzuBoAABCImpoapaamev+Ot4SwI3kvXcXFxRF2AACIMP6GoDBAGQAA2BphBwAA2BphBwAA2BpjdgAAiBAej0f19fVWlxEy0dHRcrlc7T4OYQcAgDBnmqbKy8t1+vRpq0sJuYSEBCUnJ7drHjzCDgAAYa4x6CQmJqpbt26dYgJc0zT1+eefq7KyUpLUv3//Nh+LsAMAQBjzeDzeoNOnTx+rywmp2NhYSVJlZaUSExPbfEmLAcoAAISxxjE63bp1s7gSazSed3vGKhF2AACIAJ3h0lVzgnHehB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AFzEMQx6PJ6DNMAyrywUQZL/73e/Up08f1dbWNnl+2rRpmjVrlkVVtR1hB0AThmEoIyNdUVFRAW0ZGekEHsBmbrzxRnk8Hr3yyive5yorK7V161bdfvvtFlbWNsygDKAJ0zTldh/TqU+P+J2t1OPxqFf6IJmmGaLqAIRCbGysbr31Vm3YsEE33nijJOn3v/+90tLSNHHiRGuLawPCDoBmuVyuoKw2DCAyzZ07V2PHjtXx48d1ySWXaOPGjbrtttsicnJDwg4AALjIqFGjdOWVV+p3v/udsrOztX//fm3dutXqstqEsAMAAJr1gx/8QPn5+Tp+/LiysrKUmppqdUltwgBlAADQrFtvvVXHjh3TM888E5EDkxsRdgAAQLPi4+M1Y8YM9ejRQ9OmTbO6nDYj7AAAgBYdP35cM2fOVExMjNWltBljdgAAwEVOnTqlHTt2aMeOHXryySetLqddCDsA2s3j8fht43A45HTSmQxEilGjRunUqVP613/9V1122WVWl9MuhB0AbWYYhlwuZ0Dd26mpA3T06KcEHiBCHD161OoSgoawA6DNTNOUx2Oo6ujHiopq+Z8TZloGYCXCDoB2Y7ZlAOGM/mQAAGBrhB0AAGBrhB0AAGBrjNkBACBClZaW6uTJkyF5r759+yotLS0k7xVshB0AACJQaWmphg4dqs8//zwk79etWzcdPHiwXYHnj3/8o9avX6+SkhJVVVXpgw8+0MiRI4NXZAsIOwAQQQzDCOgWfiZxtL+TJ0/q888/17O/fkKXDR7coe916O9/1x0Lf6yTJ0+2K+ycO3dO11xzjW666SbNnTs3iBX6RtgBgAhhGIZSU1NVVlbmt21KSorcbjeBpxO4bPBgjbpihNVlBGTWrFmSQj9hIWEHACKEaZoqKyvT46uf9hliDMPQ4rvnMYkj8P8RdgAgwjidTnpsgFaw9LdlxYoVcjgcTbYhQ4Z4958/f165ubnq06ePevTooRkzZqiioqLJMUpLSzV16lR169ZNiYmJuuuuu9TQ0BDqUwEAAF/y/PPPq0ePHt7tv/7rvyyrxfKeneHDh2v79u3ex19eX2fJkiXaunWrXnjhBcXHx2vhwoWaPn263nrrLUkX1tuZOnWqkpOTtWvXLp04cULf//73FR0drYcffjjk5wKEu0AGtwaygjkA+PPP//zPGjdunPfxJZdcYlktloedqKgoJScnX/R8dXW1nn32WW3atEnXXnutJGnDhg0aOnSodu/erfHjx+v111/XgQMHtH37diUlJWnkyJF64IEHdM8992jFihXq0qVLs+9ZW1ur2tpa7+OampqOOTkgjBiGoYyMdLndxwJqz3gPAO3Rs2dP9ezZ0+oyJIXBDMp///vflZKSokGDBmnmzJkqLS2VJJWUlKi+vl5ZWVnetkOGDFFaWpqKi4slScXFxRoxYoSSkpK8bXJyclRTU6P9+/e3+J6rVq1SfHy8d0tNTe2gswPCh2macruP6dSnR1Rz7NMWt5NH/m51qQBsqqqqSnv27NGBAwckSYcOHdKePXtUXl7eoe9rac/OuHHjtHHjRl122WU6ceKEfvGLX+jrX/+69u3bp/LycnXp0kUJCQlNXpOUlOT9UMrLy5sEncb9jftasnTpUuXl5Xkf19TUEHjQafhboZzVy4HIcujvHf8FJVjv8corr2jOnDnex7fccoskafny5VqxYkVQ3qM5loadKVOmeP/7iiuu0Lhx45Senq7/+I//UGxsbIe9b0xMjGJiYjrs+AAAdLS+ffuqW7duumPhj0Pyft26dVPfvn3bdYzbbrtNt912W3AKagXLx+x8WUJCgr761a/q8OHDuu6661RXV6fTp0836d2pqKjwjvFJTk7WO++80+QYjXdrNTcOCAAAu0hLS9PBgwdZGysAYRV2zp49q48//lizZs3S6NGjFR0draKiIs2YMUPShWt7paWlyszMlCRlZmbqoYceUmVlpRITEyVJhYWFiouL07Bhwyw7DwAAQiEtLS1iA0goWRp2fvrTn+rb3/620tPTVVZWpuXLl8vlcum73/2u4uPjdccddygvL0+9e/dWXFycFi1apMzMTI0fP16SlJ2drWHDhmnWrFlavXq1ysvLtWzZMuXm5nKZCgAASLI47Bw7dkzf/e539Y9//EP9+vXTNddco927d6tfv36SpDVr1sjpdGrGjBmqra1VTk6OnnzySe/rXS6XtmzZogULFigzM1Pdu3fX7NmztXLlSqtOCQAAhBlLw87mzZt97u/atasKCgpUUFDQYpv09HRt27Yt2KUBAACbCKsxOwCA0ApkVu1GDoeDNbkQkfipBYBOyjAMpaamKioqKqAtNTVVhmFYXTbQavTsAEAnZZqmysrK9Pjqp/322BiGocV3z2MZEUQkwg4AdHJOp5PLU7A1wg4AABGqtLSUSQUDQNgBACAClZaWasiQIfriiy9C8n6xsbH66KOPWhV43nzzTT366KMqKSnRiRMn9PLLL2vatGkdV2QLCDsAAESgkydP6osvvtDsmXOVnJTSoe9VXlGm555/RidPnmxV2Dl37pyuvPJK3X777Zo+fXoHVugbYQcAgAiWnJSitAHpVpfRrClTpjRZ9NsqjEgDAAC2RtgBAAC2RtgBAAC2xpgdACHj8Xj8tmFJAgDBRtgB0OEMw5DL5VRMTIzftqmpA3T06KcEHgBBQ9gB0OFM05THY6jq6MeKimr5nx2Px6Ne6YNYkgCwibNnz+rw4cPex5988on27Nmj3r17h3SCQsIOgJBxuVxyuVxWlwHYSnlFWdi+x3vvvadJkyZ5H+fl5UmSZs+erY0bNwajtIAQdgAAiEB9+/ZVbGysnnv+mZC8X2xsrPr27duq10ycODEsemoJOwAARKC0tDR99NFHrI0VAMIOAAARKi0tLWIDSChxuwMAALA1wg4AALA1LmMBNmAYht9BgIFM6AcgfIXDQF8rBOO8CTtAhDMMQxkZ6XK7jwXUvrP+gwlEqujoaEnS559/rtjYWIurCb3PP/9c0v9+Dm1B2AEinGmacruP6dSnR3zOYVNXV6e+gwaHsDJYzV9vHr19kcHlcikhIUGVlZWSpG7dusnhcFhcVcczTVOff/65KisrlZCQ0K45ugg7gE34m7CPyfw6D8MwJCmg5TkkevsiQXJysiR5A09nkpCQ4D3/tiLsAIBN5T+yXi4fy3M0NDRoyb3zJcJO2HM4HOrfv78SExNVX19vdTkhEx0dHZQvaoQdALApp9Ppc0FVFluNPCy50jb8pAMAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFuLsroAAEDk8Hg8fts4HA45nXyXRvgg7AAA/DIMQ5IUExPjt21KSorcbjeBB2GDsAMACFj+I+vlimr5T4dhGFp89zyZphnCqgDfCDsAgIA5nU56bBBx+IkFAAC2RtgBAAC2RtgBAAC2RtgBAAC2RtgBAAC2xt1YABAGDMPwe7t2IBP6AbgYYQcALGYYhlJTU1VWVhZQe+awAVonbMLOI488oqVLl2rx4sXKz8+XJJ0/f14/+clPtHnzZtXW1ionJ0dPPvmkkpKSvK8rLS3VggUL9Ne//lU9evTQ7NmztWrVKkX5mPQKAMKJaZoqKyvT46uf9jmHTUNDg5bcO18i7ACtEhZjdt5991099dRTuuKKK5o8v2TJEv35z3/WCy+8oJ07d6qsrEzTp0/37vd4PJo6darq6uq0a9cuPffcc9q4caPuv//+UJ8CALRb44R9vjYArWf5b87Zs2c1c+ZMPfPMM+rVq5f3+erqaj377LN67LHHdO2112r06NHasGGDdu3apd27d0uSXn/9dR04cEC///3vNXLkSE2ZMkUPPPCACgoKVFdX1+J71tbWqqampskGAADsyfKwk5ubq6lTpyorK6vJ8yUlJaqvr2/y/JAhQ5SWlqbi4mJJUnFxsUaMGNHkslZOTo5qamq0f//+Ft9z1apVio+P926pqalBPisAABAuLA07mzdv1vvvv69Vq1ZdtK+8vFxdunRRQkJCk+eTkpJUXl7ubfPloNO4v3FfS5YuXarq6mrv5na723kmAAAgXFk2itftdmvx4sUqLCxU165dQ/reMTExiomJCel7AgAAa1jWs1NSUqLKykpdddVVioqKUlRUlHbu3KknnnhCUVFRSkpKUl1dnU6fPt3kdRUVFUpOTpYkJScnq6Ki4qL9jfsAAAAsCzuTJ0/W3r17tWfPHu82ZswYzZw50/vf0dHRKioq8r7m0KFDKi0tVWZmpiQpMzNTe/fuVWVlpbdNYWGh4uLiNGzYsJCfEwAACD+WXcbq2bOnLr/88ibPde/eXX369PE+f8cddygvL0+9e/dWXFycFi1apMzMTI0fP16SlJ2drWHDhmnWrFlavXq1ysvLtWzZMuXm5nKZCgAASAqjSQWbs2bNGjmdTs2YMaPJpIKNXC6XtmzZogULFigzM1Pdu3fX7NmztXLlSgurBgAA4SSsws6OHTuaPO7atasKCgpUUFDQ4mvS09O1bdu2Dq4MAABEKsvn2QEAAOhIhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrYTWpIADYiWEYMk3TbzuPxxOCaoDOi7ADAB3AMAylpqaqrKws4NcEEowAtB5hBwA6gGmaKisr0+Orn5bT6XvEQENDg5bcO18i7AAdgrADAB3I6XT6DTv+9gNoH37DAACArdGzAwAIukAGXTscDnq1EBKEHQBA0BiGIUmKiYnx2zYlJUVut5vAgw5H2AEABF3+I+vlimr5T4xhGFp89zzuQENIEHYAAEEXyMBsIFT4SQQAALZG2AEAALZG2AEAALZG2AEAALZG2AEAALZG2AEAALZG2AEAALZG2AEAALZG2AEAALZG2AEAALbGchFAGDMMw+/aQYGsLg0AnRlhBwhThmEoIyNdbvexgNqzoCIANI+wA4Qp0zTldh/TqU+PyOVytdiurq5OfQcNDmFlABBZCDtAmHO5XD7Djq99AAAGKAMAAJsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFuLsroAAPi/PB6P3zYOh0NOJ9/XAPhH2AEQNgzDkMvlVExMjN+2qakDdPTopwQeAH4RdgCEDdM05fEYqjr6saKiWv7nyePxqFf6IJmmGcLqAEQqwg6AsONyueRyuawuA4BN0P8LAABsjbADAABsrU1hZ9CgQfrHP/5x0fOnT5/WoEGD2l0UAABAsLQp7Bw9erTZW0Nra2t1/PjxdhcFAAAQLK0aoPzKK694//u1115TfHy897HH41FRUZEyMjKCVhwAAEB7tapnZ9q0aZo2bZocDodmz57tfTxt2jTdcsstKiws1K9+9auAj7du3TpdccUViouLU1xcnDIzM/WXv/zFu//8+fPKzc1Vnz591KNHD82YMUMVFRVNjlFaWqqpU6eqW7duSkxM1F133aWGhobWnBYAALCxVvXsGIYhSRo4cKDeffdd9e3bt11vPmDAAD3yyCMaPHiwTNPUc889pxtuuEEffPCBhg8friVLlmjr1q164YUXFB8fr4ULF2r69Ol66623JF3oTZo6daqSk5O1a9cunThxQt///vcVHR2thx9+uF21AQAAe2jTPDuffPJJUN7829/+dpPHDz30kNatW6fdu3drwIABevbZZ7Vp0yZde+21kqQNGzZo6NCh2r17t8aPH6/XX39dBw4c0Pbt25WUlKSRI0fqgQce0D333KMVK1aoS5cuQakTAABErjZPKlhUVKSioiJVVlZ6e3wa/fa3v2318Twej1544QWdO3dOmZmZKikpUX19vbKysrxthgwZorS0NBUXF2v8+PEqLi7WiBEjlJSU5G2Tk5OjBQsWaP/+/Ro1alSz71VbW6va2lrv45qamlbXCwAAIkOb7sb6xS9+oezsbBUVFenkyZM6depUk6019u7dqx49eigmJkbz58/Xyy+/rGHDhqm8vFxdunRRQkJCk/ZJSUkqLy+XJJWXlzcJOo37G/e1ZNWqVYqPj/duqampraoZAABEjjb17Kxfv14bN27UrFmz2l3AZZddpj179qi6ulovvviiZs+erZ07d7b7uL4sXbpUeXl53sc1NTUEHgAAbKpNYaeurk5f+9rXglJAly5d9JWvfEWSNHr0aL377rt6/PHHdfPNN6uurk6nT59u0rtTUVGh5ORkSVJycrLeeeedJsdrvFursU1zYmJiAlpVGQAARL42Xcb6wQ9+oE2bNgW7FkkX7viqra3V6NGjFR0draKiIu++Q4cOqbS0VJmZmZKkzMxM7d27V5WVld42hYWFiouL07BhwzqkPgBA8Hg8Hr/b/x0XCrRWm3p2zp8/r6efflrbt2/XFVdcoejo6Cb7H3vssYCOs3TpUk2ZMkVpaWk6c+aMNm3apB07dngnLLzjjjuUl5en3r17Ky4uTosWLVJmZqbGjx8vScrOztawYcM0a9YsrV69WuXl5Vq2bJlyc3PpuQGAMNYYYAL5tzolJUVut1tOJ8s5om3aFHb+9re/aeTIkZKkffv2NdnncDgCPk5lZaW+//3v68SJE4qPj9cVV1yh1157Tdddd50kac2aNXI6nZoxY4Zqa2uVk5OjJ5980vt6l8ulLVu2aMGCBcrMzFT37t01e/ZsrVy5si2nBQAIsfxH1ssV1fKfIsMwtPjueTJNM4RVwW7aFHb++te/BuXNn332WZ/7u3btqoKCAhUUFLTYJj09Xdu2bQtKPQCA0HI6nfTYoMPxEwYAAGytTT07kyZN8nm56o033mhzQQAAAMHUprDTOF6nUX19vfbs2aN9+/Zp9uzZwagLAAAgKNoUdtasWdPs8ytWrNDZs2fbVRAAAEAwBXXMzve+9702rYsFAADQUYIadoqLi9W1a9dgHhIAAKBd2nQZa/r06U0em6apEydO6L333tPPf/7zoBQGAP54PJ6A2jkcDm5vBjqxNoWd+Pj4Jo+dTqcuu+wyrVy5UtnZ2UEpDABaYhiGXC5nwDOlp6YO0NGjnxJ4gE6qTWFnw4YNwa4DAAJmmqY8HkNVRz9WlI/Zd6ULvT+90gcxAy/QibUp7DQqKSnRwYMHJUnDhw/XqFGjglIUAATC5XLJ5XJZXQaAMNemsFNZWalbbrlFO3bsUEJCgiTp9OnTmjRpkjZv3qx+/foFs0YAAIA2a9MF7EWLFunMmTPav3+/qqqqVFVVpX379qmmpkY//vGPg10jAABAm7WpZ+fVV1/V9u3bNXToUO9zw4YNU0FBAQOUAQBAWGlTz45hGIqOjr7o+ejoaBmG0e6iAAAAgqVNYefaa6/V4sWLVVZW5n3u+PHjWrJkiSZPnhy04gAAANqrTWHn17/+tWpqapSRkaFLL71Ul156qQYOHKiamhqtXbs22DUCQFgxDEMej8fvBiA8tGnMTmpqqt5//31t375dH330kSRp6NChysrKCmpxABBuDMNQampqk55tX5jfB7Beq8LOG2+8oYULF2r37t2Ki4vTddddp+uuu06SVF1dreHDh2v9+vX6+te/3iHFAoDVTNNUWVmZHl/9tM8ZmRsaGrTk3vkSYQewXKsuY+Xn52vu3LmKi4u7aF98fLx++MMf6rHHHgtacQAQrpxOp98NQHho1W/jhx9+qOuvv77F/dnZ2SopKWl3UQAAAMHSqrBTUVHR7C3njaKiovTZZ5+1uygAAIBgaVXYueSSS7Rv374W9//tb39T//79210UAABAsLQq7HzrW9/Sz3/+c50/f/6ifV988YWWL1+uf/qnfwpacQAAAO3Vqruxli1bpj/+8Y/66le/qoULF+qyyy6TJH300UcqKCiQx+PRfffd1yGFAgAAtEWrwk5SUpJ27dqlBQsWaOnSpd75IxwOh3JyclRQUKCkpKQOKRQAAKAtWj2pYHp6urZt26ZTp07p8OHDMk1TgwcPVq9evTqiPgAAgHZp0wzKktSrVy+NHTs2mLUAAAAEHbNeAQAAWyPsAAAAWyPsAAAAWyPsAAAAWyPsAAAAWyPsAAAAWyPsAAAAWyPsAAAAWyPsAAAAW2vzDMoAAISKx+MJqJ3D4ZDTyfd4NEXYAQCELcMwJEkxMTEBtU9JSZHb7SbwoAnCDgAg7OU/sl6uKN9/sgzD0OK758k0zRBVhUhB2AEAhD2n00lvDdqMnxwAAGBrhB0AAGBrhB0AAGBrhB0AAGBrDFAGQswwjIDuFgl0XhEAgG+EHSCEDMNQRka63O5jAb+G22gBoH0IO0AImaYpt/uYTn16RC6Xy2fburo69R00OESVAYB9EXYAC7hcLr9hx99+AEBgGKAMAABsjZ4dAJ1CIAO+TdOUw+Fo93EAhBfCDgBbMwxDLpczoIUku0RHq66+PqDjMnAciByEHQC2ZpqmPB5DVUc/VpSPhSQbGhrUO+NSrXlkvd92S+6dLxF2gIhB2AHQKfgbFN7YU+NvwUkWowQiD7+1AADA1iwNO6tWrdLYsWPVs2dPJSYmatq0aTp06FCTNufPn1dubq769OmjHj16aMaMGaqoqGjSprS0VFOnTlW3bt2UmJiou+66Sw0NDaE8FQAAEKYsDTs7d+5Ubm6udu/ercLCQtXX1ys7O1vnzp3ztlmyZIn+/Oc/64UXXtDOnTtVVlam6dOne/d7PB5NnTpVdXV12rVrl5577jlt3LhR999/vxWnBAAAwoylY3ZeffXVJo83btyoxMRElZSU6Bvf+Iaqq6v17LPPatOmTbr22mslSRs2bNDQoUO1e/dujR8/Xq+//roOHDig7du3KykpSSNHjtQDDzyge+65RytWrFCXLl2sODUAABAmwmrMTnV1tSSpd+/ekqSSkhLV19crKyvL22bIkCFKS0tTcXGxJKm4uFgjRoxQUlKSt01OTo5qamq0f//+Zt+ntrZWNTU1TTYAAGBPYRN2DMPQnXfeqQkTJujyyy+XJJWXl6tLly5KSEho0jYpKUnl5eXeNl8OOo37G/c1Z9WqVYqPj/duqampQT4bAAAQLsIm7OTm5mrfvn3avHlzh7/X0qVLVV1d7d3cbneHvycAALBGWMyzs3DhQm3ZskVvvvmmBgwY4H0+OTlZdXV1On36dJPenYqKCiUnJ3vbvPPOO02O13i3VmOb/ysmJiag2VQBAEDks7RnxzRNLVy4UC+//LLeeOMNDRw4sMn+0aNHKzo6WkVFRd7nDh06pNLSUmVmZkqSMjMztXfvXlVWVnrbFBYWKi4uTsOGDQvNiQAAgLBlac9Obm6uNm3apD/96U/q2bOnd4xNfHy8YmNjFR8frzvuuEN5eXnq3bu34uLitGjRImVmZmr8+PGSpOzsbA0bNkyzZs3S6tWrVV5ermXLlik3N5feGwAAYG3YWbdunSRp4sSJTZ7fsGGDbrvtNknSmjVr5HQ6NWPGDNXW1ionJ0dPPvmkt63L5dKWLVu0YMECZWZmqnv37po9e7ZWrlwZqtMAAABhzNKwE8iqwV27dlVBQYEKCgpabJOenq5t27YFszQAAGATYXM3FgAAQEcg7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsLi1XPAQAIFo/H47eNw+GQ08n3/c6CsAMAsAXDMCQpoEWgU1JS5Ha7CTydBGEHAGAr+Y+slyuq5T9vhmFo8d3zAlqfEfZA2AEA2IrT6aTHBk0QdgB0CoZh+Pwm33gJBID9EHYA2FpjwHnooYd8fttvDDtc2gDsh7ADoFPImTxV0dHRLe6vr6/XL59+ViLsALZD2AHQKTgcDjkcDp/7AdgTI7gAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtces5ECT+ZuiVAluNGQAQXIQdIAgMw1BGRrrc7mMBtWeWXgAIHcIOEASmacrtPqZTnx6Ry+VqsV1dXZ36DhocwsoAAIQdIIhcLpfPsONrHwCgYzBAGQAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2Bq3ngPAlxiGIcMwfO4HEFkIOwCgCyHG5XIq72c/8tvW5XJeaB+CugC0H2EHACSZMuXxGHrt319UdHR0i+0aGhqUfcuMEFYGoL0IOwDwJU6n0+dM11zGAiIPA5QBAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtMakg4IdhGDJN02cbj8cTomoAAK1F2AF8MAxDGRnpcruPBdTeXyiCfbBgKBA5CDuAD6Zpyu0+plOfHvG5hEBdXZ36DhocwspgFRYMBSIPYQcIgMvl8hl2fO2DvbBgKBB5CDsA0AYsGApEDu7GAgAAtkbYAQAAtkbYAQAAtsaYHQBApxTI/FgOh0NOJ/0CkY6wAwDoVBoHj8fExPhtm5KSIrfbTeCJcJaGnTfffFOPPvqoSkpKdOLECb388suaNm2ad79pmlq+fLmeeeYZnT59WhMmTNC6des0ePD/zmdSVVWlRYsW6c9//rOcTqdmzJihxx9/XD169LDgjACEUiCzW3NXFFqS/8h6uaJa/jNoGIYW3z2PyUJtwNKwc+7cOV155ZW6/fbbNX369Iv2r169Wk888YSee+45DRw4UD//+c+Vk5OjAwcOqGvXrpKkmTNn6sSJEyosLFR9fb3mzJmjefPmadOmTaE+HQAh0vjH56GHHvL7jbuhoeHCazq8KkQap9NJj00nYWnYmTJliqZMmdLsPtM0lZ+fr2XLlumGG26QJP3ud79TUlKS/vM//1O33HKLDh48qFdffVXvvvuuxowZI0lau3atvvWtb+mXv/ylUlJSmj12bW2tamtrvY9ramqCfGYAQiFn8lSfE/tJ0vna83rsNxskvp0DnVbYRtpPPvlE5eXlysrK8j4XHx+vcePGqbi4WJJUXFyshIQEb9CRpKysLDmdTr399tstHnvVqlWKj4/3bqmpqR13IgA6jMPhCGgD0LmFbdgpLy+XJCUlJTV5PikpybuvvLxciYmJTfZHRUWpd+/e3jbNWbp0qaqrq72b2+0OcvUAACBcdMq7sWJiYgIahQ8AACJf2PbsJCcnS5IqKiqaPF9RUeHdl5ycrMrKyib7GxoaVFVV5W0DAAA6t7ANOwMHDlRycrKKioq8z9XU1Ojtt99WZmamJCkzM1OnT59WSUmJt80bb7whwzA0bty4kNcMAADCj6WXsc6ePavDhw97H3/yySfas2ePevfurbS0NN1555168MEHNXjwYO+t5ykpKd65eIYOHarrr79ec+fO1fr161VfX6+FCxfqlltuafFOLAAA0LlYGnbee+89TZo0yfs4Ly9PkjR79mxt3LhRd999t86dO6d58+bp9OnTuuaaa/Tqq69659iRpOeff14LFy7U5MmTvZMKPvHEEyE/FwAAEJ4sDTsTJ070OTOlw+HQypUrtXLlyhbb9O7dmwkEAQBAi8J2zA4AAEAwEHYAAICtEXYAAICtEXYAAICtdcoZlAGEN8MwfN68YBhGCKtpH8Mw/NYbSecDRCLCDoCw0RhwHnroITmdLXc8NzQ0XGgfkqraxjAMuVxO5f3sRwG1dzodamhokMPHeROKgLYh7AAIOzmTpyo6OrrF/edrz+ux32yQfPT+WM2UKY/H0Gv//qLPc5Gkuro6TZ11c0DByOVyXghSwSoU6AQIOwDCjsPhkMPh8Lk/UjidTrlcvqOJw+kIKBg1NDQo+5YZwS4RsD3CDjotf+NCJMnj8YSoms7BTmNxOoK/YNTZPx+grQg76JQMw1BGRrrc7mMBtfcXiuCbncbiAIg8hB10SqZpyu0+plOfHvH5Tbqurk59Bw0OYWX2ZoexOAAiD2EHnZrL5fIZdvyNtUDr2GksDjqPQC5nOxwOn72WsBZhBwCAZjSOkYqJifHbNiUlRW63m8ATpgg7AAD4kP/IermiWv5zaRiGFt89j7F9YYywAwCAD06nkx6bCMf/ewAAwNbo2QEAmzIMQw4fc/Mwbw86C8IOAEQYf4uLNjQ0yOVy6s575/s9FstPoDMg7ABAhGjt4qIsPwFcQNgBgAgR6OKitXW1mnLrTSw/Afx/hB0AiDD+Qgx3DgFN8RsBAABsjbADAABsjbADAABsjbADAABsjbADAABsjbADAABsjVvPYSuGYQS08rDH4wlBNZ2Hv8+d+VwAWImwA9swDEMZGelyu48F/JpAghFa1vj5PfTQQz7ndmloaLjQPiRVAUBThB3YhmmacruP6dSnR3xOuCZJdXV16jtocIgqs7+cyVN9zuh7vva8HvvNBolwCcAChB3Yjsvl8ht2/O1H6zgcDjkcDp/7AcAqDFAGAAC2RtgBAAC2xmUsAOjkDMPwe8ccd9T5F+hdng6Hg8VaQ4ywAwCdlGEYcrmcyvvZjwJq73I5L7ymg+uKNI1BMCYmJqD2KSkpcrvdBJ4QIuwAQCdlypTHY+i1f3/R59100oXpA7JvmeG3F6gz9wDlP7Jerijff1YNw9Diu+cx7UWIEXYAoJNzOp1+71BsaGgIuBeos/YAOZ1OemvCFGEHQLMCmY26M3+L72wC7QVq7AECwglhB0ATgc6KLDEzcmfkrxeIAIxwRNgB0Cx/syJLzIwMIDIQdgA0y9+syI1tACDcMZIKAADYGmEHAADYGpexYKlA7viRLgya9XfJJNDZSwEAnQthB5YxDEMZGelyu4/5bdslOlp19fUBHZfJugDYQaBfBll+wj/CDixjmqbc7mM69ekRn7ey1tXVqe+gwao6+rGifMxO2tgOAMKdv57oC18GM1RWVub3WCw/4R9hB5ZzuVw+w07jvkDbwTd/3xaZJwXoOK1dRyv/X5/yO68Ry0/4R9gBOolAJwtkokAEA2to+eZvHa2GhgYtuXe+nFyiCgrCDmATgfbY+JsskIkC0R6tWUnd6XSooaFBDh9/zO0aivyto0XACS7CDhDhWttjIz+TBTJRINoj0DW06urqNHXWzSwsipAg7KBDBHIXAbeKBxc9Nggn/tbQcjgdLCyKkCHsIKhz3TQe79JBg+Q+5v+W8sbjomWBXp7yt7wDPTYIRywsilAg7HRyHTXXjSRuFW8nBhQDQHAQdjq5YM918+W2nfFWcX+9MI37AulNY0Ax8L/83d3V2Kbxfx1BvBMs2MfrCIEMC+jMkw/aJuwUFBTo0UcfVXl5ua688kqtXbtWV199tdVlRYxgzXXz5badSWt7Yfy1+3JbBhSjM2vN3V3ShX9/7rx3fgDt/A96bnzvYB2vI7Rm3p7OPPmgLcLOH/7wB+Xl5Wn9+vUaN26c8vPzlZOTo0OHDikxMdHq8izDIOHQC7QXJvvab6lLly4+j0WPDRD43V2SVFtXqym33hTwoOdA5gLyeAy9vvklnz3a4TCI2t+8PR01+WCkLGlhi7Dz2GOPae7cuZozZ44kaf369dq6dat++9vf6t5777W0to74QQj0EgiDhIMn2IOE/bX7clsA/gcyN7YJpG1DQ0OreoscDkdAg6itvNQWqLq6uoB73/21i6QlLSI+7NTV1amkpERLly71Pud0OpWVlaXi4uJmX1NbW6va2lrv4+rqaklSTU1NUGszDEOXDx+u44H8IPTvrw/27PH7g2AYhkaNGhXQD5ckffzB+3JFtXzM+vp6Db5qrKqqqhTl45tQfd2Fgcn+2rWmrVXtWtO2rrZOkvTz++/3GT4ae8iqz5xRtM9B2Rd+7mrOnPH7LTXQtla1o0ZqDKd2rT2mx2Poxac3+P79r6/TTfNuD+B4dXI6HQGHJ6czsEtjDodUU1OtaB+9wI3vHcjxnE6nunXrFlCNDocj4C/BD97/S7mcvsPgfSt/ourq6qAPc2j8u+23VjPCHT9+3JRk7tq1q8nzd911l3n11Vc3+5rly5ebunDzChsbGxsbG1uEb26322dWiPienbZYunSp8vLyvI8Nw1BVVZX69Olji0sHNTU1Sk1NldvtVlxcnNXlhA0+l4vxmTSPz6V5fC4X4zNpXqg+F9M0debMGaWkpPhsF/Fhp2/fvnK5XKqoqGjyfEVFhZKTk5t9TUxMzEUj1xMSEjqqRMvExcXxy9cMPpeL8Zk0j8+leXwuF+MzaV4oPpf4+Hi/bSL+/rMuXbpo9OjRKioq8j5nGIaKioqUmZlpYWUAACAcRHzPjiTl5eVp9uzZGjNmjK6++mrl5+fr3Llz3ruzAABA52WLsHPzzTfrs88+0/3336/y8nKNHDlSr776qpKSkqwuzRIxMTFavnx5QJNMdSZ8LhfjM2ken0vz+FwuxmfSvHD7XBymyQQrAADAviJ+zA4AAIAvhB0AAGBrhB0AAGBrhB0AAGBrhJ1OYOvWrRo3bpxiY2PVq1cvTZs2zeqSwkZtba1Gjhwph8OhPXv2WF2OpY4ePao77rhDAwcOVGxsrC699FItX75cdXV1VpcWcgUFBcrIyFDXrl01btw4vfPOO1aXZJlVq1Zp7Nix6tmzpxITEzVt2jQdOnTI6rLCziOPPCKHw6E777zT6lIsd/z4cX3ve99Tnz59FBsbqxEjRui9996ztCbCjs299NJLmjVrlubMmaMPP/xQb731lm699Varywobd999t99pxjuLjz76SIZh6KmnntL+/fu1Zs0arV+/Xj/72c+sLi2k/vCHPygvL0/Lly/X+++/ryuvvFI5OTmqrKy0ujRL7Ny5U7m5udq9e7cKCwtVX1+v7OxsnTt3zurSwsa7776rp556SldccYXVpVju1KlTmjBhgqKjo/WXv/xFBw4c0K9+9Sv16tXL2sKCsxwnwlF9fb15ySWXmL/5zW+sLiUsbdu2zRwyZIi5f/9+U5L5wQcfWF1S2Fm9erU5cOBAq8sIqauvvtrMzc31PvZ4PGZKSoq5atUqC6sKH5WVlaYkc+fOnVaXEhbOnDljDh482CwsLDS/+c1vmosXL7a6JEvdc8895jXXXGN1GRehZ8fG3n//fR0/flxOp1OjRo1S//79NWXKFO3bt8/q0ixXUVGhuXPn6t/+7d/UrVs3q8sJW9XV1erdu7fVZYRMXV2dSkpKlJWV5X3O6XQqKytLxcXFFlYWPqqrqyWpU/1c+JKbm6upU6c2+ZnpzF555RWNGTNGN954oxITEzVq1Cg988wzVpfFZSw7O3LkiCRpxYoVWrZsmbZs2aJevXpp4sSJqqqqsrg665imqdtuu03z58/XmDFjrC4nbB0+fFhr167VD3/4Q6tLCZmTJ0/K4/FcNPt6UlKSysvLLaoqfBiGoTvvvFMTJkzQ5ZdfbnU5ltu8ebPef/99rVq1yupSwsaRI0e0bt06DR48WK+99poWLFigH//4x3ruuecsrYuwE4HuvfdeORwOn1vj+AtJuu+++zRjxgyNHj1aGzZskMPh0AsvvGDxWQRfoJ/L2rVrdebMGS1dutTqkkMi0M/ly44fP67rr79eN954o+bOnWtR5Qg3ubm52rdvnzZv3mx1KZZzu91avHixnn/+eXXt2tXqcsKGYRi66qqr9PDDD2vUqFGaN2+e5s6dq/Xr11taly3WxupsfvKTn+i2227z2WbQoEE6ceKEJGnYsGHe52NiYjRo0CCVlpZ2ZImWCPRzeeONN1RcXHzRmi1jxozRzJkzLf8GEmyBfi6NysrKNGnSJH3ta1/T008/3cHVhZe+ffvK5XKpoqKiyfMVFRVKTk62qKrwsHDhQm3ZskVvvvmmBgwYYHU5lispKVFlZaWuuuoq73Mej0dvvvmmfv3rX6u2tlYul8vCCq3Rv3//Jn9zJGno0KF66aWXLKroAsJOBOrXr5/69evnt93o0aMVExOjQ4cO6ZprrpEk1dfX6+jRo0pPT+/oMkMu0M/liSee0IMPPuh9XFZWppycHP3hD3/QuHHjOrJESwT6uUgXenQmTZrk7QV0OjtX52+XLl00evRoFRUVeadoMAxDRUVFWrhwobXFWcQ0TS1atEgvv/yyduzYoYEDB1pdUliYPHmy9u7d2+S5OXPmaMiQIbrnnns6ZdCRpAkTJlw0NcH//M//WP43h7BjY3FxcZo/f76WL1+u1NRUpaen69FHH5Uk3XjjjRZXZ520tLQmj3v06CFJuvTSSzv1N9bjx49r4sSJSk9P1y9/+Ut99tln3n2dqVcjLy9Ps2fP1pgxY3T11VcrPz9f586d05w5c6wuzRK5ubnatGmT/vSnP6lnz57esUvx8fGKjY21uDrr9OzZ86JxS927d1efPn069XimJUuW6Gtf+5oefvhh3XTTTXrnnXf09NNPW95LTNixuUcffVRRUVGaNWuWvvjiC40bN05vvPGG9XMeIOwUFhbq8OHDOnz48EWhzzRNi6oKvZtvvlmfffaZ7r//fpWXl2vkyJF69dVXLxq03FmsW7dOkjRx4sQmz2/YsMHv5VF0PmPHjtXLL7+spUuXauXKlRo4cKDy8/M1c+ZMS+tymJ3pXzEAANDpdK4L8gAAoNMh7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7ACApLq6OqtLANBBCDsAwtqLL76oESNGKDY2Vn369FFWVpbOnTsnSfrtb3+r4cOHKyYmRv3799fChQu9rystLdUNN9ygHj16KC4uTjfddJMqKiq8+1esWKGRI0fqN7/5jQYOHKiuXbtKkk6fPq0f/OAH6tevn+Li4nTttdfqww8/DO1JAwgqwg6AsHXixAl997vf1e23366DBw9qx44dmj59ukzT1Lp165Sbm6t58+Zp7969euWVV/SVr3xFkmQYhm644QZVVVVp586dKiws1JEjR3TzzTc3Of7hw4f10ksv6Y9//KP27NkjSbrxxhtVWVmpv/zlLyopKdFVV12lyZMnq6qqKtSnDyBIWPUcQNh6//33NXr0aB09elTp6elN9l1yySWaM2eOHnzwwYteV1hYqClTpuiTTz5RamqqJOnAgQMaPny43nnnHY0dO1YrVqzQww8/rOPHj6tfv36SpP/+7//W1KlTVVlZqZiYGO/xvvKVr+juu+/WvHnzOvBsAXSUKKsLAICWXHnllZo8ebJGjBihnJwcZWdn6zvf+Y7q6+tVVlamyZMnN/u6gwcPKjU11Rt0JGnYsGFKSEjQwYMHNXbsWElSenq6N+hI0ocffqizZ8+qT58+TY73xRdf6OOPP+6AMwQQCoQdAGHL5XKpsLBQu3bt0uuvv661a9fqvvvuU1FRUVCO37179yaPz549q/79+2vHjh0XtU1ISAjKewIIPcIOgLDmcDg0YcIETZgwQffff7/S09NVWFiojIwMFRUVadKkSRe9ZujQoXK73XK73U0uY50+fVrDhg1r8b2uuuoqlZeXKyoqShkZGR11SgBCjLADIGy9/fbbKioqUnZ2thITE/X222/rs88+09ChQ7VixQrNnz9fiYmJmjJlis6cOaO33npLixYtUlZWlkaMGKGZM2cqPz9fDQ0N+tGPfqRvfvObGjNmTIvvl5WVpczMTE2bNk2rV6/WV7/6VZWVlWnr1q36l3/5F5+vBRC+CDsAwlZcXJzefPNN5efnq6amRunp6frVr36lKVOmSJLOnz+vNWvW6Kc//an69u2r73znO5Iu9Ab96U9/0qJFi/SNb3xDTqdT119/vdauXevz/RwOh7Zt26b77rtPc+bM0Weffabk5GR94xvfUFJSUoefL4COwd1YAADA1phnBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2Nr/AwKdXcya05rFAAAAAElFTkSuQmCC", 61 | "text/plain": [ 62 | "
" 63 | ] 64 | }, 65 | "metadata": {}, 66 | "output_type": "display_data" 67 | } 68 | ], 69 | "source": [ 70 | "import seaborn as sns\n", 71 | "import pandas as pd\n", 72 | "import warnings\n", 73 | "import matplotlib.pyplot as plt\n", 74 | "warnings.filterwarnings(\"ignore\", \"is_categorical_dtype\")\n", 75 | "warnings.filterwarnings(\"ignore\", \"use_inf_as_na\")\n", 76 | "\n", 77 | "score = clf.decision_function(X)\n", 78 | "df = pd.DataFrame({'score': score, 'y': y})\n", 79 | "sns.histplot(df, x=\"score\", hue=\"y\")\n", 80 | "plt.show()" 81 | ] 82 | } 83 | ], 84 | "metadata": { 85 | "colab": { 86 | "provenance": [] 87 | }, 88 | "kernelspec": { 89 | "display_name": "Python 3", 90 | "name": "python3" 91 | }, 92 | "language_info": { 93 | "name": "python" 94 | } 95 | }, 96 | "nbformat": 4, 97 | "nbformat_minor": 0 98 | } 99 | -------------------------------------------------------------------------------- /doc/source/figs/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/softmin/ReHLine-python/08597596bc569b094fb6df2026f741a183b78aa1/doc/source/figs/logo.png -------------------------------------------------------------------------------- /doc/source/figs/res.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/softmin/ReHLine-python/08597596bc569b094fb6df2026f741a183b78aa1/doc/source/figs/res.png -------------------------------------------------------------------------------- /doc/source/figs/tab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/softmin/ReHLine-python/08597596bc569b094fb6df2026f741a183b78aa1/doc/source/figs/tab.png -------------------------------------------------------------------------------- /doc/source/getting_started.rst: -------------------------------------------------------------------------------- 1 | Getting Started 2 | =============== 3 | 4 | This page provides a starter example to introduce users to the ``rehline`` package and showcase its primary features, facilitating exploration and familiarization. 5 | 6 | To proceed, ensure that you have already installed ``rehline``: 7 | 8 | .. code:: bash 9 | 10 | pip install rehline 11 | 12 | -------------------------------- 13 | 14 | ``rehline`` is a versatile solver for machine learning problems, particularly effective for Empirical Risk Minimization (ERM) with `non-smooth` objectives. We will use ERM as our starting example to demonstrate that: 15 | 16 | .. admonition:: Note 17 | :class: tip 18 | 19 | With ``rehline``, you can easily transform different `loss functions` and add `constraints` to your ERM with no tears! 20 | 21 | Let's begin by generating a toy dataset and splitting it into training and test sets using scikit-learn's `make_regression`. 22 | 23 | .. code:: python 24 | 25 | # Import necessary libraries 26 | import numpy as np 27 | from sklearn.datasets import make_regression 28 | from sklearn.model_selection import train_test_split 29 | from sklearn.preprocessing import StandardScaler 30 | 31 | np.random.seed(1024) 32 | # Generate toy data 33 | n, d = 1000, 5 34 | scaler = StandardScaler() 35 | X, y = make_regression(n_samples=n, n_features=d, noise=1.0) 36 | # Normalize X and add intercept 37 | X = scaler.fit_transform(X) 38 | X = np.hstack((X, np.ones((n, 1)))) 39 | 40 | # Split data into training and test sets 41 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=50) 42 | 43 | Quantile Regression 44 | ------------------- 45 | 46 | Next, let's use ``rehline`` to fit a quantile regression (QR) at quantile level 0.95 (:math:`\kappa=0.95`). 47 | 48 | The ridge-regularized QR solves the following optimization problem: 49 | 50 | .. math:: 51 | 52 | \min_{\beta \in \mathbb{R}^{d}} \ C \sum_{i=1}^n \rho_\kappa ( y_i - x_i^\intercal \beta ) + \frac{1}{2} \| \beta \|^2, 53 | 54 | where :math:`\rho_\kappa(u) = u \cdot (\kappa - \mathbf{1}(u < 0))` is the `check loss`, :math:`x_i \in \mathbb{R}^d` is a feature vector, and :math:`y_i \in \mathbb{R}` is the response variable. 55 | 56 | Since the `check loss` is a piecewise linear quadratic function (PLQ), it can be solved using ``rehline.plqERM_Ridge``: 57 | 58 | .. code:: python 59 | 60 | from rehline import plqERM_Ridge 61 | # Define a QR estimator 62 | clf = plqERM_Ridge(loss={'name': 'QR', 'qt': 0.95}, C=1.0) 63 | clf.fit(X=X_train, y=y_train) 64 | # Make predictions 65 | q_predict = clf.decision_function(X_test) 66 | 67 | # Plot results 68 | import matplotlib.pyplot as plt 69 | plt.scatter(x=X_test[:, 0], y=y_test, label='y_true') 70 | plt.scatter(x=X_test[:, 0], y=q_predict, alpha=0.5, label='q_95') 71 | plt.legend(loc="upper left") 72 | plt.show() 73 | 74 | Huber Regression 75 | ---------------- 76 | 77 | If you prefer Huber regression, it is also a PLQ function. 78 | 79 | The ridge-regularized Huber minimization solves the following optimization problem: 80 | 81 | .. math:: 82 | 83 | \min_{\mathbf{\beta}} C \sum_{i=1}^n H_\kappa( y_i - \mathbf{x}_i^\intercal \mathbf{\beta} ) + \frac{1}{2} \| \mathbf{\beta} \|_2^2, 84 | 85 | where :math:`H_\kappa(\cdot)` is the Huber loss defined as follows: 86 | 87 | .. math:: 88 | \begin{equation*} 89 | H_\kappa(z) = 90 | \begin{cases} 91 | z^2/2, & 0 < |z| \leq \kappa, \\ 92 | \kappa ( |z| - \kappa/2 ), & |z| > \kappa. 93 | \end{cases} 94 | \end{equation*} 95 | 96 | .. code:: python 97 | 98 | from rehline import plqERM_Ridge 99 | # Define a Huber estimator 100 | clf = plqERM_Ridge(loss={'name': 'huber', 'tau': 0.5}, C=1.0) 101 | clf.fit(X=X_train, y=y_train) 102 | # Make predictions 103 | y_huber = clf.decision_function(X_test) 104 | 105 | # Plot results 106 | import matplotlib.pyplot as plt 107 | plt.scatter(x=X_test[:, 0], y=y_test, label='y_true') 108 | plt.scatter(x=X_test[:, 0], y=y_huber, alpha=0.5, label='y_huber') 109 | plt.legend(loc="upper left") 110 | plt.show() 111 | 112 | Fairness Constraints 113 | -------------------- 114 | 115 | You have now learned that the fitted Huber regression requires a fairness constraint for the first feature :math:`\mathbf{X}_{1}`. Specifically, the correlation between the predicted :math:`\hat{Y}` and :math:`\mathbf{X}_{1}` must be less than `tol=0.1`, that is, 116 | 117 | .. math:: 118 | 119 | \min_{\mathbf{\beta}} C \sum_{i=1}^n H_\kappa( y_i - \mathbf{x}_i^\intercal \mathbf{\beta} ) + \frac{1}{2} \| \mathbf{\beta} \|_2^2, \quad \text{s.t.} \quad \Big | \frac{1}{n} \sum_{i=1}^n \mathbf{z}_i \mathbf{\beta}^\intercal \mathbf{x}_i \Big| \leq \mathbf{\rho} 120 | 121 | With `rehline`, you can easily add a `fairness constraint` to your ERM. 122 | 123 | .. code:: python 124 | 125 | from rehline import plqERM_Ridge 126 | from scipy.stats import pearsonr 127 | # Define a Huber estimator with fairness constraint 128 | clf = plqERM_Ridge(loss={'name': 'huber', 'tau': 0.5}, 129 | constraint=[{'name': 'fair', 'sen_idx': [0], 'tol_sen': 0.1}], 130 | C=1.0, 131 | max_iter=10000) 132 | clf.fit(X=X_train, y=y_train) 133 | # Make predictions 134 | y_huber_fair = clf.decision_function(X_test) 135 | 136 | # Plot results 137 | import matplotlib.pyplot as plt 138 | plt.scatter(x=X_test[:, 0], y=y_test, label='y_true') 139 | plt.scatter(x=X_test[:, 0], y=y_huber, alpha=0.5, label='y_huber') 140 | plt.scatter(x=X_test[:, 0], y=y_huber_fair, alpha=0.5, label='y_huber_fair') 141 | plt.legend(loc="upper left") 142 | plt.show() 143 | 144 | .. nblinkgallery:: 145 | :caption: Related Examples 146 | :name: rst-link-gallery 147 | 148 | examples/QR.ipynb 149 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | .. dnn-inference documentation master file, created by 2 | sphinx-quickstart on Sun Aug 8 20:28:09 2021. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | 🦌 ReHLine 7 | ========== 8 | 9 | .. image:: figs/logo.png 10 | :width: 18% 11 | :align: right 12 | 13 | .. -*- mode: rst -*- 14 | 15 | |PyPi|_ |MIT|_ |Python3|_ |downloads|_ 16 | 17 | .. |PyPi| image:: https://badge.fury.io/py/rehline.svg 18 | .. _PyPi: https://pypi.org/project/rehline/ 19 | 20 | .. |MIT| image:: https://img.shields.io/pypi/l/dnn-inference.svg 21 | .. _MIT: https://opensource.org/licenses/MIT 22 | 23 | .. |Python3| image:: https://img.shields.io/badge/python-3-blue.svg 24 | .. _Python3: www.python.org 25 | 26 | .. |downloads| image:: https://pepy.tech/badge/rehline 27 | .. _downloads: https://pepy.tech/project/rehline 28 | 29 | 30 | **ReHLine** is designed to be a computationally efficient and practically useful software package for large-scale ERMs. 31 | 32 | - Homepage: `https://rehline.github.io/ `_ 33 | - GitHub repo: `https://github.com/softmin/ReHLine-python `_ 34 | - Documentation: `https://rehline-python.readthedocs.io `_ 35 | - PyPi: `https://pypi.org/project/rehline `_ 36 | - Paper: `NeurIPS | 2023 `_ 37 | .. - Open Source: `MIT license `_ 38 | 39 | The proposed **ReHLine** solver has appealing exhibits appealing properties: 40 | 41 | .. list-table:: 42 | :widths: 20 80 43 | 44 | * - **Flexible losses** 45 | - It applies to ANY convex piecewise linear-quadratic loss function, including the hinge loss, the squared-hinge the check loss, the Huber loss, etc. 46 | * - **Flexible constraints** 47 | - It supports linear equality and inequality constraints on the parameter vector. 48 | * - **Super-Efficient** 49 | - The optimization algorithm has a provable **LINEAR** convergence rate, and the per-iteration computational complexity is **LINEAR** in the sample size. 50 | 51 | ✨ New Features: Scikit-Learn Compatible Estimators 52 | --------------------------------------------------- 53 | 54 | We are excited to introduce full scikit-learn compatibility! `ReHLine` now provides `plq_Ridge_Classifier` and `plq_Ridge_Regressor` estimators that integrate seamlessly with the entire scikit-learn ecosystem. 55 | 56 | This means you can: 57 | - Drop `ReHLine` estimators directly into your existing scikit-learn `Pipeline`. 58 | - Perform robust hyperparameter tuning using `GridSearchCV`. 59 | - Use standard scikit-learn evaluation metrics and cross-validation tools. 60 | 61 | 🔨 Installation 62 | --------------- 63 | 64 | Install ``rehline`` using ``pip`` 65 | 66 | .. code:: bash 67 | 68 | pip install rehline 69 | 70 | Reference 71 | --------- 72 | If you use this code please star 🌟 the repository and cite the following paper: 73 | 74 | .. code:: bib 75 | 76 | @article{daiqiu2023rehline, 77 | title={ReHLine: Regularized Composite ReLU-ReHU Loss Minimization with Linear Computation and Linear Convergence}, 78 | author={Dai, Ben and Yixuan Qiu}, 79 | journal={Advances in Neural Information Processing Systems}, 80 | year={2023}, 81 | } 82 | 83 | 84 | .. toctree:: 85 | :maxdepth: 2 86 | :hidden: 87 | 88 | getting_started 89 | tutorials 90 | example 91 | benchmark 92 | -------------------------------------------------------------------------------- /doc/source/tutorials.rst: -------------------------------------------------------------------------------- 1 | Tutorials 2 | --------- 3 | 4 | `ReHLine` is designed to address the regularized ReLU-ReHU minimization problem, named *ReHLine optimization*, of the following form: 5 | 6 | .. math:: 7 | 8 | \min_{\mathbf{\beta} \in \mathbb{R}^d} \sum_{i=1}^n \sum_{l=1}^L \text{ReLU}( u_{li} \mathbf{x}_i^\intercal \mathbf{\beta} + v_{li}) + \sum_{i=1}^n \sum_{h=1}^H {\text{ReHU}}_{\tau_{hi}}( s_{hi} \mathbf{x}_i^\intercal \mathbf{\beta} + t_{hi}) + \frac{1}{2} \| \mathbf{\beta} \|_2^2, \ \text{ s.t. } \mathbf{A} \mathbf{\beta} + \mathbf{b} \geq \mathbf{0}, 9 | 10 | 11 | where :math:`\mathbf{U} = (u_{li}),\mathbf{V} = (v_{li}) \in \mathbb{R}^{L \times n}` 12 | and :math:`\mathbf{S} = (s_{hi}),\mathbf{T} = (t_{hi}),\mathbf{\tau} = (\tau_{hi}) \in \mathbb{R}^{H \times n}` 13 | are the ReLU-ReHU loss parameters, and :math:`(\mathbf{A},\mathbf{b})` are the constraint parameters. 14 | This formulation has a wide range of applications spanning various fields, including statistics, 15 | machine learning, computational biology, and social studies. 16 | Some popular examples include SVMs with fairness constraints (FairSVM), 17 | elastic net regularized quantile regression (ElasticQR), 18 | and ridge regularized Huber minimization (RidgeHuber). 19 | 20 | See `Manual ReHLine Formulation`_ documentation for more details and examples on converting your problem to ReHLine formulation. 21 | 22 | 23 | Moreover, the following specific classes of formulations can be directly solved by `ReHLine`. 24 | 25 | List of Tutorials 26 | ================= 27 | 28 | .. list-table:: 29 | :align: left 30 | :widths: 10 10 20 31 | :header-rows: 1 32 | 33 | * - tutorials 34 | - | API 35 | - | description 36 | * - `Manual ReHLine Formulation <./tutorials/ReHLine_manual.rst>`_ 37 | - | `ReHLine <./autoapi/rehline/index.html#rehline.ReHLine>`_ 38 | - | ReHLine minimization with manual parameter settings. 39 | 40 | * - `ReHLine: Empirical Risk Minimization <./tutorials/ReHLine_ERM.rst>`_ 41 | - | `plqERM_Ridge <./autoapi/rehline/index.html#rehline.plqERM_Ridge>`_ 42 | - | Empirical Risk Minimization (ERM) with a piecewise linear-quadratic (PLQ) objective with a ridge penalty. 43 | 44 | * - `ReHLine: Scikit-learn Compatible Estimators <./tutorials/ReHLine_sklearn.rst>`_ 45 | - | `plq_Ridge_Classifier <./autoapi/rehline/index.html#rehline.plq_Ridge_Classifier>`_ `plq_Ridge_Regressor <./autoapi/rehline/index.html#rehline.plq_Ridge_Regressor>`_ 46 | - | Scikit-learn compatible estimators framework for empirical risk minimization problem. 47 | 48 | * - `ReHLine: Ridge Composite Quantile Regression <./examples/CQR.ipynb>`_ 49 | - | `CQR_Ridge <./autoapi/rehline/index.html#rehline.CQR_Ridge>`_ 50 | - | Composite Quantile Regression (CQR) with a ridge penalty. 51 | 52 | * - `ReHLine: Matrix Factorization <./tutorials/ReHLine_MF.rst>`_ 53 | - | `plqMF_Ridge <./autoapi/rehline/index.html#rehline.plqMF_Ridge>`_ 54 | - | Matrix Factorization (MF) with a piecewise linear-quadratic (PLQ) objective with a ridge penalty. 55 | 56 | .. toctree:: 57 | :maxdepth: 2 58 | :hidden: 59 | 60 | ./tutorials/ReHLine_manual 61 | ./tutorials/ReHLine_ERM 62 | ./tutorials/loss 63 | ./tutorials/constraint 64 | ./tutorials/ReHLine_sklearn 65 | ./tutorials/warmstart 66 | 67 | -------------------------------------------------------------------------------- /doc/source/tutorials/ReHLine_ERM.rst: -------------------------------------------------------------------------------- 1 | ReHLine: Empirical Risk Minimization 2 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 3 | 4 | The objective function is given by the following PLQ formulation, where :math:`\phi` is a convex piecewise linear function and :math:`\lambda` is a positive regularization parameter. 5 | 6 | .. math:: 7 | 8 | \min_{\pmb{\beta} \in \mathbb{R}^d} \sum_{i=1}^n \text{PLQ}(y_i, \mathbf{x}_i^T \pmb{\beta}) + \frac{1}{2} \| \pmb{\beta} \|_2^2, \ \text{ s.t. } \ 9 | \mathbf{A} \pmb{\beta} + \mathbf{b} \geq \mathbf{0}, 10 | 11 | where :math:`\text{PLQ}(\cdot, \cdot)` is a convex piecewise linear quadratic function, see `Loss <./loss.rst>`_ for build-in loss functions, and :math:`\mathbf{A}` is a :math:`K \times d` matrix, and :math:`\mathbf{b}` is a :math:`K`-dimensional vector for linear constraints, see `Constraints <./constraint.rst>`_ for more details. 12 | 13 | For example, it supports the following loss functions and constraints. 14 | 15 | .. image:: ../figs/tab.png 16 | 17 | Example 18 | ------- 19 | 20 | .. nblinkgallery:: 21 | :caption: Emprical Risk Minimization 22 | :name: rst-link-gallery 23 | 24 | ../examples/QR.ipynb 25 | ../examples/CQR.ipynb 26 | ../examples/SVM.ipynb 27 | ../examples/FairSVM.ipynb 28 | -------------------------------------------------------------------------------- /doc/source/tutorials/ReHLine_MF.rst: -------------------------------------------------------------------------------- 1 | ReHLine: Matrix Factorization 2 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -------------------------------------------------------------------------------- /doc/source/tutorials/ReHLine_manual.rst: -------------------------------------------------------------------------------- 1 | Manual ReHLine Formulation 2 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 3 | 4 | `ReHLine` is designed to address the regularized ReLU-ReHU minimization problem, named *ReHLine optimization*, of the following form: 5 | 6 | .. math:: 7 | 8 | \min_{\mathbf{\beta} \in \mathbb{R}^d} \sum_{i=1}^n \sum_{l=1}^L \text{ReLU}( u_{li} \mathbf{x}_i^\intercal \mathbf{\beta} + v_{li}) + \sum_{i=1}^n \sum_{h=1}^H {\text{ReHU}}_{\tau_{hi}}( s_{hi} \mathbf{x}_i^\intercal \mathbf{\beta} + t_{hi}) + \frac{1}{2} \| \mathbf{\beta} \|_2^2, \ \text{ s.t. } \mathbf{A} \mathbf{\beta} + \mathbf{b} \geq \mathbf{0}, 9 | 10 | 11 | where :math:`\mathbf{U} = (u_{li}),\mathbf{V} = (v_{li}) \in \mathbb{R}^{L \times n}` 12 | and :math:`\mathbf{S} = (s_{hi}),\mathbf{T} = (t_{hi}),\mathbf{\tau} = (\tau_{hi}) \in \mathbb{R}^{H \times n}` 13 | are the ReLU-ReHU loss parameters, and :math:`(\mathbf{A},\mathbf{b})` are the constraint parameters. 14 | 15 | The key to using `ReHLine`` to solve any problem lies in utilizing custom ReHLine parameters to represent the problem, we illustrate this with following examples. Suppose that we have `X` and `y` as our data. 16 | 17 | .. code-block:: python 18 | 19 | ## Data 20 | ## X : [n x d] 21 | ## y : [n] 22 | import numpy as np 23 | n, d = X.shape 24 | 25 | .. note:: 26 | 27 | Most of the examples below can be directly implemented by `ReHLine: Empirical Risk Minimization <./tutorials/ReHLine_ERM.rst>`_; we are simply illustrating how to convert the problem to the ReHLine formulation. 28 | 29 | SVM 30 | --- 31 | 32 | SVMs solve the following optimization problem: 33 | 34 | .. math:: 35 | \min_{\mathbf{\beta} \in \mathbb{R}^d} \frac{C}{n} \sum_{i=1}^n ( 1 - y_i \mathbf{\beta}^\intercal \mathbf{x}_i )_+ + \frac{1}{2} \| \mathbf{\beta} \|_2^2 36 | 37 | where :math:`\mathbf{x}_i \in \mathbb{R}^d` is a feature vector, and :math:`y_i \in \{-1, 1\}` is a binary label. Note that the SVM can be rewritten as a ReHLine optimization with 38 | 39 | .. math:: 40 | \mathbf{U} \leftarrow -C \mathbf{y}^\intercal/n, \quad 41 | \mathbf{V} \leftarrow C \mathbf{1}^\intercal_n/n, 42 | 43 | where :math:`\mathbf{1}_n = (1, \cdots, 1)^\intercal` is the $n$-length one vector, :math:`\mathbf{X} \in \mathbb{R}^{n \times d}` is the feature matrix, and :math:`\mathbf{y} = (y_1, \cdots, y_n)^\intercal` is the response vector. 44 | 45 | The python implementation is: 46 | 47 | .. code-block:: python 48 | 49 | ## SVM ReHLine parameters 50 | clf = ReHLine() 51 | ## U 52 | clf.U = -(C*y).reshape(1,-1) 53 | ## V 54 | clf.V = (C*np.array(np.ones(n))).reshape(1,-1) 55 | ## Fit 56 | clf.fit(X) 57 | 58 | Smooth SVM 59 | ---------- 60 | 61 | Smoothed SVMs solve the following optimization problem: 62 | 63 | .. math:: 64 | \min_{\mathbf{\beta} \in \mathbb{R}^d} \frac{C}{n} \sum_{i=1}^n V( y_i \mathbf{\beta}^\intercal \mathbf{x}_i ) + \frac{1}{2} \| \mathbf{\beta} \|_2^2 65 | 66 | where :math:`\mathbf{x}_i \in \mathbb{R}^d` is a feature vector, and :math:`y_i \in \{-1, 1\}` is a binary label, and :math:`V(\cdot)` is the modified Huber loss or the smoothed hinge loss: 67 | 68 | .. math:: 69 | \begin{equation*} 70 | V(z) = 71 | \begin{cases} 72 | \ 0, & z \geq 1, \\ 73 | \ (1-z)^2/2, & 0 < z \leq 1, \\ 74 | \ (1/2 - z ), & z < 0. 75 | \end{cases} 76 | \end{equation*} 77 | 78 | Smoothed SVM can be rewritten as a ReHLine optimization with 79 | 80 | .. math:: 81 | \mathbf{S} \leftarrow -\sqrt{C/n} \mathbf{y}^\intercal, \quad 82 | \mathbf{T} \leftarrow \sqrt{C/n} \mathbf{1}^\intercal_n, \quad 83 | \mathbf{\tau} \leftarrow \sqrt{C/n} \mathbf{1}^\intercal_n. 84 | 85 | where :math:`\mathbf{1}_n = (1, \cdots, 1)^\intercal` is the $n$-length one vector, :math:`\mathbf{X} \in \mathbb{R}^{n \times d}` is the feature matrix, and :math:`\mathbf{y} = (y_1, \cdots, y_n)^\intercal` is the response vector. 86 | 87 | The python implementation is: 88 | 89 | .. code-block:: python 90 | 91 | ## sSVM ReHLine parameters 92 | clf = ReHLine() 93 | ## S 94 | clf.S = -(np.sqrt(C/n)*y).reshape(1,-1) 95 | ## T 96 | clf.T = (np.sqrt(C/n)*np.ones(n)).reshape(1,-1) 97 | ## Tau 98 | clf.Tau = (np.sqrt(C/n)*np.ones(n)).reshape(1,-1) 99 | ## Fit 100 | clf.fit(X) 101 | 102 | FairSVM 103 | ------- 104 | 105 | The SVM with fairness constraints (FairSVM) solves the following optimization problem: 106 | 107 | .. math:: 108 | \begin{align} 109 | & \min_{\mathbf{\beta} \in \mathbb{R}^d} \frac{C}{n} \sum_{i=1}^n ( 1 - y_i \mathbf{\beta}^\intercal \mathbf{x}_i )_+ + \frac{1}{2} \| \mathbf{\beta} \|_2^2, \nonumber \\ 110 | \text{subj. to } & \quad \frac{1}{n} \sum_{i=1}^n \mathbf{z}_i \mathbf{\beta}^\intercal \mathbf{x}_i \leq \mathbf{\rho}, \quad \frac{1}{n} \sum_{i=1}^n \mathbf{z}_i \mathbf{\beta}^\intercal \mathbf{x}_i \geq -\mathbf{\rho}, 111 | \end{align} 112 | 113 | where :math:`\mathbf{x}_i \in \mathbb{R}^d` is a feature vector, and :math:`y_i \in \{-1, 1\}` is a binary label, $\mathbf{z}_i$ is a collection of centered sensitive features 114 | 115 | .. math:: 116 | \sum_{i=1}^n z_{ij} = 0, 117 | 118 | such as gender and/or race. The constraints limit the correlation between the $d_0$-length sensitive features :math:`\mathbf{z}_ i \in \mathbb{R}^{d_0}` and the decision function :math:`\mathbf{\beta}^\intercal \mathbf{x}`, and the constants :math:`\mathbf{\rho} \in \mathbb{R}_+^{d_0}` trade-offs predictive accuracy and fairness. Note that the FairSVM can be rewritten as a ReHLine optimization with 119 | 120 | .. math:: 121 | \mathbf{U} \leftarrow -C \mathbf{y}^\intercal/n, \quad 122 | \mathbf{V} \leftarrow C \mathbf{1}^\intercal_n/n, \quad 123 | \mathbf{A} \leftarrow 124 | \begin{pmatrix} 125 | \mathbf{Z}^\intercal \mathbf{X} / n \\ 126 | -\mathbf{Z}^\intercal \mathbf{X} / n 127 | \end{pmatrix}, \quad 128 | \mathbf{b} \leftarrow 129 | \begin{pmatrix} 130 | \mathbf{\rho} \\ 131 | \mathbf{\rho} 132 | \end{pmatrix} 133 | 134 | The python implementation is: 135 | 136 | .. code-block:: python 137 | 138 | ## FairSVM ReHLine parameters 139 | clf = ReHLine() 140 | ## U 141 | clf.U = -(C*y).reshape(1,-1) 142 | ## V 143 | clf.V = (C*np.array(np.ones(n))).reshape(1,-1) 144 | ## A 145 | ## we illustrate that the first column of X as sensitive features, and tol is 0.1 146 | X_sen = X[:,0] 147 | tol_sen = 0.1 148 | clf.A = np.repeat([X_sen @ X], repeats=[2], axis=0) / n 149 | clf.A[1] = -clf.A[1] 150 | ## b 151 | clf.b = np.array([tol_sen, tol_sen]) 152 | ## Fit 153 | clf.fit(X) 154 | 155 | Ridge Huber regression 156 | ---------------------- 157 | 158 | The Ridge regularized Huber minimization (RidgeHuber) solves the following optimization problem: 159 | 160 | .. math:: 161 | \min_{\mathbf{\beta}} \frac{1}{n} \sum_{i=1}^n H_\kappa( y_i - \mathbf{x}_i^\intercal \mathbf{\beta} ) + \frac{\lambda}{2} \| \mathbf{\beta} \|_2^2, 162 | 163 | where :math:`H_\kappa(\cdot)` is the Huber loss with a given parameter :math:`\kappa`: 164 | 165 | .. math:: 166 | H_\kappa(z) = 167 | \begin{cases} 168 | z^2/2, & 0 < |z| \leq \kappa, \\ 169 | \ \kappa ( |z| - \kappa/2 ), & |z| > \kappa. 170 | \end{cases} 171 | 172 | In this case, the RidgeHuber can be rewritten as a ReHLine optimization with: 173 | 174 | .. math:: 175 | \mathbf{S} \leftarrow 176 | \begin{pmatrix} 177 | -\sqrt{\frac{1}{n\lambda}} \mathbf{1}^\intercal_n \\ 178 | \sqrt{\frac{1}{n\lambda}} \mathbf{1}^\intercal_n \\ 179 | \end{pmatrix}, \quad 180 | \mathbf{T} \leftarrow 181 | \begin{pmatrix} 182 | \sqrt{\frac{1}{n\lambda}} \mathbf{y}^\intercal \\ 183 | -\sqrt{\frac{1}{n\lambda}} \mathbf{y}^\intercal \\ 184 | \end{pmatrix}, \quad 185 | \mathbf{\tau} \leftarrow 186 | \begin{pmatrix} 187 | \kappa \sqrt{\frac{1}{n\lambda}} \mathbf{1}^\intercal_n \\ 188 | \\ 189 | \kappa \sqrt{\frac{1}{n\lambda}} \mathbf{1}^\intercal_n \\ 190 | \end{pmatrix}. 191 | 192 | The python implementation is: 193 | 194 | .. code-block:: python 195 | 196 | ## Huber ReHLine parameters 197 | clf = ReHLine() 198 | ## S 199 | clf.S = -np.repeat([np.sqrt(1/n/lam)*np.ones(n)], repeats=[2], axis=0) 200 | clf.S[1] = -clf.S[1] 201 | ## T 202 | clf.T = np.repeat([np.sqrt(1/n/lam)*y], repeats=[2], axis=0) 203 | clf.T[1] = -clf.T[1] 204 | ## Tau 205 | clf.Tau = np.repeat([kappa*np.sqrt(1/n/lam)*np.ones(n)], repeats=[2], axis=0) 206 | ## Fit 207 | clf.fit(X) -------------------------------------------------------------------------------- /doc/source/tutorials/ReHLine_sklearn.rst: -------------------------------------------------------------------------------- 1 | ReHLine with Scikit-Learn 2 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 3 | 4 | .. image:: https://scikit-learn.org/stable/_static/scikit-learn-logo-small.png 5 | :alt: scikit-learn 6 | :align: right 7 | :width: 150px 8 | 9 | `ReHLine` provides a versatile and powerful solver for empirical risk minimization problems with linear constraints. To make it even more accessible and easy to integrate into standard machine learning workflows, it now comes with a scikit-learn compatible estimator. 10 | 11 | This means you can use `ReHLine` just like any other scikit-learn estimator, allowing you to seamlessly use it with scikit-learn's rich ecosystem, including tools like `Pipeline` for building workflows and `GridSearchCV` for hyperparameter tuning. 12 | 13 | This tutorial will guide you through the process of using the `ReHLine` scikit-learn estimator, from basic usage to advanced integration with scikit-learn's powerful features. 14 | 15 | Mathematical Formulation 16 | ------------------------ 17 | 18 | The `ReHLine` solver addresses the following empirical risk minimization problem with a piecewise linear-quadratic (PLQ) loss, ridge regularization, and linear constraints. The objective function is: 19 | 20 | .. math:: 21 | 22 | \min_{\pmb{\beta} \in \mathbb{R}^d} \sum_{i=1}^n \text{PLQ}(y_i, \mathbf{x}_i^T \pmb{\beta}) + \frac{1}{2} \| \pmb{\beta} \|_2^2, \ \text{ s.t. } \ 23 | \mathbf{A} \pmb{\beta} + \mathbf{b} \geq \mathbf{0}, 24 | 25 | where: 26 | - :math:`\text{PLQ}(\cdot, \cdot)` is a convex piecewise linear-quadratic loss function. You can find built-in loss functions in the `Loss <./loss.rst>`_ section. 27 | - :math:`\mathbf{A}` is a :math:`K \times d` matrix and :math:`\mathbf{b}` is a :math:`K`-dimensional vector representing `K` linear constraints. See `Constraints <./constraint.rst>`_ for more details. 28 | 29 | For example, `ReHLine` supports the following loss functions and constraints: 30 | 31 | .. image:: ../figs/tab.png 32 | 33 | Basic Usage 34 | ----------- 35 | 36 | Here is a simple example of how to use the `plq_Ridge_Classifier` for a binary classification task. The estimator follows the standard scikit-learn API: `fit(X, y)` and `predict(X)`. 37 | 38 | .. code-block:: python 39 | 40 | import numpy as np 41 | from sklearn.datasets import make_classification 42 | from sklearn.model_selection import train_test_split 43 | from rehline import plq_Ridge_Classifier 44 | 45 | # Generate synthetic data 46 | X, y = make_classification(n_samples=100, n_features=10, random_state=42) 47 | 48 | # Split data into training and testing sets 49 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 50 | 51 | # Initialize and train the classifier 52 | # We use the SVM loss as an example 53 | clf = plq_Ridge_Classifier(loss={'name': 'svm'}, C=1.0) 54 | clf.fit(X_train, y_train) 55 | 56 | # Make predictions 57 | y_pred = clf.predict(X_test) 58 | 59 | # Print the accuracy 60 | accuracy = clf.score(X_test, y_test) 61 | print(f"Accuracy: {accuracy:.2f}") 62 | 63 | 64 | Using ReHLine with Pipelines 65 | ---------------------------- 66 | 67 | You can easily integrate `ReHLine` estimators into scikit-learn `Pipeline` objects. This is useful for chaining preprocessing steps, such as feature scaling, with the `ReHLine` estimator. 68 | 69 | .. code-block:: python 70 | 71 | from sklearn.pipeline import Pipeline 72 | from sklearn.preprocessing import StandardScaler 73 | 74 | # Create a pipeline with a scaler and the classifier 75 | pipe = Pipeline([ 76 | ('scaler', StandardScaler()), 77 | ('clf', plq_Ridge_Classifier(loss={'name': 'svm'})) 78 | ]) 79 | 80 | # The pipeline can be used as a single estimator 81 | pipe.fit(X_train, y_train) 82 | accuracy = pipe.score(X_test, y_test) 83 | print(f"Pipeline Accuracy: {accuracy:.2f}") 84 | 85 | Hyperparameter Tuning with GridSearchCV 86 | --------------------------------------- 87 | 88 | The scikit-learn compatibility also allows you to use `GridSearchCV` to find the best hyperparameters for your `ReHLine` model. 89 | 90 | .. code-block:: python 91 | 92 | from sklearn.model_selection import GridSearchCV 93 | 94 | # Define the parameter grid to search 95 | param_grid = { 96 | 'clf__C': [0.1, 1.0, 10.0], 97 | 'clf__loss': [{'name': 'svm'}, {'name': 'sSVM'}] 98 | } 99 | 100 | # Create the GridSearchCV object 101 | grid_search = GridSearchCV(pipe, param_grid, cv=5) 102 | grid_search.fit(X_train, y_train) 103 | 104 | # Print the best parameters and score 105 | print(f"Best Parameters: {grid_search.best_params_}") 106 | print(f"Best CV Score: {grid_search.best_score_:.2f}") 107 | 108 | 109 | Example 110 | ------- 111 | 112 | .. nblinkgallery:: 113 | :caption: Emprical Risk Minimization 114 | :name: rst-link-gallery 115 | 116 | ../examples/Sklearn_Mixin.ipynb 117 | -------------------------------------------------------------------------------- /doc/source/tutorials/constraint.rst: -------------------------------------------------------------------------------- 1 | Constraint 2 | ********** 3 | 4 | Supported linear constraints in ReHLine are listed in the table below. 5 | 6 | Usage 7 | ----- 8 | 9 | .. code:: python 10 | 11 | # list of 12 | # name (str): name of the custom linear constraints 13 | # loss_kwargs: more keys and values for constraint parameters 14 | constraint = [{'name': , <**constraint_kwargs>}, ...] 15 | 16 | .. list-table:: 17 | :align: left 18 | :widths: 5 20 15 19 | :header-rows: 1 20 | 21 | * - constraint 22 | - | args 23 | - | Example 24 | 25 | * - **nonnegative** 26 | - | ``name``: 'nonnegative' or '>=0' 27 | - | ``constraint=[{'name': '>=0'}]`` 28 | 29 | * - **fair** 30 | - | ``name``: 'fair' or 'fairness' 31 | | ``sen_idx``: a list contains column indices for sensitive attributes 32 | | ``tol_sen``: 1d array [p] of tolerance for fairness 33 | - | ``constraint=[{'name': 'fair', 'sen_idx': sen_idx, 'tol_sen': tol_sen}]`` 34 | 35 | * - **custom** 36 | - | ``name``: 'custom' 37 | | ``A``: 2d array [K x d] for linear constraint coefficients 38 | | ``b``: 1d array [K] of constraint intercepts 39 | - | ``constraint=[{'name': 'custom', 'A': A, 'b': b}]`` 40 | 41 | Related Examples 42 | ---------------- 43 | 44 | .. nblinkgallery:: 45 | :caption: Constraints 46 | :name: rst-link-gallery 47 | 48 | ../examples/FairSVM.ipynb 49 | -------------------------------------------------------------------------------- /doc/source/tutorials/loss.rst: -------------------------------------------------------------------------------- 1 | Loss 2 | **** 3 | 4 | Supported loss functions in ReHLine are listed in the table below. 5 | 6 | Usage 7 | ----- 8 | 9 | .. code:: python 10 | 11 | # name (str): name of the custom loss function 12 | # loss_kwargs: more keys and values for loss parameters 13 | loss = {'name': , <**loss_kwargs>} 14 | 15 | 16 | 17 | Classification loss 18 | ~~~~~~~~~~~~~~~~~~~ 19 | 20 | .. list-table:: 21 | :align: left 22 | :widths: 5 20 15 23 | :header-rows: 1 24 | 25 | * - loss 26 | - | args 27 | - | Example 28 | 29 | * - **SVM** 30 | - | ``name``: 'hinge' / 'svm' / 'SVM' 31 | - | ``loss={'name': 'SVM'}`` 32 | 33 | * - **Smooth SVM** 34 | - | ``name``: 'sSVM' / 'smooth SVM' / 'smooth hinge' 35 | - | ``loss={'name': 'sSVM'}`` 36 | 37 | 38 | Regression loss 39 | ~~~~~~~~~~~~~~~ 40 | 41 | .. list-table:: 42 | :align: left 43 | :widths: 5 20 15 44 | :header-rows: 1 45 | 46 | * - loss 47 | - | args 48 | - | Example 49 | 50 | * - **Quantile Reg** 51 | - | ``name``: 'check' / 'quantile' / 'QR' 52 | | ``qt`` (*float*): qt 53 | - | ``loss={'name': 'QR', 'qt': 0.25}`` 54 | 55 | * - **Huber** 56 | - | ``name``: 'huber' / 'Huber' 57 | - | ``loss={'name': 'huber'}`` 58 | 59 | * - **SVR** 60 | - | ``name``: 'SVR' / 'svr' 61 | | ``epsilon`` (*float*): 0.1 62 | - | ``loss={'name': 'svr', 'epsilon': 0.1}`` 63 | 64 | Related Examples 65 | ---------------- 66 | 67 | .. nblinkgallery:: 68 | :caption: Constraints 69 | :name: rst-link-gallery 70 | 71 | ../examples/QR.ipynb 72 | ../examples/SVM.ipynb 73 | -------------------------------------------------------------------------------- /doc/source/tutorials/warmstart.rst: -------------------------------------------------------------------------------- 1 | Warm-Starting with ReHLine 2 | ========================== 3 | 4 | This tutorial explains how to use warm-starting with ReHLine, a Python library for regression with hinge loss, to enhance the efficiency of solving similar optimization problems. 5 | 6 | Introduction 7 | ------------ 8 | 9 | Warm-starting is a technique used to accelerate the convergence of optimization algorithms by initializing them with a solution from a previous run. This is particularly beneficial when you have a sequence of related problems to solve. 10 | 11 | Setup 12 | ----- 13 | 14 | Before you begin, ensure you have the necessary packages installed. You need the `rehline` library, which is used for regression with hinge loss, and `numpy` for numerical operations. Install these packages using pip if you haven't already: 15 | 16 | .. code-block:: bash 17 | 18 | pip install rehline numpy 19 | 20 | Simulating the Dataset 21 | ---------------------- 22 | 23 | We first create a random dataset for classification: 24 | 25 | .. code-block:: python 26 | 27 | import numpy as np 28 | 29 | n, d, C = 1000, 3, 0.5 30 | X = np.random.randn(n, d) 31 | beta0 = np.random.randn(d) 32 | y = np.sign(X.dot(beta0) + np.random.randn(n)) 33 | 34 | - **n** is the number of samples. 35 | - **d** is the number of features. 36 | - **C** is a regularization parameter. 37 | - **X** is the feature matrix. 38 | - **y** is the target vector, generated as a sign function of a linear combination of features plus some noise. 39 | 40 | Using ReHLine Solver 41 | -------------------- 42 | 43 | The `ReHLine_solver` is tested first with a cold start and then with a warm start: 44 | 45 | .. code-block:: python 46 | 47 | from rehline._base import ReHLine_solver 48 | 49 | U = -(C*y).reshape(1,-1) 50 | V = (C*np.array(np.ones(n))).reshape(1,-1) 51 | res = ReHLine_solver(X, U, V) # Cold start 52 | res_ws = ReHLine_solver(X, U, V, Lambda=res.Lambda) # Warm start 53 | 54 | - **Cold Start**: The solver starts from scratch without any prior information. 55 | - **Warm Start**: The solver uses the solution from the cold start (`res.Lambda`) as the initial point for the next run. 56 | 57 | Using ReHLine Class 58 | ------------------- 59 | 60 | The `ReHLine` class is used to fit a model: 61 | 62 | .. code-block:: python 63 | 64 | from rehline import ReHLine 65 | 66 | clf = ReHLine(verbose=1) 67 | clf.C = C 68 | clf.U = -y.reshape(1,-1) 69 | clf.V = np.array(np.ones(n)).reshape(1,-1) 70 | clf.fit(X) # Cold start 71 | 72 | clf.C = 2*C 73 | clf.warm_start = 1 74 | clf.fit(X) # Warm start 75 | 76 | - **Cold Start**: The class is fitted with the initial data. 77 | - **Warm Start**: The class is fitted again with a different regularization parameter (`2*C`), using the previous solution as a starting point. 78 | 79 | Using plqERM_Ridge 80 | ------------------ 81 | 82 | Finally, the `plqERM_Ridge` class is tested similarly: 83 | 84 | .. code-block:: python 85 | 86 | from rehline import plqERM_Ridge 87 | 88 | clf = plqERM_Ridge(loss={'name': 'svm'}, C=C, verbose=1) 89 | clf.fit(X=X, y=y) # Cold start 90 | 91 | clf.C = 2*C 92 | clf.warm_start = 1 93 | clf.fit(X=X, y=y) # Warm start 94 | 95 | -------------------------------------------------------------------------------- /figs/res.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/softmin/ReHLine-python/08597596bc569b094fb6df2026f741a183b78aa1/figs/res.png -------------------------------------------------------------------------------- /figs/tab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/softmin/ReHLine-python/08597596bc569b094fb6df2026f741a183b78aa1/figs/tab.png -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "rehline" 3 | version = "0.1.1" 4 | description = "Regularized Composite ReLU-ReHU Loss Minimization with Linear Computation and Linear Convergence" 5 | authors = [ 6 | {name = "Ben Dai", email = "bendai@cuhk.edu.hk"}, 7 | {name = "Yixuan Qiu", email = "yixuanq@gmail.com"} 8 | ] 9 | maintainers = [ 10 | {name = "Ben Dai", email = "bendai@cuhk.edu.hk"}, 11 | ] 12 | license = {file = "LICENSE"} 13 | readme = "README.md" 14 | requires-python = ">= 3.10" 15 | dependencies = [ 16 | "numpy >= 1.23.5", 17 | "scipy >= 1.11.4", 18 | "scikit-learn >= 1.2.2" 19 | ] 20 | 21 | [pyproject.urls] 22 | homepage = "https://rehline.github.io/" 23 | repository = "https://github.com/softmin/ReHLine-python" 24 | documentation = "https://rehline-python.readthedocs.io/en/latest/index.html" 25 | 26 | [tool.setuptools] 27 | py-modules = ["build"] 28 | 29 | [tool.cibuildwheel] 30 | # Only build on CPython 31 | build = "cp*" 32 | 33 | [build-system] 34 | requires = ["requests ~= 2.31.0", "pybind11 ~= 2.13.0", "setuptools >= 69.0.0", "wheel >= 0.42.0"] 35 | build-backend = "setuptools.build_meta" 36 | -------------------------------------------------------------------------------- /rehline/__init__.py: -------------------------------------------------------------------------------- 1 | # Import from internal C++ module 2 | from ._base import (ReHLine_solver, _BaseReHLine, 3 | _make_constraint_rehline_param, _make_loss_rehline_param) 4 | from ._class import CQR_Ridge, ReHLine, plqERM_Ridge 5 | from ._internal import rehline_internal, rehline_result 6 | from ._path_sol import plqERM_Ridge_path_sol 7 | from ._sklearn_mixin import plq_Ridge_Classifier, plq_Ridge_Regressor 8 | 9 | __all__ = ("_BaseReHLine", 10 | "ReHLine_solver", 11 | "ReHLine", 12 | "plqERM_Ridge", 13 | "CQR_Ridge", 14 | "plqERM_Ridge_path_sol", 15 | "plq_Ridge_Classifier", 16 | "plq_Ridge_Regressor", 17 | "_make_loss_rehline_param", 18 | "_make_constraint_rehline_param") 19 | -------------------------------------------------------------------------------- /rehline/_base.py: -------------------------------------------------------------------------------- 1 | """Base functions for ReHLine.""" 2 | 3 | # Authors: Ben Dai 4 | # Yixuan Qiu 5 | 6 | # License: MIT License 7 | 8 | from abc import abstractmethod 9 | 10 | import numpy as np 11 | from scipy.special import huber 12 | from sklearn.base import BaseEstimator 13 | from sklearn.utils.validation import check_array, check_is_fitted, check_X_y 14 | 15 | from ._internal import rehline_internal, rehline_result 16 | 17 | 18 | class _BaseReHLine(BaseEstimator): 19 | r"""Base Class of ReHLine Formulation. 20 | 21 | .. math:: 22 | 23 | \min_{\mathbf{\beta} \in \mathbb{R}^d} \sum_{i=1}^n \sum_{l=1}^L \text{ReLU}( u_{li} \mathbf{x}_i^\intercal \mathbf{\beta} + v_{li}) + \sum_{i=1}^n \sum_{h=1}^H {\text{ReHU}}_{\tau_{hi}}( s_{hi} \mathbf{x}_i^\intercal \mathbf{\beta} + t_{hi}) + \frac{1}{2} \| \mathbf{\beta} \|_2^2, \\ \text{ s.t. } 24 | \mathbf{A} \mathbf{\beta} + \mathbf{b} \geq \mathbf{0}, 25 | 26 | where :math:`\mathbf{U} = (u_{li}),\mathbf{V} = (v_{li}) \in \mathbb{R}^{L \times n}` 27 | and :math:`\mathbf{S} = (s_{hi}),\mathbf{T} = (t_{hi}),\mathbf{\tau} = (\tau_{hi}) \in \mathbb{R}^{H \times n}` 28 | are the ReLU-ReHU loss parameters, and :math:`(\mathbf{A},\mathbf{b})` are the constraint parameters. 29 | 30 | Parameters 31 | ---------- 32 | 33 | C : float, default=1.0 34 | Regularization parameter. The strength of the regularization is 35 | inversely proportional to C. Must be strictly positive. 36 | 37 | U, V: array of shape (L, n_samples), default=np.empty(shape=(0, 0)) 38 | The parameters pertaining to the ReLU part in the loss function. 39 | 40 | Tau, S, T: array of shape (H, n_samples), default=np.empty(shape=(0, 0)) 41 | The parameters pertaining to the ReHU part in the loss function. 42 | 43 | A: array of shape (K, n_features), default=np.empty(shape=(0, 0)) 44 | The coefficient matrix in the linear constraint. 45 | 46 | b: array of shape (K, ), default=np.empty(shape=0) 47 | The intercept vector in the linear constraint. 48 | 49 | """ 50 | 51 | def __init__(self, *, C=1., 52 | U=np.empty(shape=(0,0)), V=np.empty(shape=(0,0)), 53 | Tau=np.empty(shape=(0,0)), 54 | S=np.empty(shape=(0,0)), T=np.empty(shape=(0,0)), 55 | A=np.empty(shape=(0,0)), b=np.empty(shape=(0))): 56 | self.C = C 57 | self._U = U 58 | self._V = V 59 | self._S = S 60 | self._T = T 61 | self._Tau = Tau 62 | self._A = A 63 | self._b = b 64 | self.L = self._U.shape[0] 65 | self.H = self._S.shape[0] 66 | self.K = self._A.shape[0] 67 | 68 | def get_params(self, deep=True): 69 | """Get parameters for this estimator. 70 | 71 | Override the default get_params to exclude computation-only parameters. 72 | 73 | Parameters 74 | ---------- 75 | deep : bool, default=True 76 | If True, will return the parameters for this estimator and 77 | contained subobjects that are estimators. 78 | 79 | Returns 80 | ------- 81 | params : dict 82 | Parameter names mapped to their values. 83 | """ 84 | out = dict() 85 | for key in self._get_param_names(): 86 | if key not in ['U', 'V', 'S', 'T', 'Tau', 'A', 'b', 'Lambda', 'Gamma', 'xi']: 87 | value = getattr(self, key) 88 | if deep and hasattr(value, 'get_params') and not isinstance(value, type): 89 | deep_items = value.get_params().items() 90 | out.update((key + '__' + k, val) for k, val in deep_items) 91 | out[key] = value 92 | return out 93 | 94 | def auto_shape(self): 95 | """ 96 | Automatically generate the shape of the parameters of the ReHLine loss function. 97 | """ 98 | self.L = self._U.shape[0] 99 | self.H = self._S.shape[0] 100 | self.K = self._A.shape[0] 101 | 102 | def cast_sample_weight(self, sample_weight=None): 103 | """ 104 | Cast the sample weight to the ReHLine parameters. 105 | 106 | Parameters 107 | ---------- 108 | sample_weight : array-like of shape (n_samples,), default=None 109 | Sample weights. If None, then samples are equally weighted. 110 | 111 | Returns 112 | ------- 113 | U_weight : array-like of shape (L, n_samples) 114 | Weighted ReLU coefficient matrix. 115 | 116 | V_weight : array-like of shape (L, n_samples) 117 | Weighted ReLU intercept vector. 118 | 119 | Tau_weight : array-like of shape (H, n_samples) 120 | Weighted ReHU cutpoint matrix. 121 | 122 | S_weight : array-like of shape (H, n_samples) 123 | Weighted ReHU coefficient vector. 124 | 125 | T_weight : array-like of shape (H, n_samples) 126 | Weighted ReHU intercept vector. 127 | 128 | Notes 129 | ----- 130 | This method casts the sample weight to the ReHLine parameters by multiplying 131 | the sample weight with the ReLU and ReHU parameters. If sample_weight is None, 132 | then the sample weight is set to the weight parameter C. 133 | """ 134 | 135 | self.auto_shape() 136 | 137 | sample_weight = self.C*sample_weight 138 | 139 | if self.L > 0: 140 | U_weight = self._U * sample_weight 141 | V_weight = self._V * sample_weight 142 | else: 143 | U_weight = self._U 144 | V_weight = self._V 145 | 146 | if self.H > 0: 147 | sqrt_sample_weight = np.sqrt(sample_weight) 148 | Tau_weight = self._Tau * sqrt_sample_weight 149 | S_weight = self._S * sqrt_sample_weight 150 | T_weight = self._T * sqrt_sample_weight 151 | else: 152 | Tau_weight = self._Tau 153 | S_weight = self._S 154 | T_weight = self._T 155 | 156 | return U_weight, V_weight, Tau_weight, S_weight, T_weight 157 | 158 | def call_ReLHLoss(self, score): 159 | """ 160 | Return the value of the ReHLine loss of the `score`. 161 | 162 | Parameters 163 | ---------- 164 | score : ndarray of shape (n_samples, ) 165 | The input score that will be evaluated through the ReHLine loss. 166 | 167 | Returns 168 | ------- 169 | float 170 | ReHLine loss evaluation of the given score. 171 | """ 172 | n = len(score) 173 | relu_input = np.zeros((self.L, n)) 174 | rehu_input = np.zeros((self.H, n)) 175 | if self.L > 0: 176 | relu_input = (self._U.T * score[:,np.newaxis]).T + self._V 177 | if self.H > 0: 178 | rehu_input = (self._S.T * score[:,np.newaxis]).T + self._T 179 | return np.sum(_relu(relu_input), 0) + np.sum(_rehu(rehu_input), 0) 180 | 181 | @abstractmethod 182 | def fit(self, X, y, sample_weight): 183 | """Fit model.""" 184 | 185 | @abstractmethod 186 | def decision_function(self, X): 187 | """The decision function evaluated on the given dataset 188 | 189 | Parameters 190 | ---------- 191 | X : array-like of shape (n_samples, n_features) 192 | The data matrix. 193 | 194 | Returns 195 | ------- 196 | ndarray of shape (n_samples, ) 197 | Returns the decision function of the samples. 198 | """ 199 | # Check if fit has been called 200 | check_is_fitted(self) 201 | 202 | X = check_array(X) 203 | 204 | def _relu(x): 205 | """ 206 | Evaluation of ReLU given a vector. 207 | 208 | Parameters 209 | ---------- 210 | 211 | x: {array-like} of shape (n_samples, ) 212 | Training vector, where `n_samples` is the number of samples 213 | 214 | 215 | Returns 216 | ------- 217 | array of shape (n_samples, ) 218 | An array with ReLU applied, i.e., all negative values are replaced with 0. 219 | 220 | """ 221 | return np.maximum(x, 0) 222 | 223 | 224 | def _rehu(x, cut=1): 225 | """ 226 | Evaluation of ReHU given a vector. 227 | 228 | Parameters 229 | ---------- 230 | 231 | x: {array-like} of shape (n_samples, ) 232 | Training vector, where `n_samples` is the number of samples 233 | 234 | cut: {array-like} of shape (n_samples, ) 235 | Cutpoints of ReHU, where `n_samples` is the number of samples 236 | 237 | Returns 238 | ------- 239 | array of shape (n_samples, ) 240 | The result of the ReHU function. 241 | 242 | """ 243 | n_samples = x.shape[0] 244 | cut = cut * np.ones_like(x) 245 | 246 | u = np.maximum(x, 0) 247 | return huber(cut, u) 248 | 249 | def _check_relu(relu_coef, relu_intercept): 250 | assert relu_coef.shape == relu_intercept.shape, "`relu_coef` and `relu_intercept` should be the same shape!" 251 | 252 | def _check_rehu(rehu_coef, rehu_intercept, rehu_cut): 253 | assert rehu_coef.shape == rehu_intercept.shape, "`rehu_coef` and `rehu_intercept` should be the same shape!" 254 | if len(rehu_coef) > 0: 255 | assert (rehu_cut >= 0.0).all(), "`rehu_cut` must be non-negative!" 256 | 257 | 258 | def ReHLine_solver(X, U, V, 259 | Tau=np.empty(shape=(0, 0)), 260 | S=np.empty(shape=(0, 0)), T=np.empty(shape=(0, 0)), 261 | A=np.empty(shape=(0, 0)), b=np.empty(shape=(0)), 262 | Lambda=np.empty(shape=(0, 0)), 263 | Gamma=np.empty(shape=(0, 0)), 264 | xi=np.empty(shape=(0, 0)), 265 | max_iter=1000, tol=1e-4, shrink=1, verbose=1, trace_freq=100): 266 | result = rehline_result() 267 | if len(Lambda)>0: 268 | result.Lambda = np.maximum(0, np.minimum(Lambda, 1.0)) 269 | if len(Gamma)>0: 270 | result.Gamma = np.maximum(0, np.minimum(Gamma, Tau)) 271 | if len(xi)>0: 272 | result.xi = np.maximum(xi, 0.0) 273 | rehline_internal(result, X, A, b, U, V, S, T, Tau, max_iter, tol, shrink, verbose, trace_freq) 274 | return result 275 | 276 | 277 | def _make_loss_rehline_param(loss, X, y): 278 | """The `_make_loss_rehline_param` function generates parameters for the ReHLine solver, based on the provided training data. 279 | 280 | The function supports various loss functions, including: 281 | - 'hinge' 282 | - 'svm' or 'SVM' 283 | - 'mae' or 'MAE' or 'mean absolute error' 284 | - 'check' or 'quantile' or 'quantile regression' or 'QR' 285 | - 'sSVM' or 'smooth SVM' or 'smooth hinge' 286 | - 'TV' 287 | - 'huber' or 'Huber' 288 | - 'SVR' or 'svr' 289 | - Custom loss functions (manual setup required) 290 | 291 | Parameters 292 | ---------- 293 | loss : dict 294 | A dictionary containing the loss function parameters. 295 | 296 | Keys: 297 | - 'name' : str, the name of the loss function (e.g. 'hinge', 'svm', 'QR', etc.) 298 | - 'loss_kwargs': more keys and values for loss parameters 299 | 300 | X : ndarray of shape (n_samples, n_features) 301 | The generated samples. 302 | 303 | y : ndarray of shape (n_samples,) 304 | The +/- labels for class membership of each sample. 305 | """ 306 | 307 | # n, d = X.shape 308 | n = len(y) 309 | 310 | ## initialization of ReHLine params 311 | U=np.empty(shape=(0,0)) 312 | V=np.empty(shape=(0,0)) 313 | Tau=np.empty(shape=(0,0)) 314 | S=np.empty(shape=(0,0)) 315 | T=np.empty(shape=(0,0)) 316 | 317 | # _dummy_X = False 318 | 319 | if (loss['name'] == 'hinge') or (loss['name'] == 'svm')\ 320 | or (loss['name'] == 'SVM'): 321 | U = -y.reshape(1,-1) 322 | V = (np.array(np.ones(n))).reshape(1,-1) 323 | 324 | elif (loss['name'] == 'check') \ 325 | or (loss['name'] == 'quantile') \ 326 | or (loss['name'] == 'quantile regression') \ 327 | or (loss['name'] == 'QR'): 328 | 329 | qt = loss['qt'] 330 | 331 | U = np.ones((2, n)) 332 | V = np.ones((2, n)) 333 | 334 | U[0] = - qt*U[0] 335 | U[1] = (1-qt)*U[1] 336 | V[0] = qt*V[0]*y 337 | V[1] = -(1-qt)*V[1]*y 338 | 339 | # elif (loss['name'] == 'CQR') \ 340 | 341 | # n_qt = len(loss['qt']) 342 | # U = np.ones((2, n*n_qt)) 343 | # V = np.ones((2, n*n_qt)) 344 | # X_fake = np.zeros((n*n_qt, d+n_qt)) 345 | 346 | # for l,qt_tmp in enumerate(loss['qt']): 347 | # U[0,l*n:(l+1)*n] = - (qt_tmp*U[0,l*n:(l+1)*n]) 348 | # U[1,l*n:(l+1)*n] = ((1.-qt_tmp)*U[1,l*n:(l+1)*n]) 349 | 350 | # V[0,l*n:(l+1)*n] = qt_tmp*V[0,l*n:(l+1)*n]*y 351 | # V[1,l*n:(l+1)*n] = - (1.-qt_tmp)*V[1,l*n:(l+1)*n]*y 352 | 353 | # X_fake[l*n:(l+1)*n,:d] = X 354 | # X_fake[l*n:(l+1)*n,d+l] = 1. 355 | 356 | elif (loss['name'] == 'sSVM') \ 357 | or (loss['name'] == 'smooth SVM') \ 358 | or (loss['name'] == 'smooth hinge'): 359 | S = np.ones((1, n)) 360 | T = np.ones((1, n)) 361 | Tau = np.ones((1, n)) 362 | S[0] = - y 363 | 364 | elif loss['name'] == 'TV': 365 | U = np.ones((2, n)) 366 | V = np.ones((2, n)) 367 | U[1] = - U[1] 368 | 369 | V[0] = - X.dot(y) 370 | V[1] = X.dot(y) 371 | 372 | elif (loss['name'] == 'huber') or (loss['name'] == 'Huber'): 373 | S = np.ones((2, n)) 374 | T = np.ones((2, n)) 375 | Tau = loss['tau'] * np.ones((2, n)) 376 | 377 | S[0] = -S[0] 378 | T[0] = y 379 | T[1] = -y 380 | 381 | elif (loss['name'] in ['SVR', 'svr']): 382 | U = np.ones((2, n)) 383 | V = np.ones((2, n)) 384 | U[1] = -U[1] 385 | 386 | V[0] = -(y + loss['epsilon']) 387 | V[1] = (y - loss['epsilon']) 388 | 389 | 390 | elif (loss['name'] == 'MAE') \ 391 | or (loss['name'] == 'mae') \ 392 | or (loss['name'] == 'mean absolute error'): 393 | U = np.array([[1.0] * n, [-1.0] * n]) 394 | V = np.array([-y , y]) 395 | 396 | 397 | else: 398 | raise Exception("Sorry, ReHLine currently does not support this loss function, \ 399 | but you can manually set ReHLine params to solve the problem via `ReHLine` class.") 400 | 401 | return U, V, Tau, S, T 402 | 403 | def _make_constraint_rehline_param(constraint, X, y=None): 404 | """The `_make_constraint_rehline_param` function generates constraint parameters for the ReHLine solver. 405 | 406 | Parameters 407 | ---------- 408 | constraint : list of dict 409 | A list of dictionaries, where each dictionary represents a constraint. 410 | Each dictionary must contain a 'name' key, which specifies the type of constraint. 411 | The following constraint types are supported: 412 | * 'nonnegative' or '>=0': A non-negativity constraint. 413 | * 'fair' or 'fairness': A fairness constraint using 'sen_idx' and 'tol_sen'. 414 | * 'custom': A custom constraint, where the user must provide the constraint matrix 'A' and vector 'b'. 415 | 416 | X : array-like of shape (n_samples, n_features) 417 | The design matrix. 418 | 419 | y : array-like of shape (n_samples,), default=None 420 | The target variable. Not used in this function. 421 | 422 | Returns 423 | ------- 424 | A : array-like of shape (n_constraints, n_features) 425 | The constraint matrix. 426 | 427 | b : array-like of shape (n_constraints,) 428 | The constraint vector. 429 | """ 430 | 431 | n, d = X.shape 432 | 433 | ## initialization 434 | A = np.empty(shape=(0, 0)) 435 | b = np.empty(shape=(0)) 436 | 437 | for constr_tmp in constraint: 438 | if (constr_tmp['name'] == 'nonnegative') or (constr_tmp['name'] == '>=0'): 439 | A_tmp = np.identity(d) 440 | b_tmp = np.zeros(d) 441 | 442 | elif (constr_tmp['name'] == 'fair') or (constr_tmp['name'] == 'fairness'): 443 | sen_idx = constr_tmp['sen_idx'] # list of indices 444 | tol_sen = constr_tmp['tol_sen'] 445 | tol_sen = np.array(tol_sen).reshape(-1) 446 | 447 | X_sen = X[:, sen_idx] 448 | X_sen = X_sen.reshape(n, -1) 449 | 450 | assert X_sen.shape[1] == len(tol_sen), "dim of X_sen and len of tol_sen must be equal" 451 | 452 | A_tmp = np.repeat(X_sen.T @ X, repeats=[2], axis=0) / n 453 | A_tmp[::2] = -A_tmp[::2] 454 | b_tmp = np.repeat(tol_sen, repeats=[2], axis=0) 455 | 456 | elif (constr_tmp['name'] == 'custom'): 457 | A_tmp = constr_tmp['A'] 458 | b_tmp = constr_tmp['b'] 459 | 460 | else: 461 | raise Exception("Sorry, ReHLine currently does not support this constraint, \ 462 | but you can add it by manually setting A and b via {'name': 'custom', 'A': A, 'b': b}") 463 | 464 | A = np.vstack([A, A_tmp]) if A.size else A_tmp 465 | b = np.hstack([b, b_tmp]) if b.size else b_tmp 466 | 467 | return A, b 468 | 469 | 470 | def _make_penalty_rehline_param(self, penalty=None, X=None): 471 | """The `_make_penalty_rehline_param` function generates penalty parameters for the ReHLine solver. 472 | """ 473 | raise Exception("Sorry, `_make_penalty_rehline_param` feature is currently under development.") 474 | 475 | 476 | def _cast_sample_bias(U, V, Tau, S, T, sample_bias=None): 477 | """Cast sample bias to ReHLine parameters by injecting bias into V and T. 478 | 479 | This function modifies the ReHLine parameters to incorporate individual 480 | sample biases through linear transformations of the intercept parameters. 481 | 482 | Parameters 483 | ---------- 484 | U : array-like of shape (L, n_samples) 485 | ReLU coefficient matrix. 486 | 487 | V : array-like of shape (L, n_samples) 488 | ReLU intercept vector. 489 | 490 | Tau : array-like of shape (H, n_samples) 491 | ReHU cutpoint matrix. 492 | 493 | S : array-like of shape (H, n_samples) 494 | ReHU coefficient vector. 495 | 496 | T : array-like of shape (H, n_samples) 497 | ReHU intercept vector. 498 | 499 | sample_bias : array-like of shape (n_samples, 1) 500 | Individual sample bias vector. If None, parameters are returned unchanged. 501 | 502 | Returns 503 | ------- 504 | U_bias : array-like of shape (L, n_samples) 505 | Biased coefficient matrix, actually doesn't change 506 | 507 | V_bias : array-like of shape (L, n_samples) 508 | Biased ReLU intercept vector: V + U * sample_bias 509 | 510 | Tau_bias : array-like of shape (H, n_samples) 511 | Biased ReHU cutpoint matrix, actually doesn't change 512 | 513 | S_bias : array-like of shape (H, n_samples) 514 | Biased ReHU coefficient vector, actually doesn't change 515 | 516 | T_bias : array-like of shape (H, n_samples) 517 | Biased ReHU intercept vector: T + S * sample_bias 518 | 519 | Notes 520 | ----- 521 | The transformation applies the sample bias through: 522 | - V_bias = V + U ⊙ sample_bias 523 | - T_bias = T + S ⊙ sample_bias 524 | 525 | where ⊙ denotes element-wise multiplication with broadcasting. 526 | """ 527 | if sample_bias is None: 528 | return U, V, Tau, S, T 529 | 530 | else: 531 | sample_bias = sample_bias.reshape(1, -1) 532 | U_bias = U 533 | V_bias = V + (U * sample_bias if U.shape[0] > 0 else 0) 534 | Tau_bias = Tau 535 | S_bias = S 536 | T_bias = T + (S * sample_bias if S.shape[0] > 0 else 0) 537 | 538 | return U_bias, V_bias, Tau_bias, S_bias, T_bias 539 | 540 | 541 | def _cast_sample_weight(U, V, Tau, S, T, C=1.0, sample_weight=None): 542 | """Apply sample weights and regularization to ReHLine parameters. 543 | 544 | Parameters 545 | ---------- 546 | U : array-like of shape (L, n_samples) 547 | ReLU coefficient matrix. 548 | 549 | V : array-like of shape (L, n_samples) 550 | ReLU intercept vector. 551 | 552 | Tau : array-like of shape (H, n_samples) 553 | ReHU cutpoint matrix. 554 | 555 | S : array-like of shape (H, n_samples) 556 | ReHU coefficient vector. 557 | 558 | T : array-like of shape (H, n_samples) 559 | ReHU intercept vector. 560 | 561 | C : float, default=1.0 562 | Regularization parameter. The strength of the regularization is 563 | inversely proportional to C. Must be strictly positive. 564 | 565 | sample_weight : array-like of shape (n_samples,), default=None 566 | Individual sample weight. If None, then samples are equally weighted. 567 | 568 | Returns 569 | ------- 570 | U_weight : array-like of shape (L, n_samples) 571 | Weighted ReLU coefficient matrix. 572 | 573 | V_weight : array-like of shape (L, n_samples) 574 | Weighted ReLU intercept vector. 575 | 576 | Tau_weight : array-like of shape (H, n_samples) 577 | Weighted ReHU cutpoint matrix. 578 | 579 | S_weight : array-like of shape (H, n_samples) 580 | Weighted ReHU coefficient vector. 581 | 582 | T_weight : array-like of shape (H, n_samples) 583 | Weighted ReHU intercept vector. 584 | 585 | Notes 586 | ----- 587 | This function casts the sample weight to the ReHLine parameters by multiplying 588 | the sample weight with the ReLU and ReHU parameters. If sample_weight is None, 589 | then the sample weight is set to the regularization parameter C. 590 | """ 591 | sample_weight = C * sample_weight 592 | 593 | if U.shape[0] > 0: 594 | U_weight = U * sample_weight 595 | V_weight = V * sample_weight 596 | else: 597 | U_weight = U 598 | V_weight = V 599 | 600 | if S.shape[0] > 0: 601 | sqrt_sample_weight = np.sqrt(sample_weight) 602 | Tau_weight = Tau * sqrt_sample_weight 603 | S_weight = S * sqrt_sample_weight 604 | T_weight = T * sqrt_sample_weight 605 | else: 606 | Tau_weight = Tau 607 | S_weight = S 608 | T_weight = T 609 | 610 | return U_weight, V_weight, Tau_weight, S_weight, T_weight 611 | 612 | 613 | # def append_l1(self, X, l1_pen=1.0): 614 | # r""" 615 | # This function appends the l1 penalty to the ReHLine problem. The formulation becomes: 616 | 617 | # .. math:: 618 | 619 | # \min_{\mathbf{\beta} \in \mathbb{R}^d} \sum_{i=1}^n \sum_{l=1}^L \text{ReLU}( u_{li} \mathbf{x}_i^\intercal \mathbf{\beta} + v_{li}) + \sum_{i=1}^n \sum_{h=1}^H {\text{ReHU}}_{\tau_{hi}}( s_{hi} \mathbf{x}_i^\intercal \mathbf{\beta} + t_{hi}) + \frac{1}{2} \| \mathbf{\beta} \|_2^2 + \lambda_1 \| \mathbf{\beta} \|_1, \\ \text{ s.t. } 620 | # \mathbf{A} \mathbf{\beta} + \mathbf{b} \geq \mathbf{0}, 621 | 622 | # where :math:`\lambda_1` is associated with `l1_pen`. 623 | 624 | # Parameters 625 | # ---------- 626 | 627 | # X : ndarray of shape (n_samples, n_features) 628 | # The generated samples. 629 | 630 | # l1_pen : float, default=1.0 631 | # The l1 penalty level, which controls the complexity or sparsity of the resulting model. 632 | 633 | # Returns 634 | # ------- 635 | 636 | # X_fake: ndarray of shape (n_samples+n_features, n_features) 637 | # The manipulated data matrix. It has been padded with 638 | # identity matrix, allowing the correctly structured data to be input 639 | # into `self.fit` or other modelling processes. 640 | 641 | # Examples 642 | # -------- 643 | 644 | # >>> import numpy as np 645 | # >>> from rehline import ReHLine 646 | 647 | # >>> # simulate classification dataset 648 | # >>> n, d, C, lam1 = 1000, 3, 0.5, 1.0 649 | # >>> np.random.seed(1024) 650 | # >>> X = np.random.randn(1000, 3) 651 | # >>> beta0 = np.random.randn(3) 652 | # >>> y = np.sign(X.dot(beta0) + np.random.randn(n)) 653 | 654 | # >>> clf = ReHLine(loss={'name': 'svm'}, C=C) 655 | # >>> clf.make_ReLHLoss(X=X, y=y, loss={'name': 'svm'}) 656 | # >>> # save and fit with the manipulated data matrix 657 | # >>> X_fake = clf.append_l1(X, l1_pen=lam1) 658 | # >>> clf.fit(X=X_fake) 659 | # >>> print('sol privided by rehline: %s' %clf.coef_) 660 | # >>> sol privided by rehline: [ 7.17796629e-01 -1.87075728e-06 2.61965622e+00] #sparse sol 661 | # >>> print(clf.decision_function([[.1,.2,.3]])) 662 | # >>> [0.85767616] 663 | # """ 664 | 665 | # n, d = X.shape 666 | # l1_pen = l1_pen*np.ones(d) 667 | # U_new = np.zeros((self.L+2, n+d)) 668 | # V_new = np.zeros((self.L+2, n+d)) 669 | # ## Block 1 670 | # if len(self.U): 671 | # U_new[:self.L, :n] = self.U 672 | # V_new[:self.L, :n] = self.V 673 | # ## Block 2 674 | # U_new[-2,n:] = l1_pen 675 | # U_new[-1,n:] = -l1_pen 676 | 677 | # if len(self.S): 678 | # S_new = np.zeros((self.H, n+d)) 679 | # T_new = np.zeros((self.H, n+d)) 680 | # Tau_new = np.zeros((self.H, n+d)) 681 | 682 | # S_new[:,:n] = self.S 683 | # T_new[:,:n] = self.T 684 | # Tau_new[:,:n] = self.Tau 685 | 686 | # self.S = S_new 687 | # self.T = T_new 688 | # self.Tau = Tau_new 689 | 690 | # ## fake X 691 | # X_fake = np.zeros((n+d, d)) 692 | # X_fake[:n,:] = X 693 | # X_fake[n:,:] = np.identity(d) 694 | 695 | # self.U = U_new 696 | # self.V = V_new 697 | # self.auto_shape() 698 | # return X_fake 699 | -------------------------------------------------------------------------------- /rehline/_class.py: -------------------------------------------------------------------------------- 1 | """ ReHLine: Regularized Composite ReLU-ReHU Loss Minimization with Linear Computation and Linear Convergence """ 2 | 3 | # Authors: Ben Dai 4 | # C++ support by Yixuan Qiu 5 | 6 | # License: MIT License 7 | 8 | import warnings 9 | 10 | import numpy as np 11 | from sklearn.base import BaseEstimator 12 | from sklearn.exceptions import ConvergenceWarning 13 | from sklearn.utils.validation import (_check_sample_weight, check_array, 14 | check_is_fitted, check_X_y) 15 | 16 | from ._base import (ReHLine_solver, _BaseReHLine, 17 | _make_constraint_rehline_param, _make_loss_rehline_param) 18 | 19 | 20 | class ReHLine(_BaseReHLine, BaseEstimator): 21 | r"""ReHLine Minimization. 22 | 23 | .. math:: 24 | 25 | \min_{\mathbf{\beta} \in \mathbb{R}^d} \sum_{i=1}^n \sum_{l=1}^L \text{ReLU}( u_{li} \mathbf{x}_i^\intercal \mathbf{\beta} + v_{li}) + \sum_{i=1}^n \sum_{h=1}^H {\text{ReHU}}_{\tau_{hi}}( s_{hi} \mathbf{x}_i^\intercal \mathbf{\beta} + t_{hi}) + \frac{1}{2} \| \mathbf{\beta} \|_2^2, \\ \text{ s.t. } 26 | \mathbf{A} \mathbf{\beta} + \mathbf{b} \geq \mathbf{0}, 27 | 28 | where :math:`\mathbf{U} = (u_{li}),\mathbf{V} = (v_{li}) \in \mathbb{R}^{L \times n}` 29 | and :math:`\mathbf{S} = (s_{hi}),\mathbf{T} = (t_{hi}),\mathbf{\tau} = (\tau_{hi}) \in \mathbb{R}^{H \times n}` 30 | are the ReLU-ReHU loss parameters, and :math:`(\mathbf{A},\mathbf{b})` are the constraint parameters. 31 | 32 | Parameters 33 | ---------- 34 | C : float, default=1.0 35 | Regularization parameter. The strength of the regularization is 36 | inversely proportional to C. Must be strictly positive. 37 | 38 | _U, _V: array of shape (L, n_samples), default=np.empty(shape=(0, 0)) 39 | The parameters pertaining to the ReLU part in the loss function. 40 | 41 | _Tau, _S, _T: array of shape (H, n_samples), default=np.empty(shape=(0, 0)) 42 | The parameters pertaining to the ReHU part in the loss function. 43 | 44 | _A: array of shape (K, n_features), default=np.empty(shape=(0, 0)) 45 | The coefficient matrix in the linear constraint. 46 | 47 | _b: array of shape (K, ), default=np.empty(shape=0) 48 | The intercept vector in the linear constraint. 49 | 50 | verbose : int, default=0 51 | Enable verbose output. 52 | 53 | max_iter : int, default=1000 54 | The maximum number of iterations to be run. 55 | 56 | tol : float, default=1e-4 57 | The tolerance for the stopping criterion. 58 | 59 | shrink : float, default=1 60 | The shrinkage of dual variables for the ReHLine algorithm. 61 | 62 | warm_start : bool, default=False 63 | Whether to use the given dual params as an initial guess for the 64 | optimization algorithm. 65 | 66 | trace_freq : int, default=100 67 | The frequency at which to print the optimization trace. 68 | 69 | Attributes 70 | ---------- 71 | coef\_ : array-like 72 | The optimized model coefficients. 73 | 74 | n_iter\_ : int 75 | The number of iterations performed by the ReHLine solver. 76 | 77 | opt_result\_ : object 78 | The optimization result object. 79 | 80 | dual_obj\_ : array-like 81 | The dual objective function values. 82 | 83 | primal_obj\_ : array-like 84 | The primal objective function values. 85 | 86 | _Lambda: array-like 87 | The optimized dual variables for ReLU parts. 88 | 89 | _Gamma: array-like 90 | The optimized dual variables for ReHU parts. 91 | 92 | _xi: array-like 93 | The optimized dual variables for linear constraints. 94 | 95 | Examples 96 | -------- 97 | 98 | >>> ## test SVM on simulated dataset 99 | >>> import numpy as np 100 | >>> from rehline import ReHLine 101 | 102 | >>> # simulate classification dataset 103 | >>> n, d, C = 1000, 3, 0.5 104 | >>> np.random.seed(1024) 105 | >>> X = np.random.randn(1000, 3) 106 | >>> beta0 = np.random.randn(3) 107 | >>> y = np.sign(X.dot(beta0) + np.random.randn(n)) 108 | 109 | >>> # Usage of ReHLine 110 | >>> n, d = X.shape 111 | >>> U = -(C*y).reshape(1,-1) 112 | >>> L = U.shape[0] 113 | >>> V = (C*np.array(np.ones(n))).reshape(1,-1) 114 | >>> clf = ReHLine(C=C) 115 | >>> clf._U, clf._V = U, V 116 | >>> clf.fit(X=X) 117 | >>> print('sol privided by rehline: %s' %clf.coef_) 118 | >>> sol privided by rehline: [ 0.7410154 -0.00615574 2.66990408] 119 | >>> print(clf.decision_function([[.1,.2,.3]])) 120 | >>> [0.87384162] 121 | 122 | References 123 | ---------- 124 | .. [1] `Dai, B., Qiu, Y,. (2023). ReHLine: Regularized Composite ReLU-ReHU Loss Minimization with Linear Computation and Linear Convergence `_ 125 | """ 126 | 127 | def __init__(self, C=1., 128 | U=np.empty(shape=(0,0)), V=np.empty(shape=(0,0)), 129 | Tau=np.empty(shape=(0,0)), 130 | S=np.empty(shape=(0,0)), T=np.empty(shape=(0,0)), 131 | A=np.empty(shape=(0,0)), b=np.empty(shape=(0)), 132 | max_iter=1000, tol=1e-4, shrink=1, warm_start=0, 133 | verbose=0, trace_freq=100): 134 | self.C = C 135 | self._U = U 136 | self._V = V 137 | self._S = S 138 | self._T = T 139 | self._Tau = Tau 140 | self._A = A 141 | self._b = b 142 | self.L = U.shape[0] 143 | self.H = S.shape[0] 144 | self.K = A.shape[0] 145 | self.max_iter = max_iter 146 | self.tol = tol 147 | self.shrink = shrink 148 | self.warm_start = warm_start 149 | self.verbose = verbose 150 | self.trace_freq = trace_freq 151 | self._Lambda = np.empty(shape=(0, 0)) 152 | self._Gamma = np.empty(shape=(0, 0)) 153 | self._xi = np.empty(shape=(0, 0)) 154 | self.coef_ = None 155 | 156 | def fit(self, X, sample_weight=None): 157 | """Fit the model based on the given training data. 158 | 159 | Parameters 160 | ---------- 161 | 162 | X: {array-like} of shape (n_samples, n_features) 163 | Training vector, where `n_samples` is the number of samples and 164 | `n_features` is the number of features. 165 | 166 | sample_weight : array-like of shape (n_samples,), default=None 167 | Array of weights that are assigned to individual 168 | samples. If not provided, then each sample is given unit weight. 169 | 170 | Returns 171 | ------- 172 | self : object 173 | An instance of the estimator. 174 | """ 175 | # X = check_array(X) 176 | sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) 177 | 178 | U_weight, V_weight, Tau_weight, S_weight, T_weight = self.cast_sample_weight(sample_weight=sample_weight) 179 | 180 | if not self.warm_start: 181 | ## remove warm_start params 182 | self._Lambda = np.empty(shape=(0, 0)) 183 | self._Gamma = np.empty(shape=(0, 0)) 184 | self._xi = np.empty(shape=(0, 0)) 185 | 186 | result = ReHLine_solver(X=X, 187 | U=U_weight, V=V_weight, 188 | Tau=Tau_weight, 189 | S=S_weight, T=T_weight, 190 | A=self._A, b=self._b, 191 | Lambda=self._Lambda, Gamma=self._Gamma, xi=self._xi, 192 | max_iter=self.max_iter, tol=self.tol, 193 | shrink=self.shrink, verbose=self.verbose, 194 | trace_freq=self.trace_freq) 195 | 196 | self.opt_result_ = result 197 | # primal solution 198 | self.coef_ = result.beta 199 | # dual solution 200 | self._Lambda = result.Lambda 201 | self._Gamma = result.Gamma 202 | self._xi = result.xi 203 | # algo convergence 204 | self.n_iter_ = result.niter 205 | self.dual_obj_ = result.dual_objfns 206 | self.primal_obj_ = result.primal_objfns 207 | 208 | if self.n_iter_ >= self.max_iter: 209 | warnings.warn( 210 | "ReHLine failed to converge, increase the number of iterations: `max_iter`.", 211 | ConvergenceWarning, 212 | ) 213 | return self 214 | 215 | def decision_function(self, X): 216 | """The decision function evaluated on the given dataset 217 | 218 | Parameters 219 | ---------- 220 | X : array-like of shape (n_samples, n_features) 221 | The data matrix. 222 | 223 | Returns 224 | ------- 225 | ndarray of shape (n_samples, ) 226 | Returns the decision function of the samples. 227 | """ 228 | # Check if fit has been called 229 | check_is_fitted(self) 230 | 231 | X = check_array(X) 232 | return np.dot(X, self.coef_) 233 | 234 | class plqERM_Ridge(_BaseReHLine, BaseEstimator): 235 | r"""Empirical Risk Minimization (ERM) with a piecewise linear-quadratic (PLQ) objective with a ridge penalty. 236 | 237 | .. math:: 238 | 239 | \min_{\mathbf{\beta} \in \mathbb{R}^d} \sum_{i=1}^n \text{PLQ}(y_i, \mathbf{x}_i^T \mathbf{\beta}) + \frac{1}{2} \| \mathbf{\beta} \|_2^2, \ \text{ s.t. } \ 240 | \mathbf{A} \mathbf{\beta} + \mathbf{b} \geq \mathbf{0}, 241 | 242 | The function supports various loss functions, including: 243 | - 'hinge', 'svm' or 'SVM' 244 | - 'check' or 'quantile' or 'quantile regression' or 'QR' 245 | - 'sSVM' or 'smooth SVM' or 'smooth hinge' 246 | - 'TV' 247 | - 'huber' or 'Huber' 248 | - 'SVR' or 'svr' 249 | 250 | The following constraint types are supported: 251 | * 'nonnegative' or '>=0': A non-negativity constraint. 252 | * 'fair' or 'fairness': A fairness constraint. 253 | * 'custom': A custom constraint, where the user must provide the constraint matrix 'A' and vector 'b'. 254 | 255 | Parameters 256 | ---------- 257 | loss : dict 258 | A dictionary specifying the loss function parameters. 259 | 260 | constraint : list of dict 261 | A list of dictionaries, where each dictionary represents a constraint. 262 | Each dictionary must contain a 'name' key, which specifies the type of constraint. 263 | 264 | C : float, default=1.0 265 | Regularization parameter. The strength of the regularization is 266 | inversely proportional to C. Must be strictly positive. 267 | `C` will be absorbed by the ReHLine parameters when `self.make_ReLHLoss` is conducted. 268 | 269 | verbose : int, default=0 270 | Enable verbose output. Note that this setting takes advantage of a 271 | per-process runtime setting in liblinear that, if enabled, may not work 272 | properly in a multithreaded context. 273 | 274 | max_iter : int, default=1000 275 | The maximum number of iterations to be run. 276 | 277 | _U, _V: array of shape (L, n_samples), default=np.empty(shape=(0, 0)) 278 | The parameters pertaining to the ReLU part in the loss function. 279 | 280 | _Tau, _S, _T: array of shape (H, n_samples), default=np.empty(shape=(0, 0)) 281 | The parameters pertaining to the ReHU part in the loss function. 282 | 283 | _A: array of shape (K, n_features), default=np.empty(shape=(0, 0)) 284 | The coefficient matrix in the linear constraint. 285 | 286 | _b: array of shape (K, ), default=np.empty(shape=0) 287 | The intercept vector in the linear constraint. 288 | 289 | Attributes 290 | ---------- 291 | coef\_ : array-like 292 | The optimized model coefficients. 293 | 294 | n_iter\_ : int 295 | The number of iterations performed by the ReHLine solver. 296 | 297 | opt_result\_ : object 298 | The optimization result object. 299 | 300 | dual_obj\_ : array-like 301 | The dual objective function values. 302 | 303 | primal_obj\_ : array-like 304 | The primal objective function values. 305 | 306 | Methods 307 | ------- 308 | fit(X, y, sample_weight=None) 309 | Fit the model based on the given training data. 310 | 311 | decision_function(X) 312 | The decision function evaluated on the given dataset. 313 | 314 | Notes 315 | ----- 316 | The `plqERM_Ridge` class is a subclass of `_BaseReHLine` and `BaseEstimator`, which suggests that it is part of a larger framework for implementing ReHLine algorithms. 317 | 318 | """ 319 | 320 | def __init__(self, loss, 321 | constraint=[], 322 | C=1., 323 | U=np.empty(shape=(0,0)), V=np.empty(shape=(0,0)), 324 | Tau=np.empty(shape=(0,0)), 325 | S=np.empty(shape=(0,0)), T=np.empty(shape=(0,0)), 326 | A=np.empty(shape=(0,0)), b=np.empty(shape=(0)), 327 | max_iter=1000, tol=1e-4, shrink=1, warm_start=0, 328 | verbose=0, trace_freq=100): 329 | self.loss = loss 330 | self.constraint = constraint 331 | self.C = C 332 | self._U = U 333 | self._V = V 334 | self._S = S 335 | self._T = T 336 | self._Tau = Tau 337 | self._A = A 338 | self._b = b 339 | self.L = U.shape[0] 340 | self.H = S.shape[0] 341 | self.K = A.shape[0] 342 | self.max_iter = max_iter 343 | self.tol = tol 344 | self.shrink = shrink 345 | self.warm_start = warm_start 346 | self.verbose = verbose 347 | self.trace_freq = trace_freq 348 | self._Lambda = np.empty(shape=(0, 0)) 349 | self._Gamma = np.empty(shape=(0, 0)) 350 | self._xi = np.empty(shape=(0, 0)) 351 | self.coef_ = None 352 | 353 | def fit(self, X, y, sample_weight=None): 354 | """Fit the model based on the given training data. 355 | 356 | Parameters 357 | ---------- 358 | 359 | X: {array-like} of shape (n_samples, n_features) 360 | Training vector, where `n_samples` is the number of samples and 361 | `n_features` is the number of features. 362 | 363 | y : array-like of shape (n_samples,) 364 | The target variable. 365 | 366 | sample_weight : array-like of shape (n_samples,), default=None 367 | Array of weights that are assigned to individual 368 | samples. If not provided, then each sample is given unit weight. 369 | 370 | Returns 371 | ------- 372 | self : object 373 | An instance of the estimator. 374 | 375 | 376 | """ 377 | n, d = X.shape 378 | 379 | ## loss -> rehline params 380 | self._U, self._V, self._Tau, self._S, self._T = _make_loss_rehline_param(loss=self.loss, X=X, y=y) 381 | 382 | ## constrain -> rehline params 383 | self._A, self._b = _make_constraint_rehline_param(constraint=self.constraint, X=X, y=y) 384 | self.auto_shape() 385 | 386 | sample_weight = _check_sample_weight(sample_weight, X, dtype=X.dtype) 387 | 388 | U_weight, V_weight, Tau_weight, S_weight, T_weight = self.cast_sample_weight(sample_weight=sample_weight) 389 | 390 | if not self.warm_start: 391 | ## remove warm_start params 392 | self._Lambda = np.empty(shape=(0, 0)) 393 | self._Gamma = np.empty(shape=(0, 0)) 394 | self._xi = np.empty(shape=(0, 0)) 395 | 396 | result = ReHLine_solver(X=X, 397 | U=U_weight, V=V_weight, 398 | Tau=Tau_weight, 399 | S=S_weight, T=T_weight, 400 | A=self._A, b=self._b, 401 | Lambda=self._Lambda, Gamma=self._Gamma, xi=self._xi, 402 | max_iter=self.max_iter, tol=self.tol, 403 | shrink=self.shrink, verbose=self.verbose, 404 | trace_freq=self.trace_freq) 405 | 406 | self.opt_result_ = result 407 | # primal solution 408 | self.coef_ = result.beta 409 | # dual solution 410 | self._Lambda = result.Lambda 411 | self._Gamma = result.Gamma 412 | self._xi = result.xi 413 | # algo convergence 414 | self.n_iter_ = result.niter 415 | self.dual_obj_ = result.dual_objfns 416 | self.primal_obj_ = result.primal_objfns 417 | 418 | if self.n_iter_ >= self.max_iter: 419 | warnings.warn( 420 | "ReHLine failed to converge, increase the number of iterations: `max_iter`.", 421 | ConvergenceWarning, 422 | ) 423 | 424 | return self 425 | 426 | def decision_function(self, X): 427 | """The decision function evaluated on the given dataset 428 | 429 | Parameters 430 | ---------- 431 | X : array-like of shape (n_samples, n_features) 432 | The data matrix. 433 | 434 | Returns 435 | ------- 436 | ndarray of shape (n_samples, ) 437 | Returns the decision function of the samples. 438 | """ 439 | # Check if fit has been called 440 | check_is_fitted(self) 441 | 442 | X = check_array(X) 443 | return np.dot(X, self.coef_) 444 | 445 | 446 | class CQR_Ridge(_BaseReHLine, BaseEstimator): 447 | r"""Composite Quantile Regressor (CQR) with a ridge penalty. 448 | 449 | It allows for the fitting of a linear regression model that minimizes a composite quantile loss function. 450 | 451 | .. math:: 452 | 453 | \min_{\mathbf{\beta} \in \mathbb{R}^d, \mathbf{\beta_0} \in \mathbb{R}^K} \sum_{k=1}^K \sum_{i=1}^n \text{PLQ}(y_i, \mathbf{x}_i^T \mathbf{\beta} + \mathbf{\beta_0k}) + \frac{1}{2} \| \mathbf{\beta} \|_2^2. 454 | 455 | 456 | Parameters 457 | ---------- 458 | quantiles : list of float (n_quantiles,) 459 | The quantiles to be estimated. 460 | 461 | C : float, default=1.0 462 | Regularization parameter. The strength of the regularization is 463 | inversely proportional to C. Must be strictly positive. 464 | `C` will be absorbed by the ReHLine parameters when `self.make_ReLHLoss` is conducted. 465 | 466 | verbose : int, default=0 467 | Enable verbose output. Note that this setting takes advantage of a 468 | per-process runtime setting in liblinear that, if enabled, may not work 469 | properly in a multithreaded context. 470 | 471 | max_iter : int, default=1000 472 | The maximum number of iterations to be run. 473 | 474 | tol : float, default=1e-4 475 | The tolerance for the stopping criterion. 476 | 477 | shrink : float, default=1 478 | The shrinkage of dual variables for the ReHLine algorithm. 479 | 480 | warm_start : bool, default=False 481 | Whether to use the given dual params as an initial guess for the 482 | optimization algorithm. 483 | 484 | trace_freq : int, default=100 485 | The frequency at which to print the optimization trace. 486 | 487 | Attributes 488 | ---------- 489 | coef\_ : array-like 490 | The optimized model coefficients. 491 | 492 | intercept\_ : array-like 493 | The optimized model intercepts. 494 | 495 | quantiles\_: array-like 496 | The quantiles to be estimated. 497 | 498 | n_iter\_ : int 499 | The number of iterations performed by the ReHLine solver. 500 | 501 | opt_result\_ : object 502 | The optimization result object. 503 | 504 | dual_obj\_ : array-like 505 | The dual objective function values. 506 | 507 | primal_obj\_ : array-like 508 | The primal objective function values. 509 | 510 | Methods 511 | ------- 512 | fit(X, y, sample_weight=None) 513 | Fit the model based on the given training data. 514 | 515 | predict(X) 516 | The prediction for the given dataset. 517 | """ 518 | 519 | def __init__(self, quantiles, 520 | C=1., 521 | max_iter=1000, tol=1e-4, shrink=1, warm_start=0, 522 | verbose=0, trace_freq=100): 523 | self.quantiles = quantiles 524 | self.C = C 525 | # self.U = U 526 | # self.V = V 527 | # self.S = S 528 | # self.T = T 529 | # self.Tau = Tau 530 | # self.A = A 531 | # self.b = b 532 | # self.L = U.shape[0] 533 | # self.H = S.shape[0] 534 | # self.K = A.shape[0] 535 | self.max_iter = max_iter 536 | self.tol = tol 537 | self.shrink = shrink 538 | self.warm_start = warm_start 539 | self.verbose = verbose 540 | self.trace_freq = trace_freq 541 | self._Lambda = np.empty(shape=(0, 0)) 542 | self._Gamma = np.empty(shape=(0, 0)) 543 | self._xi = np.empty(shape=(0, 0)) 544 | self.coef_ = None 545 | self.quantiles_ = np.array(quantiles) # consistent with coef_ and intercept_ in sklearn 546 | 547 | def fit(self, X, y, sample_weight=None): 548 | """Fit the model based on the given training data. 549 | 550 | Parameters 551 | ---------- 552 | 553 | X: {array-like} of shape (n_samples, n_features) 554 | Training vector, where `n_samples` is the number of samples and 555 | `n_features` is the number of features. 556 | 557 | y : array-like of shape (n_samples,) 558 | The target variable. 559 | 560 | sample_weight : array-like of shape (n_samples,), default=None 561 | Array of weights that are assigned to individual 562 | samples. If not provided, then each sample is given unit weight. 563 | 564 | Returns 565 | ------- 566 | self : object 567 | An instance of the estimator. 568 | 569 | 570 | """ 571 | n, d = X.shape 572 | n_qt = len(self.quantiles_) 573 | 574 | # transform X and sample_weight to fit the CQR 575 | transform_X = np.zeros((n*n_qt, d+n_qt)) 576 | 577 | for l, _ in enumerate(self.quantiles_): 578 | transform_X[l*n:(l+1)*n,:d] = X 579 | transform_X[l*n:(l+1)*n,d+l] = 1. 580 | 581 | transform_sample_weight = np.tile(sample_weight, n_qt) if sample_weight is not None else None 582 | 583 | ## loss -> rehline params 584 | self._Tau = np.empty(shape=(0,0)) 585 | self._S = np.empty(shape=(0,0)) 586 | self._T = np.empty(shape=(0,0)) 587 | self._U = np.ones((2, n*n_qt)) 588 | self._V = np.ones((2, n*n_qt)) 589 | 590 | for l,qt in enumerate(self.quantiles_): 591 | self._U[0,l*n:(l+1)*n] = - (qt*self._U[0,l*n:(l+1)*n]) 592 | self._U[1,l*n:(l+1)*n] = ((1.-qt)*self._U[1,l*n:(l+1)*n]) 593 | 594 | self._V[0,l*n:(l+1)*n] = qt*self._V[0,l*n:(l+1)*n]*y 595 | self._V[1,l*n:(l+1)*n] = - (1.-qt)*self._V[1,l*n:(l+1)*n]*y 596 | 597 | ## no constrain in CQR -> empty rehline params A and b 598 | self._A, self._b = np.empty(shape=(0, 0)), np.empty(shape=(0)) 599 | self.auto_shape() 600 | 601 | transform_sample_weight = _check_sample_weight(transform_sample_weight, transform_X, dtype=transform_X.dtype) 602 | 603 | U_weight, V_weight, Tau_weight, S_weight, T_weight = self.cast_sample_weight(sample_weight=transform_sample_weight) 604 | 605 | if not self.warm_start: 606 | ## remove warm_start params 607 | self._Lambda = np.empty(shape=(0, 0)) 608 | self._Gamma = np.empty(shape=(0, 0)) 609 | self._xi = np.empty(shape=(0, 0)) 610 | 611 | result = ReHLine_solver(X=transform_X, 612 | U=U_weight, V=V_weight, 613 | Tau=Tau_weight, 614 | S=S_weight, T=T_weight, 615 | A=self._A, b=self._b, 616 | Lambda=self._Lambda, Gamma=self._Gamma, xi=self._xi, 617 | max_iter=self.max_iter, tol=self.tol, 618 | shrink=self.shrink, verbose=self.verbose, 619 | trace_freq=self.trace_freq) 620 | 621 | self.opt_result_ = result 622 | # primal solution 623 | self.coef_ = result.beta[:-n_qt] 624 | self.intercept_ = result.beta[-n_qt:] 625 | # dual solution 626 | self._Lambda = result.Lambda 627 | self._Gamma = result.Gamma 628 | self._xi = result.xi 629 | # algo convergence 630 | self.n_iter_ = result.niter 631 | self.dual_obj_ = result.dual_objfns 632 | self.primal_obj_ = result.primal_objfns 633 | 634 | if self.n_iter_ >= self.max_iter: 635 | warnings.warn( 636 | "ReHLine failed to converge, increase the number of iterations: `max_iter`.", 637 | ConvergenceWarning, 638 | ) 639 | 640 | return self 641 | 642 | def predict(self, X): 643 | """The decision function evaluated on the given dataset 644 | 645 | Parameters 646 | ---------- 647 | X : array-like of shape (n_samples, n_features) 648 | The data matrix. 649 | 650 | Returns 651 | ------- 652 | ndarray of shape (n_samples, n_quantiles) 653 | Returns the decision function of the samples. 654 | """ 655 | # Check if fit has been called 656 | check_is_fitted(self) 657 | X = check_array(X) 658 | 659 | n, d = X.shape 660 | n_qt = len(self.quantiles_) 661 | 662 | # transform X for CQR prediction 663 | transform_X = np.zeros((n*n_qt, d+n_qt)) 664 | 665 | for l, _ in enumerate(self.quantiles_): 666 | transform_X[l*n:(l+1)*n,:d] = X 667 | transform_X[l*n:(l+1)*n,d+l] = 1. 668 | 669 | # get beta by concatenating coef_ and intercept_ 670 | beta = np.concatenate((self.coef_, self.intercept_)) 671 | return np.dot(transform_X, beta).reshape(-1, n_qt, order='F') 672 | 673 | 674 | # # ReHLine estimator with an option of additional linear term 675 | # class ReHLineLinear(ReHLine, _BaseReHLine, BaseEstimator): 676 | # r"""ReHLine Minimization with additional linear terms. 677 | 678 | # .. math:: 679 | 680 | # \min_{\mathbf{\beta} \in \mathbb{R}^d} \sum_{i=1}^n \sum_{l=1}^L \text{ReLU}( u_{li} \mathbf{x}_i^\intercal \mathbf{\beta} + v_{li}) + \sum_{i=1}^n \sum_{h=1}^H {\text{ReHU}}_{\tau_{hi}}( s_{hi} \mathbf{x}_i^\intercal \mathbf{\beta} + t_{hi}) + \mathbf{\mu}^T \mathbf{\beta} + \frac{1}{2} \| \mathbf{\beta} \|_2^2, \\ \text{ s.t. } 681 | # \mathbf{A} \mathbf{\beta} + \mathbf{b} \geq \mathbf{0}, 682 | 683 | # where :math:`\mathbf{U} = (u_{li}),\mathbf{V} = (v_{li}) \in \mathbb{R}^{L \times n}` 684 | # and :math:`\mathbf{S} = (s_{hi}),\mathbf{T} = (t_{hi}),\mathbf{\tau} = (\tau_{hi}) \in \mathbb{R}^{H \times n}` 685 | # are the ReLU-ReHU loss parameters, and :math:`(\mathbf{A},\mathbf{b})` are the constraint parameters. 686 | 687 | # Parameters 688 | # ---------- 689 | 690 | # C : float, default=1.0 691 | # Regularization parameter. The strength of the regularization is 692 | # inversely proportional to C. Must be strictly positive. 693 | # `C` will be absorbed by the ReHLine parameters when `self.make_ReLHLoss` is conducted. 694 | 695 | # verbose : int, default=0 696 | # Enable verbose output. Note that this setting takes advantage of a 697 | # per-process runtime setting in liblinear that, if enabled, may not work 698 | # properly in a multithreaded context. 699 | 700 | # max_iter : int, default=1000 701 | # The maximum number of iterations to be run. 702 | 703 | # U, V: array of shape (L, n_samples), default=np.empty(shape=(0, 0)) 704 | # The parameters pertaining to the ReLU part in the loss function. 705 | 706 | # Tau, S, T: array of shape (H, n_samples), default=np.empty(shape=(0, 0)) 707 | # The parameters pertaining to the ReHU part in the loss function. 708 | 709 | # mu: array of shape (n_features, ), default=np.empty(shape=0) 710 | # The parameters pertaining to the linear part in the loss function. 711 | 712 | # A: array of shape (K, n_features), default=np.empty(shape=(0, 0)) 713 | # The coefficient matrix in the linear constraint. 714 | 715 | # b: array of shape (K, ), default=np.empty(shape=0) 716 | # The intercept vector in the linear constraint. 717 | 718 | 719 | # Attributes 720 | # ---------- 721 | 722 | # coef_ : array of shape (n_features,) 723 | # Weights assigned to the features (coefficients in the primal 724 | # problem). 725 | 726 | # n_iter_: int 727 | # Maximum number of iterations run across all classes. 728 | 729 | # """ 730 | # def __init__(self, *, mu=np.empty(shape=(0))): 731 | # _BaseReHLine().__init__(C, U, V, Tau, S, T, A, b, mu, max_iter, tol, shrink, verbose, trace_freq) 732 | 733 | # # @override 734 | # def fit(self, X, sample_weight=None): 735 | # """Fit the model based on the given training data. 736 | 737 | # Parameters 738 | # ---------- 739 | 740 | # X: {array-like} of shape (n_samples, n_features) 741 | # Training vector, where `n_samples` is the number of samples and 742 | # `n_features` is the number of features. 743 | 744 | # sample_weight : array-like of shape (n_samples,), default=None 745 | # Array of weights that are assigned to individual 746 | # samples. If not provided, then each sample is given unit weight. 747 | 748 | # Returns 749 | # ------- 750 | # self : object 751 | # An instance of the estimator. 752 | # """ 753 | 754 | # # X = check_array(X) 755 | # if sample_weight is None: 756 | # sample_weight = np.ones(X.shape[0]) 757 | 758 | # if self.L > 0: 759 | # U_weight = self.U * sample_weight 760 | # V_weight = self.V * sample_weight 761 | # else: 762 | # U_weight = self.U 763 | # V_weight = self.V 764 | 765 | # if self.H > 0: 766 | # sqrt_sample_weight = np.sqrt(sample_weight) 767 | # Tau_weight = self.Tau * sqrt_sample_weight 768 | # S_weight = self.S * sqrt_sample_weight 769 | # T_weight = self.T * sqrt_sample_weight 770 | # else: 771 | # Tau_weight = self.Tau 772 | # S_weight = self.S 773 | # T_weight = self.T 774 | 775 | 776 | # Xtmu = X @ self.mu 777 | # if self.mu.size > 0: 778 | # # v_li' = v_li + u_li * (x_i.T @ mu) 779 | # V_weight = V_weight + U_weight * Xtmu.reshape(1, -1) if self.L > 0 else V_weight 780 | # # t_hi' = t_hi + s_hi * (x_i.T @ mu) 781 | # T_weight = T_weight + S_weight * Xtmu.reshape(1, -1) if self.H > 0 else T_weight 782 | # b_shifted = self.b + self.A @ self.mu 783 | # else: 784 | # b_shifted = self.b 785 | 786 | # result = ReHLine_solver(X=X, 787 | # U=U_weight, V=V_weight, 788 | # Tau=Tau_weight, 789 | # S=S_weight, T=T_weight, 790 | # A=self.A, b=b_shifted, 791 | # max_iter=self.max_iter, tol=self.tol, 792 | # shrink=self.shrink, verbose=self.verbose, 793 | # trace_freq=self.trace_freq) 794 | 795 | # # unshift results 796 | # result.beta = result.beta + self.mu 797 | # result.dual_objfns = [dual_func + 0.5*self.mu.T @ self.mu for dual_func in result.dual_objfns] 798 | # result.primal_objfns = [primal_func - 0.5*self.mu.T @ self.mu for primal_func in result.primal_objfns] 799 | 800 | # self.coef_ = result.beta 801 | # self.opt_result_ = result 802 | # self.n_iter_ = result.niter 803 | # self.dual_obj_ = result.dual_objfns 804 | # self.primal_obj_ = result.primal_objfns 805 | -------------------------------------------------------------------------------- /rehline/_data.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | from sklearn.datasets import make_classification 4 | from sklearn.preprocessing import StandardScaler 5 | 6 | 7 | def make_fair_classification(n_samples=100, n_features=5, ind_sensitive=0): 8 | """ 9 | Generate a random binary fair classification problem. 10 | 11 | Parameters 12 | ---------- 13 | n_samples : int, default=100 14 | The number of samples. 15 | 16 | n_features : int, default=5 17 | The total number of features. 18 | 19 | ind_sensitive : int, default=0 20 | The index of the sensitive feature. 21 | 22 | Returns 23 | ------- 24 | X : ndarray of shape (n_samples, n_features) 25 | The generated samples. 26 | 27 | y : ndarray of shape (n_samples,) 28 | The +/- labels for class membership of each sample. 29 | 30 | X_sen: ndarray of shape (n_samples,) 31 | The centered samples of the sensitive feature. 32 | """ 33 | 34 | X, y = make_classification(n_samples, n_features) 35 | y = 2*y - 1 36 | 37 | scaler = StandardScaler() 38 | X = scaler.fit_transform(X) 39 | 40 | X_sen = X[:, ind_sensitive] 41 | 42 | return X, y, X_sen -------------------------------------------------------------------------------- /rehline/_loss.py: -------------------------------------------------------------------------------- 1 | """ ReHLoss: Convert a piecewise quadratic loss function to a ReHLoss. """ 2 | 3 | # Authors: Ben Dai 4 | # Yixuan Qiu 5 | 6 | # License: MIT License 7 | 8 | 9 | import numpy as np 10 | 11 | from ._base import _check_rehu, _check_relu, _rehu, _relu 12 | 13 | 14 | class ReHLoss(object): 15 | """ 16 | A ReHLine loss function composed of one or multiple ReLU and ReHU components. 17 | 18 | Parameters 19 | ---------- 20 | 21 | relu_coef : {array-like} of shape (n_relu, n_samples) 22 | ReLU coeff matrix, where `n_loss` is the number of losses and 23 | `n_relu` is the number of relus. 24 | 25 | relu_intercept : {array-like} of shape (n_relu, n_samples) 26 | ReLU intercept matrix, where `n_loss` is the number of losses and 27 | `n_relu` is the number of relus. 28 | 29 | rehu_coef : {array-like} of shape (n_rehu, n_samples) 30 | ReHU coeff matrix, where `n_loss` is the number of losses and 31 | `n_relu` is the number of rehus. 32 | 33 | rehu_intercept : {array-like} of shape (n_rehu, n_samples) 34 | ReHU coeff matrix, where `n_loss` is the number of losses and 35 | `n_relu` is the number of rehus. 36 | 37 | rehu_cut : {array-like} of shape (n_rehu, n_samples) 38 | ReHU cutpoints matrix, where `n_loss` is the number of losses and 39 | `n_relu` is the number of rehus. 40 | 41 | Example 42 | ------- 43 | 44 | >>> import numpy as np 45 | >>> L, H, n = 3, 2, 100 46 | >>> relu_coef, relu_intercept = np.random.randn(L,n), np.random.randn(L,n) 47 | >>> rehu_cut, rehu_coef, rehu_intercept = abs(np.random.randn(H,n)), np.random.randn(H,n), np.random.randn(H,n) 48 | >>> x = np.random.randn(n) 49 | >>> random_loss = ReHLoss(relu_coef, relu_intercept, rehu_coef, rehu_intercept, rehu_cut) 50 | >>> random_loss(x) 51 | """ 52 | 53 | def __init__(self, relu_coef, relu_intercept, 54 | rehu_coef=np.empty(shape=(0,0)), rehu_intercept=np.empty(shape=(0,0)), rehu_cut=1): 55 | self.relu_coef = relu_coef 56 | self.relu_intercept = relu_intercept 57 | self.rehu_cut = rehu_cut * np.ones_like(rehu_coef) 58 | self.rehu_coef = rehu_coef 59 | self.rehu_intercept = rehu_intercept 60 | self.H = rehu_coef.shape[0] 61 | self.L = relu_coef.shape[0] 62 | self.n = relu_coef.shape[1] 63 | 64 | def __call__(self, x): 65 | """Evaluate ReHLoss given a data matrix 66 | 67 | Parameters 68 | ---------- 69 | x: {array-like} of shape (n_samples, ) 70 | Training vector, where `n_samples` is the number of samples 71 | 72 | For ERM question, the input of this function is np.dot(X, self.coef_) rather than a single X. 73 | """ 74 | if (self.L > 0) and (self.H > 0): 75 | assert self.relu_coef.shape[1] == self.rehu_coef.shape[1], "n_samples for `relu_coef` and `rehu_coef` should be the same shape!" 76 | 77 | _check_relu(self.relu_coef, self.relu_intercept) 78 | _check_rehu(self.rehu_coef, self.rehu_intercept, self.rehu_cut) 79 | 80 | self.L, self.H, self.n = self.relu_coef.shape[0], self.rehu_coef.shape[0], self.relu_coef.shape[1] 81 | 82 | ans = 0 83 | if len(self.relu_coef) > 0: 84 | relu_input = (self.relu_coef.T * x[:,np.newaxis]).T + self.relu_intercept 85 | ans += np.sum(_relu(relu_input), 0).sum() 86 | if len(self.rehu_coef) > 0: 87 | rehu_input = (self.rehu_coef.T * x[:,np.newaxis]).T + self.rehu_intercept 88 | ans += np.sum(_rehu(rehu_input, cut=self.rehu_cut), 0).sum() 89 | 90 | return ans 91 | -------------------------------------------------------------------------------- /rehline/_path_sol.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import numpy as np 4 | 5 | from ._base import _make_loss_rehline_param 6 | from ._class import plqERM_Ridge 7 | from ._class import CQR_Ridge 8 | from ._loss import ReHLoss 9 | 10 | 11 | def plqERM_Ridge_path_sol( 12 | X, 13 | y, 14 | *, 15 | loss, 16 | constraint=[], 17 | eps=1e-3, 18 | n_Cs=100, 19 | Cs=None, 20 | max_iter=5000, 21 | tol=1e-4, 22 | verbose=0, 23 | shrink=1, 24 | warm_start=False, 25 | return_time=True, 26 | ): 27 | """ 28 | Compute the PLQ Empirical Risk Minimization (ERM) path over a range of regularization parameters. 29 | This function evaluates the model's performance for different values of the regularization parameter 30 | and provides structured benchmarking output. 31 | 32 | Parameters 33 | ---------- 34 | X : ndarray of shape (n_samples, n_features) 35 | Training input samples. 36 | 37 | y : ndarray of shape (n_samples,) 38 | Target values corresponding to each input sample. 39 | 40 | loss : dict 41 | Dictionary describing the PLQ loss function parameters. Used to construct the loss object internally. 42 | 43 | constraint : list of dict, optional (default=[]) 44 | List of constraints applied to the optimization problem. Each constraint should be represented 45 | as a dictionary compatible with the solver. 46 | 47 | 48 | eps : float, default=1e-3 49 | Defines the length of the regularization path when `Cs` is not provided. 50 | The values of `C` will range from `10^log10(eps)` to `10^-log10(eps)`. 51 | 52 | n_Cs : int, default=100 53 | Number of regularization values to evaluate if `Cs` is not provided. 54 | 55 | Cs : array-like of shape (n_Cs,), optional 56 | Explicit values of regularization strength `C` to use. If `None`, the values are generated 57 | logarithmically between 1e-2 and 1e3. 58 | 59 | max_iter : int, default=5000 60 | Maximum number of iterations allowed for the optimization solver at each `C`. 61 | 62 | tol : float, default=1e-4 63 | Tolerance for solver convergence. 64 | 65 | verbose : int, default=0 66 | Controls verbosity level of output. Set to higher values (e.g., 1 or 2) for detailed progress logs. 67 | When verbose = 1, only print path results table; 68 | when verbose = 2, print path results table and path solution plot. 69 | 70 | shrink : float, default=1 71 | Shrinkage factor for the solver, potentially influencing convergence behavior. 72 | 73 | warm_start : bool, default=False 74 | If True, reuse the previous solution to warm-start the next solver step, speeding up convergence. 75 | 76 | return_time : bool, default=True 77 | If True, return timing information for each value of `C`. 78 | 79 | Returns 80 | ------- 81 | Cs : ndarray of shape (n_Cs,) 82 | Array of regularization parameters used in the path. 83 | 84 | times : list of float 85 | Time in seconds taken to fit the model at each `C`. Returned only if `return_time=True`. 86 | 87 | n_iters : list of int 88 | Number of iterations used by the solver at each regularization value. 89 | 90 | obj_values : list of float 91 | Final objective values (including loss and regularization terms) at each `C`. 92 | 93 | L2_norms : list of float 94 | L2 norm of the coefficients (excluding bias) at each `C`. 95 | 96 | coefs : ndarray of shape (n_features, n_Cs) 97 | Learned model coefficients at each regularization strength. 98 | 99 | Example 100 | ------- 101 | 102 | >>> # generate data 103 | >>> np.random.seed(42) 104 | >>> n, d, C = 1000, 5, 0.5 105 | >>> X = np.random.randn(n, d) 106 | >>> beta0 = np.random.randn(d) 107 | >>> y = np.sign(X.dot(beta0) + np.random.randn(n)) 108 | >>> # define loss function 109 | >>> loss = {'name': 'svm'} 110 | >>> Cs = np.logspace(-1,3,15) 111 | >>> constraint = [{'name': 'nonnegative'}] 112 | 113 | 114 | >>> # calculate 115 | >>> Cs, times, n_iters, losses, norms, coefs = plqERM_Ridge_path_sol( 116 | ... X, y, loss=loss, Cs=Cs, max_iter=100000,tol=1e-4,verbose=2, 117 | ... warm_start=False, constraint=constraint, return_time=True 118 | ... ) 119 | 120 | """ 121 | 122 | n_samples, n_features = X.shape 123 | 124 | if Cs is None: 125 | log_eps = np.log10(eps) 126 | Cs = np.logspace(log_eps, -log_eps, n_Cs) 127 | 128 | # Sort Cs to ensure computation starts from the smallest value 129 | Cs = np.sort(Cs) 130 | n_Cs = len(Cs) 131 | coefs = np.zeros((n_features, n_Cs)) 132 | n_iters = [] 133 | times = [] 134 | obj_values = [] 135 | L2_norms = [] 136 | 137 | 138 | if return_time: 139 | total_start = time.time() 140 | 141 | U, V, Tau, S, T = _make_loss_rehline_param(loss, X, y) 142 | loss_obj = ReHLoss(U, V, S, T, Tau) 143 | 144 | # Lambda_ws = np.empty(shape=(0, 0)) 145 | # Gamma_ws = np.empty(shape=(0, 0)) 146 | # xi_ws = np.empty(shape=(0, 0)) 147 | 148 | clf = plqERM_Ridge( 149 | loss=loss, constraint=constraint, C=Cs[0], 150 | max_iter=max_iter, tol=tol, shrink=shrink, 151 | verbose=1*(verbose>=2), # ben: if verbose is 1, then the fit function will not show the progress 152 | warm_start=warm_start 153 | ) 154 | 155 | for i, C in enumerate(Cs): 156 | if return_time: 157 | start_time = time.time() 158 | 159 | clf.C = C 160 | 161 | # clf = plqERM_Ridge( 162 | # loss=loss, constraint=constraint, C=C, 163 | # max_iter=max_iter, tol=tol, shrink=shrink, verbose=verbose, 164 | # warm_start=warm_start 165 | # ) 166 | 167 | # if (warm_start and (i>0)): 168 | # clf.Lambda = Lambda_ws 169 | # clf.Gamma = Gamma_ws 170 | # clf.xi = xi_ws 171 | 172 | clf.fit(X, y) 173 | coefs[:, i] = clf.coef_ 174 | 175 | # Compute loss function parameters for ReHLoss 176 | l2_norm = np.linalg.norm(clf.coef_) ** 2 177 | score = clf.decision_function(X) 178 | total_obj = loss_obj(score) + 0.5*l2_norm 179 | obj_values.append(round(total_obj, 4)) 180 | L2_norms.append(round(np.linalg.norm(clf.coef_), 4)) 181 | 182 | # if warm_start: 183 | # Lambda_ws = clf.Lambda 184 | # Gamma_ws = clf.Gamma 185 | # xi_ws = clf.xi 186 | 187 | if return_time: 188 | elapsed_time = time.time() - start_time 189 | times.append(elapsed_time) 190 | 191 | n_iters.append(clf.n_iter_) 192 | 193 | if return_time: 194 | total_time = time.time() - total_start 195 | avg_time_per_iter = total_time / sum(n_iters) if sum(n_iters) > 0 else float("inf") 196 | 197 | 198 | if verbose >= 1: 199 | print("\nPLQ ERM Path Solution Results") 200 | print("=" * 90) 201 | print(f"{'C Value':<15}{'Iterations':<15}{'Time (s)':<20}{'Loss':<20}{'L2 Norm':<20}") 202 | print("-" * 90) 203 | 204 | for C, iters, t, loss_val, l2 in zip(Cs, n_iters, times, obj_values, L2_norms): 205 | if return_time: 206 | print(f"{C:<15.4g}{iters:<15}{t:<20.6f}{loss_val:<20.6f}{l2:<20.6f}") 207 | else: 208 | print(f"{C:<15.4g}{iters:<15}{loss_val:<20.6f}{l2:<20.6f}") 209 | 210 | print("=" * 90) 211 | print(f"{'Total Time':<12}{total_time:.6f} sec") 212 | print(f"{'Avg Time/Iter':<12}{avg_time_per_iter:.6f} sec") 213 | print("=" * 90) 214 | 215 | 216 | if return_time: 217 | return Cs, times, n_iters, obj_values, L2_norms, coefs 218 | else: 219 | return Cs, n_iters, obj_values, L2_norms, coefs 220 | 221 | 222 | 223 | def CQR_Ridge_path_sol( 224 | X, 225 | y, 226 | *, 227 | quantiles, 228 | eps=1e-5, 229 | n_Cs=50, 230 | Cs=None, 231 | max_iter=5000, 232 | tol=1e-4, 233 | verbose=0, 234 | shrink=1, 235 | warm_start=False, 236 | return_time=True, 237 | ): 238 | """ 239 | Compute the regularization path for Composite Quantile Regression (CQR) with ridge penalty. 240 | 241 | This function fits a series of CQR models using different values of the regularization parameter `C`. 242 | It reuses a single estimator and modifies `C` in-place before refitting. 243 | 244 | Parameters 245 | ---------- 246 | X : ndarray of shape (n_samples, n_features) 247 | Feature matrix. 248 | 249 | y : ndarray of shape (n_samples,) 250 | Response vector. 251 | 252 | quantiles : list of float 253 | Quantile levels (e.g. [0.1, 0.5, 0.9]). 254 | 255 | eps : float, default=1e-5 256 | Log-scaled lower bound for generated `C` values (used if `Cs` is None). 257 | 258 | n_Cs : int, default=50 259 | Number of `C` values to generate. 260 | 261 | Cs : array-like or None, default=None 262 | Explicit values of regularization strength. If None, use `eps` and `n_Cs` to generate them. 263 | 264 | max_iter : int, default=5000 265 | Maximum number of solver iterations. 266 | 267 | tol : float, default=1e-4 268 | Solver convergence tolerance. 269 | 270 | verbose : int, default=0 271 | Verbosity level. 272 | 273 | shrink : float, default=1 274 | Shrinkage parameter passed to solver. 275 | 276 | warm_start : bool, default=False 277 | Use previous dual solution to initialize the next fit. 278 | 279 | return_time : bool, default=True 280 | Whether to return a list of fit durations. 281 | 282 | Returns 283 | ------- 284 | Cs : ndarray 285 | List of regularization strengths. 286 | 287 | models : list 288 | List of fitted model objects. 289 | 290 | coefs : ndarray of shape (n_Cs, n_quantiles, n_features) 291 | Coefficient matrices per quantile and `C`. 292 | 293 | intercepts : ndarray of shape (n_Cs, n_quantiles) 294 | Intercepts per quantile and `C`. 295 | 296 | fit_times : list of float, optional 297 | Elapsed fit times (if `return_time=True`). 298 | 299 | 300 | Example 301 | ------- 302 | >>> from sklearn.datasets import make_friedman1 303 | >>> from sklearn.preprocessing import StandardScaler 304 | >>> import numpy as np 305 | >>> from rehline import CQR_Ridge_path_sol 306 | 307 | >>> # Generate the data 308 | >>> X, y = make_friedman1(n_samples=500, n_features=6, noise=1.0, random_state=42) 309 | >>> X = StandardScaler().fit_transform(X) 310 | >>> y = y / y.std() 311 | 312 | >>> # Set quantiles and Cs 313 | >>> quantiles = [0.1, 0.5, 0.9] 314 | >>> Cs = np.logspace(-5, 0, 30) 315 | 316 | >>> # Fit CQR path 317 | >>> Cs, models, coefs, intercepts, fit_times = CQR_Ridge_path_sol( 318 | ... X, y, 319 | ... quantiles=quantiles, 320 | ... Cs=Cs, 321 | ... max_iter=100000, 322 | ... tol=1e-4, 323 | ... verbose=1, 324 | ... warm_start=True, 325 | ... return_time=True 326 | ... ) 327 | 328 | """ 329 | 330 | if Cs is None: 331 | log_Cs = np.linspace(np.log10(eps), np.log10(10), n_Cs) 332 | Cs = np.power(10.0, log_Cs) 333 | else: 334 | Cs = np.array(Cs) 335 | 336 | models = [] 337 | fit_times = [] 338 | coefs = [] 339 | intercepts = [] 340 | 341 | clf = CQR_Ridge( 342 | quantiles=quantiles, 343 | C=Cs[0], 344 | max_iter=max_iter, 345 | tol=tol, 346 | shrink=shrink, 347 | verbose=verbose, 348 | warm_start=warm_start, 349 | ) 350 | 351 | for i, C in enumerate(Cs): 352 | clf.C = C 353 | 354 | if return_time: 355 | start = time.time() 356 | 357 | clf.fit(X, y) 358 | 359 | d = X.shape[1] 360 | n_qt = len(quantiles) 361 | 362 | coef_matrix = np.tile(clf.coef_, (n_qt, 1)) 363 | intercept_vector = clf.intercept_ 364 | 365 | models.append(clf) 366 | coefs.append(coef_matrix) 367 | intercepts.append(intercept_vector) 368 | 369 | if return_time: 370 | elapsed = time.time() - start 371 | fit_times.append(elapsed) 372 | if verbose >= 1: 373 | print(f"[OK] C={C:.3e}, time={elapsed:.3f}s") 374 | 375 | coefs = np.array(coefs) # (n_Cs, n_quantiles, n_features) 376 | intercepts = np.array(intercepts) # (n_Cs, n_quantiles) 377 | 378 | if return_time: 379 | return Cs, models, coefs, intercepts, fit_times 380 | else: 381 | return Cs, models, coefs, intercepts 382 | -------------------------------------------------------------------------------- /rehline/_sklearn_mixin.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn.base import ClassifierMixin, RegressorMixin 3 | from sklearn.utils.validation import check_array, check_is_fitted, check_X_y 4 | from sklearn.utils.multiclass import check_classification_targets 5 | from sklearn.preprocessing import LabelEncoder 6 | from sklearn.utils.class_weight import compute_class_weight 7 | from sklearn.utils._tags import ClassifierTags, RegressorTags 8 | 9 | from ._class import plqERM_Ridge 10 | 11 | 12 | 13 | class plq_Ridge_Classifier(plqERM_Ridge, ClassifierMixin): 14 | """ 15 | Empirical Risk Minimization (ERM) Classifier with a Piecewise Linear-Quadratic (PLQ) loss 16 | and ridge penalty, compatible with the scikit-learn API. 17 | 18 | This wrapper makes ``plqERM_Ridge`` behave as a classifier: 19 | - Accepts arbitrary binary labels in the original label space. 20 | - Computes class weights on original labels (if ``class_weight`` is set). 21 | - Encodes labels with ``LabelEncoder`` into {0,1}, then maps to {-1,+1} for training. 22 | - Supports optional intercept fitting (via an augmented constant feature). 23 | - Provides standard methods ``fit``, ``predict``, and ``decision_function``. 24 | - Integrates with scikit-learn ecosystem (e.g., GridSearchCV, Pipeline). 25 | 26 | Parameters 27 | ---------- 28 | loss : dict 29 | Dictionary specifying the loss function parameters. Examples include: 30 | - {'name': 'svm'} 31 | - {'name': 'sSVM'} 32 | - {'name': 'huber'} 33 | and other PLQ losses supported by ``plqERM_Ridge``. 34 | 35 | constraint : list of dict, default=[] 36 | Optional constraints. Each dictionary must include a ``'name'`` key. 37 | Examples: {'name': 'nonnegative'}, {'name': 'fair'}, {'name': 'custom'}. 38 | 39 | C : float, default=1.0 40 | Inverse regularization strength. 41 | 42 | _U, _V, _Tau, _S, _T : ndarray, default empty 43 | Parameters for the PLQ representation of the loss function. 44 | Typically built internally by helper functions. 45 | 46 | _A : ndarray of shape (K, n_features), default empty 47 | Linear-constraint coefficient matrix. 48 | 49 | _b : ndarray of shape (K,), default empty 50 | Linear-constraint intercept vector. 51 | 52 | max_iter : int, default=1000 53 | Maximum number of iterations for the ReHLine solver. 54 | 55 | tol : float, default=1e-4 56 | Convergence tolerance. 57 | 58 | shrink : int, default=1 59 | Shrinkage parameter for the solver. 60 | 61 | warm_start : int, default=0 62 | Whether to reuse the previous solution for initialization. 63 | 64 | verbose : int, default=0 65 | Verbosity level for the solver. 66 | 67 | trace_freq : int, default=100 68 | Frequency (in iterations) at which solver progress is traced 69 | when ``verbose > 0``. 70 | 71 | fit_intercept : bool, default=True 72 | Whether to fit an intercept term. If True, a constant feature column is added 73 | to ``X`` during training. The last learned coefficient is extracted as 74 | ``intercept_``. 75 | 76 | intercept_scaling : float, default=1.0 77 | Value used for the constant feature column when ``fit_intercept=True``. 78 | Matches the convention used in scikit-learn's ``LinearSVC``. 79 | 80 | class_weight : dict, 'balanced', or None, default=None 81 | Class weights applied like in LinearSVC: 82 | - 'balanced' uses n_samples / (n_classes * n_j). 83 | - dict maps label -> weight in the ORIGINAL label space. 84 | 85 | Attributes 86 | ---------- 87 | coef_ : ndarray of shape (n_features,) 88 | Coefficients excluding the intercept. 89 | 90 | intercept_ : float 91 | Intercept term. 0.0 if ``fit_intercept=False``. 92 | 93 | classes_ : ndarray of shape (2,) 94 | Unique class labels in the original label space. 95 | 96 | _label_encoder : LabelEncoder 97 | Encodes original labels into {0,1} for internal training. 98 | """ 99 | 100 | def __init__(self, 101 | loss, 102 | constraint=[], 103 | C=1., 104 | U=np.empty((0, 0)), V=np.empty((0, 0)), 105 | Tau=np.empty((0, 0)), S=np.empty((0, 0)), T=np.empty((0, 0)), 106 | A=np.empty((0, 0)), b=np.empty((0,)), 107 | max_iter=1000, tol=1e-4, shrink=1, warm_start=0, 108 | verbose=0, trace_freq=100, 109 | fit_intercept=True, 110 | intercept_scaling=1.0, 111 | class_weight=None): 112 | 113 | self.loss = loss 114 | self.constraint = constraint 115 | self.C = C 116 | self._U = U 117 | self._V = V 118 | self._S = S 119 | self._T = T 120 | self._Tau = Tau 121 | self._A = A 122 | self._b = b 123 | self.L = U.shape[0] 124 | self.H = S.shape[0] 125 | self.K = A.shape[0] 126 | self.max_iter = max_iter 127 | self.tol = tol 128 | self.shrink = shrink 129 | self.warm_start = warm_start 130 | self.verbose = verbose 131 | self.trace_freq = trace_freq 132 | self._Lambda = np.empty((0, 0)) 133 | self._Gamma = np.empty((0, 0)) 134 | self._xi = np.empty((0, 0)) 135 | self.coef_ = None 136 | 137 | self.fit_intercept = fit_intercept 138 | self.intercept_scaling = float(intercept_scaling) 139 | self.class_weight = class_weight 140 | 141 | self._label_encoder = None 142 | self.classes_ = None 143 | 144 | def fit(self, X, y, sample_weight=None): 145 | """ 146 | Fit the classifier to training data. 147 | 148 | Parameters 149 | ---------- 150 | X : array-like of shape (n_samples, n_features) 151 | Training features. 152 | 153 | y : array-like of shape (n_samples,) 154 | Target labels. 155 | 156 | sample_weight : array-like of shape (n_samples,), default=None 157 | Per-sample weights. If None, uniform weights are used. 158 | 159 | Returns 160 | ------- 161 | self : object 162 | Fitted estimator. 163 | """ 164 | # Validate input (dense only) and set n_features_in_ 165 | X, y = check_X_y( 166 | X, y, 167 | accept_sparse=False, 168 | dtype=np.float64, 169 | order="C", 170 | ) 171 | self.n_features_in_ = X.shape[1] 172 | 173 | check_classification_targets(y) 174 | 175 | # Establish classes_ on ORIGINAL labels 176 | self.classes_ = np.unique(y) 177 | if self.classes_.size != 2: 178 | raise ValueError( 179 | f"plqERMClassifier currently supports only binary classification, " 180 | f"but received {self.classes_.size} classes: {self.classes_}." 181 | ) 182 | 183 | # Compute class weights on original labels 184 | if self.class_weight is not None: 185 | cw_vec = compute_class_weight( 186 | class_weight=self.class_weight, 187 | classes=self.classes_, 188 | y=y, 189 | ) 190 | cw_map = {c: w for c, w in zip(self.classes_, cw_vec)} 191 | sw_cw = np.asarray([cw_map[yi] for yi in y], dtype=np.float64) 192 | sample_weight = sw_cw if sample_weight is None else (np.asarray(sample_weight) * sw_cw) 193 | 194 | # Encode -> {0,1} -> {-1,+1} 195 | le = LabelEncoder().fit(self.classes_) 196 | self._label_encoder = le 197 | y01 = le.transform(y) 198 | y_pm = 2 * y01 - 1 199 | 200 | # Add constant column for intercept 201 | X_aug = X 202 | if self.fit_intercept: 203 | col = np.full((X.shape[0], 1), self.intercept_scaling, dtype=X.dtype) 204 | X_aug = np.hstack([X, col]) 205 | 206 | super().fit(X_aug, y_pm, sample_weight=sample_weight) 207 | 208 | # Split intercept 209 | if self.fit_intercept: 210 | self.intercept_ = float(self.coef_[-1]) 211 | self.coef_ = self.coef_[:-1].copy() 212 | else: 213 | self.intercept_ = 0.0 214 | 215 | return self 216 | 217 | def decision_function(self, X): 218 | """ 219 | Compute the decision function for samples in X. 220 | 221 | Parameters 222 | ---------- 223 | X : array-like of shape (n_samples, n_features) 224 | Input samples. 225 | 226 | Returns 227 | ------- 228 | ndarray of shape (n_samples,) 229 | Continuous scores for each sample. 230 | """ 231 | check_is_fitted(self, attributes=["coef_", "intercept_", "_label_encoder", "classes_"]) 232 | X = check_array(X, accept_sparse=False, dtype=np.float64, order="C") 233 | return X @ self.coef_ + self.intercept_ 234 | 235 | def predict(self, X): 236 | """ 237 | Predict class labels for samples in X. 238 | 239 | Parameters 240 | ---------- 241 | X : array-like of shape (n_samples, n_features) 242 | Input samples. 243 | 244 | Returns 245 | ------- 246 | y_pred : ndarray of shape (n_samples,) 247 | Predicted class labels in the original label space. 248 | """ 249 | scores = self.decision_function(X) 250 | pred01 = (scores >= 0).astype(int) 251 | return self._label_encoder.inverse_transform(pred01) 252 | 253 | def __sklearn_tags__(self): 254 | """ 255 | Return scikit-learn estimator tags for compatibility. 256 | """ 257 | tags = super().__sklearn_tags__() 258 | tags.estimator_type = "classifier" 259 | tags.classifier_tags = ClassifierTags() 260 | tags.target_tags.required = True 261 | tags.input_tags.sparse = False 262 | return tags 263 | 264 | 265 | class plq_Ridge_Regressor(plqERM_Ridge, RegressorMixin): 266 | """ 267 | Empirical Risk Minimization (ERM) regressor with a Piecewise Linear-Quadratic (PLQ) loss 268 | and a ridge penalty, implemented as a scikit-learn compatible estimator. 269 | 270 | This wrapper adds standard sklearn conveniences while delegating loss/constraint construction 271 | to :class:`plqERM_Ridge` (via `_make_loss_rehline_param` / `_make_constraint_rehline_param`). 272 | 273 | Key behavior 274 | ------------ 275 | - **Intercept handling**: if ``fit_intercept=True``, a constant column (value = ``intercept_scaling``) 276 | is appended to the right of the design matrix before calling the base solver. The last learned 277 | coefficient is then split out as ``intercept_``. 278 | → The column indices of the original features reamin; therefore, ``sen_idx`` in the constraint ``fair`` follow the original index. 279 | - **Constraint handling**: constraints are passed through unchanged; the base class will call 280 | ``_make_constraint_rehline_param(constraint, X, y)`` on the matrix given to `fit`. 281 | With your updated implementation, ``fair`` must be specified as 282 | ``{'name': 'fair', 'sen_idx': list[int], 'tol_sen': list[float]}``. 283 | 284 | Parameters 285 | ---------- 286 | loss : dict, default={'name': 'QR', 'qt': 0.5} 287 | PLQ loss configuration (e.g., median Quantile Regression). Examples: 288 | ``{'name': 'QR', 'qt': 0.5}``, ``{'name': 'huber', 'tau': 1.0}``, 289 | ``{'name': 'SVR', 'epsilon': 0.1}``. 290 | Required keys depend on the chosen loss and are consumed by the underlying solver. 291 | constraint : list of dict, default=[] 292 | Constraint specifications. Supported by your updated `_make_constraint_rehline_param`: 293 | - ``{'name': 'nonnegative'}`` or ``{'name': '>=0'}`` 294 | - ``{'name': 'fair', 'sen_idx': list[int], 'tol_sen': list[float]}`` 295 | - ``{'name': 'custom', 'A': ndarray[K, d], 'b': ndarray[K]}`` 296 | Note: when ``fit_intercept=True``, a constant column is appended **as the last column**; 297 | since you index sensitive columns by ``sen_idx`` on the *original* features, indices stay valid. 298 | C : float, default=1.0 299 | Regularization parameter (absorbed by ReHLine parameters inside the solver). 300 | _U, _V, _Tau, _S, _T : ndarray, default empty 301 | Advanced PLQ parameters for the underlying ReHLine formulation (usually left as defaults). 302 | _A, _b : ndarray, default empty 303 | Optional linear constraint matrices (used only if ``constraint`` contains ``{'name': 'custom'}``). 304 | (Your `_make_constraint_rehline_param` is responsible for validating their shapes.) 305 | max_iter : int, default=1000 306 | Maximum iterations for the ReHLine solver. 307 | tol : float, default=1e-4 308 | Convergence tolerance for the ReHLine solver. 309 | shrink : int, default=1 310 | Shrink parameter passed to the solver (see solver docs). 311 | warm_start : int, default=0 312 | Warm start flag passed to the solver (see solver docs). 313 | verbose : int, default=0 314 | Verbosity for the solver (0: silent). 315 | trace_freq : int, default=100 316 | Iteration frequency to trace solver internals (if ``verbose`` is enabled). 317 | fit_intercept : bool, default=True 318 | If ``True``, append a constant column (value = ``intercept_scaling``) to the design matrix 319 | before calling the solver. The learned last coefficient is then split as ``intercept_``. 320 | intercept_scaling : float, default=1.0 321 | Scaling applied to the appended constant column when ``fit_intercept=True``. 322 | 323 | Attributes 324 | ---------- 325 | coef_ : ndarray of shape (n_features,) 326 | Learned linear coefficients (excluding the intercept term). 327 | intercept_ : float 328 | Intercept term extracted from the last coefficient when ``fit_intercept=True``, otherwise 0.0. 329 | n_features_in_ : int 330 | Number of input features seen during :meth:`fit` (before intercept augmentation). 331 | 332 | Notes 333 | ----- 334 | This estimator **does not support sparse input**. If you need sparse support, convert inputs to dense 335 | or wrap this estimator in a scikit-learn :class:`~sklearn.pipeline.Pipeline` with a transformer that 336 | densifies data (at the cost of memory). 337 | """ 338 | 339 | def __init__(self, 340 | loss={'name': 'QR', 'qt': 0.5}, 341 | constraint=[], 342 | C=1., 343 | U=np.empty((0, 0)), V=np.empty((0, 0)), 344 | Tau=np.empty((0, 0)), S=np.empty((0, 0)), T=np.empty((0, 0)), 345 | A=np.empty((0, 0)), b=np.empty((0,)), 346 | max_iter=1000, tol=1e-4, shrink=1, warm_start=0, 347 | verbose=0, trace_freq=100, 348 | fit_intercept=True, 349 | intercept_scaling=1.0): 350 | 351 | self.loss = loss 352 | self.constraint = constraint 353 | self.C = C 354 | self._U = U 355 | self._V = V 356 | self._S = S 357 | self._T = T 358 | self._Tau = Tau 359 | self._A = A 360 | self._b = b 361 | self.L = U.shape[0] 362 | self.H = S.shape[0] 363 | self.K = A.shape[0] 364 | self.max_iter = max_iter 365 | self.tol = tol 366 | self.shrink = shrink 367 | self.warm_start = warm_start 368 | self.verbose = verbose 369 | self.trace_freq = trace_freq 370 | self._Lambda = np.empty((0, 0)) 371 | self._Gamma = np.empty((0, 0)) 372 | self._xi = np.empty((0, 0)) 373 | self.coef_ = None 374 | 375 | self.fit_intercept = fit_intercept 376 | self.intercept_scaling = float(intercept_scaling) 377 | 378 | 379 | def fit(self, X, y, sample_weight=None): 380 | """ 381 | If ``fit_intercept=True``, a constant column (value = ``intercept_scaling``) is appended 382 | to the **right** of ``X`` before calling the base solver. The base class 383 | (:class:`plqERM_Ridge`) will construct the loss and constraints via its internal helpers 384 | on the matrix passed here. After solving, the last coefficient is split as 385 | ``intercept_`` and removed from ``coef_``. 386 | 387 | Parameters 388 | ---------- 389 | X : ndarray of shape (n_samples, n_features) 390 | Training design matrix (dense). Sparse inputs are not supported. 391 | y : ndarray of shape (n_samples,) 392 | Target values. 393 | sample_weight : ndarray of shape (n_samples,), default=None 394 | Optional per-sample weights; forwarded to the underlying solver. 395 | 396 | Returns 397 | ------- 398 | self : object 399 | Fitted estimator. 400 | 401 | """ 402 | 403 | # Dense-only validation 404 | X, y = check_X_y(X, y, accept_sparse=False, dtype=np.float64, order="C") 405 | self.n_features_in_ = X.shape[1] 406 | 407 | # Intercept augmentation (append as last column so original indices stay the same) 408 | X_aug = X 409 | if self.fit_intercept: 410 | col = np.full((X.shape[0], 1), self.intercept_scaling, dtype=X.dtype) 411 | X_aug = np.hstack([X, col]) 412 | 413 | super().fit(X_aug, y, sample_weight=sample_weight) 414 | 415 | # Split intercept from coefficients to match sklearn's linear model API 416 | if self.fit_intercept: 417 | self.intercept_ = float(self.coef_[-1]) 418 | self.coef_ = self.coef_[:-1].copy() 419 | else: 420 | self.intercept_ = 0.0 421 | 422 | return self 423 | 424 | def decision_function(self, X): 425 | """Compute f(X) = X @ coef_ + intercept_. 426 | 427 | Parameters 428 | ---------- 429 | X : ndarray of shape (n_samples, n_features) 430 | Input data (dense). Must have the same number of features as seen in :meth:`fit`. 431 | 432 | Returns 433 | ------- 434 | scores : ndarray of shape (n_samples,) 435 | Predicted real-valued scores.、 436 | """ 437 | check_is_fitted(self, attributes=["coef_", "intercept_"]) 438 | X = check_array(X, accept_sparse=False, dtype=np.float64, order="C") 439 | return X @ self.coef_ + self.intercept_ 440 | 441 | def predict(self, X): 442 | """ 443 | Predict targets as the linear decision function. 444 | Parameters 445 | ---------- 446 | X : ndarray of shape (n_samples, n_features) 447 | Input data (dense). 448 | 449 | Returns 450 | ------- 451 | y_pred : ndarray of shape (n_samples,) 452 | Predicted target values (real-valued). 453 | """ 454 | return self.decision_function(X) 455 | 456 | def __sklearn_tags__(self): 457 | """ 458 | Estimator tags: regressor, requires y, dense-only. 459 | Parameters 460 | ---------- 461 | X : ndarray of shape (n_samples, n_features) 462 | Input data (dense). 463 | 464 | Returns 465 | ------- 466 | y_pred : ndarray of shape (n_samples,) 467 | Predicted target values (real-valued). 468 | """ 469 | 470 | tags = super().__sklearn_tags__() 471 | tags.estimator_type = "regressor" 472 | tags.regressor_tags = RegressorTags() 473 | tags.input_tags.sparse = False 474 | tags.target_tags.required = True 475 | return tags 476 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | emoji 2 | renku_sphinx_theme 3 | matplotlib 4 | numpy 5 | pandas 6 | scikit_learn 7 | scipy 8 | setuptools 9 | sphinx-autodoc-typehints 10 | numpydoc 11 | nbsphinx 12 | sphinx-autoapi 13 | furo -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import zipfile 3 | from pathlib import Path 4 | 5 | import requests 6 | from pybind11.setup_helpers import Pybind11Extension, build_ext 7 | from setuptools import setup 8 | 9 | __version__ = "0.1.1" 10 | 11 | # The main interface is through Pybind11Extension. 12 | # * You can add cxx_std=11/14/17, and then build_ext can be removed. 13 | # * You can set include_pybind11=false to add the include directory yourself, 14 | # say from a submodule. 15 | # 16 | # Note: 17 | # Sort input source files if you glob sources to ensure bit-for-bit 18 | # reproducible builds (https://github.com/pybind/python_example/pull/53) 19 | 20 | # The directory that contains setup.py 21 | SETUP_DIRECTORY = Path(__file__).resolve().parent 22 | 23 | # Download Eigen source files 24 | # Modified from https://github.com/tohtsky/irspack/blob/main/setup.py 25 | class get_eigen_include(object): 26 | EIGEN3_URL = "https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.zip" 27 | EIGEN3_DIRNAME = "eigen-3.4.0" 28 | 29 | def __str__(self) -> str: 30 | # Test whether the environment variable EIGEN3_INCLUDE_DIR is set 31 | # If yes, directly return this directory 32 | eigen_include_dir = os.environ.get("EIGEN3_INCLUDE_DIR", None) 33 | if eigen_include_dir is not None: 34 | return eigen_include_dir 35 | 36 | # If the directory already exists (e.g. from previous setup), 37 | # directly return it 38 | target_dir = SETUP_DIRECTORY / self.EIGEN3_DIRNAME 39 | if target_dir.exists(): 40 | return target_dir.name 41 | 42 | # Filename for the downloaded Eigen source package 43 | download_target_dir = SETUP_DIRECTORY / "eigen3.zip" 44 | response = requests.get(self.EIGEN3_URL, stream=True) 45 | with download_target_dir.open("wb") as ofs: 46 | for chunk in response.iter_content(chunk_size=1024): 47 | ofs.write(chunk) 48 | # Unzip package 49 | with zipfile.ZipFile(download_target_dir) as ifs: 50 | ifs.extractall() 51 | 52 | return target_dir.name 53 | 54 | ext_modules = [ 55 | Pybind11Extension("rehline._internal", 56 | ["src/rehline.cpp"], 57 | include_dirs=[get_eigen_include()], 58 | # Example: passing in the version to the compiled code 59 | define_macros=[('VERSION_INFO', __version__)], 60 | ), 61 | ] 62 | 63 | setup( 64 | name="rehline", 65 | version=__version__, 66 | author=["Ben Dai", "Yixuan Qiu"], 67 | author_email="bendai@cuhk.edu.hk", 68 | url="https://rehline-python.readthedocs.io/en/latest/", 69 | description="Regularized Composite ReLU-ReHU Loss Minimization with Linear Computation and Linear Convergence", 70 | packages=["rehline"], 71 | # install_requires=["requests", "pybind11", "numpy", "scipy", "scikit-learn"], 72 | ext_modules=ext_modules, 73 | # extras_require={"test": "pytest"}, 74 | # Currently, build_ext only provides an optional "highest supported C++ 75 | # level" feature, but in the future it may provide more features. 76 | cmdclass={"build_ext": build_ext}, 77 | zip_safe=False, 78 | python_requires=">= 3.10", 79 | ) 80 | 81 | ## build .so file 82 | ## $ c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) ./src/rehline.cpp -o _internal$(python3-config --extension-suffix) -------------------------------------------------------------------------------- /src/rehline.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "rehline.h" 10 | 11 | namespace py = pybind11; 12 | 13 | using Matrix = Eigen::Matrix; 14 | using MapMat = Eigen::Ref; 15 | using Vector = Eigen::VectorXd; 16 | using MapVec = Eigen::Ref; 17 | 18 | using ReHLineResult = rehline::ReHLineResult; 19 | 20 | void rehline_internal( 21 | ReHLineResult& result, 22 | const MapMat& X, const MapMat& A, const MapVec& b, 23 | const MapMat& U, const MapMat& V, 24 | const MapMat& S, const MapMat& T, const MapMat& Tau, 25 | int max_iter, double tol, int shrink = 1, 26 | int verbose = 0, int trace_freq = 100 27 | ) 28 | { 29 | rehline::rehline_solver(result, X, A, b, U, V, S, T, Tau, 30 | max_iter, tol, shrink, verbose, trace_freq); 31 | } 32 | 33 | PYBIND11_MODULE(_internal, m) { 34 | py::class_(m, "rehline_result") 35 | .def(py::init<>()) 36 | .def_readwrite("beta", &ReHLineResult::beta) 37 | .def_readwrite("xi", &ReHLineResult::xi) 38 | .def_readwrite("Lambda", &ReHLineResult::Lambda) 39 | .def_readwrite("Gamma", &ReHLineResult::Gamma) 40 | .def_readwrite("niter", &ReHLineResult::niter) 41 | .def_readwrite("dual_objfns", &ReHLineResult::dual_objfns) 42 | .def_readwrite("primal_objfns", &ReHLineResult::primal_objfns); 43 | 44 | // https://hopstorawpointers.blogspot.com/2018/06/pybind11-and-python-sub-modules.html 45 | m.attr("__name__") = "rehline._internal"; 46 | m.doc() = "rehline"; 47 | m.def("rehline_internal", &rehline_internal); 48 | } 49 | -------------------------------------------------------------------------------- /tests/_test_cqr.py: -------------------------------------------------------------------------------- 1 | ## Test CQR on simulated dataset 2 | import numpy as np 3 | 4 | from rehline import CQR_Ridge 5 | 6 | 7 | def test_CQR_Ridge(): 8 | # Simulate dataset 9 | np.random.seed(42) 10 | X = np.random.randn(1000, 2) 11 | beta = np.array([1, 2]) 12 | y = X @ beta + np.random.randn(1000) 13 | sample_weight = np.random.rand(1000) 14 | 15 | # Define loss parameters 16 | quantiles = [0.05, 0.5, 0.95] 17 | n_qt = len(quantiles) 18 | 19 | # Initialize and fit the model 20 | cqr = CQR_Ridge(quantiles=quantiles) 21 | cqr.fit(X, y, sample_weight=sample_weight) 22 | 23 | # Check if the model coefficients are not None 24 | assert cqr.coef_.shape == (len(beta),), "Model shape is incorrect" 25 | assert cqr.intercept_.shape == (n_qt,), "Intercept shape is incorrect" 26 | assert cqr.quantiles_.shape == (n_qt,), "Quantiles shape is incorrect" 27 | 28 | # Check decision function output 29 | pred = cqr.predict(X[:5]) 30 | assert pred.shape == (5,n_qt), "Decision function output shape is incorrect" 31 | 32 | print(f"Coefficients of CQR_Ridge: {cqr.coef_}; Intercepts of CQR_Ridge: {cqr.intercept_}") 33 | 34 | print("All tests passed.") 35 | 36 | if __name__ == "__main__": 37 | test_CQR_Ridge() 38 | -------------------------------------------------------------------------------- /tests/_test_fairsvm.py: -------------------------------------------------------------------------------- 1 | ## Test SVM on simulated dataset 2 | import numpy as np 3 | from sklearn.datasets import make_classification 4 | from sklearn.preprocessing import StandardScaler 5 | from rehline import plqERM_Ridge 6 | 7 | np.random.seed(1024) 8 | # simulate classification dataset 9 | n, d, C = 100, 5, 0.5 10 | X, y = make_classification(n, d) 11 | y = 2*y - 1 12 | 13 | scaler = StandardScaler() 14 | X = scaler.fit_transform(X) 15 | sen_idx = [0] 16 | 17 | ## solution provided by ReHLine 18 | # build-in hinge loss for svm 19 | clf = plqERM_Ridge(loss={'name': 'svm'}, C=C) 20 | 21 | # specific the param of FairSVM 22 | X_sen = X[:,sen_idx] 23 | A = np.repeat([X_sen.flatten() @ X], repeats=[2], axis=0) / n 24 | A[1] = -A[1] 25 | # suppose the fair tolerance is 0.01 26 | b = np.array([.01, .01]) 27 | clf._A, clf._b = A, b 28 | clf.fit(X=X, y=y) 29 | 30 | print('solution privided by rehline: %s' %clf.coef_) 31 | score = X@clf.coef_ 32 | cor_sen = np.mean(score * X_sen) 33 | print('correlation btw score and X_sen is: %.3f' %cor_sen) 34 | -------------------------------------------------------------------------------- /tests/_test_path_sol.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import matplotlib.pyplot as plt 4 | import numpy as np 5 | ## load datasets 6 | from sklearn.datasets import make_hastie_10_2 7 | 8 | from rehline import plqERM_Ridge_path_sol 9 | 10 | X, y = make_hastie_10_2() 11 | 12 | # define loss function 13 | loss = {'name': 'svm'} 14 | Cs = np.logspace(-20, 10, 50, base=2) # Define a range of C values for the path 15 | 16 | # calculate with warm_start=False 17 | # Cs_values_cold, times_cold, n_iters_cold, loss_values_cold, L2_norms_cold, coefs_cold = plqERM_Ridge_path_sol( 18 | # X, y, 19 | # loss=loss, 20 | # Cs=Cs, 21 | # max_iter=5000000, 22 | # tol=1e-4, 23 | # verbose=0, 24 | # warm_start=False, 25 | # constraint=[], 26 | # return_time=True, 27 | # ) 28 | 29 | # calculate with warm_start=True 30 | Cs_values_warm, times_warm, n_iters_warm, loss_values_warm, L2_norms_warm, coefs_warm = plqERM_Ridge_path_sol( 31 | X, y, 32 | loss=loss, 33 | Cs=Cs, 34 | max_iter=1000000, 35 | tol=1e-4, 36 | verbose=1, 37 | warm_start=True, 38 | constraint=[], 39 | return_time=True, 40 | ) 41 | 42 | 43 | # # Plot Cs vs times comparison 44 | # plt.figure(figsize=(10, 6)) 45 | # plt.plot(Cs, times_warm, 'o-', label='Warm Start') 46 | # plt.plot(Cs, times_cold, 's-', label='Cold Start') 47 | # plt.xscale('log', base=2) 48 | # plt.xlabel('C values') 49 | # plt.ylabel('Time (seconds)') 50 | # plt.title('Computation Time vs. C Parameter') 51 | # plt.legend() 52 | # plt.grid(True) 53 | # plt.show() 54 | 55 | 56 | # # Print table comparing number of iterations 57 | # print("\nComparison of Number of Iterations:") 58 | # print("-" * 50) 59 | # print(f"{'C Value':^15} | {'Cold Start (iterations)':^20} | {'Warm Start (iterations)':^20}") 60 | # print("-" * 50) 61 | # for i, C in enumerate(Cs): 62 | # print(f"{C:^15.4f} | {n_iters_cold[i]:^20d} | {n_iters_warm[i]:^20d}") 63 | -------------------------------------------------------------------------------- /tests/_test_sklearn_mixin.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn.datasets import make_classification, make_regression 3 | from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score 4 | from sklearn.pipeline import Pipeline 5 | from sklearn.preprocessing import StandardScaler 6 | from sklearn.metrics import accuracy_score, classification_report, mean_squared_error, r2_score 7 | 8 | ## test classifier 9 | from rehline import plq_Ridge_Classifier 10 | # generate the dataset 11 | X, y = make_classification( 12 | n_samples=2000, 13 | n_features=20, 14 | n_informative=8, 15 | n_redundant=4, 16 | n_repeated=0, 17 | n_classes=2, 18 | weights=[0.7, 0.3], # imbalance 19 | class_sep=1.2, 20 | flip_y=0.01, 21 | random_state=42, 22 | ) 23 | 24 | X_train, X_test, y_train, y_test = train_test_split( 25 | X, y, test_size=0.25, stratify=y, random_state=42 26 | ) 27 | 28 | # set the pipeline 29 | pipe = Pipeline([ 30 | ("scaler", StandardScaler()), 31 | ("clf", plq_Ridge_Classifier(loss={"name": "svm"})), 32 | ]) 33 | 34 | # set the parameter grid 35 | param_grid = { 36 | "clf__loss": [{"name": "svm"}, {"name": "sSVM"}], 37 | "clf__C": [0.1, 1.0, 3.0], 38 | "clf__fit_intercept": [True, False], 39 | "clf__intercept_scaling": [0.5, 1.0, 2.0], 40 | "clf__max_iter": [5000, 10000], 41 | "clf__class_weight": [None, "balanced", {0: 1.0, 1: 2.0}], 42 | "clf__constraint": [ 43 | [], # no constraint 44 | [{"name": "nonnegative"}], 45 | [{"name": "fair", "sen_idx": [0], "tol_sen": 0.1}], 46 | ], 47 | } 48 | 49 | # cross_val_score function 50 | cv_scores = cross_val_score( 51 | pipe, 52 | X_train, y_train, 53 | cv=5, 54 | scoring="accuracy", 55 | n_jobs=-1, 56 | ) 57 | print("CV scores:", cv_scores) 58 | 59 | # perform GridSearchCV to tune the hyperparameter 60 | grid = GridSearchCV( 61 | estimator=pipe, 62 | param_grid=param_grid, 63 | scoring="accuracy", 64 | cv=5, 65 | n_jobs=-1, 66 | refit=True, 67 | verbose=1, 68 | ) 69 | 70 | grid.fit(X_train, y_train) 71 | print("Best params:", grid.best_params_) 72 | print("Best CV accuracy:", grid.best_score_) 73 | best_model = grid.best_estimator_ 74 | y_pred = best_model.predict(X_test) 75 | test_acc = accuracy_score(y_test, y_pred) 76 | 77 | print("Test accuracy:", test_acc) 78 | print("\nClassification report:\n", classification_report(y_test, y_pred, digits=4)) 79 | 80 | ## test regressor 81 | from rehline import plq_Ridge_Regressor 82 | 83 | # generate the data 84 | X, y = make_regression( 85 | n_samples=1500, 86 | n_features=15, 87 | n_informative=10, 88 | noise=10.0, 89 | random_state=42 90 | ) 91 | 92 | X_train, X_test, y_train, y_test = train_test_split( 93 | X, y, test_size=0.25, random_state=42 94 | ) 95 | 96 | # set the pipeline 97 | pipe = Pipeline([ 98 | ("scaler", StandardScaler()), 99 | ("reg", plq_Ridge_Regressor(loss={"name": "QR", "qt": 0.5})), 100 | ]) 101 | 102 | # set the param_grid 103 | param_grid = { 104 | "reg__loss": [ 105 | {"name": "QR", "qt": 0.5}, 106 | {"name": "huber", "tau": 1.0}, # Huber needs tau 107 | {"name": "SVR", "epsilon": 0.1}, # SVR needs epsilon 108 | ], 109 | "reg__C": [0.1, 1.0, 10.0], 110 | "reg__fit_intercept": [True, False], 111 | "reg__intercept_scaling": [0.5, 1.0], 112 | "reg__max_iter": [5000, 8000], 113 | "reg__constraint": [ 114 | [], # no constraint 115 | [{"name": "nonnegative"}], 116 | [{"name": "fair", "sen_idx": [0], "tol_sen": 0.1}], 117 | ], 118 | } 119 | 120 | # cross_val_score function 121 | cv_scores = cross_val_score( 122 | pipe, 123 | X_train, y_train, 124 | cv=5, 125 | scoring="r2", 126 | n_jobs=-1, 127 | ) 128 | print("CV R^2 scores:", cv_scores) 129 | print("Mean CV R^2:", np.mean(cv_scores)) 130 | 131 | # use GridSearchCV to tune the hyperparameters 132 | grid = GridSearchCV( 133 | estimator=pipe, 134 | param_grid=param_grid, 135 | scoring="r2", 136 | cv=5, 137 | n_jobs=-1, 138 | refit=True, 139 | verbose=1, 140 | ) 141 | 142 | grid.fit(X_train, y_train) 143 | # print the best parameters and the best CV R^2 score 144 | print("Best params:", grid.best_params_) 145 | print("Best CV R^2:", grid.best_score_) 146 | # use the best estimator to fit and predict the model 147 | best_model = grid.best_estimator_ 148 | y_pred = best_model.predict(X_test) 149 | print("Test R^2:", r2_score(y_test, y_pred)) 150 | print("Test MSE:", mean_squared_error(y_test, y_pred)) -------------------------------------------------------------------------------- /tests/_test_svm.py: -------------------------------------------------------------------------------- 1 | ## Test SVM on simulated dataset 2 | import numpy as np 3 | 4 | from rehline import ReHLine, plqERM_Ridge 5 | 6 | np.random.seed(1024) 7 | # simulate classification dataset 8 | n, d, C = 1000, 3, 0.5 9 | X = np.random.randn(1000, 3) 10 | beta0 = np.random.randn(3) 11 | y = np.sign(X.dot(beta0) + np.random.randn(n)) 12 | 13 | ## solution provided by sklearn 14 | from sklearn.svm import LinearSVC 15 | 16 | clf = LinearSVC(C=C, loss='hinge', fit_intercept=False, 17 | random_state=0, tol=1e-6, max_iter=1000000) 18 | clf.fit(X, y) 19 | sol = clf.coef_.flatten() 20 | 21 | print('solution privided by liblinear: %s' %sol) 22 | 23 | 24 | print('solution privided by rehline: %s' %clf.coef_) 25 | print(clf.decision_function([[.1,.2,.3]])) 26 | 27 | ## solution provided by plqERM_Ridge 28 | 29 | clf = plqERM_Ridge(loss={'name': 'svm'}, C=C) 30 | clf.fit(X=X, y=y) 31 | 32 | print('solution privided by plqERM_Ridge: %s' %clf.coef_) 33 | print(clf.decision_function([[.1,.2,.3]])) 34 | 35 | # manually specify params 36 | # n, d = X.shape 37 | # U = -(C*y).reshape(1,-1) 38 | # L = U.shape[0] 39 | # V = (C*np.array(np.ones(n))).reshape(1,-1) 40 | 41 | # clf = ReHLine(C=C) 42 | # clf._U, clf._V = U, V 43 | # clf.fit(X=X) 44 | 45 | # print('solution privided by rehline: %s' %clf.coef_) 46 | # print(clf.decision_function([[.1,.2,.3]])) -------------------------------------------------------------------------------- /tests/_test_svr.py: -------------------------------------------------------------------------------- 1 | ## Test SVR on simulated dataset 2 | import numpy as np 3 | 4 | from rehline import ReHLine, plqERM_Ridge 5 | 6 | np.random.seed(1024) 7 | # simulate regression dataset 8 | n, d, C = 10000, 5, 0.5 9 | X = np.random.randn(n, d) 10 | beta0 = np.random.randn(d) 11 | print(beta0) 12 | y = X.dot(beta0) + np.random.randn(n) 13 | new_sample = np.random.randn(d) 14 | 15 | ## solution provided by sklearn 16 | from sklearn.svm import LinearSVR 17 | 18 | reg = LinearSVR(C=C, loss='epsilon_insensitive', fit_intercept=False, epsilon=1e-5, 19 | random_state=0, tol=1e-6, max_iter=1000000, dual='auto') 20 | reg.fit(X, y) 21 | sol = reg.coef_.flatten() 22 | 23 | print('solution privided by liblinear: %s' %sol) 24 | print(reg.predict([new_sample])) 25 | 26 | ## solution provided by ReHLine 27 | # build-in loss 28 | loss_dict = {'name': 'svr', 'epsilon': 1e-5} 29 | reg = plqERM_Ridge(loss=loss_dict, C=C) 30 | reg.fit(X=X, y=y) 31 | 32 | print('solution privided by rehline: %s' %reg.coef_) 33 | print(reg.decision_function([new_sample])) 34 | 35 | # manually specify params 36 | n, d = X.shape 37 | 38 | U = np.ones((2, n))*C 39 | V = np.ones((2, n)) 40 | U[1] = -U[1] 41 | V[0] = -C*(y + loss_dict['epsilon']) 42 | V[1] = C*(y - loss_dict['epsilon']) 43 | 44 | reg = ReHLine(C=C) 45 | reg.U, reg.V = U, V 46 | reg.fit(X=X) 47 | 48 | print('solution privided by rehline (manually specified params): %s' %reg.coef_) 49 | print(reg.decision_function([new_sample])) 50 | 51 | # Output: 52 | # [-0.26024832 -0.29394989 0.05549916 2.24410393 -1.47306613] 53 | # solution privided by liblinear: [-0.266752 -0.28534044 0.05883864 2.24027556 -1.4996596 ] 54 | # [3.65233956] 55 | # solution privided by rehline: [-0.26742525 -0.28575844 0.05840424 2.24040812 -1.50018069] 56 | # [3.65242688] 57 | # solution privided by rehline (manually specified params): [-0.26742525 -0.28575844 0.05840424 2.24040812 -1.50018069] 58 | # [3.65242688] -------------------------------------------------------------------------------- /tests/_test_warmstart.py: -------------------------------------------------------------------------------- 1 | """ 2 | ReHLine-python: test warmstart 3 | v.0.0.6 - 2024-11-15 4 | """ 5 | # Authors: Ben Dai 6 | 7 | import numpy as np 8 | 9 | from rehline import ReHLine, plqERM_Ridge 10 | from rehline._base import ReHLine_solver 11 | 12 | 13 | def test_warmstart(): 14 | np.random.seed(1024) 15 | 16 | # Simulate classification dataset 17 | n, d, C = 1000, 3, 0.5 18 | X = np.random.randn(n, d) 19 | beta0 = np.random.randn(d) 20 | y = np.sign(X.dot(beta0) + np.random.randn(n)) 21 | 22 | # Test `ReHLine_solver` 23 | print("\nTesting ReHLine_solver") 24 | print("------------------------") 25 | 26 | # Test cold start 27 | print("Cold start:") 28 | U = -(C*y).reshape(1,-1) 29 | V = (C*np.array(np.ones(n))).reshape(1,-1) 30 | res = ReHLine_solver(X, U, V) 31 | print(f"Lambda shape: {res.Lambda.shape}") 32 | 33 | # Test warm start 34 | print("\nWarm start:") 35 | res_ws = ReHLine_solver(X, U, V, Lambda=res.Lambda) 36 | print(f"Lambda shape: {res_ws.Lambda.shape}") 37 | 38 | # Test `ReHLine` 39 | print("\nTesting ReHLine") 40 | print("-----------------") 41 | 42 | print("Cold start:") 43 | clf = ReHLine(verbose=1) 44 | clf.C = C 45 | clf.U = -y.reshape(1,-1) 46 | clf.V = np.array(np.ones(n)).reshape(1,-1) 47 | clf.fit(X) 48 | print(f"Coefficients: {clf.coef_}") 49 | 50 | print("\nWarm start:") 51 | clf.C = 2*C 52 | clf.warm_start = 1 53 | clf.fit(X) 54 | print(f"Coefficients: {clf.coef_}") 55 | 56 | # Test `plqERM_Ridge` 57 | print("\nTesting plqERM_Ridge") 58 | print("---------------------") 59 | 60 | print("Cold start:") 61 | clf = plqERM_Ridge(loss={'name': 'svm'}, C=C, verbose=1) 62 | clf.fit(X=X, y=y) 63 | print(f"Coefficients: {clf.coef_}") 64 | 65 | print("\nWarm start:") 66 | clf.C = 2*C 67 | clf.warm_start = 1 68 | clf.fit(X=X, y=y) 69 | print(f"Coefficients: {clf.coef_}") 70 | 71 | if __name__ == "__main__": 72 | test_warmstart() 73 | -------------------------------------------------------------------------------- /to-do.md: -------------------------------------------------------------------------------- 1 | # To-do list 2 | 3 | ## src 4 | - [x] warmstarting 5 | - [ ] GridSearchCV 6 | 7 | ## Class 8 | - [ ] sklearn Classifier and Regressor Estimator 9 | - [ ] Elastic Net ERM 10 | 11 | ## Loss 12 | - [ ] MAE 13 | - [ ] TV 14 | - [ ] Hinge_squared 15 | 16 | ## Constraint 17 | - [ ] box constraints 18 | - [ ] Monotonic constraints --------------------------------------------------------------------------------