├── .gitignore ├── BFV_theory ├── BFV_theory.ipynb ├── images │ ├── HE.drawio │ ├── HE.png │ └── bfv │ │ ├── ModularNumbers.drawio │ │ ├── ModularNumbers.pdf │ │ ├── ModularNumbers.png │ │ ├── PolynomialTorus.drawio │ │ ├── PolynomialTorus.png │ │ ├── addition_noise │ │ ├── e1e2.png │ │ ├── eu1u2.png │ │ ├── plot.png │ │ └── se1e2.png │ │ └── multiplication_noise │ │ └── plot.png ├── test_error_addition.ipynb └── test_error_multiplication.ipynb ├── Experiments ├── EncryptedProcessing.py ├── LeNet1.pt ├── LeNet1_Approx_single_square.pt ├── LeNet1_single_tanh.pt ├── MemoryAndTime │ ├── .directory │ ├── Encrypted │ │ └── experiment.py │ └── Plain │ │ └── experiment.py └── ModelExperiments.py ├── Experiments_FashionMNIST ├── EncryptedProcessing.py ├── LeNet1.pt ├── LeNet1_Approx_single_square.pt ├── LeNet1_single_tanh.pt ├── MemoryAndTime │ ├── .directory │ ├── Encrypted │ │ └── experiment.py │ └── Plain │ │ └── experiment.py └── ModelExperiments.py ├── Experiments_MNIST ├── EncryptedProcessing.py ├── LeNet1.pt ├── LeNet1_Approx_single_square.pt ├── LeNet1_single_tanh.pt ├── MemoryAndTime │ ├── .directory │ ├── Encrypted │ │ └── experiment.py │ └── Plain │ │ └── experiment.py └── ModelExperiments.py ├── HE-ML ├── HE-ML.ipynb ├── HE_processing.drawio ├── HE_processing.png ├── LeNet1.pt └── LeNet1_Approx.pt ├── README.md └── WCCI2022 ├── Tutorial_complete.ipynb └── Tutorial_to_fill.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | /data/ 131 | /HE-ML/data/ 132 | /Experiments/data/ 133 | -------------------------------------------------------------------------------- /BFV_theory/images/HE.drawio: -------------------------------------------------------------------------------- 1 | 5V1tb6JKFP41Jns/3AZmQOBja727X7bpbm+yyX5jcVQSZCxgq/31OwiozIyiBecF26TFgx7gOWfOGzxxAEeL9dfEX86/4wmKBsCYrAfwcQDAENrkby7YFALogEIwS8JJITL3gpfwA5VCo5SuwglKa2/MMI6ycFkXBjiOUZDVZH6S4Pf626Y4qh916c8QI3gJ/IiV/gon2byQusDZy7+hcDavjmwOvWLPwq/eXKpI5/4Evxei7cXB8QCOEoyzYmuxHqEox67CpUDgvyN7dyeWoDg75wNPq43nT9PxU/rDeEHPbz8ff3z8W2p586NVecHlyWabCgEUT+5zIMmrGMdE+DDPFhF5ZZJN9iRKjWhSw7U8pa8IL1CWbMgb3vdo2qWt5wdAVrIERX4WvtWt4ZdGne3U7Y7wjENyJsAo/Q8OSz2l+wHPqKtI8SoJUPmpQ/QaFJlDSlHmJzOUMYrIxsFl70Vb41xgKMAY6v/NMiR+mmMdB8lmmYU4ZoyXoXVWt5gfhbOYbAfEYighgjeUZLmi+3LHIpxM8o8/JCgNP/w/W1UGeb3Mr217tfbDwH7Mda0ynBZL9qQz5IdA65PuUO6tnJiB+cBdTJPjL8A47ho1W1wKPGSA/4YX5DdZzsOgX+BD9wzweYv1auBbDPiV7TWGGRiwDrNrMTBDkSjbDMoWmwa0Q5kO2LJRHl6UaoPIT1MSYLTKtsB07ty6c+8W7KUJl6fLEZtznRuwmGPduR0ZjFUl2F4ux17DKI9aIdmY5RvjSkIOsBNqH+tMj6pyAZu4gchY592qJXbrUhVLVKv5wBQO1B9m40hmkJXcTbaRBj2Ame6epcPMtsED0IOoAZvdWWzUYLveCm2tcabzpHSc2QbXG2oPMzRVCxtsh8uArHslDwEpv4G3+2koxc+ee55WSzcLV67wTV4TTRWWj70sLKGhWmHJ644L4Kd460EBjnCy3TN8XeHCCnA6hTDPdntRYSBgH5itUNAXy1HRUPogymT75B5MVaGr2FTVZJvgHoxVLUsxb64U9zq3e/ks1Nj/WKez8NnJvUGv4PkdYJvo3pnSsq07xzuopzoyZZNe0abkNeo3MQGEjmKFGuD08rdhCguqZgq23bdN7WGmKy/p7T5g2/3qeS2NYaYrL/kw85rxHkQN80jqlBY1OJ12ibbWONOJUjrObF/suPrDfMbMW2zY4N0c7lmVbxtXGcY2qBU8jK285vaGsZarWGEJeZ1z8zDWMIIgnyvTw1gb9HUYS0dD6eMryGuU9ceZvqPNw1nsCuF0wWMwcL3Bfb4A/MWSXHj8J11ur58j0t0gyjk+2wv3saqV7/i8e9/6+zPdpYnEefTzaf59FLw+/159WL9egW+muCMmFVW/ckCSdg+CehbM+vQTBZQiaF2tbOUaik24qjKpmNVwljscYVLtYBbBpOICrxeTqhX4NJOKC/61mFRc8DnZ1h59IZLcBQyy/Y/2oJteveaEQzYlCMWcl3nJKVlW+V97xIFlNyI+FIl4ZxwrdfNwwYs6es8efpbg3KDXssVm6c64Vwpb0qnfsvfsjix5Wq9oS+rMymqZj6iK2WCLAKEtis6srHZ5ylbMEsqystrBTLGydpFGwIyJD7OqrKx2MNOduHSY2ZZakRFTO5xhszuLjRrKsrLa4UznSek4q8rKajeTMFULG52xstSt8U/Tp5hSvBtWFtM5XLnC15qV1W5NGaoVljfEyuo0GvJGV2KjoaqsrJaD8OaZrFiYVWVltYKZfjZYOszdsbIUzu05e+ogCXunk/BFpKyjakVP77rjZKlrSJo75XRjyNNqhRuS16TfxPSPZmRJL9K0ZmS1y1NQNVOoysjqtOqS3uory8jqtOqSD7OyjKx2OJtH7kVLixrKMrLa4UwnSuk4q8rIagfzGfNusWGjM0aWujU+TZ1q6Ko+y8hq6AGvXONrzchqt6ZcxQrLW2JkdRoNpY+u1GVkdXo3m4ez2BXC64K3j8Xaoxi99uURWafZvR2hsCvLu+q0dpXv3sryrjrtxa6IM3m5/3asokTaf8UYHP8F -------------------------------------------------------------------------------- /BFV_theory/images/HE.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/BFV_theory/images/HE.png -------------------------------------------------------------------------------- /BFV_theory/images/bfv/ModularNumbers.drawio: -------------------------------------------------------------------------------- 1 | 5ZpNb9s4EIZ/jY8lyOH3sUmb9JB2dxFg97gQLNoWKluuLNfO/vod2dRXGRlpUQOS5YvFIUVJ84gczivO+P36+JhH29XnLHbpDGh8nPEPMwAmBcW/0vJytlgqz4ZlnsS+UWN4Tv5z3ujPW+6T2O06DYssS4tk2zXOs83GzYuOLcrz7NBttsjS7lW30dIFhud5lIbWf5K4WHkrU7ap+OSS5cpf2oA+V6yjqrF/kt0qirNDy8Q/zvh9nmXF+Wh9vHdp6bzKL+fzHnpq6xvL3aZ4ywlPiy/Z87+fn9Z/P/4V//E1Ut/o0zvfy/co3fsH9jdbvFQewF7Q2Vi4O6ySwj1vo3lZc0DeaFsV6xRLDA+j3fZMYJEcHV70Lo52q/LgVLvINoXHCwLL4e1X9+Lywh1bJv84jy5buyJ/wSa+FqwgXEptrNBcGYH9nnrwrxpTQDRjRglBldZW+Tfv0HBUwIkGwYSUwipetVi1gPY0ifx7tazvqnE9Hnjv/wQJuCaJRZKm91ma5aeOOD39rgNFWiBKUS6p0ZpZ3kUiLGESpACuwRjGeYAENEFmQjDGqJFcKwiZ9LX57VD4FKBoDaRkIaXlOIkMnYm4ESaWCiJBYKSToDhTHSZCMUIxyAjrh4sYNhR5I1DAUIKRAt1eHmoKo6aiboSKFpoAV/X0xbpBxVhiOMVKoYVEb4ZxflBU9I1QMdIQUcYWP1y6UQWsIW0oSg2birkRKgIsseI0ThABTlmjHiv2RqhwUMRY42O91HbUY6VKzkeP5fJg0Vhbh3rMV/TAqVw1mR/MYDkn5HXCwuzAqdxKYn853I+Nyq1k9peXxmObwcLcngZY0DFF1/+7Is++usrhm2zjfmDgTVGaLDdYnKO7HdrvSjcn8yh97yvWSRynfcDzbL+JT3LmtUQaJYkUTEvBjazVynrhRgmlHKdFv1AIVRrDumoo7yNZC6KKXkvNZKEgwKZDUllcVlhdk+wOSmCKWNnGMGyUoYoQzpU3i9JQRnBVTqtfVxCCE+gGAhs2yVB54BMiKSkB8+t666BIhmqFmBBJVsbCvjGJEy5RrIE58EAZKhxyOiQ1UPRzX6DUFAhvJRUwaJIQiiJqOiQvLl61kASzhuZj/LBBhjqKng5IcZpcaR/JcQ3JUHsx0yHJAZeuynplAJeqIw6TEOo1djoky6/SZszrVQiFHTYhZYfzMk1sRmKlw40yi4RXpJ0JaTunT0ayLz4ClNHz4p7DQbF8bS/C3VhEgbLcktkfHrzM/hsoK8NJ9eWmUmApDVgqUbdq82OGCH0tZG/YqDDf59/rjbduE78vN0KXENJot0vml0i+yY8u7mya7o9bLFxVSBr6q7K95izf+Z9Zgjfx1l0OPdtMq/532T6fO99lwyO4ikWQ1PCeFRTmM6Q9Sln3IkWUL10RXARZRC+tZtuywa7/STX8EPk7u7zx4Nxh80LVGF57x7DY7Dg/N2/27fOP/wM= -------------------------------------------------------------------------------- /BFV_theory/images/bfv/ModularNumbers.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/BFV_theory/images/bfv/ModularNumbers.pdf -------------------------------------------------------------------------------- /BFV_theory/images/bfv/ModularNumbers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/BFV_theory/images/bfv/ModularNumbers.png -------------------------------------------------------------------------------- /BFV_theory/images/bfv/PolynomialTorus.drawio: -------------------------------------------------------------------------------- 1 | 7Z1LcxxHdoV/DZeqyPdjOdJ45IVk2ZbDs3T0ECCAEIjmgNCQ9K93NtAPFG4BQt2pCtQ5ZS4koLuJR319s26e/DL5zv/w8euPt5tPlz9vz86v3zlz9vWd//M752wMpv1v98i3h0eqiQ8PXNxene1fdHrg16v/Pd8/uP97F79fnZ1/7r3wbru9vrv61H/w/fbm5vz9Xe+xze3t9kv/ZR+21/3v+mlzcS4e+PX95lo++ters7vL/aM21dMT/3p+dXG5/9bF5YcnPm4OL97/Jp8vN2fbL48e8v/yzv9wu93ePXz08esP59e7i3e4Lg9/7y/PPHv8wW7Pb+5e8xd++vBv21//5+efPv73j/9x9stvm/R389N3+6/yj8317/tfeP/D3n07XIH2VdrFbp98/+Xy6u7810+b97tnvjTe7bHLu4/X7TPbPtx8/vRA4MPV1/P2Tb//sL252xN1oX3++e52+9vxMvr2iPwdDj/Q+e3d+ddHD+1/px/Ptx/P726/tZfsn3XZdrnGXGrIPpXQvtP9V/h24NFla0sKwaSca9q/+76cWCbnu+yCDTGGmvzhFZePoD7zks3+vXVx/KFOl799sCcwgoaflcbV9fUP2+vt7f0X8ub+zwClCZjEbDprjI+m5Gyr7xFJ1XWlWB9j9e09670g4nLXkIVgrTUl+pycRPLcayZnEkiYtEve1RTawBpd8jb1mPj2pGljWqg2uhicD8uGEkmguFg651O77A2QycZBU0kkVNpQ36WcjsNXn4o1tSvetCdDDrFdTXlLWRSVTEKl+NiVdrXrvlz693lnS/cYSkrLplJIqASTO1/u68TV4kL/voJWK5WEijeh8+328XCvj7lC18phLgiP5eViaeNad7zVZ1fywqnMO3dcSrHEsJv7HScsti6ciiOh8vLtHo0Ky8z+5dYYbQST00grsLQLc9e//g8x1uGC32xvzp8w2D+0ub66uGmfvm+X+7w9/v3uMl+931z/af/Ex6uzs+vngN9uf7852+H980woU81djDm3UiqtymIf5R/nZm1y2orvlLv550gev0YycwVnVk49ZYHRkiymdLld5MOfft5mq+0eQzjcIZaKUs5X/YpQ/pMp3aJIyjluWBHJF4sy2tQle4IpQ/BFkZTz4rgeki/fKJNznX/Uii57dHVyKp3WQ7KNmV0t9kiyDzLFrvWap65n2SDl7DuvB6QvpnPZPEcSqyTljL2sh6TLuWst6n4+6erTeT7UfdLJaX5dEUrTZwfWrjq50n8InldB7+VCBJtFuoFoZ0XZzsv3x8VmO5vN2b+Xs79d/PI5/pL+68Pn3/7zt3985+TkQ4C8aFf00+uv21EX3Pzt8BXMi9cz2toFf5zLGdu7ns50bX7Xrneq0do2QZCto3WlCzH66GvYLbuZJK/ow2tSKqbW3AbNKXLPwSs668LN2ebz5f1b2/bf2jYOvrVfRv760WuXStvoS7j/75M52sOzpc3kotk5fAN4fGeN89nGEqwPYYjO4Eumf7vPCecVKwXzckq2TaVPnILs52IbeXY3I9OavZptqxXJ4rnXTA7jrZdt3gTGt8dv+Da+t0vsTA1BWgKLYvXWoua8rB5oeJNTdff/7d+C7lG2G5GtJZmQB6a8i2L11v7mTKwwYby1tjlv4VTXmdYB76vG9ce40hlXYiimROtTrAsn9dYq58xDnA2tjw6nG1J/9cvtWvBiTE2thTON2MLvR29teM7ccO9o5FNZPVHXwQrrrb3PmYfAztRyIpWfTI7A6urNbdA3LazdIBmDSbb6kGsouSyc1ltbom9ZWtV3KSVfg3dtDPRRWorLYsWdPfxBf4EGizubeLFthxsEZTYBsLI0L+CYOlsfjZz9JYq6S8Jzq0kTSvFhQMawqb1BHt0mnwXc2k9fW3tjTfJz8YUUieflW3y3Wz06ZIvpyXJiwgIM6RfPfDs1pofUGyykkJ7x3B1SH+kfB5mLIgrpG89dpP11gmoar2BcqQ1I63vk0v+iiEJ6xzPfV3dqxqP76pPO2MT+KLzsQRhTR37LxthaBwUYUlOeOeHL0CUrQyIAXXnuvqlHtJTO20c32QHfaElAZZCE4L3OTLQ/6u72otuYQmgdUzA114UjHVCZVx9I3C+snGSnZJ4cVbPwYVdObgC2FszcKbVrnx5ZnwNK1BKnOsNbLGXJfn3n0vWO5+ed25wudh+aw2Ptmzx6ePnvhGfM9qlq2+9MxoPtHmrsD9/RdCmb5HIytaYycFrIkoR3/4oF7rmF9/YrdqEf2aYnlrtUdxclufvFW+5Hziu03A8Hx0CsXo4Hhaa5I60lT0YD1HP3slNggsUlunsk030ELFAaSKr7+NJhUt09kuuuGOW4XHePJLsr+m4q2d0j2e6KUZDKdg9ItvvklYUmegYk233q0gITqAN3BMFluwfuhILKdg8yoCBbTx4PmMl2D/S2+3i+VLZ7oLfdFbdTbNs90Nvumg4J2XYP9La7pkiRbfdAb7sr7qtMtnukt92nbozBbPdIb7srEj5o2/2g2PDa7pq+qUcUy3aP/La7gmjoQnhs0j45yBvLfo8yf2Kz35ULLS/Y7/cJxYvy4JIIywSKTIZXNFKYMvwwXxlAnWT4T5ubHuj099+3d/sr/d3n+0v9p/aCdpG/np48ifJCqbfPKfWnR++/JaNprxhHmEx7O3BwiwA8t2qfS+wO1+k4Gucu+ccnystyXZJrbwfORxHX8W1l+xPqFdr2duCwkwn5TLx8qkAF5tvbgaNK1sAD1Li3A+eQUOHicu7twKkiHLhQeSBZcYryYfLu7YENKywy894OHN9BxYvLvbcDR3NQ4eKy7+3AvyNIhYvMv7cOaYff5OUFJnVbRx5LcDn4duAMIipcVBa+Hfh3U8kWuhWImTx8O3BQENk6t4IwlYlvB04xkmPu2hCDu/h24PAjMhlf1Ssh2/h24AglMh1fVajIPr4dOGmJTMjX3F+ZjHw7cD4TmZI/eZMM5uTbgUOfyKR8Te4HbeXbgdOhyLR8VQfVY4rl5duBI6TYxHwNU+hz6O3AUVNsKr520QX2JHo7cCQVmX2v6ZmI9Hs7cIzVfP69W7F/rxk8qAT81xzCNbeAX7w469573z/sXtbrovz7eY/HmsS/P5Jeo38/cCTWhHymXjkdjwrNvx84wmoNPFD9+4EDqahwkfn3A4dNceBC5QHl7YwvHyr/fuAUKCpYbP79wKFOVLzI/PuBI5qocJH59wPnL1HhYvPvB45XouLF5d8PnJRERYvMvx849ogKF5d/P3CIEdt69njEVP59lDkH2+L2eMJc/n2U0Ykcc9eGGN2/jzJiYfPvNb0StH+fZAzD5t9rChXav08yq2Hz7xX3Vyr//qBEEPv3UzfJaP59kpEQm3+vyP2w/fskgyM2/17TQfWYgvn3SYZLdP69gim2f59kBkXn3ysXXXD9+yRjJzb/XtEzMfn3SaZO8/n3fs3+vWLwoPLv0ytso/kPwBf+fdglFUD+fZ5VAprEvz+SXqN/n6Gsn/Go0Pz7gzexMh6o/n2GsnrG4yLz7zOU1jMCFyoPKG9nfPlQ+fcZaq+RYqwj8+8z1GYjRRfO5d9nqL1IirGQy7/PULuRJq8uOOm0QO1Gmrq80ITuQh5LkPn3hTy14PLviwwt2NazxyOm8u+LzDnYFrfHE+by74uMTuSYuzbE6P59kRELm3+v6ZWg/fsiYxg2/15TqND+fZFZDZt/r7i/Uvn3ReY7bP791E0ymn9fZSTE5t8rcj9s/77K4IjNv9d0UD2mYP79YeWG2b9XMMX276vMoOj8e+WiC65/X2XsxObfK3omJv++ytRpPv8+rNm/VwweVP59fYW9Mrd/n3LojDXHP/3SjbXVbsBR8eusgskkKv4R+hpV/ApllIxHBabiOwNljEzGA1TFdwZKGRmPi0vFdwefjA4XKg+onSzjy4dJxXcGah+LYqzjUvGdgdrnoujCqVR8Z6B2uijGQioV3xmojS6TVxeaf+oM1E6XqcsLzO1ucz9uWlwq/nFplBUXlYp/XPMkVvHHI2ZS8Z2VOQfbOvd4wlQqvrMyOpFj7toQg6v4zsqIhU3F1/RKyCr+0ZkiVvE1hYqs4rcfRjBlU/EV91cmFf+ovRGr+FM3yWAqvrMyEmJT8RW5H7SK76wMjthUfE0H1WOKpeIfwwlmFV/BFFrFP/mSxCq+ctEFVsU/JhDEKr6iZyJS8Y9xxKCKf7Ll43MS/dLfC6NseUV9M9nyxznvW9ry7Wo+Pa0+BajT6o/zyuUq8ifSK1TkjzMiiMVNBSo0Rd4hmRzT8UBV5J2ckVPhIlPkPZLKMQYXKg+kHSaK8qFS5L2cAFPBYlPkPdL+E00XzqXIe6QdKJqxkEuR90gbUKavLjgv1CPtQJm8vNCca08eS5Ap8p48teBS5L0MLciWnBWIqRT5MGAVrJ4wlyIfZHQix9y1IUZX5I//RDitIq/qlaAV+SBjGDJFXlWo0Ip8kFkNmSKvub9SKfJB5jtkivzkTTKaIh9kJESmyGtyP2xFPsjgiEyRV3VQPaZginyQ4RKbIq9hiq3IB5lBsSny2kUXXEU+ytiJTJHX9ExMinyUqdN8p9WnFZ9Wrxk8qPz7g1T9lv59m84K/95g+fdxVgloEv/+SHqN/n2Esn7Go0Lz7yOU1jMZD1T/PkJZPeNxkfn3EUrrGYELlQeUtzO+fKj8+ygjEypYbP59gtpspOjCufz7BLUXSTEWcvn3hykzKy42/z5B7UaaurzQhO5EHkuQ+feJPLXg8u+TDC3Y1rPHI6by75PMOdgWt8cT5vLvk4xO5Ji7NsTo/n2SEQubf6/plaD9+yxjGDb/XlOo0P59llkNm3+vuL9S+feHCTSxfz91k4zm32cZCbH594rcD9u/zzI4YvPvNR1UjymYf59luETn3yuYYvv3WWZQdP69ctEF17/PMnZi8+8VPROTf59l6jR0RH1ewxH1mvqmUuTLKwSTuRV5F4QiHyqWIl9mFT8mUeSPpNeoyBeZBEzIZ+rFzfGo0BT5AmVyTMYDVZEvUCrHeFxkinyBUjlG4ELlAbXDZHz5UCnyBWp/iWKsI1PkC9T+E0UXzqXIF6gdKIqxkEuRr1AbUCavLjgvtM4aRLw9Ly5F/jBas9IiU+QreWrBpchXGVqwLTmPR0ylyNcBq2D1hLkU+SqjEznmrg0xuiJfZcTCpshreiVoRb7KGIZNkdcUKrQiX2VWw6bIK+6vTIq8NzLfYVPkp26SwRR5b2QkxKbIK3I/aEX+GFMSK/KaDqrHFEuR90aGS3SKvIIptCLfBhMJ9f8jCmxF3hsZO7Ep8oqeiUiR9wOL1EOKfFmFIq+o73GK/IDdvSRHvr0XxbtBIJ7dkTfCkffeIzny3swq6kzhyJ9Ir9CR9wbJzFGgAnPkvUVSb6bjAerIe4tk3ihwcTny3soEhQMXKg8kt0ZRPkyOvLdI+4E0Yx2XI+8t0oYgTRdO5ch7i7RfSDMWUjny3iLtGJq+utDEUG+RdgxNXl5g0rW35LEElyPvHXlqQeXIeydDC7I1ZwViJkfeH8qP15FXEKZy5L2T0Ykcc9eGGNyR905GLGSOvKpXQnbkvZMxDJkjrypUZEfeO5nVkDnymvsrlSPvZL5D5shP3iSjOfJORkJkjrwm98N25J0MjsgceVUH1WMK5sh7GS6xOfIaptiOvJcZFJsjr110wXXkvYydyBx5Tc/E5Mh7mToNOfJ1DY68pr6ZjpH3/hVC0OyKfBSKvM1Qx8h7P6unM4kifyS9RkXeQ4k541GhKfIeyryZjAeqIu+hxJvxuMgUeQ9l3ozABcojQKk148uHSpEPUNuBFGMdmSIfZGBBxYtMkQ9Q24UUYyGXIh9ekQ8g42JT5APUhqGpywvNuQ7ksQSZIh/IUwsuRT7I0IJtyXk8YipFPsicg239eTxhLkU+yuhEjrlrQ4yuyEcZsbAp8ppeCVqRP/577LyKvKZQoRX5KLMaNkVecX+lUuSjzHfYFPmpm2Q0RT7KSIhNkVfkftiKfJTBEZsir+mgekzBFPkowyU6RV7BFFuRjzKDolPklYsuuIp8lLETmyKv6JmYFPkkU6chRX43Pq/AkVcUOJUjn15hBM3tyLc7SXe4mZyuIpIif/jKC1bk03O41qDIJygxZzwqNEU+yWRmDTxQFfkEJd6Mx0WmyCco82YELlQeUGrN+PKhUuQT1HYgxVhHpsgnqP1Aii6cS5HPUNuFFGMhlyKfoTYMTV5dcF7o4f3GyotLkc/ksQSZIp/JUwsuRT7L0IJtyXk8YipFPsucg239eTxhLkU+y+hEjrlrQ4yuyGcZsbAp8ppeCVqRzzKGYVPkNYUKrcgXmdWwKfKK+yuVIl9kvsOmyE/dJKMp8kVGQmyKvCL3w1bkiwyO2BR5TQfVYwqmyBcZLtEp8gqm2Ip8kRkUnSKvXHTBVeSLjJ3YFHlFz8SkyBeZOg0q8nYViryiwDEU+fbp7XZ79+i5H9t1u/x5e3a+e8X/AQ== -------------------------------------------------------------------------------- /BFV_theory/images/bfv/PolynomialTorus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/BFV_theory/images/bfv/PolynomialTorus.png -------------------------------------------------------------------------------- /BFV_theory/images/bfv/addition_noise/e1e2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/BFV_theory/images/bfv/addition_noise/e1e2.png -------------------------------------------------------------------------------- /BFV_theory/images/bfv/addition_noise/eu1u2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/BFV_theory/images/bfv/addition_noise/eu1u2.png -------------------------------------------------------------------------------- /BFV_theory/images/bfv/addition_noise/plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/BFV_theory/images/bfv/addition_noise/plot.png -------------------------------------------------------------------------------- /BFV_theory/images/bfv/addition_noise/se1e2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/BFV_theory/images/bfv/addition_noise/se1e2.png -------------------------------------------------------------------------------- /BFV_theory/images/bfv/multiplication_noise/plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/BFV_theory/images/bfv/multiplication_noise/plot.png -------------------------------------------------------------------------------- /Experiments/EncryptedProcessing.py: -------------------------------------------------------------------------------- 1 | from functools import reduce 2 | 3 | from Pyfhel import Pyfhel 4 | 5 | import torch 6 | import torchvision 7 | import torchvision.transforms as transforms 8 | import torch.nn as nn 9 | 10 | import numpy as np 11 | 12 | import logging as log 13 | log.basicConfig(filename='experiments.log', 14 | format='%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s', 15 | datefmt='%Y-%m-%d %H:%M:%S', encoding='utf-8', level=log.DEBUG) 16 | 17 | transform = transforms.ToTensor() 18 | 19 | test_set = torchvision.datasets.MNIST( 20 | root='./data', 21 | train=False, 22 | download=True, 23 | transform=transform 24 | ) 25 | 26 | model_file = "../LeNet1_Approx_single_square.pt" 27 | log.info(f"Loading model from file {model_file}...") 28 | 29 | 30 | class Square(nn.Module): 31 | def __init__(self): 32 | super().__init__() 33 | 34 | def forward(self, t): 35 | return torch.pow(t, 2) 36 | 37 | model = torch.load(model_file) 38 | model.eval() 39 | 40 | log.info(model) 41 | 42 | 43 | # Code for matrix encoding/encryption 44 | def encode_matrix(HE, matrix): 45 | try: 46 | return np.array(list(map(HE.encodeFrac, matrix))) 47 | except TypeError: 48 | return np.array([encode_matrix(HE, m) for m in matrix]) 49 | 50 | 51 | def decode_matrix(HE, matrix): 52 | try: 53 | return np.array(list(map(HE.decodeFrac, matrix))) 54 | except TypeError: 55 | return np.array([decode_matrix(HE, m) for m in matrix]) 56 | 57 | 58 | def encrypt_matrix(HE, matrix): 59 | try: 60 | return np.array(list(map(HE.encryptFrac, matrix))) 61 | except TypeError: 62 | return np.array([encrypt_matrix(HE, m) for m in matrix]) 63 | 64 | 65 | def decrypt_matrix(HE, matrix): 66 | try: 67 | return np.array(list(map(HE.decryptFrac, matrix))) 68 | except TypeError: 69 | return np.array([decrypt_matrix(HE, m) for m in matrix]) 70 | 71 | 72 | # Code for encoded CNN 73 | class ConvolutionalLayer: 74 | def __init__(self, HE, weights, stride=(1, 1), padding=(0, 0), bias=None): 75 | self.HE = HE 76 | self.weights = encode_matrix(HE, weights) 77 | self.stride = stride 78 | self.padding = padding 79 | self.bias = bias 80 | if bias is not None: 81 | self.bias = encode_matrix(HE, bias) 82 | 83 | def __call__(self, t): 84 | t = apply_padding(t, self.padding) 85 | result = np.array([[np.sum([convolute2d(image_layer, filter_layer, self.stride) 86 | for image_layer, filter_layer in zip(image, _filter)], axis=0) 87 | for _filter in self.weights] 88 | for image in t]) 89 | 90 | if self.bias is not None: 91 | return np.array([[layer + bias for layer, bias in zip(image, self.bias)] for image in result]) 92 | else: 93 | return result 94 | 95 | 96 | def convolute2d(image, filter_matrix, stride): 97 | x_d = len(image[0]) 98 | y_d = len(image) 99 | x_f = len(filter_matrix[0]) 100 | y_f = len(filter_matrix) 101 | 102 | y_stride = stride[0] 103 | x_stride = stride[1] 104 | 105 | x_o = ((x_d - x_f) // x_stride) + 1 106 | y_o = ((y_d - y_f) // y_stride) + 1 107 | 108 | def get_submatrix(matrix, x, y): 109 | index_row = y * y_stride 110 | index_column = x * x_stride 111 | return matrix[index_row: index_row + y_f, index_column: index_column + x_f] 112 | 113 | return np.array( 114 | [[np.sum(get_submatrix(image, x, y) * filter_matrix) for x in range(0, x_o)] for y in range(0, y_o)]) 115 | 116 | def apply_padding(t, padding): 117 | y_p = padding[0] 118 | x_p = padding[1] 119 | zero = t[0][0][y_p+1][x_p+1] - t[0][0][y_p+1][x_p+1] 120 | return [[np.pad(mat, ((y_p, y_p), (x_p, x_p)), 'constant', constant_values=zero) for mat in layer] for layer in t] 121 | 122 | 123 | class LinearLayer: 124 | def __init__(self, HE, weights, bias=None): 125 | self.HE = HE 126 | self.weights = encode_matrix(HE, weights) 127 | self.bias = bias 128 | if bias is not None: 129 | self.bias = encode_matrix(HE, bias) 130 | 131 | def __call__(self, t): 132 | result = np.array([[np.sum(image * row) for row in self.weights] for image in t]) 133 | if self.bias is not None: 134 | result = np.array([row + self.bias for row in result]) 135 | return result 136 | 137 | 138 | class SquareLayer: 139 | def __init__(self, HE): 140 | self.HE = HE 141 | 142 | def __call__(self, image): 143 | return square(self.HE, image) 144 | 145 | 146 | def square(HE, image): 147 | try: 148 | return np.array(list(map(lambda x: HE.power(x, 2), image))) 149 | except TypeError: 150 | return np.array([square(HE, m) for m in image]) 151 | 152 | 153 | class FlattenLayer: 154 | def __call__(self, image): 155 | dimension = image.shape 156 | return image.reshape(dimension[0], dimension[1]*dimension[2]*dimension[3]) 157 | 158 | 159 | class AveragePoolLayer: 160 | def __init__(self, HE, kernel_size, stride=(1, 1), padding=(0, 0)): 161 | self.HE = HE 162 | self.kernel_size = kernel_size 163 | self.stride = stride 164 | self.padding = padding 165 | 166 | def __call__(self, t): 167 | t = apply_padding(t, self.padding) 168 | return np.array([[_avg(self.HE, layer, self.kernel_size, self.stride) for layer in image] for image in t]) 169 | 170 | 171 | def _avg(HE, image, kernel_size, stride): 172 | x_s = stride[1] 173 | y_s = stride[0] 174 | 175 | x_k = kernel_size[1] 176 | y_k = kernel_size[0] 177 | 178 | x_d = len(image[0]) 179 | y_d = len(image) 180 | 181 | x_o = ((x_d - x_k) // x_s) + 1 182 | y_o = ((y_d - y_k) // y_s) + 1 183 | 184 | denominator = HE.encodeFrac(1 / (x_k * y_k)) 185 | 186 | def get_submatrix(matrix, x, y): 187 | index_row = y * y_s 188 | index_column = x * x_s 189 | return matrix[index_row: index_row + y_k, index_column: index_column + x_k] 190 | 191 | return [[np.sum(get_submatrix(image, x, y)) * denominator for x in range(0, x_o)] for y in range(0, y_o)] 192 | 193 | 194 | # We can now define a function to "convert" a PyTorch model to a list of sequential HE-ready-to-be-used layers: 195 | def build_from_pytorch(HE, net): 196 | # Define builders for every possible layer 197 | 198 | def conv_layer(layer): 199 | if layer.bias is None: 200 | bias = None 201 | else: 202 | bias = layer.bias.detach().numpy() 203 | 204 | return ConvolutionalLayer(HE, weights=layer.weight.detach().numpy(), 205 | stride=layer.stride, 206 | padding=layer.padding, 207 | bias=bias) 208 | 209 | def lin_layer(layer): 210 | if layer.bias is None: 211 | bias = None 212 | else: 213 | bias = layer.bias.detach().numpy() 214 | return LinearLayer(HE, layer.weight.detach().numpy(), 215 | bias) 216 | 217 | def avg_pool_layer(layer): 218 | # This proxy is required because in PyTorch an AvgPool2d can have kernel_size, stride and padding either of 219 | # type (int, int) or int, unlike in Conv2d 220 | kernel_size = (layer.kernel_size, layer.kernel_size) if isinstance(layer.kernel_size, int) else layer.kernel_size 221 | stride = (layer.stride, layer.stride) if isinstance(layer.stride, int) else layer.stride 222 | padding = (layer.padding, layer.padding) if isinstance(layer.padding, int) else layer.padding 223 | 224 | return AveragePoolLayer(HE, kernel_size, stride, padding) 225 | 226 | def flatten_layer(layer): 227 | return FlattenLayer() 228 | 229 | def square_layer(layer): 230 | return SquareLayer(HE) 231 | 232 | # Maps every PyTorch layer type to the correct builder 233 | options = {"Conv": conv_layer, 234 | "Line": lin_layer, 235 | "Flat": flatten_layer, 236 | "AvgP": avg_pool_layer, 237 | "Squa": square_layer 238 | } 239 | 240 | encoded_layers = [options[str(layer)[0:4]](layer) for layer in net] 241 | return encoded_layers 242 | 243 | 244 | log.info(f"Run the experiments...") 245 | 246 | from joblib import Parallel, delayed, parallel_backend 247 | import time 248 | n_threads = 8 249 | 250 | log.info(f"I will use {n_threads} threads.") 251 | 252 | p = 953983721 253 | m = 4096 254 | 255 | log.info(f"Using encryption parameters: m = {m}, p = {p}") 256 | 257 | HE = Pyfhel() 258 | HE.contextGen(p=p, m=m) 259 | HE.keyGen() 260 | relinKeySize=3 261 | HE.relinKeyGen(bitCount=5, size=relinKeySize) 262 | 263 | model.to("cpu") 264 | model_encoded = build_from_pytorch(HE, model) 265 | 266 | 267 | experiment_loader = torch.utils.data.DataLoader( 268 | test_set, 269 | batch_size=n_threads, 270 | shuffle=True 271 | ) 272 | 273 | 274 | def enc_and_process(image): 275 | encrypted_image = encrypt_matrix(HE, image.unsqueeze(0).numpy()) 276 | 277 | for layer in model_encoded: 278 | encrypted_image = layer(encrypted_image) 279 | 280 | result = decrypt_matrix(HE, encrypted_image) 281 | return result 282 | 283 | 284 | def check_net(): 285 | total_correct = 0 286 | n_batch = 0 287 | 288 | for batch in experiment_loader: 289 | images, labels = batch 290 | with parallel_backend('multiprocessing'): 291 | preds = Parallel(n_jobs=n_threads)(delayed(enc_and_process)(image) for image in images) 292 | 293 | preds = reduce(lambda x, y: np.concatenate((x, y)), preds) 294 | preds = torch.Tensor(preds) 295 | 296 | for image in preds: 297 | for value in image: 298 | if value > 100000: 299 | log.warning("WARNING: probably you are running out of NB.") 300 | 301 | total_correct += preds.argmax(dim=1).eq(labels).sum().item() 302 | n_batch = n_batch + 1 303 | if n_batch % 5 == 0 or n_batch == 1: 304 | log.info(f"Done {n_batch} batches.") 305 | log.info(f"This means we processed {n_threads * n_batch} images.") 306 | log.info(f"Correct images for now: {total_correct}") 307 | log.info("---------------------------") 308 | 309 | return total_correct 310 | 311 | 312 | starting_time = time.time() 313 | 314 | log.info(f"Start experiment...") 315 | 316 | correct = check_net() 317 | 318 | total_time = time.time() - starting_time 319 | log.info(f"Total corrects on the entire test set: {correct}") 320 | log.info("Time: ", total_time) 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | -------------------------------------------------------------------------------- /Experiments/LeNet1.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/Experiments/LeNet1.pt -------------------------------------------------------------------------------- /Experiments/LeNet1_Approx_single_square.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/Experiments/LeNet1_Approx_single_square.pt -------------------------------------------------------------------------------- /Experiments/LeNet1_single_tanh.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/Experiments/LeNet1_single_tanh.pt -------------------------------------------------------------------------------- /Experiments/MemoryAndTime/.directory: -------------------------------------------------------------------------------- 1 | [Dolphin] 2 | HeaderColumnWidths=495,114,235 3 | Timestamp=2021,7,22,10,58,4.162 4 | Version=4 5 | ViewMode=1 6 | -------------------------------------------------------------------------------- /Experiments/MemoryAndTime/Encrypted/experiment.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from Pyfhel import Pyfhel 4 | 5 | import torch 6 | import torchvision 7 | import torchvision.transforms as transforms 8 | import torch.nn as nn 9 | 10 | import numpy as np 11 | 12 | import logging as log 13 | 14 | from memory_profiler import profile 15 | 16 | log.basicConfig(filename='experiments.log', 17 | format='%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s', 18 | datefmt='%Y-%m-%d %H:%M:%S', encoding='utf-8', level=log.DEBUG) 19 | 20 | # Clear processing 21 | 22 | # Encrypted processing 23 | 24 | transform = transforms.ToTensor() 25 | 26 | test_set = torchvision.datasets.MNIST( 27 | root='../data', 28 | train=False, 29 | download=True, 30 | transform=transform 31 | ) 32 | 33 | class Square(nn.Module): 34 | def __init__(self): 35 | super().__init__() 36 | 37 | def forward(self, t): 38 | return torch.pow(t, 2) 39 | 40 | model_file = "../LeNet1_Approx_single_square.pt" 41 | model = torch.load(model_file) 42 | model.eval() 43 | 44 | # Code for matrix encoding/encryption 45 | def encode_matrix(HE, matrix): 46 | try: 47 | return np.array(list(map(HE.encodeFrac, matrix))) 48 | except TypeError: 49 | return np.array([encode_matrix(HE, m) for m in matrix]) 50 | 51 | 52 | def decode_matrix(HE, matrix): 53 | try: 54 | return np.array(list(map(HE.decodeFrac, matrix))) 55 | except TypeError: 56 | return np.array([decode_matrix(HE, m) for m in matrix]) 57 | 58 | 59 | def encrypt_matrix(HE, matrix): 60 | try: 61 | return np.array(list(map(HE.encryptFrac, matrix))) 62 | except TypeError: 63 | return np.array([encrypt_matrix(HE, m) for m in matrix]) 64 | 65 | 66 | def decrypt_matrix(HE, matrix): 67 | try: 68 | return np.array(list(map(HE.decryptFrac, matrix))) 69 | except TypeError: 70 | return np.array([decrypt_matrix(HE, m) for m in matrix]) 71 | 72 | 73 | # Code for encoded CNN 74 | class ConvolutionalLayer: 75 | def __init__(self, HE, weights, stride=(1, 1), padding=(0, 0), bias=None): 76 | self.HE = HE 77 | self.weights = encode_matrix(HE, weights) 78 | self.stride = stride 79 | self.padding = padding 80 | self.bias = bias 81 | if bias is not None: 82 | self.bias = encode_matrix(HE, bias) 83 | 84 | def __call__(self, t): 85 | t = apply_padding(t, self.padding) 86 | result = np.array([[np.sum([convolute2d(image_layer, filter_layer, self.stride) 87 | for image_layer, filter_layer in zip(image, _filter)], axis=0) 88 | for _filter in self.weights] 89 | for image in t]) 90 | 91 | if self.bias is not None: 92 | return np.array([[layer + bias for layer, bias in zip(image, self.bias)] for image in result]) 93 | else: 94 | return result 95 | 96 | 97 | def convolute2d(image, filter_matrix, stride): 98 | x_d = len(image[0]) 99 | y_d = len(image) 100 | x_f = len(filter_matrix[0]) 101 | y_f = len(filter_matrix) 102 | 103 | y_stride = stride[0] 104 | x_stride = stride[1] 105 | 106 | x_o = ((x_d - x_f) // x_stride) + 1 107 | y_o = ((y_d - y_f) // y_stride) + 1 108 | 109 | def get_submatrix(matrix, x, y): 110 | index_row = y * y_stride 111 | index_column = x * x_stride 112 | return matrix[index_row: index_row + y_f, index_column: index_column + x_f] 113 | 114 | return np.array( 115 | [[np.sum(get_submatrix(image, x, y) * filter_matrix) for x in range(0, x_o)] for y in range(0, y_o)]) 116 | 117 | def apply_padding(t, padding): 118 | y_p = padding[0] 119 | x_p = padding[1] 120 | zero = t[0][0][y_p+1][x_p+1] - t[0][0][y_p+1][x_p+1] 121 | return [[np.pad(mat, ((y_p, y_p), (x_p, x_p)), 'constant', constant_values=zero) for mat in layer] for layer in t] 122 | 123 | 124 | class LinearLayer: 125 | def __init__(self, HE, weights, bias=None): 126 | self.HE = HE 127 | self.weights = encode_matrix(HE, weights) 128 | self.bias = bias 129 | if bias is not None: 130 | self.bias = encode_matrix(HE, bias) 131 | 132 | def __call__(self, t): 133 | result = np.array([[np.sum(image * row) for row in self.weights] for image in t]) 134 | if self.bias is not None: 135 | result = np.array([row + self.bias for row in result]) 136 | return result 137 | 138 | 139 | class SquareLayer: 140 | def __init__(self, HE): 141 | self.HE = HE 142 | 143 | def __call__(self, image): 144 | return square(self.HE, image) 145 | 146 | 147 | def square(HE, image): 148 | try: 149 | return np.array(list(map(lambda x: HE.power(x, 2), image))) 150 | except TypeError: 151 | return np.array([square(HE, m) for m in image]) 152 | 153 | 154 | class FlattenLayer: 155 | def __call__(self, image): 156 | dimension = image.shape 157 | return image.reshape(dimension[0], dimension[1]*dimension[2]*dimension[3]) 158 | 159 | 160 | class AveragePoolLayer: 161 | def __init__(self, HE, kernel_size, stride=(1, 1), padding=(0, 0)): 162 | self.HE = HE 163 | self.kernel_size = kernel_size 164 | self.stride = stride 165 | self.padding = padding 166 | 167 | def __call__(self, t): 168 | t = apply_padding(t, self.padding) 169 | return np.array([[_avg(self.HE, layer, self.kernel_size, self.stride) for layer in image] for image in t]) 170 | 171 | 172 | def _avg(HE, image, kernel_size, stride): 173 | x_s = stride[1] 174 | y_s = stride[0] 175 | 176 | x_k = kernel_size[1] 177 | y_k = kernel_size[0] 178 | 179 | x_d = len(image[0]) 180 | y_d = len(image) 181 | 182 | x_o = ((x_d - x_k) // x_s) + 1 183 | y_o = ((y_d - y_k) // y_s) + 1 184 | 185 | denominator = HE.encodeFrac(1 / (x_k * y_k)) 186 | 187 | def get_submatrix(matrix, x, y): 188 | index_row = y * y_s 189 | index_column = x * x_s 190 | return matrix[index_row: index_row + y_k, index_column: index_column + x_k] 191 | 192 | return [[np.sum(get_submatrix(image, x, y)) * denominator for x in range(0, x_o)] for y in range(0, y_o)] 193 | 194 | 195 | # We can now define a function to "convert" a PyTorch model to a list of sequential HE-ready-to-be-used layers: 196 | def build_from_pytorch(HE, net): 197 | # Define builders for every possible layer 198 | 199 | def conv_layer(layer): 200 | if layer.bias is None: 201 | bias = None 202 | else: 203 | bias = layer.bias.detach().numpy() 204 | 205 | return ConvolutionalLayer(HE, weights=layer.weight.detach().numpy(), 206 | stride=layer.stride, 207 | padding=layer.padding, 208 | bias=bias) 209 | 210 | def lin_layer(layer): 211 | if layer.bias is None: 212 | bias = None 213 | else: 214 | bias = layer.bias.detach().numpy() 215 | return LinearLayer(HE, layer.weight.detach().numpy(), 216 | bias) 217 | 218 | def avg_pool_layer(layer): 219 | # This proxy is required because in PyTorch an AvgPool2d can have kernel_size, stride and padding either of 220 | # type (int, int) or int, unlike in Conv2d 221 | kernel_size = (layer.kernel_size, layer.kernel_size) if isinstance(layer.kernel_size, int) else layer.kernel_size 222 | stride = (layer.stride, layer.stride) if isinstance(layer.stride, int) else layer.stride 223 | padding = (layer.padding, layer.padding) if isinstance(layer.padding, int) else layer.padding 224 | 225 | return AveragePoolLayer(HE, kernel_size, stride, padding) 226 | 227 | def flatten_layer(layer): 228 | return FlattenLayer() 229 | 230 | def square_layer(layer): 231 | return SquareLayer(HE) 232 | 233 | # Maps every PyTorch layer type to the correct builder 234 | options = {"Conv": conv_layer, 235 | "Line": lin_layer, 236 | "Flat": flatten_layer, 237 | "AvgP": avg_pool_layer, 238 | "Squa": square_layer 239 | } 240 | 241 | encoded_layers = [options[str(layer)[0:4]](layer) for layer in net] 242 | return encoded_layers 243 | 244 | 245 | p = 953983721 246 | m = 4096 247 | 248 | log.info(f"Using encryption parameters: m = {m}, p = {p}") 249 | 250 | 251 | 252 | experiment_loader = torch.utils.data.DataLoader( 253 | test_set, 254 | batch_size=1, 255 | shuffle=True 256 | ) 257 | 258 | 259 | def enc_and_process(image, HE, model_encoded): 260 | encrypted_image = encrypt_matrix(HE, image.unsqueeze(0).numpy()) 261 | 262 | for layer in model_encoded: 263 | encrypted_image = layer(encrypted_image) 264 | 265 | result = decrypt_matrix(HE, encrypted_image) 266 | return result 267 | 268 | @profile 269 | def experiment_encrypted(): 270 | HE = Pyfhel() 271 | HE.contextGen(p=p, m=m) 272 | HE.keyGen() 273 | relinKeySize = 3 274 | HE.relinKeyGen(bitCount=5, size=relinKeySize) 275 | 276 | model.to("cpu") 277 | model_encoded = build_from_pytorch(HE, model) 278 | 279 | for batch in experiment_loader: 280 | image, labels = batch 281 | preds = enc_and_process(image[0], HE, model_encoded) 282 | return 283 | 284 | 285 | if __name__ == '__main__': 286 | 287 | log.info("Starting experiment...") 288 | starting_time = time.time() 289 | experiment_encrypted() 290 | total_time = time.time() - starting_time 291 | log.info(f"The encrypted processing of one image required {total_time}") 292 | 293 | 294 | 295 | -------------------------------------------------------------------------------- /Experiments/MemoryAndTime/Plain/experiment.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from Pyfhel import Pyfhel 4 | 5 | import torch 6 | import torchvision 7 | import torchvision.transforms as transforms 8 | import torch.nn as nn 9 | 10 | import numpy as np 11 | 12 | import logging as log 13 | 14 | from memory_profiler import profile 15 | 16 | device = 'cpu' 17 | 18 | log.basicConfig(filename='experiments.log', 19 | format='%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s', 20 | datefmt='%Y-%m-%d %H:%M:%S', encoding='utf-8', level=log.DEBUG) 21 | 22 | class Square(nn.Module): 23 | def __init__(self): 24 | super().__init__() 25 | 26 | def forward(self, t): 27 | return torch.pow(t, 2) 28 | 29 | transform = transforms.ToTensor() 30 | 31 | test_set = torchvision.datasets.MNIST( 32 | root='../../data', 33 | train=False, 34 | download=True, 35 | transform=transform 36 | ) 37 | 38 | test_loader = torch.utils.data.DataLoader( 39 | test_set, 40 | batch_size=1, 41 | shuffle=True 42 | ) 43 | 44 | def forward_one_image(network, device): 45 | network.eval() 46 | 47 | with torch.no_grad(): 48 | for batch in test_loader: # Get Batch 49 | images, labels = batch 50 | images, labels = images.to(device), labels.to(device) 51 | 52 | preds = network(images) # Pass Batch 53 | return 54 | 55 | 56 | lenet1 = torch.load("../../LeNet1_Approx_single_square.pt") 57 | lenet1.eval() 58 | lenet1.to(device) 59 | 60 | lenet1_singletanh = torch.load("../../LeNet1_single_tanh.pt") 61 | lenet1_singletanh.eval() 62 | lenet1_singletanh.to(device) 63 | 64 | lenet1_singlesquare = torch.load("../../LeNet1_single_tanh.pt") 65 | lenet1_singlesquare.eval() 66 | lenet1_singlesquare.to(device) 67 | 68 | n_experiments = 1000 69 | 70 | 71 | # @profile 72 | def experiment_LeNet1(): 73 | for i in range(0, n_experiments): 74 | forward_one_image(lenet1, device) 75 | 76 | 77 | # @profile 78 | def experiment_LeNet1_singletanh(): 79 | for i in range(0, n_experiments): 80 | forward_one_image(lenet1_singletanh, device) 81 | 82 | 83 | # @profile 84 | def experiment_LeNet1_singlesquare(): 85 | for i in range(0, n_experiments): 86 | forward_one_image(lenet1_singlesquare, device) 87 | 88 | 89 | if __name__ == '__main__': 90 | 91 | log.info("Starting experiment...") 92 | starting_time = time.time() 93 | experiment_LeNet1() 94 | t1 = time.time() 95 | log.info(f"The processing of one image for LeNet-1 required {(t1-starting_time)/n_experiments} seconds") 96 | experiment_LeNet1_singletanh() 97 | t2 = time.time() 98 | log.info(f"The processing of one image for LeNet-1 (single tanh) required {(t2-t1)/n_experiments} seconds") 99 | experiment_LeNet1_singlesquare() 100 | t3 = time.time() 101 | log.info(f"The processing of one image for approx LeNet-1 (single square) required {(t3-t2)/n_experiments} seconds") 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /Experiments/ModelExperiments.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | import torchvision.transforms as transforms 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | import torch.optim as optim 7 | 8 | import numpy as np 9 | 10 | import logging as log 11 | log.basicConfig(filename='ModelExperiments.log', 12 | format='%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s', 13 | datefmt='%Y-%m-%d %H:%M:%S', encoding='utf-8', level=log.DEBUG) 14 | 15 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 16 | 17 | transform = transforms.ToTensor() 18 | 19 | train_set = torchvision.datasets.MNIST( 20 | root = './data', 21 | train=True, 22 | download=True, 23 | transform=transform 24 | ) 25 | 26 | test_set = torchvision.datasets.MNIST( 27 | root = './data', 28 | train=False, 29 | download=True, 30 | transform=transform 31 | ) 32 | 33 | train_loader = torch.utils.data.DataLoader( 34 | train_set, 35 | batch_size=50, 36 | shuffle=True 37 | ) 38 | 39 | test_loader = torch.utils.data.DataLoader( 40 | test_set, 41 | batch_size=50, 42 | shuffle=True 43 | ) 44 | 45 | class Square(nn.Module): 46 | def __init__(self): 47 | super().__init__() 48 | 49 | def forward(self, t): 50 | return torch.pow(t, 2) 51 | 52 | 53 | def get_num_correct(preds, labels): 54 | return preds.argmax(dim=1).eq(labels).sum().item() 55 | 56 | 57 | def train_net(network, epochs, device): 58 | optimizer = optim.Adam(network.parameters(), lr=0.001) 59 | for epoch in range(epochs): 60 | 61 | total_loss = 0 62 | total_correct = 0 63 | 64 | for batch in train_loader: # Get Batch 65 | images, labels = batch 66 | images, labels = images.to(device), labels.to(device) 67 | 68 | preds = network(images) # Pass Batch 69 | loss = F.cross_entropy(preds, labels) # Calculate Loss 70 | 71 | optimizer.zero_grad() 72 | loss.backward() # Calculate Gradients 73 | optimizer.step() # Update Weights 74 | 75 | total_loss += loss.item() 76 | total_correct += get_num_correct(preds, labels) 77 | 78 | 79 | def test_net(network, device): 80 | network.eval() 81 | total_loss = 0 82 | total_correct = 0 83 | 84 | with torch.no_grad(): 85 | for batch in test_loader: # Get Batch 86 | images, labels = batch 87 | images, labels = images.to(device), labels.to(device) 88 | 89 | preds = network(images) # Pass Batch 90 | loss = F.cross_entropy(preds, labels) # Calculate Loss 91 | 92 | total_loss += loss.item() 93 | total_correct += get_num_correct(preds, labels) 94 | 95 | accuracy = round(100. * (total_correct / len(test_loader.dataset)), 4) 96 | 97 | return total_correct / len(test_loader.dataset) 98 | 99 | 100 | experiments = 10 101 | 102 | # Initial LeNet-1 103 | accuracies = [] 104 | for i in range(0, experiments): 105 | LeNet1 = nn.Sequential( 106 | nn.Conv2d(1, 4, kernel_size=5), 107 | nn.Tanh(), 108 | nn.AvgPool2d(kernel_size=2), 109 | 110 | nn.Conv2d(4, 12, kernel_size=5), 111 | nn.Tanh(), 112 | nn.AvgPool2d(kernel_size=2), 113 | 114 | nn.Flatten(), 115 | 116 | nn.Linear(192, 10), 117 | ) 118 | 119 | LeNet1.to(device) 120 | train_net(LeNet1, 15, device) 121 | acc = test_net(LeNet1, device) 122 | accuracies.append(acc) 123 | 124 | m = np.array(accuracies) 125 | log.info(f"Results for LeNet-1:") 126 | log.info(f"Mean accuracy on test set: {np.mean(m)}") 127 | log.info(f"Var: {np.var(m)}") 128 | 129 | # Optional: save the last trained LeNet-1: 130 | torch.save(LeNet1, "LeNet1.pt") 131 | 132 | 133 | # LeNet-1 with a single tanh 134 | accuracies = [] 135 | for i in range(0, experiments): 136 | LeNet1_singletanh = nn.Sequential( 137 | nn.Conv2d(1, 4, kernel_size=5), 138 | nn.Tanh(), 139 | nn.AvgPool2d(kernel_size=2), 140 | 141 | nn.Conv2d(4, 12, kernel_size=5), 142 | # nn.Tanh(), 143 | nn.AvgPool2d(kernel_size=2), 144 | 145 | nn.Flatten(), 146 | 147 | nn.Linear(192, 10), 148 | ) 149 | 150 | LeNet1_singletanh.to(device) 151 | train_net(LeNet1_singletanh, 15, device) 152 | acc = test_net(LeNet1_singletanh, device) 153 | accuracies.append(acc) 154 | 155 | m = np.array(accuracies) 156 | log.info(f"Results for LeNet-1 (single tanh):") 157 | log.info(f"Mean accuracy on test set: {np.mean(m)}") 158 | log.info(f"Var: {np.var(m)}") 159 | 160 | # Optional: save the last trained LeNet-1 (single tanh): 161 | torch.save(LeNet1_singletanh, "LeNet1_single_tanh.pt") 162 | 163 | 164 | # Approximated LeNet-1 (single square) 165 | accuracies = [] 166 | for i in range(0, experiments): 167 | Approx_LeNet1 = nn.Sequential( 168 | nn.Conv2d(1, 4, kernel_size=5), 169 | Square(), 170 | nn.AvgPool2d(kernel_size=2), 171 | 172 | nn.Conv2d(4, 12, kernel_size=5), 173 | # nn.Tanh(), 174 | nn.AvgPool2d(kernel_size=2), 175 | 176 | nn.Flatten(), 177 | 178 | nn.Linear(192, 10), 179 | ) 180 | 181 | Approx_LeNet1.to(device) 182 | train_net(Approx_LeNet1, 15, device) 183 | acc = test_net(Approx_LeNet1, device) 184 | accuracies.append(acc) 185 | 186 | m = np.array(accuracies) 187 | log.info(f"Results for approximated LeNet-1 (single square):") 188 | log.info(f"Mean accuracy on test set: {np.mean(m)}") 189 | log.info(f"Var: {np.var(m)}") 190 | 191 | # Optional: save the last trained approximated LeNet-1: 192 | torch.save(Approx_LeNet1, "LeNet1_Approx_single_square.pt") 193 | 194 | # Approximated LeNet-1 (single square) - the one saved and used by the encrypted processing 195 | model = torch.load("LeNet1_Approx_single_square.pt") 196 | model.eval() 197 | model.to(device) 198 | acc = test_net(model, device) 199 | log.info(f"Results for approximated LeNet-1 (single square) - the one saved to file: {acc}") 200 | 201 | 202 | 203 | 204 | 205 | -------------------------------------------------------------------------------- /Experiments_FashionMNIST/EncryptedProcessing.py: -------------------------------------------------------------------------------- 1 | from functools import reduce 2 | 3 | from Pyfhel import Pyfhel 4 | 5 | import torch 6 | import torchvision 7 | import torchvision.transforms as transforms 8 | import torch.nn as nn 9 | 10 | import numpy as np 11 | 12 | import logging as log 13 | log.basicConfig(filename='experiments.log', 14 | format='%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s', 15 | datefmt='%Y-%m-%d %H:%M:%S', encoding='utf-8', level=log.DEBUG) 16 | 17 | transform = transforms.ToTensor() 18 | 19 | test_set = torchvision.datasets.FashionMNIST( 20 | root='./data', 21 | train=False, 22 | download=True, 23 | transform=transform 24 | ) 25 | 26 | model_file = "../LeNet1_Approx_single_square.pt" 27 | log.info(f"Loading model from file {model_file}...") 28 | 29 | 30 | class Square(nn.Module): 31 | def __init__(self): 32 | super().__init__() 33 | 34 | def forward(self, t): 35 | return torch.pow(t, 2) 36 | 37 | model = torch.load(model_file) 38 | model.eval() 39 | 40 | log.info(model) 41 | 42 | 43 | # Code for matrix encoding/encryption 44 | def encode_matrix(HE, matrix): 45 | try: 46 | return np.array(list(map(HE.encodeFrac, matrix))) 47 | except TypeError: 48 | return np.array([encode_matrix(HE, m) for m in matrix]) 49 | 50 | 51 | def decode_matrix(HE, matrix): 52 | try: 53 | return np.array(list(map(HE.decodeFrac, matrix))) 54 | except TypeError: 55 | return np.array([decode_matrix(HE, m) for m in matrix]) 56 | 57 | 58 | def encrypt_matrix(HE, matrix): 59 | try: 60 | return np.array(list(map(HE.encryptFrac, matrix))) 61 | except TypeError: 62 | return np.array([encrypt_matrix(HE, m) for m in matrix]) 63 | 64 | 65 | def decrypt_matrix(HE, matrix): 66 | try: 67 | return np.array(list(map(HE.decryptFrac, matrix))) 68 | except TypeError: 69 | return np.array([decrypt_matrix(HE, m) for m in matrix]) 70 | 71 | 72 | # Code for encoded CNN 73 | class ConvolutionalLayer: 74 | def __init__(self, HE, weights, stride=(1, 1), padding=(0, 0), bias=None): 75 | self.HE = HE 76 | self.weights = encode_matrix(HE, weights) 77 | self.stride = stride 78 | self.padding = padding 79 | self.bias = bias 80 | if bias is not None: 81 | self.bias = encode_matrix(HE, bias) 82 | 83 | def __call__(self, t): 84 | t = apply_padding(t, self.padding) 85 | result = np.array([[np.sum([convolute2d(image_layer, filter_layer, self.stride) 86 | for image_layer, filter_layer in zip(image, _filter)], axis=0) 87 | for _filter in self.weights] 88 | for image in t]) 89 | 90 | if self.bias is not None: 91 | return np.array([[layer + bias for layer, bias in zip(image, self.bias)] for image in result]) 92 | else: 93 | return result 94 | 95 | 96 | def convolute2d(image, filter_matrix, stride): 97 | x_d = len(image[0]) 98 | y_d = len(image) 99 | x_f = len(filter_matrix[0]) 100 | y_f = len(filter_matrix) 101 | 102 | y_stride = stride[0] 103 | x_stride = stride[1] 104 | 105 | x_o = ((x_d - x_f) // x_stride) + 1 106 | y_o = ((y_d - y_f) // y_stride) + 1 107 | 108 | def get_submatrix(matrix, x, y): 109 | index_row = y * y_stride 110 | index_column = x * x_stride 111 | return matrix[index_row: index_row + y_f, index_column: index_column + x_f] 112 | 113 | return np.array( 114 | [[np.sum(get_submatrix(image, x, y) * filter_matrix) for x in range(0, x_o)] for y in range(0, y_o)]) 115 | 116 | def apply_padding(t, padding): 117 | y_p = padding[0] 118 | x_p = padding[1] 119 | zero = t[0][0][y_p+1][x_p+1] - t[0][0][y_p+1][x_p+1] 120 | return [[np.pad(mat, ((y_p, y_p), (x_p, x_p)), 'constant', constant_values=zero) for mat in layer] for layer in t] 121 | 122 | 123 | class LinearLayer: 124 | def __init__(self, HE, weights, bias=None): 125 | self.HE = HE 126 | self.weights = encode_matrix(HE, weights) 127 | self.bias = bias 128 | if bias is not None: 129 | self.bias = encode_matrix(HE, bias) 130 | 131 | def __call__(self, t): 132 | result = np.array([[np.sum(image * row) for row in self.weights] for image in t]) 133 | if self.bias is not None: 134 | result = np.array([row + self.bias for row in result]) 135 | return result 136 | 137 | 138 | class SquareLayer: 139 | def __init__(self, HE): 140 | self.HE = HE 141 | 142 | def __call__(self, image): 143 | return square(self.HE, image) 144 | 145 | 146 | def square(HE, image): 147 | try: 148 | return np.array(list(map(lambda x: HE.power(x, 2), image))) 149 | except TypeError: 150 | return np.array([square(HE, m) for m in image]) 151 | 152 | 153 | class FlattenLayer: 154 | def __call__(self, image): 155 | dimension = image.shape 156 | return image.reshape(dimension[0], dimension[1]*dimension[2]*dimension[3]) 157 | 158 | 159 | class AveragePoolLayer: 160 | def __init__(self, HE, kernel_size, stride=(1, 1), padding=(0, 0)): 161 | self.HE = HE 162 | self.kernel_size = kernel_size 163 | self.stride = stride 164 | self.padding = padding 165 | 166 | def __call__(self, t): 167 | t = apply_padding(t, self.padding) 168 | return np.array([[_avg(self.HE, layer, self.kernel_size, self.stride) for layer in image] for image in t]) 169 | 170 | 171 | def _avg(HE, image, kernel_size, stride): 172 | x_s = stride[1] 173 | y_s = stride[0] 174 | 175 | x_k = kernel_size[1] 176 | y_k = kernel_size[0] 177 | 178 | x_d = len(image[0]) 179 | y_d = len(image) 180 | 181 | x_o = ((x_d - x_k) // x_s) + 1 182 | y_o = ((y_d - y_k) // y_s) + 1 183 | 184 | denominator = HE.encodeFrac(1 / (x_k * y_k)) 185 | 186 | def get_submatrix(matrix, x, y): 187 | index_row = y * y_s 188 | index_column = x * x_s 189 | return matrix[index_row: index_row + y_k, index_column: index_column + x_k] 190 | 191 | return [[np.sum(get_submatrix(image, x, y)) * denominator for x in range(0, x_o)] for y in range(0, y_o)] 192 | 193 | 194 | # We can now define a function to "convert" a PyTorch model to a list of sequential HE-ready-to-be-used layers: 195 | def build_from_pytorch(HE, net): 196 | # Define builders for every possible layer 197 | 198 | def conv_layer(layer): 199 | if layer.bias is None: 200 | bias = None 201 | else: 202 | bias = layer.bias.detach().numpy() 203 | 204 | return ConvolutionalLayer(HE, weights=layer.weight.detach().numpy(), 205 | stride=layer.stride, 206 | padding=layer.padding, 207 | bias=bias) 208 | 209 | def lin_layer(layer): 210 | if layer.bias is None: 211 | bias = None 212 | else: 213 | bias = layer.bias.detach().numpy() 214 | return LinearLayer(HE, layer.weight.detach().numpy(), 215 | bias) 216 | 217 | def avg_pool_layer(layer): 218 | # This proxy is required because in PyTorch an AvgPool2d can have kernel_size, stride and padding either of 219 | # type (int, int) or int, unlike in Conv2d 220 | kernel_size = (layer.kernel_size, layer.kernel_size) if isinstance(layer.kernel_size, int) else layer.kernel_size 221 | stride = (layer.stride, layer.stride) if isinstance(layer.stride, int) else layer.stride 222 | padding = (layer.padding, layer.padding) if isinstance(layer.padding, int) else layer.padding 223 | 224 | return AveragePoolLayer(HE, kernel_size, stride, padding) 225 | 226 | def flatten_layer(layer): 227 | return FlattenLayer() 228 | 229 | def square_layer(layer): 230 | return SquareLayer(HE) 231 | 232 | # Maps every PyTorch layer type to the correct builder 233 | options = {"Conv": conv_layer, 234 | "Line": lin_layer, 235 | "Flat": flatten_layer, 236 | "AvgP": avg_pool_layer, 237 | "Squa": square_layer 238 | } 239 | 240 | encoded_layers = [options[str(layer)[0:4]](layer) for layer in net] 241 | return encoded_layers 242 | 243 | 244 | log.info(f"Run the experiments...") 245 | 246 | from joblib import Parallel, delayed, parallel_backend 247 | import time 248 | n_threads = 8 249 | 250 | log.info(f"I will use {n_threads} threads.") 251 | 252 | p = 953983721 253 | m = 4096 254 | 255 | log.info(f"Using encryption parameters: m = {m}, p = {p}") 256 | 257 | HE = Pyfhel() 258 | HE.contextGen(p=p, m=m) 259 | HE.keyGen() 260 | relinKeySize=3 261 | HE.relinKeyGen(bitCount=5, size=relinKeySize) 262 | 263 | model.to("cpu") 264 | model_encoded = build_from_pytorch(HE, model) 265 | 266 | 267 | experiment_loader = torch.utils.data.DataLoader( 268 | test_set, 269 | batch_size=n_threads, 270 | shuffle=True 271 | ) 272 | 273 | 274 | def enc_and_process(image): 275 | encrypted_image = encrypt_matrix(HE, image.unsqueeze(0).numpy()) 276 | 277 | for layer in model_encoded: 278 | encrypted_image = layer(encrypted_image) 279 | 280 | result = decrypt_matrix(HE, encrypted_image) 281 | return result 282 | 283 | 284 | def check_net(): 285 | total_correct = 0 286 | n_batch = 0 287 | 288 | for batch in experiment_loader: 289 | images, labels = batch 290 | with parallel_backend('multiprocessing'): 291 | preds = Parallel(n_jobs=n_threads)(delayed(enc_and_process)(image) for image in images) 292 | 293 | preds = reduce(lambda x, y: np.concatenate((x, y)), preds) 294 | preds = torch.Tensor(preds) 295 | 296 | for image in preds: 297 | for value in image: 298 | if value > 100000: 299 | log.warning("WARNING: probably you are running out of NB.") 300 | 301 | total_correct += preds.argmax(dim=1).eq(labels).sum().item() 302 | n_batch = n_batch + 1 303 | if n_batch % 5 == 0 or n_batch == 1: 304 | log.info(f"Done {n_batch} batches.") 305 | log.info(f"This means we processed {n_threads * n_batch} images.") 306 | log.info(f"Correct images for now: {total_correct}") 307 | log.info("---------------------------") 308 | 309 | return total_correct 310 | 311 | 312 | starting_time = time.time() 313 | 314 | log.info(f"Start experiment...") 315 | 316 | correct = check_net() 317 | 318 | total_time = time.time() - starting_time 319 | log.info(f"Total corrects on the entire test set: {correct}") 320 | log.info("Time: ", total_time) 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | -------------------------------------------------------------------------------- /Experiments_FashionMNIST/LeNet1.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/Experiments_FashionMNIST/LeNet1.pt -------------------------------------------------------------------------------- /Experiments_FashionMNIST/LeNet1_Approx_single_square.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/Experiments_FashionMNIST/LeNet1_Approx_single_square.pt -------------------------------------------------------------------------------- /Experiments_FashionMNIST/LeNet1_single_tanh.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/Experiments_FashionMNIST/LeNet1_single_tanh.pt -------------------------------------------------------------------------------- /Experiments_FashionMNIST/MemoryAndTime/.directory: -------------------------------------------------------------------------------- 1 | [Dolphin] 2 | HeaderColumnWidths=495,114,235 3 | Timestamp=2021,7,22,10,58,4.162 4 | Version=4 5 | ViewMode=1 6 | -------------------------------------------------------------------------------- /Experiments_FashionMNIST/MemoryAndTime/Encrypted/experiment.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from Pyfhel import Pyfhel 4 | 5 | import torch 6 | import torchvision 7 | import torchvision.transforms as transforms 8 | import torch.nn as nn 9 | 10 | import numpy as np 11 | 12 | import logging as log 13 | 14 | from memory_profiler import profile 15 | 16 | log.basicConfig(filename='experiments.log', 17 | format='%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s', 18 | datefmt='%Y-%m-%d %H:%M:%S', encoding='utf-8', level=log.DEBUG) 19 | 20 | # Clear processing 21 | 22 | # Encrypted processing 23 | 24 | transform = transforms.ToTensor() 25 | 26 | test_set = torchvision.datasets.MNIST( 27 | root='../data', 28 | train=False, 29 | download=True, 30 | transform=transform 31 | ) 32 | 33 | class Square(nn.Module): 34 | def __init__(self): 35 | super().__init__() 36 | 37 | def forward(self, t): 38 | return torch.pow(t, 2) 39 | 40 | model_file = "../LeNet1_Approx_single_square.pt" 41 | model = torch.load(model_file) 42 | model.eval() 43 | 44 | # Code for matrix encoding/encryption 45 | def encode_matrix(HE, matrix): 46 | try: 47 | return np.array(list(map(HE.encodeFrac, matrix))) 48 | except TypeError: 49 | return np.array([encode_matrix(HE, m) for m in matrix]) 50 | 51 | 52 | def decode_matrix(HE, matrix): 53 | try: 54 | return np.array(list(map(HE.decodeFrac, matrix))) 55 | except TypeError: 56 | return np.array([decode_matrix(HE, m) for m in matrix]) 57 | 58 | 59 | def encrypt_matrix(HE, matrix): 60 | try: 61 | return np.array(list(map(HE.encryptFrac, matrix))) 62 | except TypeError: 63 | return np.array([encrypt_matrix(HE, m) for m in matrix]) 64 | 65 | 66 | def decrypt_matrix(HE, matrix): 67 | try: 68 | return np.array(list(map(HE.decryptFrac, matrix))) 69 | except TypeError: 70 | return np.array([decrypt_matrix(HE, m) for m in matrix]) 71 | 72 | 73 | # Code for encoded CNN 74 | class ConvolutionalLayer: 75 | def __init__(self, HE, weights, stride=(1, 1), padding=(0, 0), bias=None): 76 | self.HE = HE 77 | self.weights = encode_matrix(HE, weights) 78 | self.stride = stride 79 | self.padding = padding 80 | self.bias = bias 81 | if bias is not None: 82 | self.bias = encode_matrix(HE, bias) 83 | 84 | def __call__(self, t): 85 | t = apply_padding(t, self.padding) 86 | result = np.array([[np.sum([convolute2d(image_layer, filter_layer, self.stride) 87 | for image_layer, filter_layer in zip(image, _filter)], axis=0) 88 | for _filter in self.weights] 89 | for image in t]) 90 | 91 | if self.bias is not None: 92 | return np.array([[layer + bias for layer, bias in zip(image, self.bias)] for image in result]) 93 | else: 94 | return result 95 | 96 | 97 | def convolute2d(image, filter_matrix, stride): 98 | x_d = len(image[0]) 99 | y_d = len(image) 100 | x_f = len(filter_matrix[0]) 101 | y_f = len(filter_matrix) 102 | 103 | y_stride = stride[0] 104 | x_stride = stride[1] 105 | 106 | x_o = ((x_d - x_f) // x_stride) + 1 107 | y_o = ((y_d - y_f) // y_stride) + 1 108 | 109 | def get_submatrix(matrix, x, y): 110 | index_row = y * y_stride 111 | index_column = x * x_stride 112 | return matrix[index_row: index_row + y_f, index_column: index_column + x_f] 113 | 114 | return np.array( 115 | [[np.sum(get_submatrix(image, x, y) * filter_matrix) for x in range(0, x_o)] for y in range(0, y_o)]) 116 | 117 | def apply_padding(t, padding): 118 | y_p = padding[0] 119 | x_p = padding[1] 120 | zero = t[0][0][y_p+1][x_p+1] - t[0][0][y_p+1][x_p+1] 121 | return [[np.pad(mat, ((y_p, y_p), (x_p, x_p)), 'constant', constant_values=zero) for mat in layer] for layer in t] 122 | 123 | 124 | class LinearLayer: 125 | def __init__(self, HE, weights, bias=None): 126 | self.HE = HE 127 | self.weights = encode_matrix(HE, weights) 128 | self.bias = bias 129 | if bias is not None: 130 | self.bias = encode_matrix(HE, bias) 131 | 132 | def __call__(self, t): 133 | result = np.array([[np.sum(image * row) for row in self.weights] for image in t]) 134 | if self.bias is not None: 135 | result = np.array([row + self.bias for row in result]) 136 | return result 137 | 138 | 139 | class SquareLayer: 140 | def __init__(self, HE): 141 | self.HE = HE 142 | 143 | def __call__(self, image): 144 | return square(self.HE, image) 145 | 146 | 147 | def square(HE, image): 148 | try: 149 | return np.array(list(map(lambda x: HE.power(x, 2), image))) 150 | except TypeError: 151 | return np.array([square(HE, m) for m in image]) 152 | 153 | 154 | class FlattenLayer: 155 | def __call__(self, image): 156 | dimension = image.shape 157 | return image.reshape(dimension[0], dimension[1]*dimension[2]*dimension[3]) 158 | 159 | 160 | class AveragePoolLayer: 161 | def __init__(self, HE, kernel_size, stride=(1, 1), padding=(0, 0)): 162 | self.HE = HE 163 | self.kernel_size = kernel_size 164 | self.stride = stride 165 | self.padding = padding 166 | 167 | def __call__(self, t): 168 | t = apply_padding(t, self.padding) 169 | return np.array([[_avg(self.HE, layer, self.kernel_size, self.stride) for layer in image] for image in t]) 170 | 171 | 172 | def _avg(HE, image, kernel_size, stride): 173 | x_s = stride[1] 174 | y_s = stride[0] 175 | 176 | x_k = kernel_size[1] 177 | y_k = kernel_size[0] 178 | 179 | x_d = len(image[0]) 180 | y_d = len(image) 181 | 182 | x_o = ((x_d - x_k) // x_s) + 1 183 | y_o = ((y_d - y_k) // y_s) + 1 184 | 185 | denominator = HE.encodeFrac(1 / (x_k * y_k)) 186 | 187 | def get_submatrix(matrix, x, y): 188 | index_row = y * y_s 189 | index_column = x * x_s 190 | return matrix[index_row: index_row + y_k, index_column: index_column + x_k] 191 | 192 | return [[np.sum(get_submatrix(image, x, y)) * denominator for x in range(0, x_o)] for y in range(0, y_o)] 193 | 194 | 195 | # We can now define a function to "convert" a PyTorch model to a list of sequential HE-ready-to-be-used layers: 196 | def build_from_pytorch(HE, net): 197 | # Define builders for every possible layer 198 | 199 | def conv_layer(layer): 200 | if layer.bias is None: 201 | bias = None 202 | else: 203 | bias = layer.bias.detach().numpy() 204 | 205 | return ConvolutionalLayer(HE, weights=layer.weight.detach().numpy(), 206 | stride=layer.stride, 207 | padding=layer.padding, 208 | bias=bias) 209 | 210 | def lin_layer(layer): 211 | if layer.bias is None: 212 | bias = None 213 | else: 214 | bias = layer.bias.detach().numpy() 215 | return LinearLayer(HE, layer.weight.detach().numpy(), 216 | bias) 217 | 218 | def avg_pool_layer(layer): 219 | # This proxy is required because in PyTorch an AvgPool2d can have kernel_size, stride and padding either of 220 | # type (int, int) or int, unlike in Conv2d 221 | kernel_size = (layer.kernel_size, layer.kernel_size) if isinstance(layer.kernel_size, int) else layer.kernel_size 222 | stride = (layer.stride, layer.stride) if isinstance(layer.stride, int) else layer.stride 223 | padding = (layer.padding, layer.padding) if isinstance(layer.padding, int) else layer.padding 224 | 225 | return AveragePoolLayer(HE, kernel_size, stride, padding) 226 | 227 | def flatten_layer(layer): 228 | return FlattenLayer() 229 | 230 | def square_layer(layer): 231 | return SquareLayer(HE) 232 | 233 | # Maps every PyTorch layer type to the correct builder 234 | options = {"Conv": conv_layer, 235 | "Line": lin_layer, 236 | "Flat": flatten_layer, 237 | "AvgP": avg_pool_layer, 238 | "Squa": square_layer 239 | } 240 | 241 | encoded_layers = [options[str(layer)[0:4]](layer) for layer in net] 242 | return encoded_layers 243 | 244 | 245 | p = 953983721 246 | m = 4096 247 | 248 | log.info(f"Using encryption parameters: m = {m}, p = {p}") 249 | 250 | 251 | 252 | experiment_loader = torch.utils.data.DataLoader( 253 | test_set, 254 | batch_size=1, 255 | shuffle=True 256 | ) 257 | 258 | 259 | def enc_and_process(image, HE, model_encoded): 260 | encrypted_image = encrypt_matrix(HE, image.unsqueeze(0).numpy()) 261 | 262 | for layer in model_encoded: 263 | encrypted_image = layer(encrypted_image) 264 | 265 | result = decrypt_matrix(HE, encrypted_image) 266 | return result 267 | 268 | @profile 269 | def experiment_encrypted(): 270 | HE = Pyfhel() 271 | HE.contextGen(p=p, m=m) 272 | HE.keyGen() 273 | relinKeySize = 3 274 | HE.relinKeyGen(bitCount=5, size=relinKeySize) 275 | 276 | model.to("cpu") 277 | model_encoded = build_from_pytorch(HE, model) 278 | 279 | for batch in experiment_loader: 280 | image, labels = batch 281 | preds = enc_and_process(image[0], HE, model_encoded) 282 | return 283 | 284 | 285 | if __name__ == '__main__': 286 | 287 | log.info("Starting experiment...") 288 | starting_time = time.time() 289 | experiment_encrypted() 290 | total_time = time.time() - starting_time 291 | log.info(f"The encrypted processing of one image required {total_time}") 292 | 293 | 294 | 295 | -------------------------------------------------------------------------------- /Experiments_FashionMNIST/MemoryAndTime/Plain/experiment.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from Pyfhel import Pyfhel 4 | 5 | import torch 6 | import torchvision 7 | import torchvision.transforms as transforms 8 | import torch.nn as nn 9 | 10 | import numpy as np 11 | 12 | import logging as log 13 | 14 | from memory_profiler import profile 15 | 16 | device = 'cpu' 17 | 18 | log.basicConfig(filename='experiments.log', 19 | format='%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s', 20 | datefmt='%Y-%m-%d %H:%M:%S', encoding='utf-8', level=log.DEBUG) 21 | 22 | class Square(nn.Module): 23 | def __init__(self): 24 | super().__init__() 25 | 26 | def forward(self, t): 27 | return torch.pow(t, 2) 28 | 29 | transform = transforms.ToTensor() 30 | 31 | test_set = torchvision.datasets.MNIST( 32 | root='../../data', 33 | train=False, 34 | download=True, 35 | transform=transform 36 | ) 37 | 38 | test_loader = torch.utils.data.DataLoader( 39 | test_set, 40 | batch_size=1, 41 | shuffle=True 42 | ) 43 | 44 | def forward_one_image(network, device): 45 | network.eval() 46 | 47 | with torch.no_grad(): 48 | for batch in test_loader: # Get Batch 49 | images, labels = batch 50 | images, labels = images.to(device), labels.to(device) 51 | 52 | preds = network(images) # Pass Batch 53 | return 54 | 55 | 56 | lenet1 = torch.load("../../LeNet1_Approx_single_square.pt") 57 | lenet1.eval() 58 | lenet1.to(device) 59 | 60 | lenet1_singletanh = torch.load("../../LeNet1_single_tanh.pt") 61 | lenet1_singletanh.eval() 62 | lenet1_singletanh.to(device) 63 | 64 | lenet1_singlesquare = torch.load("../../LeNet1_single_tanh.pt") 65 | lenet1_singlesquare.eval() 66 | lenet1_singlesquare.to(device) 67 | 68 | n_experiments = 1000 69 | 70 | 71 | # @profile 72 | def experiment_LeNet1(): 73 | for i in range(0, n_experiments): 74 | forward_one_image(lenet1, device) 75 | 76 | 77 | # @profile 78 | def experiment_LeNet1_singletanh(): 79 | for i in range(0, n_experiments): 80 | forward_one_image(lenet1_singletanh, device) 81 | 82 | 83 | # @profile 84 | def experiment_LeNet1_singlesquare(): 85 | for i in range(0, n_experiments): 86 | forward_one_image(lenet1_singlesquare, device) 87 | 88 | 89 | if __name__ == '__main__': 90 | 91 | log.info("Starting experiment...") 92 | starting_time = time.time() 93 | experiment_LeNet1() 94 | t1 = time.time() 95 | log.info(f"The processing of one image for LeNet-1 required {(t1-starting_time)/n_experiments} seconds") 96 | experiment_LeNet1_singletanh() 97 | t2 = time.time() 98 | log.info(f"The processing of one image for LeNet-1 (single tanh) required {(t2-t1)/n_experiments} seconds") 99 | experiment_LeNet1_singlesquare() 100 | t3 = time.time() 101 | log.info(f"The processing of one image for approx LeNet-1 (single square) required {(t3-t2)/n_experiments} seconds") 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /Experiments_FashionMNIST/ModelExperiments.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | import torchvision.transforms as transforms 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | import torch.optim as optim 7 | 8 | import numpy as np 9 | 10 | import logging as log 11 | log.basicConfig(filename='ModelExperiments.log', 12 | format='%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s', 13 | datefmt='%Y-%m-%d %H:%M:%S', level=log.DEBUG) 14 | 15 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 16 | 17 | transform = transforms.ToTensor() 18 | 19 | train_set = torchvision.datasets.FashionMNIST( 20 | root = './data', 21 | train=True, 22 | download=True, 23 | transform=transform 24 | ) 25 | 26 | test_set = torchvision.datasets.FashionMNIST( 27 | root = './data', 28 | train=False, 29 | download=True, 30 | transform=transform 31 | ) 32 | 33 | train_loader = torch.utils.data.DataLoader( 34 | train_set, 35 | batch_size=50, 36 | shuffle=True 37 | ) 38 | 39 | test_loader = torch.utils.data.DataLoader( 40 | test_set, 41 | batch_size=50, 42 | shuffle=True 43 | ) 44 | 45 | class Square(nn.Module): 46 | def __init__(self): 47 | super().__init__() 48 | 49 | def forward(self, t): 50 | return torch.pow(t, 2) 51 | 52 | 53 | def get_num_correct(preds, labels): 54 | return preds.argmax(dim=1).eq(labels).sum().item() 55 | 56 | 57 | def train_net(network, epochs, device): 58 | optimizer = optim.Adam(network.parameters(), lr=0.001) 59 | for epoch in range(epochs): 60 | 61 | total_loss = 0 62 | total_correct = 0 63 | 64 | for batch in train_loader: # Get Batch 65 | images, labels = batch 66 | images, labels = images.to(device), labels.to(device) 67 | 68 | preds = network(images) # Pass Batch 69 | loss = F.cross_entropy(preds, labels) # Calculate Loss 70 | 71 | optimizer.zero_grad() 72 | loss.backward() # Calculate Gradients 73 | optimizer.step() # Update Weights 74 | 75 | total_loss += loss.item() 76 | total_correct += get_num_correct(preds, labels) 77 | 78 | 79 | def test_net(network, device): 80 | network.eval() 81 | total_loss = 0 82 | total_correct = 0 83 | 84 | with torch.no_grad(): 85 | for batch in test_loader: # Get Batch 86 | images, labels = batch 87 | images, labels = images.to(device), labels.to(device) 88 | 89 | preds = network(images) # Pass Batch 90 | loss = F.cross_entropy(preds, labels) # Calculate Loss 91 | 92 | total_loss += loss.item() 93 | total_correct += get_num_correct(preds, labels) 94 | 95 | accuracy = round(100. * (total_correct / len(test_loader.dataset)), 4) 96 | 97 | return total_correct / len(test_loader.dataset) 98 | 99 | 100 | experiments = 10 101 | 102 | # Initial LeNet-1 103 | accuracies = [] 104 | for i in range(0, experiments): 105 | LeNet1 = nn.Sequential( 106 | nn.Conv2d(1, 4, kernel_size=5), 107 | nn.Tanh(), 108 | nn.AvgPool2d(kernel_size=2), 109 | 110 | nn.Conv2d(4, 12, kernel_size=5), 111 | nn.Tanh(), 112 | nn.AvgPool2d(kernel_size=2), 113 | 114 | nn.Flatten(), 115 | 116 | nn.Linear(192, 10), 117 | ) 118 | 119 | LeNet1.to(device) 120 | train_net(LeNet1, 15, device) 121 | acc = test_net(LeNet1, device) 122 | accuracies.append(acc) 123 | 124 | m = np.array(accuracies) 125 | log.info(f"Results for LeNet-1:") 126 | log.info(f"Mean accuracy on test set: {np.mean(m)}") 127 | log.info(f"Var: {np.var(m)}") 128 | 129 | # Optional: save the last trained LeNet-1: 130 | torch.save(LeNet1, "LeNet1.pt") 131 | 132 | # LeNet-1 with two squares 133 | accuracies = [] 134 | for i in range(0, experiments): 135 | LeNet1 = nn.Sequential( 136 | nn.Conv2d(1, 4, kernel_size=5), 137 | Square(), 138 | nn.AvgPool2d(kernel_size=2), 139 | 140 | nn.Conv2d(4, 12, kernel_size=5), 141 | Square(), 142 | nn.AvgPool2d(kernel_size=2), 143 | 144 | nn.Flatten(), 145 | 146 | nn.Linear(192, 10), 147 | ) 148 | 149 | LeNet1.to(device) 150 | train_net(LeNet1, 15, device) 151 | acc = test_net(LeNet1, device) 152 | accuracies.append(acc) 153 | 154 | m = np.array(accuracies) 155 | log.info(f"Results for LeNet-1 (two squares):") 156 | log.info(f"Mean accuracy on test set: {np.mean(m)}") 157 | log.info(f"Var: {np.var(m)}") 158 | 159 | # LeNet-1 with a single tanh 160 | accuracies = [] 161 | for i in range(0, experiments): 162 | LeNet1_singletanh = nn.Sequential( 163 | nn.Conv2d(1, 4, kernel_size=5), 164 | nn.Tanh(), 165 | nn.AvgPool2d(kernel_size=2), 166 | 167 | nn.Conv2d(4, 12, kernel_size=5), 168 | # nn.Tanh(), 169 | nn.AvgPool2d(kernel_size=2), 170 | 171 | nn.Flatten(), 172 | 173 | nn.Linear(192, 10), 174 | ) 175 | 176 | LeNet1_singletanh.to(device) 177 | train_net(LeNet1_singletanh, 15, device) 178 | acc = test_net(LeNet1_singletanh, device) 179 | accuracies.append(acc) 180 | 181 | m = np.array(accuracies) 182 | log.info(f"Results for LeNet-1 (single tanh):") 183 | log.info(f"Mean accuracy on test set: {np.mean(m)}") 184 | log.info(f"Var: {np.var(m)}") 185 | 186 | # Optional: save the last trained LeNet-1 (single tanh): 187 | torch.save(LeNet1_singletanh, "LeNet1_single_tanh.pt") 188 | 189 | 190 | # Approximated LeNet-1 (single square) 191 | accuracies = [] 192 | for i in range(0, experiments): 193 | Approx_LeNet1 = nn.Sequential( 194 | nn.Conv2d(1, 4, kernel_size=5), 195 | Square(), 196 | nn.AvgPool2d(kernel_size=2), 197 | 198 | nn.Conv2d(4, 12, kernel_size=5), 199 | # nn.Tanh(), 200 | nn.AvgPool2d(kernel_size=2), 201 | 202 | nn.Flatten(), 203 | 204 | nn.Linear(192, 10), 205 | ) 206 | 207 | Approx_LeNet1.to(device) 208 | train_net(Approx_LeNet1, 15, device) 209 | acc = test_net(Approx_LeNet1, device) 210 | accuracies.append(acc) 211 | 212 | m = np.array(accuracies) 213 | log.info(f"Results for approximated LeNet-1 (single square):") 214 | log.info(f"Mean accuracy on test set: {np.mean(m)}") 215 | log.info(f"Var: {np.var(m)}") 216 | 217 | # Optional: save the last trained approximated LeNet-1: 218 | torch.save(Approx_LeNet1, "LeNet1_Approx_single_square.pt") 219 | 220 | # Approximated LeNet-1 (single square) - the one saved and used by the encrypted processing 221 | model = torch.load("LeNet1_Approx_single_square.pt") 222 | model.eval() 223 | model.to(device) 224 | acc = test_net(model, device) 225 | log.info(f"Results for approximated LeNet-1 (single square) - the one saved to file: {acc}") 226 | 227 | 228 | 229 | 230 | 231 | -------------------------------------------------------------------------------- /Experiments_MNIST/EncryptedProcessing.py: -------------------------------------------------------------------------------- 1 | from functools import reduce 2 | 3 | from Pyfhel import Pyfhel 4 | 5 | import torch 6 | import torchvision 7 | import torchvision.transforms as transforms 8 | import torch.nn as nn 9 | 10 | import numpy as np 11 | 12 | import logging as log 13 | log.basicConfig(filename='experiments.log', 14 | format='%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s', 15 | datefmt='%Y-%m-%d %H:%M:%S', encoding='utf-8', level=log.DEBUG) 16 | 17 | transform = transforms.ToTensor() 18 | 19 | test_set = torchvision.datasets.MNIST( 20 | root='./data', 21 | train=False, 22 | download=True, 23 | transform=transform 24 | ) 25 | 26 | model_file = "../LeNet1_Approx_single_square.pt" 27 | log.info(f"Loading model from file {model_file}...") 28 | 29 | 30 | class Square(nn.Module): 31 | def __init__(self): 32 | super().__init__() 33 | 34 | def forward(self, t): 35 | return torch.pow(t, 2) 36 | 37 | model = torch.load(model_file) 38 | model.eval() 39 | 40 | log.info(model) 41 | 42 | 43 | # Code for matrix encoding/encryption 44 | def encode_matrix(HE, matrix): 45 | try: 46 | return np.array(list(map(HE.encodeFrac, matrix))) 47 | except TypeError: 48 | return np.array([encode_matrix(HE, m) for m in matrix]) 49 | 50 | 51 | def decode_matrix(HE, matrix): 52 | try: 53 | return np.array(list(map(HE.decodeFrac, matrix))) 54 | except TypeError: 55 | return np.array([decode_matrix(HE, m) for m in matrix]) 56 | 57 | 58 | def encrypt_matrix(HE, matrix): 59 | try: 60 | return np.array(list(map(HE.encryptFrac, matrix))) 61 | except TypeError: 62 | return np.array([encrypt_matrix(HE, m) for m in matrix]) 63 | 64 | 65 | def decrypt_matrix(HE, matrix): 66 | try: 67 | return np.array(list(map(HE.decryptFrac, matrix))) 68 | except TypeError: 69 | return np.array([decrypt_matrix(HE, m) for m in matrix]) 70 | 71 | 72 | # Code for encoded CNN 73 | class ConvolutionalLayer: 74 | def __init__(self, HE, weights, stride=(1, 1), padding=(0, 0), bias=None): 75 | self.HE = HE 76 | self.weights = encode_matrix(HE, weights) 77 | self.stride = stride 78 | self.padding = padding 79 | self.bias = bias 80 | if bias is not None: 81 | self.bias = encode_matrix(HE, bias) 82 | 83 | def __call__(self, t): 84 | t = apply_padding(t, self.padding) 85 | result = np.array([[np.sum([convolute2d(image_layer, filter_layer, self.stride) 86 | for image_layer, filter_layer in zip(image, _filter)], axis=0) 87 | for _filter in self.weights] 88 | for image in t]) 89 | 90 | if self.bias is not None: 91 | return np.array([[layer + bias for layer, bias in zip(image, self.bias)] for image in result]) 92 | else: 93 | return result 94 | 95 | 96 | def convolute2d(image, filter_matrix, stride): 97 | x_d = len(image[0]) 98 | y_d = len(image) 99 | x_f = len(filter_matrix[0]) 100 | y_f = len(filter_matrix) 101 | 102 | y_stride = stride[0] 103 | x_stride = stride[1] 104 | 105 | x_o = ((x_d - x_f) // x_stride) + 1 106 | y_o = ((y_d - y_f) // y_stride) + 1 107 | 108 | def get_submatrix(matrix, x, y): 109 | index_row = y * y_stride 110 | index_column = x * x_stride 111 | return matrix[index_row: index_row + y_f, index_column: index_column + x_f] 112 | 113 | return np.array( 114 | [[np.sum(get_submatrix(image, x, y) * filter_matrix) for x in range(0, x_o)] for y in range(0, y_o)]) 115 | 116 | def apply_padding(t, padding): 117 | y_p = padding[0] 118 | x_p = padding[1] 119 | zero = t[0][0][y_p+1][x_p+1] - t[0][0][y_p+1][x_p+1] 120 | return [[np.pad(mat, ((y_p, y_p), (x_p, x_p)), 'constant', constant_values=zero) for mat in layer] for layer in t] 121 | 122 | 123 | class LinearLayer: 124 | def __init__(self, HE, weights, bias=None): 125 | self.HE = HE 126 | self.weights = encode_matrix(HE, weights) 127 | self.bias = bias 128 | if bias is not None: 129 | self.bias = encode_matrix(HE, bias) 130 | 131 | def __call__(self, t): 132 | result = np.array([[np.sum(image * row) for row in self.weights] for image in t]) 133 | if self.bias is not None: 134 | result = np.array([row + self.bias for row in result]) 135 | return result 136 | 137 | 138 | class SquareLayer: 139 | def __init__(self, HE): 140 | self.HE = HE 141 | 142 | def __call__(self, image): 143 | return square(self.HE, image) 144 | 145 | 146 | def square(HE, image): 147 | try: 148 | return np.array(list(map(lambda x: HE.power(x, 2), image))) 149 | except TypeError: 150 | return np.array([square(HE, m) for m in image]) 151 | 152 | 153 | class FlattenLayer: 154 | def __call__(self, image): 155 | dimension = image.shape 156 | return image.reshape(dimension[0], dimension[1]*dimension[2]*dimension[3]) 157 | 158 | 159 | class AveragePoolLayer: 160 | def __init__(self, HE, kernel_size, stride=(1, 1), padding=(0, 0)): 161 | self.HE = HE 162 | self.kernel_size = kernel_size 163 | self.stride = stride 164 | self.padding = padding 165 | 166 | def __call__(self, t): 167 | t = apply_padding(t, self.padding) 168 | return np.array([[_avg(self.HE, layer, self.kernel_size, self.stride) for layer in image] for image in t]) 169 | 170 | 171 | def _avg(HE, image, kernel_size, stride): 172 | x_s = stride[1] 173 | y_s = stride[0] 174 | 175 | x_k = kernel_size[1] 176 | y_k = kernel_size[0] 177 | 178 | x_d = len(image[0]) 179 | y_d = len(image) 180 | 181 | x_o = ((x_d - x_k) // x_s) + 1 182 | y_o = ((y_d - y_k) // y_s) + 1 183 | 184 | denominator = HE.encodeFrac(1 / (x_k * y_k)) 185 | 186 | def get_submatrix(matrix, x, y): 187 | index_row = y * y_s 188 | index_column = x * x_s 189 | return matrix[index_row: index_row + y_k, index_column: index_column + x_k] 190 | 191 | return [[np.sum(get_submatrix(image, x, y)) * denominator for x in range(0, x_o)] for y in range(0, y_o)] 192 | 193 | 194 | # We can now define a function to "convert" a PyTorch model to a list of sequential HE-ready-to-be-used layers: 195 | def build_from_pytorch(HE, net): 196 | # Define builders for every possible layer 197 | 198 | def conv_layer(layer): 199 | if layer.bias is None: 200 | bias = None 201 | else: 202 | bias = layer.bias.detach().numpy() 203 | 204 | return ConvolutionalLayer(HE, weights=layer.weight.detach().numpy(), 205 | stride=layer.stride, 206 | padding=layer.padding, 207 | bias=bias) 208 | 209 | def lin_layer(layer): 210 | if layer.bias is None: 211 | bias = None 212 | else: 213 | bias = layer.bias.detach().numpy() 214 | return LinearLayer(HE, layer.weight.detach().numpy(), 215 | bias) 216 | 217 | def avg_pool_layer(layer): 218 | # This proxy is required because in PyTorch an AvgPool2d can have kernel_size, stride and padding either of 219 | # type (int, int) or int, unlike in Conv2d 220 | kernel_size = (layer.kernel_size, layer.kernel_size) if isinstance(layer.kernel_size, int) else layer.kernel_size 221 | stride = (layer.stride, layer.stride) if isinstance(layer.stride, int) else layer.stride 222 | padding = (layer.padding, layer.padding) if isinstance(layer.padding, int) else layer.padding 223 | 224 | return AveragePoolLayer(HE, kernel_size, stride, padding) 225 | 226 | def flatten_layer(layer): 227 | return FlattenLayer() 228 | 229 | def square_layer(layer): 230 | return SquareLayer(HE) 231 | 232 | # Maps every PyTorch layer type to the correct builder 233 | options = {"Conv": conv_layer, 234 | "Line": lin_layer, 235 | "Flat": flatten_layer, 236 | "AvgP": avg_pool_layer, 237 | "Squa": square_layer 238 | } 239 | 240 | encoded_layers = [options[str(layer)[0:4]](layer) for layer in net] 241 | return encoded_layers 242 | 243 | 244 | log.info(f"Run the experiments...") 245 | 246 | from joblib import Parallel, delayed, parallel_backend 247 | import time 248 | n_threads = 8 249 | 250 | log.info(f"I will use {n_threads} threads.") 251 | 252 | p = 953983721 253 | m = 4096 254 | 255 | log.info(f"Using encryption parameters: m = {m}, p = {p}") 256 | 257 | HE = Pyfhel() 258 | HE.contextGen(p=p, m=m) 259 | HE.keyGen() 260 | relinKeySize=3 261 | HE.relinKeyGen(bitCount=5, size=relinKeySize) 262 | 263 | model.to("cpu") 264 | model_encoded = build_from_pytorch(HE, model) 265 | 266 | 267 | experiment_loader = torch.utils.data.DataLoader( 268 | test_set, 269 | batch_size=n_threads, 270 | shuffle=True 271 | ) 272 | 273 | 274 | def enc_and_process(image): 275 | encrypted_image = encrypt_matrix(HE, image.unsqueeze(0).numpy()) 276 | 277 | for layer in model_encoded: 278 | encrypted_image = layer(encrypted_image) 279 | 280 | result = decrypt_matrix(HE, encrypted_image) 281 | return result 282 | 283 | 284 | def check_net(): 285 | total_correct = 0 286 | n_batch = 0 287 | 288 | for batch in experiment_loader: 289 | images, labels = batch 290 | with parallel_backend('multiprocessing'): 291 | preds = Parallel(n_jobs=n_threads)(delayed(enc_and_process)(image) for image in images) 292 | 293 | preds = reduce(lambda x, y: np.concatenate((x, y)), preds) 294 | preds = torch.Tensor(preds) 295 | 296 | for image in preds: 297 | for value in image: 298 | if value > 100000: 299 | log.warning("WARNING: probably you are running out of NB.") 300 | 301 | total_correct += preds.argmax(dim=1).eq(labels).sum().item() 302 | n_batch = n_batch + 1 303 | if n_batch % 5 == 0 or n_batch == 1: 304 | log.info(f"Done {n_batch} batches.") 305 | log.info(f"This means we processed {n_threads * n_batch} images.") 306 | log.info(f"Correct images for now: {total_correct}") 307 | log.info("---------------------------") 308 | 309 | return total_correct 310 | 311 | 312 | starting_time = time.time() 313 | 314 | log.info(f"Start experiment...") 315 | 316 | correct = check_net() 317 | 318 | total_time = time.time() - starting_time 319 | log.info(f"Total corrects on the entire test set: {correct}") 320 | log.info("Time: ", total_time) 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | -------------------------------------------------------------------------------- /Experiments_MNIST/LeNet1.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/Experiments_MNIST/LeNet1.pt -------------------------------------------------------------------------------- /Experiments_MNIST/LeNet1_Approx_single_square.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/Experiments_MNIST/LeNet1_Approx_single_square.pt -------------------------------------------------------------------------------- /Experiments_MNIST/LeNet1_single_tanh.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/Experiments_MNIST/LeNet1_single_tanh.pt -------------------------------------------------------------------------------- /Experiments_MNIST/MemoryAndTime/.directory: -------------------------------------------------------------------------------- 1 | [Dolphin] 2 | HeaderColumnWidths=495,114,235 3 | Timestamp=2021,7,22,10,58,4.162 4 | Version=4 5 | ViewMode=1 6 | -------------------------------------------------------------------------------- /Experiments_MNIST/MemoryAndTime/Encrypted/experiment.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from Pyfhel import Pyfhel 4 | 5 | import torch 6 | import torchvision 7 | import torchvision.transforms as transforms 8 | import torch.nn as nn 9 | 10 | import numpy as np 11 | 12 | import logging as log 13 | 14 | from memory_profiler import profile 15 | 16 | log.basicConfig(filename='experiments.log', 17 | format='%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s', 18 | datefmt='%Y-%m-%d %H:%M:%S', encoding='utf-8', level=log.DEBUG) 19 | 20 | # Clear processing 21 | 22 | # Encrypted processing 23 | 24 | transform = transforms.ToTensor() 25 | 26 | test_set = torchvision.datasets.MNIST( 27 | root='../data', 28 | train=False, 29 | download=True, 30 | transform=transform 31 | ) 32 | 33 | class Square(nn.Module): 34 | def __init__(self): 35 | super().__init__() 36 | 37 | def forward(self, t): 38 | return torch.pow(t, 2) 39 | 40 | model_file = "../LeNet1_Approx_single_square.pt" 41 | model = torch.load(model_file) 42 | model.eval() 43 | 44 | # Code for matrix encoding/encryption 45 | def encode_matrix(HE, matrix): 46 | try: 47 | return np.array(list(map(HE.encodeFrac, matrix))) 48 | except TypeError: 49 | return np.array([encode_matrix(HE, m) for m in matrix]) 50 | 51 | 52 | def decode_matrix(HE, matrix): 53 | try: 54 | return np.array(list(map(HE.decodeFrac, matrix))) 55 | except TypeError: 56 | return np.array([decode_matrix(HE, m) for m in matrix]) 57 | 58 | 59 | def encrypt_matrix(HE, matrix): 60 | try: 61 | return np.array(list(map(HE.encryptFrac, matrix))) 62 | except TypeError: 63 | return np.array([encrypt_matrix(HE, m) for m in matrix]) 64 | 65 | 66 | def decrypt_matrix(HE, matrix): 67 | try: 68 | return np.array(list(map(HE.decryptFrac, matrix))) 69 | except TypeError: 70 | return np.array([decrypt_matrix(HE, m) for m in matrix]) 71 | 72 | 73 | # Code for encoded CNN 74 | class ConvolutionalLayer: 75 | def __init__(self, HE, weights, stride=(1, 1), padding=(0, 0), bias=None): 76 | self.HE = HE 77 | self.weights = encode_matrix(HE, weights) 78 | self.stride = stride 79 | self.padding = padding 80 | self.bias = bias 81 | if bias is not None: 82 | self.bias = encode_matrix(HE, bias) 83 | 84 | def __call__(self, t): 85 | t = apply_padding(t, self.padding) 86 | result = np.array([[np.sum([convolute2d(image_layer, filter_layer, self.stride) 87 | for image_layer, filter_layer in zip(image, _filter)], axis=0) 88 | for _filter in self.weights] 89 | for image in t]) 90 | 91 | if self.bias is not None: 92 | return np.array([[layer + bias for layer, bias in zip(image, self.bias)] for image in result]) 93 | else: 94 | return result 95 | 96 | 97 | def convolute2d(image, filter_matrix, stride): 98 | x_d = len(image[0]) 99 | y_d = len(image) 100 | x_f = len(filter_matrix[0]) 101 | y_f = len(filter_matrix) 102 | 103 | y_stride = stride[0] 104 | x_stride = stride[1] 105 | 106 | x_o = ((x_d - x_f) // x_stride) + 1 107 | y_o = ((y_d - y_f) // y_stride) + 1 108 | 109 | def get_submatrix(matrix, x, y): 110 | index_row = y * y_stride 111 | index_column = x * x_stride 112 | return matrix[index_row: index_row + y_f, index_column: index_column + x_f] 113 | 114 | return np.array( 115 | [[np.sum(get_submatrix(image, x, y) * filter_matrix) for x in range(0, x_o)] for y in range(0, y_o)]) 116 | 117 | def apply_padding(t, padding): 118 | y_p = padding[0] 119 | x_p = padding[1] 120 | zero = t[0][0][y_p+1][x_p+1] - t[0][0][y_p+1][x_p+1] 121 | return [[np.pad(mat, ((y_p, y_p), (x_p, x_p)), 'constant', constant_values=zero) for mat in layer] for layer in t] 122 | 123 | 124 | class LinearLayer: 125 | def __init__(self, HE, weights, bias=None): 126 | self.HE = HE 127 | self.weights = encode_matrix(HE, weights) 128 | self.bias = bias 129 | if bias is not None: 130 | self.bias = encode_matrix(HE, bias) 131 | 132 | def __call__(self, t): 133 | result = np.array([[np.sum(image * row) for row in self.weights] for image in t]) 134 | if self.bias is not None: 135 | result = np.array([row + self.bias for row in result]) 136 | return result 137 | 138 | 139 | class SquareLayer: 140 | def __init__(self, HE): 141 | self.HE = HE 142 | 143 | def __call__(self, image): 144 | return square(self.HE, image) 145 | 146 | 147 | def square(HE, image): 148 | try: 149 | return np.array(list(map(lambda x: HE.power(x, 2), image))) 150 | except TypeError: 151 | return np.array([square(HE, m) for m in image]) 152 | 153 | 154 | class FlattenLayer: 155 | def __call__(self, image): 156 | dimension = image.shape 157 | return image.reshape(dimension[0], dimension[1]*dimension[2]*dimension[3]) 158 | 159 | 160 | class AveragePoolLayer: 161 | def __init__(self, HE, kernel_size, stride=(1, 1), padding=(0, 0)): 162 | self.HE = HE 163 | self.kernel_size = kernel_size 164 | self.stride = stride 165 | self.padding = padding 166 | 167 | def __call__(self, t): 168 | t = apply_padding(t, self.padding) 169 | return np.array([[_avg(self.HE, layer, self.kernel_size, self.stride) for layer in image] for image in t]) 170 | 171 | 172 | def _avg(HE, image, kernel_size, stride): 173 | x_s = stride[1] 174 | y_s = stride[0] 175 | 176 | x_k = kernel_size[1] 177 | y_k = kernel_size[0] 178 | 179 | x_d = len(image[0]) 180 | y_d = len(image) 181 | 182 | x_o = ((x_d - x_k) // x_s) + 1 183 | y_o = ((y_d - y_k) // y_s) + 1 184 | 185 | denominator = HE.encodeFrac(1 / (x_k * y_k)) 186 | 187 | def get_submatrix(matrix, x, y): 188 | index_row = y * y_s 189 | index_column = x * x_s 190 | return matrix[index_row: index_row + y_k, index_column: index_column + x_k] 191 | 192 | return [[np.sum(get_submatrix(image, x, y)) * denominator for x in range(0, x_o)] for y in range(0, y_o)] 193 | 194 | 195 | # We can now define a function to "convert" a PyTorch model to a list of sequential HE-ready-to-be-used layers: 196 | def build_from_pytorch(HE, net): 197 | # Define builders for every possible layer 198 | 199 | def conv_layer(layer): 200 | if layer.bias is None: 201 | bias = None 202 | else: 203 | bias = layer.bias.detach().numpy() 204 | 205 | return ConvolutionalLayer(HE, weights=layer.weight.detach().numpy(), 206 | stride=layer.stride, 207 | padding=layer.padding, 208 | bias=bias) 209 | 210 | def lin_layer(layer): 211 | if layer.bias is None: 212 | bias = None 213 | else: 214 | bias = layer.bias.detach().numpy() 215 | return LinearLayer(HE, layer.weight.detach().numpy(), 216 | bias) 217 | 218 | def avg_pool_layer(layer): 219 | # This proxy is required because in PyTorch an AvgPool2d can have kernel_size, stride and padding either of 220 | # type (int, int) or int, unlike in Conv2d 221 | kernel_size = (layer.kernel_size, layer.kernel_size) if isinstance(layer.kernel_size, int) else layer.kernel_size 222 | stride = (layer.stride, layer.stride) if isinstance(layer.stride, int) else layer.stride 223 | padding = (layer.padding, layer.padding) if isinstance(layer.padding, int) else layer.padding 224 | 225 | return AveragePoolLayer(HE, kernel_size, stride, padding) 226 | 227 | def flatten_layer(layer): 228 | return FlattenLayer() 229 | 230 | def square_layer(layer): 231 | return SquareLayer(HE) 232 | 233 | # Maps every PyTorch layer type to the correct builder 234 | options = {"Conv": conv_layer, 235 | "Line": lin_layer, 236 | "Flat": flatten_layer, 237 | "AvgP": avg_pool_layer, 238 | "Squa": square_layer 239 | } 240 | 241 | encoded_layers = [options[str(layer)[0:4]](layer) for layer in net] 242 | return encoded_layers 243 | 244 | 245 | p = 953983721 246 | m = 4096 247 | 248 | log.info(f"Using encryption parameters: m = {m}, p = {p}") 249 | 250 | 251 | 252 | experiment_loader = torch.utils.data.DataLoader( 253 | test_set, 254 | batch_size=1, 255 | shuffle=True 256 | ) 257 | 258 | 259 | def enc_and_process(image, HE, model_encoded): 260 | encrypted_image = encrypt_matrix(HE, image.unsqueeze(0).numpy()) 261 | 262 | for layer in model_encoded: 263 | encrypted_image = layer(encrypted_image) 264 | 265 | result = decrypt_matrix(HE, encrypted_image) 266 | return result 267 | 268 | @profile 269 | def experiment_encrypted(): 270 | HE = Pyfhel() 271 | HE.contextGen(p=p, m=m) 272 | HE.keyGen() 273 | relinKeySize = 3 274 | HE.relinKeyGen(bitCount=5, size=relinKeySize) 275 | 276 | model.to("cpu") 277 | model_encoded = build_from_pytorch(HE, model) 278 | 279 | for batch in experiment_loader: 280 | image, labels = batch 281 | preds = enc_and_process(image[0], HE, model_encoded) 282 | return 283 | 284 | 285 | if __name__ == '__main__': 286 | 287 | log.info("Starting experiment...") 288 | starting_time = time.time() 289 | experiment_encrypted() 290 | total_time = time.time() - starting_time 291 | log.info(f"The encrypted processing of one image required {total_time}") 292 | 293 | 294 | 295 | -------------------------------------------------------------------------------- /Experiments_MNIST/MemoryAndTime/Plain/experiment.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from Pyfhel import Pyfhel 4 | 5 | import torch 6 | import torchvision 7 | import torchvision.transforms as transforms 8 | import torch.nn as nn 9 | 10 | import numpy as np 11 | 12 | import logging as log 13 | 14 | from memory_profiler import profile 15 | 16 | device = 'cpu' 17 | 18 | log.basicConfig(filename='experiments.log', 19 | format='%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s', 20 | datefmt='%Y-%m-%d %H:%M:%S', encoding='utf-8', level=log.DEBUG) 21 | 22 | class Square(nn.Module): 23 | def __init__(self): 24 | super().__init__() 25 | 26 | def forward(self, t): 27 | return torch.pow(t, 2) 28 | 29 | transform = transforms.ToTensor() 30 | 31 | test_set = torchvision.datasets.MNIST( 32 | root='../../data', 33 | train=False, 34 | download=True, 35 | transform=transform 36 | ) 37 | 38 | test_loader = torch.utils.data.DataLoader( 39 | test_set, 40 | batch_size=1, 41 | shuffle=True 42 | ) 43 | 44 | def forward_one_image(network, device): 45 | network.eval() 46 | 47 | with torch.no_grad(): 48 | for batch in test_loader: # Get Batch 49 | images, labels = batch 50 | images, labels = images.to(device), labels.to(device) 51 | 52 | preds = network(images) # Pass Batch 53 | return 54 | 55 | 56 | lenet1 = torch.load("../../LeNet1_Approx_single_square.pt") 57 | lenet1.eval() 58 | lenet1.to(device) 59 | 60 | lenet1_singletanh = torch.load("../../LeNet1_single_tanh.pt") 61 | lenet1_singletanh.eval() 62 | lenet1_singletanh.to(device) 63 | 64 | lenet1_singlesquare = torch.load("../../LeNet1_single_tanh.pt") 65 | lenet1_singlesquare.eval() 66 | lenet1_singlesquare.to(device) 67 | 68 | n_experiments = 1000 69 | 70 | 71 | # @profile 72 | def experiment_LeNet1(): 73 | for i in range(0, n_experiments): 74 | forward_one_image(lenet1, device) 75 | 76 | 77 | # @profile 78 | def experiment_LeNet1_singletanh(): 79 | for i in range(0, n_experiments): 80 | forward_one_image(lenet1_singletanh, device) 81 | 82 | 83 | # @profile 84 | def experiment_LeNet1_singlesquare(): 85 | for i in range(0, n_experiments): 86 | forward_one_image(lenet1_singlesquare, device) 87 | 88 | 89 | if __name__ == '__main__': 90 | 91 | log.info("Starting experiment...") 92 | starting_time = time.time() 93 | experiment_LeNet1() 94 | t1 = time.time() 95 | log.info(f"The processing of one image for LeNet-1 required {(t1-starting_time)/n_experiments} seconds") 96 | experiment_LeNet1_singletanh() 97 | t2 = time.time() 98 | log.info(f"The processing of one image for LeNet-1 (single tanh) required {(t2-t1)/n_experiments} seconds") 99 | experiment_LeNet1_singlesquare() 100 | t3 = time.time() 101 | log.info(f"The processing of one image for approx LeNet-1 (single square) required {(t3-t2)/n_experiments} seconds") 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /Experiments_MNIST/ModelExperiments.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | import torchvision.transforms as transforms 4 | import torch.nn as nn 5 | import torch.nn.functional as F 6 | import torch.optim as optim 7 | 8 | import numpy as np 9 | 10 | import logging as log 11 | log.basicConfig(filename='ModelExperiments.log', 12 | format='%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s', 13 | datefmt='%Y-%m-%d %H:%M:%S', encoding='utf-8', level=log.DEBUG) 14 | 15 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 16 | 17 | transform = transforms.ToTensor() 18 | 19 | train_set = torchvision.datasets.MNIST( 20 | root = './data', 21 | train=True, 22 | download=True, 23 | transform=transform 24 | ) 25 | 26 | test_set = torchvision.datasets.MNIST( 27 | root = './data', 28 | train=False, 29 | download=True, 30 | transform=transform 31 | ) 32 | 33 | train_loader = torch.utils.data.DataLoader( 34 | train_set, 35 | batch_size=50, 36 | shuffle=True 37 | ) 38 | 39 | test_loader = torch.utils.data.DataLoader( 40 | test_set, 41 | batch_size=50, 42 | shuffle=True 43 | ) 44 | 45 | class Square(nn.Module): 46 | def __init__(self): 47 | super().__init__() 48 | 49 | def forward(self, t): 50 | return torch.pow(t, 2) 51 | 52 | 53 | def get_num_correct(preds, labels): 54 | return preds.argmax(dim=1).eq(labels).sum().item() 55 | 56 | 57 | def train_net(network, epochs, device): 58 | optimizer = optim.Adam(network.parameters(), lr=0.001) 59 | for epoch in range(epochs): 60 | 61 | total_loss = 0 62 | total_correct = 0 63 | 64 | for batch in train_loader: # Get Batch 65 | images, labels = batch 66 | images, labels = images.to(device), labels.to(device) 67 | 68 | preds = network(images) # Pass Batch 69 | loss = F.cross_entropy(preds, labels) # Calculate Loss 70 | 71 | optimizer.zero_grad() 72 | loss.backward() # Calculate Gradients 73 | optimizer.step() # Update Weights 74 | 75 | total_loss += loss.item() 76 | total_correct += get_num_correct(preds, labels) 77 | 78 | 79 | def test_net(network, device): 80 | network.eval() 81 | total_loss = 0 82 | total_correct = 0 83 | 84 | with torch.no_grad(): 85 | for batch in test_loader: # Get Batch 86 | images, labels = batch 87 | images, labels = images.to(device), labels.to(device) 88 | 89 | preds = network(images) # Pass Batch 90 | loss = F.cross_entropy(preds, labels) # Calculate Loss 91 | 92 | total_loss += loss.item() 93 | total_correct += get_num_correct(preds, labels) 94 | 95 | accuracy = round(100. * (total_correct / len(test_loader.dataset)), 4) 96 | 97 | return total_correct / len(test_loader.dataset) 98 | 99 | 100 | experiments = 10 101 | 102 | # Initial LeNet-1 103 | accuracies = [] 104 | for i in range(0, experiments): 105 | LeNet1 = nn.Sequential( 106 | nn.Conv2d(1, 4, kernel_size=5), 107 | nn.Tanh(), 108 | nn.AvgPool2d(kernel_size=2), 109 | 110 | nn.Conv2d(4, 12, kernel_size=5), 111 | nn.Tanh(), 112 | nn.AvgPool2d(kernel_size=2), 113 | 114 | nn.Flatten(), 115 | 116 | nn.Linear(192, 10), 117 | ) 118 | 119 | LeNet1.to(device) 120 | train_net(LeNet1, 15, device) 121 | acc = test_net(LeNet1, device) 122 | accuracies.append(acc) 123 | 124 | m = np.array(accuracies) 125 | log.info(f"Results for LeNet-1:") 126 | log.info(f"Mean accuracy on test set: {np.mean(m)}") 127 | log.info(f"Var: {np.var(m)}") 128 | 129 | # Optional: save the last trained LeNet-1: 130 | torch.save(LeNet1, "LeNet1.pt") 131 | 132 | 133 | # LeNet-1 with a single tanh 134 | accuracies = [] 135 | for i in range(0, experiments): 136 | LeNet1_singletanh = nn.Sequential( 137 | nn.Conv2d(1, 4, kernel_size=5), 138 | nn.Tanh(), 139 | nn.AvgPool2d(kernel_size=2), 140 | 141 | nn.Conv2d(4, 12, kernel_size=5), 142 | # nn.Tanh(), 143 | nn.AvgPool2d(kernel_size=2), 144 | 145 | nn.Flatten(), 146 | 147 | nn.Linear(192, 10), 148 | ) 149 | 150 | LeNet1_singletanh.to(device) 151 | train_net(LeNet1_singletanh, 15, device) 152 | acc = test_net(LeNet1_singletanh, device) 153 | accuracies.append(acc) 154 | 155 | m = np.array(accuracies) 156 | log.info(f"Results for LeNet-1 (single tanh):") 157 | log.info(f"Mean accuracy on test set: {np.mean(m)}") 158 | log.info(f"Var: {np.var(m)}") 159 | 160 | # Optional: save the last trained LeNet-1 (single tanh): 161 | torch.save(LeNet1_singletanh, "LeNet1_single_tanh.pt") 162 | 163 | 164 | # Approximated LeNet-1 (single square) 165 | accuracies = [] 166 | for i in range(0, experiments): 167 | Approx_LeNet1 = nn.Sequential( 168 | nn.Conv2d(1, 4, kernel_size=5), 169 | Square(), 170 | nn.AvgPool2d(kernel_size=2), 171 | 172 | nn.Conv2d(4, 12, kernel_size=5), 173 | # nn.Tanh(), 174 | nn.AvgPool2d(kernel_size=2), 175 | 176 | nn.Flatten(), 177 | 178 | nn.Linear(192, 10), 179 | ) 180 | 181 | Approx_LeNet1.to(device) 182 | train_net(Approx_LeNet1, 15, device) 183 | acc = test_net(Approx_LeNet1, device) 184 | accuracies.append(acc) 185 | 186 | m = np.array(accuracies) 187 | log.info(f"Results for approximated LeNet-1 (single square):") 188 | log.info(f"Mean accuracy on test set: {np.mean(m)}") 189 | log.info(f"Var: {np.var(m)}") 190 | 191 | # Optional: save the last trained approximated LeNet-1: 192 | torch.save(Approx_LeNet1, "LeNet1_Approx_single_square.pt") 193 | 194 | # Approximated LeNet-1 (single square) - the one saved and used by the encrypted processing 195 | model = torch.load("LeNet1_Approx_single_square.pt") 196 | model.eval() 197 | model.to(device) 198 | acc = test_net(model, device) 199 | log.info(f"Results for approximated LeNet-1 (single square) - the one saved to file: {acc}") 200 | 201 | 202 | 203 | 204 | 205 | -------------------------------------------------------------------------------- /HE-ML/HE-ML.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "37db485d-fcec-4c20-943e-239a74a70cf4", 6 | "metadata": {}, 7 | "source": [ 8 | "# Homomorphic Encrypted LeNet-1\n", 9 | "This notebook will show a very practical example of running the famous LeNet-1 DL model directly on encrypted data.\n", 10 | "\n", 11 | "![scheme](HE_processing.png)" 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "id": "c1b6b3e7-d428-48e4-b9db-3609ceb94b25", 17 | "metadata": {}, 18 | "source": [ 19 | "## Homomorphic encryption operations\n", 20 | "First of all, we will look at Pyfhel, a Python library which wraps SEAL, one of the most used frameworks for HE.\n", 21 | "Pyfhel supports the BFV scheme, so, it is the one that we will use." 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 1, 27 | "id": "7c74dbc3-b890-4a87-9e8e-0094b9fc17b4", 28 | "metadata": {}, 29 | "outputs": [ 30 | { 31 | "name": "stdout", 32 | "output_type": "stream", 33 | "text": [ 34 | "\n", 35 | "Expected sum: 125.028207448, decrypted sum: 125.02820744784549\n", 36 | "Expected sub: 129.286137812, decrypted sum: 129.28613781183958\n", 37 | "Expected mul: -270.7131931708334, decrypted sum: -270.7131931686308\n" 38 | ] 39 | } 40 | ], 41 | "source": [ 42 | "from Pyfhel import Pyfhel, PyPtxt, PyCtxt\n", 43 | "\n", 44 | "HE = Pyfhel()\n", 45 | "HE.contextGen(p=65537, m=4096)\n", 46 | "HE.keyGen()\n", 47 | "\n", 48 | "print(HE)\n", 49 | "\n", 50 | "a = 127.15717263\n", 51 | "b = -2.128965182\n", 52 | "ctxt1 = HE.encryptFrac(a)\n", 53 | "ctxt2 = HE.encryptFrac(b)\n", 54 | "\n", 55 | "ctxtSum = ctxt1 + ctxt2\n", 56 | "ctxtSub = ctxt1 - ctxt2\n", 57 | "ctxtMul = ctxt1 * ctxt2\n", 58 | "\n", 59 | "resSum = HE.decryptFrac(ctxtSum)\n", 60 | "resSub = HE.decryptFrac(ctxtSub) \n", 61 | "resMul = HE.decryptFrac(ctxtMul)\n", 62 | "\n", 63 | "print(f\"Expected sum: {a+b}, decrypted sum: {resSum}\")\n", 64 | "print(f\"Expected sub: {a-b}, decrypted sum: {resSub}\")\n", 65 | "print(f\"Expected mul: {a*b}, decrypted sum: {resMul}\")" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": 2, 71 | "id": "274ad1bb-c26e-4f29-9301-997c04f43bcb", 72 | "metadata": {}, 73 | "outputs": [ 74 | { 75 | "name": "stdout", 76 | "output_type": "stream", 77 | "text": [ 78 | "82\n" 79 | ] 80 | } 81 | ], 82 | "source": [ 83 | "m1 = HE.encryptFrac(a)\n", 84 | "print(HE.noiseLevel(m1))\n", 85 | "\n", 86 | "m2 = HE.encodeFrac(2)" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": 3, 92 | "id": "a09497da-60be-4c1f-9d2a-9bd2b9390e1e", 93 | "metadata": {}, 94 | "outputs": [ 95 | { 96 | "name": "stdout", 97 | "output_type": "stream", 98 | "text": [ 99 | "81\n", 100 | "54\n", 101 | "82\n", 102 | "82\n" 103 | ] 104 | } 105 | ], 106 | "source": [ 107 | "print(HE.noiseLevel(m1+m1))\n", 108 | "print(HE.noiseLevel(m1*m1))\n", 109 | "print(HE.noiseLevel(m1+m2))\n", 110 | "print(HE.noiseLevel(m1*m2))" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "id": "ba6d0840-41a8-417a-97db-faab642806d6", 116 | "metadata": {}, 117 | "source": [ 118 | "Before starting, let's note that:\n", 119 | " 1. We will use the fractional encoder to encode (and encrypt) the values in our examples. BFV was born for integers, so, CKKS should be used if the use case involves fractional values. However it is a more complex scheme, and for this example BFV is sufficient.\n", 120 | " 2. We will not use batching (also called *packing*). While batching can greatly speed up the computations, it introduces limitations which make the encrypted ML much more complex. For this example, we will encrypt/encode each number with a polynomial." 121 | ] 122 | }, 123 | { 124 | "cell_type": "markdown", 125 | "id": "1180b17e-7644-4ce8-99a6-c1906758e2eb", 126 | "metadata": {}, 127 | "source": [ 128 | "## LeNet-1\n", 129 | "The LeNet-1 is a small CNN developed by LeCun et al. It is composed of 5 layers: a convolutional layer with 4 kernels of size 5x5 and tanh activation, an average pooling layer with kernel of size 2, another convolutional layer with 16 kernels of size 5x5 and tanh activation, another average pooling layer with kernel of size 2, and a fully connected layers with size 192x10. \n", 130 | "\n", 131 | "The highest value in the output tensor corresponds to the label LeNet-1 associated to the input image. \n", 132 | "\n", 133 | "For this tutorial we will use the MNIST dataset." 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": 4, 139 | "id": "1fc122e5-a460-4a73-bed7-012b1105081b", 140 | "metadata": {}, 141 | "outputs": [], 142 | "source": [ 143 | "import torch\n", 144 | "import torchvision\n", 145 | "import torchvision.transforms as transforms\n", 146 | "import torch.nn as nn\n", 147 | "import torch.nn.functional as F\n", 148 | "import torch.optim as optim\n", 149 | "\n", 150 | "import numpy as np" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": 5, 156 | "id": "89940a02-261f-4fc7-98e7-cea0a70be181", 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [ 160 | "device = 'cuda' if torch.cuda.is_available() else 'cpu'" 161 | ] 162 | }, 163 | { 164 | "cell_type": "code", 165 | "execution_count": 6, 166 | "id": "8fcf92dc-4929-4adc-b97e-2a1bf946bfeb", 167 | "metadata": {}, 168 | "outputs": [], 169 | "source": [ 170 | "transform = transforms.ToTensor()\n", 171 | "\n", 172 | "train_set = torchvision.datasets.MNIST(\n", 173 | " root = './data',\n", 174 | " train=True,\n", 175 | " download=True,\n", 176 | " transform=transform\n", 177 | ")\n", 178 | "\n", 179 | "test_set = torchvision.datasets.MNIST(\n", 180 | " root = './data',\n", 181 | " train=False,\n", 182 | " download=True,\n", 183 | " transform=transform\n", 184 | ")\n", 185 | "\n", 186 | "train_loader = torch.utils.data.DataLoader(\n", 187 | " train_set,\n", 188 | " batch_size=50,\n", 189 | " shuffle=True\n", 190 | ")\n", 191 | "\n", 192 | "test_loader = torch.utils.data.DataLoader(\n", 193 | " test_set,\n", 194 | " batch_size=50,\n", 195 | " shuffle=True\n", 196 | ")" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": 7, 202 | "id": "6641a660-43a4-4ed4-895a-ac7d1841dec4", 203 | "metadata": {}, 204 | "outputs": [], 205 | "source": [ 206 | "def get_num_correct(preds, labels):\n", 207 | " return preds.argmax(dim=1).eq(labels).sum().item()\n", 208 | "\n", 209 | "def train_net(network, epochs, device):\n", 210 | " optimizer = optim.Adam(network.parameters(), lr=0.001)\n", 211 | " for epoch in range(epochs):\n", 212 | "\n", 213 | " total_loss = 0\n", 214 | " total_correct = 0\n", 215 | "\n", 216 | " for batch in train_loader: # Get Batch\n", 217 | " images, labels = batch \n", 218 | " images, labels = images.to(device), labels.to(device)\n", 219 | "\n", 220 | " preds = network(images) # Pass Batch\n", 221 | " loss = F.cross_entropy(preds, labels) # Calculate Loss\n", 222 | "\n", 223 | " optimizer.zero_grad()\n", 224 | " loss.backward() # Calculate Gradients\n", 225 | " optimizer.step() # Update Weights\n", 226 | "\n", 227 | " total_loss += loss.item()\n", 228 | " total_correct += get_num_correct(preds, labels)\n", 229 | "\n", 230 | " \n", 231 | "def test_net(network, device):\n", 232 | " network.eval()\n", 233 | " total_loss = 0\n", 234 | " total_correct = 0\n", 235 | " \n", 236 | " with torch.no_grad():\n", 237 | " for batch in test_loader: # Get Batch\n", 238 | " images, labels = batch \n", 239 | " images, labels = images.to(device), labels.to(device)\n", 240 | "\n", 241 | " preds = network(images) # Pass Batch\n", 242 | " loss = F.cross_entropy(preds, labels) # Calculate Loss\n", 243 | "\n", 244 | " total_loss += loss.item()\n", 245 | " total_correct += get_num_correct(preds, labels)\n", 246 | "\n", 247 | " accuracy = round(100. * (total_correct / len(test_loader.dataset)), 4)\n", 248 | "\n", 249 | " return total_correct / len(test_loader.dataset)" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": 8, 255 | "id": "c8f74067-df4d-4b70-ae4c-ec6970c8baa2", 256 | "metadata": {}, 257 | "outputs": [], 258 | "source": [ 259 | "train = True # If set to false, it will load models previously trained and saved." 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": 9, 265 | "id": "03a4f779-4a5f-4a56-94c1-dba733942fd0", 266 | "metadata": {}, 267 | "outputs": [], 268 | "source": [ 269 | "experiments = 1" 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": 10, 275 | "id": "f8480f18-5742-45a6-b4b7-5839b2f96763", 276 | "metadata": {}, 277 | "outputs": [], 278 | "source": [ 279 | "if train:\n", 280 | " accuracies = []\n", 281 | " for i in range(0, experiments):\n", 282 | " LeNet1 = nn.Sequential(\n", 283 | " nn.Conv2d(1, 4, kernel_size=5),\n", 284 | " nn.Tanh(),\n", 285 | " nn.AvgPool2d(kernel_size=2),\n", 286 | "\n", 287 | " nn.Conv2d(4, 12, kernel_size=5),\n", 288 | " nn.Tanh(),\n", 289 | " nn.AvgPool2d(kernel_size=2),\n", 290 | "\n", 291 | " nn.Flatten(),\n", 292 | "\n", 293 | " nn.Linear(192, 10),\n", 294 | " )\n", 295 | " \n", 296 | " LeNet1.to(device)\n", 297 | " train_net(LeNet1, 15, device)\n", 298 | " acc = test_net(LeNet1, device)\n", 299 | " accuracies.append(acc)\n", 300 | " \n", 301 | " torch.save(LeNet1, \"LeNet1.pt\")\n", 302 | "else:\n", 303 | " LeNet1 = torch.load(\"LeNet1.pt\")\n", 304 | " LeNet1.eval()\n", 305 | " LeNet1.to(device)" 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": 11, 311 | "id": "19fbca30-aa89-4d74-a476-3c9107fbf851", 312 | "metadata": {}, 313 | "outputs": [ 314 | { 315 | "name": "stdout", 316 | "output_type": "stream", 317 | "text": [ 318 | "Mean accuracy on test set: 0.9875\n", 319 | "Var: 0.0\n" 320 | ] 321 | } 322 | ], 323 | "source": [ 324 | "m = np.array(accuracies)\n", 325 | "print(f\"Mean accuracy on test set: {np.mean(m)}\")\n", 326 | "print(f\"Var: {np.var(m)}\")" 327 | ] 328 | }, 329 | { 330 | "cell_type": "markdown", 331 | "id": "025dfbc9-38d0-41a7-a140-9606bfeff53f", 332 | "metadata": {}, 333 | "source": [ 334 | "## Approximating\n", 335 | "As we know, there are some operations that cannot be performed homomorphically on encrypted values. Most notably, these operations are division and comparison. It is possible to perform only linear functions.\n", 336 | "\n", 337 | "Consequently, in the LeNet-1 scheme we used, we can not use `tanh()`. This is because we cannot apply its non-linearities.\n", 338 | "\n", 339 | "\n", 340 | "One of the most common approach is to replace it with a simple polynomial function, for example a square layer (which simply performs $x \\rightarrow x^2$).\n", 341 | "\n", 342 | "We define the model with all the non-linearities removed **approximated**. This model can be re-trained, and it will be ready to be used on encrypted values." 343 | ] 344 | }, 345 | { 346 | "cell_type": "code", 347 | "execution_count": 12, 348 | "id": "52ea6449-1108-446f-ba2d-8170edcb641f", 349 | "metadata": {}, 350 | "outputs": [], 351 | "source": [ 352 | "class Square(nn.Module):\n", 353 | " def __init__(self):\n", 354 | " super().__init__()\n", 355 | "\n", 356 | " def forward(self, t):\n", 357 | " return torch.pow(t, 2)\n", 358 | "\n", 359 | "LeNet1_Approx = nn.Sequential(\n", 360 | " nn.Conv2d(1, 4, kernel_size=5),\n", 361 | " Square(),\n", 362 | " nn.AvgPool2d(kernel_size=2),\n", 363 | " \n", 364 | " nn.Conv2d(4, 12, kernel_size=5),\n", 365 | " Square(),\n", 366 | " nn.AvgPool2d(kernel_size=2),\n", 367 | " \n", 368 | " nn.Flatten(),\n", 369 | " \n", 370 | " nn.Linear(192, 10),\n", 371 | ")" 372 | ] 373 | }, 374 | { 375 | "cell_type": "code", 376 | "execution_count": 13, 377 | "id": "1ccaff6b-7cd5-49d2-9b5c-e8a881005fc1", 378 | "metadata": {}, 379 | "outputs": [], 380 | "source": [ 381 | "if train:\n", 382 | " approx_accuracies = []\n", 383 | " for i in range(0, experiments):\n", 384 | " LeNet1_Approx = nn.Sequential(\n", 385 | " nn.Conv2d(1, 4, kernel_size=5),\n", 386 | " Square(),\n", 387 | " nn.AvgPool2d(kernel_size=2),\n", 388 | "\n", 389 | " nn.Conv2d(4, 12, kernel_size=5),\n", 390 | " Square(),\n", 391 | " nn.AvgPool2d(kernel_size=2),\n", 392 | "\n", 393 | " nn.Flatten(),\n", 394 | "\n", 395 | " nn.Linear(192, 10),\n", 396 | " )\n", 397 | " \n", 398 | " LeNet1_Approx.to(device)\n", 399 | " train_net(LeNet1_Approx, 15, device)\n", 400 | " acc = test_net(LeNet1_Approx, device)\n", 401 | " approx_accuracies.append(acc)\n", 402 | " \n", 403 | " torch.save(LeNet1, \"LeNet1_Approx.pt\")\n", 404 | "\n", 405 | "else:\n", 406 | " LeNet1_Approx = torch.load(\"LeNet1_Approx.pt\")\n", 407 | " LeNet1_Approx.eval()\n", 408 | " LeNet1_Approx.to(device)" 409 | ] 410 | }, 411 | { 412 | "cell_type": "code", 413 | "execution_count": 14, 414 | "id": "c5434fcf-e01c-40c8-9ea8-bc9086e8beef", 415 | "metadata": {}, 416 | "outputs": [ 417 | { 418 | "name": "stdout", 419 | "output_type": "stream", 420 | "text": [ 421 | "Mean: 0.9862\n", 422 | "Var: 0.0\n" 423 | ] 424 | } 425 | ], 426 | "source": [ 427 | "m = np.array(approx_accuracies)\n", 428 | "print(f\"Mean: {np.mean(m)}\")\n", 429 | "print(f\"Var: {np.var(m)}\")" 430 | ] 431 | }, 432 | { 433 | "cell_type": "markdown", 434 | "id": "5bb96de6-7053-40ce-bb88-a1532270683f", 435 | "metadata": {}, 436 | "source": [ 437 | "We can see that replacing `tanh()` with `square()` did not impact the accuracy of the model dramatically. Usually this is not the case, and approximating DL models may worsen the performance badly. This is one of the challenges that HE-ML will have to consider: the creation of DL models keeping in mind the HE constraints from the beginning.\n", 438 | "\n", 439 | "In any case, now the network is HE-compatible." 440 | ] 441 | }, 442 | { 443 | "cell_type": "markdown", 444 | "id": "874fb0fa-2122-46d3-8dcf-dc528959da8c", 445 | "metadata": {}, 446 | "source": [ 447 | "## Encoding\n", 448 | "From the applicative point of view, we have two options on how we want our Torch model to run on encrypted values:\n", 449 | " 1. Modify Torch layers code in order to be fully compatible with arrays of Pyfhel ciphertexts/encoded values;\n", 450 | " 2. Create the code for the general blocks of LeNet-1 (convolutional layer, linear layer, square layer, flatten...)\n", 451 | " \n", 452 | "We opt for the second path, having already done this in our previous work: https://github.com/AlexMV12/PyCrCNN\n", 453 | "\n", 454 | "Let's remember that, in order to be used with the encrypted values, also the weights of the models will have to be **encoded**. This means that each value in the weights of each layer will be encoded in a polynomial.\n", 455 | "\n", 456 | "First, we define some useful functions to encrypt/encode matrices:" 457 | ] 458 | }, 459 | { 460 | "cell_type": "code", 461 | "execution_count": 15, 462 | "id": "cf13fcad-5ac0-4a00-8ad6-43fbf0e731c2", 463 | "metadata": {}, 464 | "outputs": [], 465 | "source": [ 466 | "def encode_matrix(HE, matrix):\n", 467 | " try:\n", 468 | " return np.array(list(map(HE.encodeFrac, matrix)))\n", 469 | " except TypeError:\n", 470 | " return np.array([encode_matrix(HE, m) for m in matrix])\n", 471 | "\n", 472 | "\n", 473 | "def decode_matrix(HE, matrix):\n", 474 | " try:\n", 475 | " return np.array(list(map(HE.decodeFrac, matrix)))\n", 476 | " except TypeError:\n", 477 | " return np.array([decode_matrix(HE, m) for m in matrix])\n", 478 | "\n", 479 | "\n", 480 | "def encrypt_matrix(HE, matrix):\n", 481 | " try:\n", 482 | " return np.array(list(map(HE.encryptFrac, matrix)))\n", 483 | " except TypeError:\n", 484 | " return np.array([encrypt_matrix(HE, m) for m in matrix])\n", 485 | "\n", 486 | "\n", 487 | "def decrypt_matrix(HE, matrix):\n", 488 | " try:\n", 489 | " return np.array(list(map(HE.decryptFrac, matrix)))\n", 490 | " except TypeError:\n", 491 | " return np.array([decrypt_matrix(HE, m) for m in matrix])" 492 | ] 493 | }, 494 | { 495 | "cell_type": "markdown", 496 | "id": "921c4f49-c53e-484b-9f6a-2930aea4cba0", 497 | "metadata": {}, 498 | "source": [ 499 | "Then, the actual code for the convolutional, linear, square, flatten and average pooling layer is required:" 500 | ] 501 | }, 502 | { 503 | "cell_type": "code", 504 | "execution_count": 16, 505 | "id": "4dbe6430-31e5-4528-aa68-f7131cea4856", 506 | "metadata": {}, 507 | "outputs": [], 508 | "source": [ 509 | "class ConvolutionalLayer:\n", 510 | " def __init__(self, HE, weights, stride=(1, 1), padding=(0, 0), bias=None):\n", 511 | " self.HE = HE\n", 512 | " self.weights = encode_matrix(HE, weights)\n", 513 | " self.stride = stride\n", 514 | " self.padding = padding\n", 515 | " self.bias = bias\n", 516 | " if bias is not None:\n", 517 | " self.bias = encode_matrix(HE, bias)\n", 518 | "\n", 519 | " def __call__(self, t):\n", 520 | " t = apply_padding(t, self.padding)\n", 521 | " result = np.array([[np.sum([convolute2d(image_layer, filter_layer, self.stride)\n", 522 | " for image_layer, filter_layer in zip(image, _filter)], axis=0)\n", 523 | " for _filter in self.weights]\n", 524 | " for image in t])\n", 525 | "\n", 526 | " if self.bias is not None:\n", 527 | " return np.array([[layer + bias for layer, bias in zip(image, self.bias)] for image in result])\n", 528 | " else:\n", 529 | " return result\n", 530 | "\n", 531 | "\n", 532 | "def convolute2d(image, filter_matrix, stride):\n", 533 | " x_d = len(image[0])\n", 534 | " y_d = len(image)\n", 535 | " x_f = len(filter_matrix[0])\n", 536 | " y_f = len(filter_matrix)\n", 537 | "\n", 538 | " y_stride = stride[0]\n", 539 | " x_stride = stride[1]\n", 540 | "\n", 541 | " x_o = ((x_d - x_f) // x_stride) + 1\n", 542 | " y_o = ((y_d - y_f) // y_stride) + 1\n", 543 | "\n", 544 | " def get_submatrix(matrix, x, y):\n", 545 | " index_row = y * y_stride\n", 546 | " index_column = x * x_stride\n", 547 | " return matrix[index_row: index_row + y_f, index_column: index_column + x_f]\n", 548 | "\n", 549 | " return np.array(\n", 550 | " [[np.sum(get_submatrix(image, x, y) * filter_matrix) for x in range(0, x_o)] for y in range(0, y_o)])\n", 551 | "\n", 552 | "def apply_padding(t, padding):\n", 553 | " y_p = padding[0]\n", 554 | " x_p = padding[1]\n", 555 | " zero = t[0][0][y_p+1][x_p+1] - t[0][0][y_p+1][x_p+1]\n", 556 | " return [[np.pad(mat, ((y_p, y_p), (x_p, x_p)), 'constant', constant_values=zero) for mat in layer] for layer in t]" 557 | ] 558 | }, 559 | { 560 | "cell_type": "code", 561 | "execution_count": 17, 562 | "id": "d80ce96f-22fe-4256-890c-cd752893b949", 563 | "metadata": {}, 564 | "outputs": [], 565 | "source": [ 566 | "class LinearLayer:\n", 567 | " def __init__(self, HE, weights, bias=None):\n", 568 | " self.HE = HE\n", 569 | " self.weights = encode_matrix(HE, weights)\n", 570 | " self.bias = bias\n", 571 | " if bias is not None:\n", 572 | " self.bias = encode_matrix(HE, bias)\n", 573 | "\n", 574 | " def __call__(self, t):\n", 575 | " result = np.array([[np.sum(image * row) for row in self.weights] for image in t])\n", 576 | " if self.bias is not None:\n", 577 | " result = np.array([row + self.bias for row in result])\n", 578 | " return result" 579 | ] 580 | }, 581 | { 582 | "cell_type": "code", 583 | "execution_count": 18, 584 | "id": "85a3370e-b2e4-4eac-89fc-2071f41aff68", 585 | "metadata": {}, 586 | "outputs": [], 587 | "source": [ 588 | "class SquareLayer:\n", 589 | " def __init__(self, HE):\n", 590 | " self.HE = HE\n", 591 | "\n", 592 | " def __call__(self, image):\n", 593 | " return square(self.HE, image)\n", 594 | "\n", 595 | "\n", 596 | "def square(HE, image):\n", 597 | " try:\n", 598 | " return np.array(list(map(lambda x: HE.power(x, 2), image)))\n", 599 | " except TypeError:\n", 600 | " return np.array([square(HE, m) for m in image])" 601 | ] 602 | }, 603 | { 604 | "cell_type": "code", 605 | "execution_count": 19, 606 | "id": "28b4e4e2-c07e-447a-b427-5140808177ba", 607 | "metadata": {}, 608 | "outputs": [], 609 | "source": [ 610 | "class FlattenLayer:\n", 611 | " def __call__(self, image):\n", 612 | " dimension = image.shape\n", 613 | " return image.reshape(dimension[0], dimension[1]*dimension[2]*dimension[3])" 614 | ] 615 | }, 616 | { 617 | "cell_type": "code", 618 | "execution_count": 20, 619 | "id": "13b04196-0347-436e-8f7f-169d5d0941ad", 620 | "metadata": {}, 621 | "outputs": [], 622 | "source": [ 623 | "class AveragePoolLayer:\n", 624 | " def __init__(self, HE, kernel_size, stride=(1, 1), padding=(0, 0)):\n", 625 | " self.HE = HE\n", 626 | " self.kernel_size = kernel_size\n", 627 | " self.stride = stride\n", 628 | " self.padding = padding\n", 629 | "\n", 630 | " def __call__(self, t):\n", 631 | " t = apply_padding(t, self.padding)\n", 632 | " return np.array([[_avg(self.HE, layer, self.kernel_size, self.stride) for layer in image] for image in t])\n", 633 | "\n", 634 | "\n", 635 | "def _avg(HE, image, kernel_size, stride):\n", 636 | " x_s = stride[1]\n", 637 | " y_s = stride[0]\n", 638 | "\n", 639 | " x_k = kernel_size[1]\n", 640 | " y_k = kernel_size[0]\n", 641 | "\n", 642 | " x_d = len(image[0])\n", 643 | " y_d = len(image)\n", 644 | "\n", 645 | " x_o = ((x_d - x_k) // x_s) + 1\n", 646 | " y_o = ((y_d - y_k) // y_s) + 1\n", 647 | "\n", 648 | " denominator = HE.encodeFrac(1 / (x_k * y_k))\n", 649 | "\n", 650 | " def get_submatrix(matrix, x, y):\n", 651 | " index_row = y * y_s\n", 652 | " index_column = x * x_s\n", 653 | " return matrix[index_row: index_row + y_k, index_column: index_column + x_k]\n", 654 | "\n", 655 | " return [[np.sum(get_submatrix(image, x, y)) * denominator for x in range(0, x_o)] for y in range(0, y_o)]" 656 | ] 657 | }, 658 | { 659 | "cell_type": "markdown", 660 | "id": "a9bfd4c8-79bb-4647-9353-5f76629f1f56", 661 | "metadata": {}, 662 | "source": [ 663 | "We can now define a function to \"convert\" a PyTorch model to a list of sequential HE-ready-to-be-used layers:" 664 | ] 665 | }, 666 | { 667 | "cell_type": "code", 668 | "execution_count": 21, 669 | "id": "886fc91b-7301-4a31-830a-05b326745c98", 670 | "metadata": {}, 671 | "outputs": [], 672 | "source": [ 673 | "def build_from_pytorch(HE, net):\n", 674 | " # Define builders for every possible layer\n", 675 | "\n", 676 | " def conv_layer(layer):\n", 677 | " if layer.bias is None:\n", 678 | " bias = None\n", 679 | " else:\n", 680 | " bias = layer.bias.detach().numpy()\n", 681 | "\n", 682 | " return ConvolutionalLayer(HE, weights=layer.weight.detach().numpy(),\n", 683 | " stride=layer.stride,\n", 684 | " padding=layer.padding,\n", 685 | " bias=bias)\n", 686 | "\n", 687 | " def lin_layer(layer):\n", 688 | " if layer.bias is None:\n", 689 | " bias = None\n", 690 | " else:\n", 691 | " bias = layer.bias.detach().numpy()\n", 692 | " return LinearLayer(HE, layer.weight.detach().numpy(),\n", 693 | " bias)\n", 694 | "\n", 695 | " def avg_pool_layer(layer):\n", 696 | " # This proxy is required because in PyTorch an AvgPool2d can have kernel_size, stride and padding either of\n", 697 | " # type (int, int) or int, unlike in Conv2d\n", 698 | " kernel_size = (layer.kernel_size, layer.kernel_size) if isinstance(layer.kernel_size, int) else layer.kernel_size\n", 699 | " stride = (layer.stride, layer.stride) if isinstance(layer.stride, int) else layer.stride\n", 700 | " padding = (layer.padding, layer.padding) if isinstance(layer.padding, int) else layer.padding\n", 701 | "\n", 702 | " return AveragePoolLayer(HE, kernel_size, stride, padding)\n", 703 | "\n", 704 | " def flatten_layer(layer):\n", 705 | " return FlattenLayer()\n", 706 | "\n", 707 | " def square_layer(layer):\n", 708 | " return SquareLayer(HE)\n", 709 | "\n", 710 | " # Maps every PyTorch layer type to the correct builder\n", 711 | " options = {\"Conv\": conv_layer,\n", 712 | " \"Line\": lin_layer,\n", 713 | " \"Flat\": flatten_layer,\n", 714 | " \"AvgP\": avg_pool_layer,\n", 715 | " \"Squa\": square_layer\n", 716 | " }\n", 717 | "\n", 718 | " encoded_layers = [options[str(layer)[0:4]](layer) for layer in net]\n", 719 | " return encoded_layers" 720 | ] 721 | }, 722 | { 723 | "cell_type": "markdown", 724 | "id": "06bde28e-7adc-4ede-876a-b5e57b63b0c4", 725 | "metadata": {}, 726 | "source": [ 727 | "## Encrypted processing\n", 728 | "\n", 729 | "Let's list the activities that we will now do:\n", 730 | " 1. Create a HE context, specifiying the encryption parameters `m` (polynomial modulus degree) and `p` (plaintext modulus). Let's remember that `q` will be chosen automatically in order to guarantee a 128-bit RSA equivalent security;\n", 731 | " 2. Convert our Torch approximated model to a list of layers able to work on matrices of encrypted values. The weights will be encoded;\n", 732 | " 3. Encrypt an image from our testing set;\n", 733 | " 4. Verify that the final classification result is correct." 734 | ] 735 | }, 736 | { 737 | "cell_type": "markdown", 738 | "id": "dc64023d-9b74-44c0-90a1-f6e7e0e3d123", 739 | "metadata": {}, 740 | "source": [ 741 | "If we look at our model, we can see that we have two **square layers**: these are the layers which have more impact on our noise!\n", 742 | "Two square layers corresponds to two ciphertext-ciphertext multiplications. Let's see if $m=4096$ gives us enough room to perform 2 encrypted multiplications." 743 | ] 744 | }, 745 | { 746 | "cell_type": "markdown", 747 | "id": "c3a03f8f-997e-4009-94b4-541f7cc56318", 748 | "metadata": {}, 749 | "source": [ 750 | "We can define a function which, after receiving $n$ and $p$ tries to encrypt and image and forward it to our approximated model (suitable encoded). This will let us see the homomorphic encryption in function." 751 | ] 752 | }, 753 | { 754 | "cell_type": "code", 755 | "execution_count": 22, 756 | "id": "3cb6d3e9-072b-4645-97da-8e0eab92e6e8", 757 | "metadata": {}, 758 | "outputs": [], 759 | "source": [ 760 | "import time\n", 761 | "\n", 762 | "def test_parameters(n, p, model):\n", 763 | " HE = Pyfhel()\n", 764 | " HE.contextGen(p=p, m=n) # what Pyfhel calls m, we call n.\n", 765 | " HE.keyGen()\n", 766 | " relinKeySize=3\n", 767 | " HE.relinKeyGen(bitCount=2, size=relinKeySize)\n", 768 | " \n", 769 | " images, labels = next(iter(test_loader))\n", 770 | "\n", 771 | " sample_image = images[0]\n", 772 | " sample_label = labels[0]\n", 773 | " \n", 774 | " model.to(\"cpu\")\n", 775 | " model_encoded = build_from_pytorch(HE, model)\n", 776 | " \n", 777 | " with torch.no_grad():\n", 778 | " expected_output = model(sample_image.unsqueeze(0))\n", 779 | " \n", 780 | " encrypted_image = encrypt_matrix(HE, sample_image.unsqueeze(0).numpy())\n", 781 | " \n", 782 | " start_time = time.time()\n", 783 | " for layer in model_encoded:\n", 784 | " encrypted_image = layer(encrypted_image)\n", 785 | " print(f\"Passed layer {layer}...\")\n", 786 | " \n", 787 | " requested_time = round(time.time() - start_time, 2)\n", 788 | " \n", 789 | " result = decrypt_matrix(HE, encrypted_image)\n", 790 | " difference = expected_output.numpy() - result\n", 791 | " \n", 792 | " print(f\"\\nThe encrypted processing of one image requested {requested_time} seconds.\")\n", 793 | " print(f\"\\nThe expected result was:\")\n", 794 | " print(expected_output)\n", 795 | " \n", 796 | " print(f\"\\nThe actual result is: \")\n", 797 | " print(result)\n", 798 | " \n", 799 | " print(f\"\\nThe error is:\")\n", 800 | " print(difference) " 801 | ] 802 | }, 803 | { 804 | "cell_type": "markdown", 805 | "id": "468be664-a5f5-4c11-ad4f-4b2b5a06f46b", 806 | "metadata": {}, 807 | "source": [ 808 | "Let's try with $n=4096$ and $p=95536$ on our approximated model." 809 | ] 810 | }, 811 | { 812 | "cell_type": "code", 813 | "execution_count": 23, 814 | "id": "0515705e-71a0-44c7-a6ab-00e2c825aeee", 815 | "metadata": {}, 816 | "outputs": [ 817 | { 818 | "name": "stdout", 819 | "output_type": "stream", 820 | "text": [ 821 | "Passed layer <__main__.ConvolutionalLayer object at 0x7f6481dc5d00>...\n", 822 | "Passed layer <__main__.SquareLayer object at 0x7f647c360be0>...\n", 823 | "Passed layer <__main__.AveragePoolLayer object at 0x7f6481db8e20>...\n", 824 | "Passed layer <__main__.ConvolutionalLayer object at 0x7f6481db8e80>...\n", 825 | "Passed layer <__main__.SquareLayer object at 0x7f6481db87f0>...\n", 826 | "Passed layer <__main__.AveragePoolLayer object at 0x7f6481db8250>...\n", 827 | "Passed layer <__main__.FlattenLayer object at 0x7f6481db86d0>...\n", 828 | "Passed layer <__main__.LinearLayer object at 0x7f6481db8ca0>...\n", 829 | "\n", 830 | "The encrypted processing of one image requested 141.85 seconds.\n", 831 | "\n", 832 | "The expected result was:\n", 833 | "tensor([[-15.5356, -16.7560, -9.6970, -6.8965, -20.3962, -9.3715, -18.1690,\n", 834 | " -21.4377, 11.4888, -11.4841]])\n", 835 | "\n", 836 | "The actual result is: \n", 837 | "[[-3.30547842e-01 -9.08289577e-02 -6.86508353e-01 -6.40577979e-01\n", 838 | " -6.80177972e-01 4.10673827e-01 1.12589991e+15 4.76321019e-03\n", 839 | " -2.83141956e-01 -4.66546431e-01]]\n", 840 | "\n", 841 | "The error is:\n", 842 | "[[-1.52050557e+01 -1.66651830e+01 -9.01046449e+00 -6.25592452e+00\n", 843 | " -1.97160470e+01 -9.78217670e+00 -1.12589991e+15 -2.14425054e+01\n", 844 | " 1.17719639e+01 -1.10175320e+01]]\n" 845 | ] 846 | } 847 | ], 848 | "source": [ 849 | "test_parameters(4096, 95536, LeNet1_Approx)" 850 | ] 851 | }, 852 | { 853 | "cell_type": "markdown", 854 | "id": "aafb43bd-ec62-4c10-b8ce-e83e0f58a4f7", 855 | "metadata": {}, 856 | "source": [ 857 | "Unfortunately, the NB is not sufficient and the decryption fails.\n", 858 | "We could try decrementing $p$, in order to reduce the NB consumption. However, the actual value is already quite low so it is difficult that it will change something.\n", 859 | "We could try incrementing $n$ to 8192, but it would result in a huge overhead in terms of memory and time.\n", 860 | "\n", 861 | "For now, it is simpler to remove a square layer from the DL model (usually removing the last one is better). This will result in a less NB demanding processing, allowing us to use $n=4096$.\n", 862 | "\n", 863 | "\n", 864 | "**Note that this does not happen on - every - image, but test a few and you will see that it happens often, making the computation unreliable. It means that we are very near the limit of NB. However, even if decryption does not fail, the results are quite unaccurate.**" 865 | ] 866 | }, 867 | { 868 | "cell_type": "code", 869 | "execution_count": 24, 870 | "id": "00e2c386-3a0f-4a1d-aeec-3d6d9733e5f0", 871 | "metadata": {}, 872 | "outputs": [ 873 | { 874 | "name": "stdout", 875 | "output_type": "stream", 876 | "text": [ 877 | "Accuracy on test set (single square layer): 0.9819\n" 878 | ] 879 | } 880 | ], 881 | "source": [ 882 | "LeNet1_Approx_singlesquare = nn.Sequential(\n", 883 | " nn.Conv2d(1, 4, kernel_size=5),\n", 884 | " Square(),\n", 885 | " nn.AvgPool2d(kernel_size=2),\n", 886 | "\n", 887 | " nn.Conv2d(4, 12, kernel_size=5),\n", 888 | "# Square(),\n", 889 | " nn.AvgPool2d(kernel_size=2),\n", 890 | "\n", 891 | " nn.Flatten(),\n", 892 | "\n", 893 | " nn.Linear(192, 10),\n", 894 | ")\n", 895 | "\n", 896 | "LeNet1_Approx_singlesquare.to(device)\n", 897 | "train_net(LeNet1_Approx_singlesquare, 15, device)\n", 898 | "acc = test_net(LeNet1_Approx_singlesquare, device)\n", 899 | "print(f\"Accuracy on test set (single square layer): {acc}\")" 900 | ] 901 | }, 902 | { 903 | "cell_type": "markdown", 904 | "id": "97a0b453-930e-40c4-9447-680c10772bf4", 905 | "metadata": {}, 906 | "source": [ 907 | "Let's try again." 908 | ] 909 | }, 910 | { 911 | "cell_type": "code", 912 | "execution_count": 25, 913 | "id": "894b9b53-e285-4f3f-be90-ab1e51fc2432", 914 | "metadata": {}, 915 | "outputs": [ 916 | { 917 | "name": "stdout", 918 | "output_type": "stream", 919 | "text": [ 920 | "Passed layer <__main__.ConvolutionalLayer object at 0x7f6481d92a00>...\n", 921 | "Passed layer <__main__.SquareLayer object at 0x7f64e535e790>...\n", 922 | "Passed layer <__main__.AveragePoolLayer object at 0x7f652416ba90>...\n", 923 | "Passed layer <__main__.ConvolutionalLayer object at 0x7f647c366ca0>...\n", 924 | "Passed layer <__main__.AveragePoolLayer object at 0x7f647c366df0>...\n", 925 | "Passed layer <__main__.FlattenLayer object at 0x7f6481dc5af0>...\n", 926 | "Passed layer <__main__.LinearLayer object at 0x7f6481dc5be0>...\n", 927 | "\n", 928 | "The encrypted processing of one image requested 133.59 seconds.\n", 929 | "\n", 930 | "The expected result was:\n", 931 | "tensor([[ -6.3078, -4.0340, 2.3669, 3.3071, -7.1296, -7.6852, -24.7246,\n", 932 | " 17.9883, -0.6826, 7.2450]])\n", 933 | "\n", 934 | "The actual result is: \n", 935 | "[[-1.68659983 0.49865329 0.98971051 0.80790324 -1.43989918 -0.27309625\n", 936 | " -0.90793463 4.30852395 -1.88644153 0.81691589]]\n", 937 | "\n", 938 | "The error is:\n", 939 | "[[ -4.6212162 -4.53261077 1.37715894 2.4992035 -5.68973472\n", 940 | " -7.41207774 -23.81671099 13.6798107 1.20383358 6.42805444]]\n" 941 | ] 942 | } 943 | ], 944 | "source": [ 945 | "test_parameters(4096, 95536, LeNet1_Approx_singlesquare)" 946 | ] 947 | }, 948 | { 949 | "cell_type": "markdown", 950 | "id": "ebad2908-ac44-437d-80de-d146f2559758", 951 | "metadata": {}, 952 | "source": [ 953 | "Now the decryption works! However, the final result are similar to the expected one, but with a discrepancy.\n", 954 | "The \"perfect\" value for $p$ can be found with a trial and error process. For our purposes, it has been set to 953983721." 955 | ] 956 | }, 957 | { 958 | "cell_type": "code", 959 | "execution_count": 26, 960 | "id": "58d01f5c-c3c2-4a89-b2f7-64aebbbdd39d", 961 | "metadata": {}, 962 | "outputs": [ 963 | { 964 | "name": "stdout", 965 | "output_type": "stream", 966 | "text": [ 967 | "Passed layer <__main__.ConvolutionalLayer object at 0x7f6481dc5850>...\n", 968 | "Passed layer <__main__.SquareLayer object at 0x7f647c366df0>...\n", 969 | "Passed layer <__main__.AveragePoolLayer object at 0x7f647c37ff40>...\n", 970 | "Passed layer <__main__.ConvolutionalLayer object at 0x7f647c37f3a0>...\n", 971 | "Passed layer <__main__.AveragePoolLayer object at 0x7f647c37f610>...\n", 972 | "Passed layer <__main__.FlattenLayer object at 0x7f647c37f1f0>...\n", 973 | "Passed layer <__main__.LinearLayer object at 0x7f647c37feb0>...\n", 974 | "\n", 975 | "The encrypted processing of one image requested 132.25 seconds.\n", 976 | "\n", 977 | "The expected result was:\n", 978 | "tensor([[-12.3960, -7.9559, 6.0188, 17.0045, -25.5208, -1.4943, -10.9638,\n", 979 | " -9.0610, 9.7218, 2.4548]])\n", 980 | "\n", 981 | "The actual result is: \n", 982 | "[[-12.35858477 -7.90092142 5.9919354 16.94333243 -25.45030745\n", 983 | " -1.48793805 -10.95613351 -9.01187793 9.67891156 2.42361685]]\n", 984 | "\n", 985 | "The error is:\n", 986 | "[[-0.03739511 -0.05502169 0.02682719 0.06116891 -0.07049219 -0.00634061\n", 987 | " -0.00768314 -0.04913719 0.04288638 0.03119973]]\n" 988 | ] 989 | } 990 | ], 991 | "source": [ 992 | "test_parameters(4096, 953983721, LeNet1_Approx_singlesquare)" 993 | ] 994 | }, 995 | { 996 | "cell_type": "markdown", 997 | "id": "fcf92d0c-b70c-45a6-8086-08cc5c21e449", 998 | "metadata": {}, 999 | "source": [ 1000 | "We are now happy with the precision of the result, which should now guarantee that the final accuracy of the encrypted processing on the whole test set is equal (or, at least, very similar) to the same obtained on unencrypted data." 1001 | ] 1002 | }, 1003 | { 1004 | "cell_type": "markdown", 1005 | "id": "c95c1027-4592-43f6-bde3-34d3118789e3", 1006 | "metadata": {}, 1007 | "source": [ 1008 | "### Computational load\n", 1009 | "Obviously, we cannot ignore the huge computational overhead generated by the encrypted processing.\n", 1010 | "\n", 1011 | "In fact, the processing of one image took about ~2min on a common desktop machine.\n", 1012 | "The computation has not been parallelized; so, it used only one thread.\n", 1013 | "\n", 1014 | "While parallelizing allows to speed up the computation, also the occupied memory is a concern: the processing of this image occupied ~700MB of RAM." 1015 | ] 1016 | }, 1017 | { 1018 | "cell_type": "code", 1019 | "execution_count": null, 1020 | "id": "e33d429c-1dfb-4cb4-8e06-59091e0083c3", 1021 | "metadata": {}, 1022 | "outputs": [], 1023 | "source": [] 1024 | }, 1025 | { 1026 | "cell_type": "code", 1027 | "execution_count": null, 1028 | "id": "f89c04ca-3ec4-4c0a-96f1-5d82801101b5", 1029 | "metadata": {}, 1030 | "outputs": [], 1031 | "source": [] 1032 | } 1033 | ], 1034 | "metadata": { 1035 | "kernelspec": { 1036 | "display_name": "pyfhel", 1037 | "language": "python", 1038 | "name": "pyfhel" 1039 | }, 1040 | "language_info": { 1041 | "codemirror_mode": { 1042 | "name": "ipython", 1043 | "version": 3 1044 | }, 1045 | "file_extension": ".py", 1046 | "mimetype": "text/x-python", 1047 | "name": "python", 1048 | "nbconvert_exporter": "python", 1049 | "pygments_lexer": "ipython3", 1050 | "version": "3.9.7" 1051 | } 1052 | }, 1053 | "nbformat": 4, 1054 | "nbformat_minor": 5 1055 | } 1056 | -------------------------------------------------------------------------------- /HE-ML/HE_processing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/HE-ML/HE_processing.png -------------------------------------------------------------------------------- /HE-ML/LeNet1.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/HE-ML/LeNet1.pt -------------------------------------------------------------------------------- /HE-ML/LeNet1_Approx.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AI-Tech-Research-Lab/Introduction-to-BFV-HE-ML/61f1534d7f859b74a5c78a980c03db7e165c5db8/HE-ML/LeNet1_Approx.pt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## WCCI 2022 2 | The notebooks for the tutorial "Privacy-preserving machine and deep learning with homomorphic encryption: an introduction" can be found in the folder "WCCI2022"! 3 | The notebook `Tutorial_to_fill.ipynb` is the suggested one. We will complete and run it during the tutorial. 4 | **You can also run it directly on Google Colab, to avoid compatibility issues at this [link](https://colab.research.google.com/drive/1g7URoO7Ew87oBWCPtAP2U5G5dC8lD1DR?usp=sharing)!**. 5 | 6 | For reference, the `Tutorial_complete.ipynb` notebook already contains the final version of the notebook with a deeper explanation. Also this one can be run on Colab: [link](https://colab.research.google.com/drive/1ojpP0rjzwcuwypCX2jnCluhoARAAnzgt?usp=sharing). 7 | 8 | # Introduction-to-BFV-HE-ML 9 | 10 | This repository contains the code used in the paper "Privacy-preserving deep learning with homomorphic encryption: an introduction". 11 | 12 | Dependencies: 13 | - [PyTorch](https://pytorch.org/get-started/locally/) 14 | - [NumPy](https://numpy.org/) 15 | - [Pyfhel](https://github.com/ibarrond/Pyfhel) 16 | 17 | The repository is organized this way: 18 | - `BFV_theory` contains a Jupyter notebook (`BFV_theory.ipynb`) which introduces the reader to the BFV Homomorphic Encryption scheme, along with a (gentle) introduction to the main math concepts of the scheme, together with an implementation of the scheme made with Python, from scratch, using NumPy. This scheme construction should help the reader gain a deeper understanding of the scheme, before using more advanced and ready-to-use libraries (like SEAL, Pyfhel, etc.); 19 | - `HE-ML` contains a Jupyter notebook (`HE-ML.ipynb`) which introduces the reader to the concept of Homomorphic-Encryption enabled Machine-Learning. In the notebook a simple CNN, the LeNet-1, will be translated into a model able to work on encrypted data, along with an explanation on the different design choices, as well as on the encryption parameters setting; 20 | - `Experiments` contains the Python scripts used to obtain the experimental results shown in the paper. Running the scripts and looking at the corresponding file `.log`, it will be possible to reproduce and check the results. 21 | -------------------------------------------------------------------------------- /WCCI2022/Tutorial_to_fill.ipynb: -------------------------------------------------------------------------------- 1 | {"cells":[{"cell_type":"markdown","source":["![image](https://drive.google.com/uc?export=view&id=113XAsG1B7eHjOf2Cn6JKZarzuaa_1rSN)\n"],"metadata":{"id":"0Xyy4mzRRDiz"},"id":"0Xyy4mzRRDiz"},{"cell_type":"markdown","id":"3Sj9RA-aRMe5","metadata":{"id":"3Sj9RA-aRMe5"},"source":["# Introduction"]},{"cell_type":"code","source":["# Please, run this while I speak. It requires some minutes.\n","!pip3 install Pyfhel==2.3.1\n","!pip3 install tenseal"],"metadata":{"id":"gdX9V1GC9_GU"},"id":"gdX9V1GC9_GU","execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## The problem"],"metadata":{"id":"mtrN_pv-qjuO"},"id":"mtrN_pv-qjuO"},{"cell_type":"markdown","id":"SErL2yV0RZUv","metadata":{"id":"SErL2yV0RZUv"},"source":["The current scenario of \"machine-learning as-a-service\" (MLaaS): \n","![image](https://drive.google.com/uc?export=view&id=1oDr0yw8i7FfjABqrWLVYiGcCYuIBOpjq)"]},{"cell_type":"markdown","id":"32zxq92YSHps","metadata":{"id":"32zxq92YSHps"},"source":["This scenario introduces **privacy problems**... \n","- Privacy of users data: ethical aspects\n","- Privacy of companies data: secrecy requirements\n","\n","\n","And leads to the main question we want to address today:\n","> How can we design software services and mobile apps providing intelligent functionalities (through machine and deep learning solutions) \n","while guaranteeing the privacy of user data?"]},{"cell_type":"markdown","id":"s3B42hZBSHkv","metadata":{"id":"s3B42hZBSHkv"},"source":["## Privacy-preserving computation in machine and deep learning: tools\n","There are two main families of tools we can use to preserve the privacy of users' data during the processing of ML and DL models:\n","1. the first is based on data anonymization and perturbation: it includes *Group-based anonymity\" and \"Differential privacy\"*;\n","2. the second is based on the (joint) computation of encrypted data: *Multi-party computation* and *Homomorphic encryption\".\n","\n","![image](https://drive.google.com/uc?export=view&id=1POmzkmAYQSZP44tnS6cm13X-aILgxK23)\n","\n","| | Ability to process encrypted data | Processing without the need of multiple rounds of communication |\n","|:-----------------------:|:---------------------------------:|:----------------------------------------------------------------:|\n","| Homomorphic Encryption | Yes | Yes |\n","| Multi-party Computation | Yes | No |\n","| Group-based Anonymity | No | Yes |\n","| Differential Privacy | No | Yes |"]},{"cell_type":"markdown","id":"Ag1LaQTfSHiM","metadata":{"id":"Ag1LaQTfSHiM"},"source":["## Homomorphic encryption\n","Homomorphic encryption (HE) is a family of encryption schemes which allows one to process directly encrypted data, without the need to decrypt them.\n","\n","![image](https://drive.google.com/uc?export=view&id=1xEhSqQKpWwn8reXm8u1MzqqYn-sn9n7W)\n","\n","In other words, an encryption function $E$ and its corresponding decryption function $D$ are homomorphic with respect to a class of functions $F$ if, for any function $f \\in F$, it is possible to construct a corresponding function $g_f$ such that $f(x) = D(g_f(E(x)))$ for a set of input $x$.\n","\n","### HE schemes\n","There are different HE schemes, all based on the same mathematical aspects, but with different peculiarities and uses:\n","\n","| Category | Computation | Example | Comment |\n","|----------------------|-------------------------------------------------------------------|------------|--------------------------------------------------|\n","| Partially HE schemes | Only one class of operations (+ or -) | RSA | |\n","| Somewhat HE schemes | Unbounded number of additions and one single multiplication | BGN | |\n","| Leveled HE schemes | A pre-determined number of additions and multiplications | BFV / CKKS | Current solutions in the HE-DL scenario |\n","| Fully HE schemes | Unbounded number of operations, often binary ones (AND, NOT, ...) | TFHE | Future (still difficult to configure and manage) |\n","\n","In the next practical example we will see how the BFV scheme works."]},{"cell_type":"markdown","id":"5LL2yUhA7zg4","metadata":{"id":"5LL2yUhA7zg4"},"source":["# Demo 1: BFV scheme from scratch"]},{"cell_type":"code","execution_count":null,"id":"Kaz0my-i730q","metadata":{"id":"Kaz0my-i730q"},"outputs":[],"source":["# Some helper functions.\n","\n","import numpy as np\n","from numpy.polynomial import Polynomial\n","from numpy.polynomial.polynomial import polypow\n","from math import floor\n","\n","# Mean and standard deviation of a Gaussian distribution that will be used in the followings.\n","# Sigma is a \"suggested\" value for safe BFV, see https://eprint.iacr.org/2019/939.pdf\n","mu, sigma = 0, 3.2\n","\n","def Poly(coeffs):\n"," \"\"\"\n"," Helper function to build polynomials, passing a dictionary of coefficients.\n"," For example, passing {0: 1, 1: -2, 2: 2} returns the polynomial\n"," 2X**2 - 2X + 1\n"," \"\"\"\n"," max_power = max(coeffs.keys())\n"," _coeffs = np.zeros(max_power + 1)\n"," \n"," for i, c in coeffs.items():\n"," _coeffs[i] = c\n"," \n"," return Polynomial(_coeffs)\n","\n","def pr(p):\n"," \"\"\" \n"," Helper function to pretty-print the polynomials, with human order\n"," of powers, removed trailing .0, etc.\n"," \"\"\"\n"," coefs = p.coef\n"," res = \"\"\n"," \n"," powers = range(len(coefs)-1, -1, -1)\n"," for power, coeff in zip(powers, reversed(coefs)):\n"," if coeff == 0:\n"," continue\n"," \n"," if int(coeff) == coeff:\n"," coeff = int(coeff)\n"," \n"," sign = \"- \" if coeff < 0 else \"+ \"\n"," \n"," if power == 0:\n"," value = abs(coeff)\n"," elif abs(coeff) != 1:\n"," value = abs(coeff)\n"," else:\n"," value = \"\"\n","\n"," power_sign = {0: \"\", 1: \"X\"}\n"," def_power_sign = f\"X**{power}\"\n"," \n"," res += f\" {sign}{value}{power_sign.get(power, def_power_sign)}\"\n"," \n"," if res[1] == \"+\":\n"," return res[3:]\n"," if res[1] == \"-\":\n"," return res[1:]\n","\n","def mod_on_coefficients(polynomial, modulo):\n"," \"\"\"\n"," Apply the modulo on the coefficients of a polynomial.\n"," \"\"\"\n"," coefs = polynomial.coef\n"," mod_coefs = []\n"," for c in coefs:\n"," mod_coefs.append(c % modulo)\n"," \n"," return Polynomial(mod_coefs)\n","\n","def round_on_nearest_integer(polynomial):\n"," \"\"\"\n"," Round the coefficients of a polynomial to the\n"," nearest integer.\n"," \"\"\"\n"," coefs = polynomial.coef\n"," round_coefs = []\n"," for c in coefs:\n"," round_coefs.append(round(c))\n"," \n"," return Polynomial(round_coefs)"]},{"cell_type":"markdown","id":"d6a687c6-65b0-4b52-9d8d-0bd8e1f2a2d0","metadata":{"tags":[],"id":"d6a687c6-65b0-4b52-9d8d-0bd8e1f2a2d0"},"source":["The Brakerski-Fan-Vercauteren (BFV) scheme is an homomorphic encryption (HE) scheme which allows the computation of some functions directly on *encrypted data*. \n","\n","In particular, it allows the computation of additions and multiplication:\n","\n","- between ciphertexts and ciphertexts;\n","- between ciphertexts and plaintexts.\n","\n","The result of the operations, once decrypted, is the same as if it were applied on the corresponding plaintexts.\n","\n","The schemes like BFV or also CKKS are based on a hard computation problem called *Ring Learning With Errors*.\n","\n","Let's understand this problem starting from the bottom.\n","\n","## Data representation\n","The data in these schemes is represented by *polynomials*. This holds both when data is encrypted (ciphertexts) and when it is unencrypted (plaintexts).\n","\n","So, the starting point are common polynomials like:"]},{"cell_type":"code","execution_count":null,"id":"f14ea393-f50b-47fa-b409-d3823ce80bd9","metadata":{"id":"f14ea393-f50b-47fa-b409-d3823ce80bd9"},"outputs":[],"source":["p = Poly( ??? )\n","print(pr(p))"]},{"cell_type":"markdown","id":"044263ee-0a31-4fe4-9457-c4c88fddb19d","metadata":{"id":"044263ee-0a31-4fe4-9457-c4c88fddb19d"},"source":["However, there is more. \n","\n","### Coefficients\n","First of all, the coefficients of the polynomials are whole numbers, and are the *remainder* relative to some other number $k$ (that is $\\bmod k$).\n","\n","Take for example $k=12$; we can see the ring as a common clock, in which each our corresponds to an element in the ring.\n","Adding $3$ to $11$ will result in $2$ instead of $14$.\n","\n","![modular](https://drive.google.com/uc?export=view&id=1tW_szr-j-7ov2Vw75pJApEzG2qf0-0hO)\n","\n","It also important to note that in this ring we could make the numbers go from $-5$ to $6$: this is arbitrary, because it is easy to see that a remainder of $-1$ is the same as a remainder of $11$. In the following this consideration will be assumed.\n","\n","### Polynomial modulus\n","The second aspect is that also the polynomials themselves are remainder: we define a special polynomial called *polynomial modulus*. Each polynomial used in the scheme is divided by this polynomial modulus and the remainder is taken.\n","\n","In the HE schemes it is common to take this polynomial modulus in the form $x^n + 1$ where $n$ is a power of $2$. For example, if we take $n=8$ we have the polynomial $x^{8} + 1$.\n","\n","Put simply, the polynomials employed in the scheme have the following two characteristics:\n","\n"," 1. Their coefficients are modulo $k$, creating a ring from $0$ to $k$;\n"," 2. The maximum degree is $n-1$;\n","\n","So, all the polynomials we are considering are of the form (assuming $n=8$):\n","$$ a_7 x^7+a_6 x^6+a_5 x^5+a_4 x^4+a_3 x^3+a_2 x^2+a_1 x+a_0 $$\n","\n","remembering that each of the 8 coefficients $a_i$ can range from $0$ to $k-1$.\n","\n","We can also give a powerful graphical representation of this:\n","\n","![torus](https://drive.google.com/uc?export=view&id=1rM24TY_qPcyJT0InhMcNW_jyYdu0w3f3)\n","\n","Each loop contains the 12 possible values of the coefficient of one of the powers of $x$ that appears in the polynomial. \n","\n","Formally: the polynomials used in the scheme are in the space $R_k = \\mathbb{Z}_k[x] / (x^n + 1)$."]},{"cell_type":"markdown","id":"d1860cda-a84d-4709-b01b-54addf812c34","metadata":{"id":"d1860cda-a84d-4709-b01b-54addf812c34"},"source":["### Practical examples\n","Let's see this in action with a multiplication between polynomials.\n","We let:\n","- $n = 8$ (hence, the polynomial modulus is $x^8 + 1$);\n","- $k = 7$\n","\n","If we multiply:\n","- a = $3 x^{4}$ and\n","- b = $4x^5$\n","\n","we obtain:\n","- $12 x^{9}$\n","\n","After each operation, we have to perform the division with the polynomial modulus and keep the remainder.\n","\n","$$ 12 x^9 \\bmod x^8 + 1 = -12x $$\n","\n","Also, the result's coefficients have to be taken $\\bmod k$.\n","\n","$$ [-12x]_k = 2x $$"]},{"cell_type":"code","execution_count":null,"id":"77c05f5f-ee00-46a2-8bd9-0e0f49b44ca1","metadata":{"id":"77c05f5f-ee00-46a2-8bd9-0e0f49b44ca1"},"outputs":[],"source":["a = Poly({ ??? })\n","b = Poly({ ??? })\n","polynomial_modulus = Poly({ ??? })\n","\n","prod = a * b\n","\n","quo, rem = ??? (prod, polynomial_modulus)\n","final_result = ??? (rem, 7)\n","\n","print(f\"A: {pr(a)}\")\n","print(f\"B: {pr(b)}\")\n","print(f\"Product of A and B: {pr(prod)}\")\n","print(f\"Polynomial modulus: {pr(polynomial_modulus)}\")\n","print(f\"----------------------\")\n","print(f\"Remainder of (A*B) / polynomial modulus: {pr(rem)}\")\n","print(f\"Apply also mod k: {pr(final_result)}\")"]},{"cell_type":"markdown","id":"2c905506-ba27-4404-a9ab-dcc4053c21cf","metadata":{"id":"2c905506-ba27-4404-a9ab-dcc4053c21cf"},"source":["The polynomials used as modulo, $x^n + 1$ are in the family of *cyclotomic polynomials*."]},{"cell_type":"markdown","id":"6c76be49-684c-4038-a940-f9db5e47f5f2","metadata":{"tags":[],"id":"6c76be49-684c-4038-a940-f9db5e47f5f2"},"source":["## Encryption using polynomials ring\n","Now we can go to the encryption/decryption phase.\n","\n","### Plaintexts, ciphertexts, and keys\n","\n","A plaintext is a polynomial :\n","- from the ring with polynomial modulus $\\Phi_n = x^n + 1$, with $n$ being a power of $2$;\n","- coefficients modulus $p$ (which will be denoted as *plaintext coefficient modulus*). \n","\n","A ciphertext is composed of (at least) **two polynomials**:\n","- from the ring with the same polynomial modulus;\n","- but with the coefficients modulus $q$ (which will be denoted as *ciphertext coefficient modulus*) which is typically much larger than $p$.\n","\n","In real cases of BFV applications, it is common to take values of $n$ of at least $2048$ while for $q$, usually very large values are used.\n","\n","To make a simpler example, we will use much smaller values, for example $n=16$, $p=7$ and $q=874$. Note that these parameter are not *secure* for real cases (and we will see why...)."]},{"cell_type":"code","execution_count":null,"id":"03e2a8e3-e880-4dce-bb74-0e22f6510a8b","metadata":{"id":"03e2a8e3-e880-4dce-bb74-0e22f6510a8b"},"outputs":[],"source":["n = ??\n","polynomial_modulus = Poly({ ??? })\n","p = ?\n","q = ???\n","\n","print(f\"Plaintext coefficient modulo: {p}\")\n","print(f\"Ciphertext coefficient modulo: {q}\")\n","print(f\"Polynomial modulus: {pr(polynomial_modulus)}\")"]},{"cell_type":"markdown","id":"7582e37d-6339-461c-acce-f6e637751ed6","metadata":{"id":"7582e37d-6339-461c-acce-f6e637751ed6"},"source":["Let's generate the private key, $k_s$: the secret key corresponds to a random polynomial of maximum degree $n$ with coefficients of either $-1, 0, 1$. \n","\n","For example:"]},{"cell_type":"code","execution_count":null,"id":"96d8725e-2b3e-4751-92f8-f6d7ea614b07","metadata":{"id":"96d8725e-2b3e-4751-92f8-f6d7ea614b07"},"outputs":[],"source":["coeffs = {}\n","for i in range(0, n):\n"," coeffs[i] = ???\n"," \n","s = Poly(coeffs)\n","print(f\"Secret key: {pr(s)}\")"]},{"cell_type":"markdown","id":"446568a5-2d83-46bf-b6fb-5eef21692920","metadata":{"id":"446568a5-2d83-46bf-b6fb-5eef21692920"},"source":["Next we generate a public key.\n","\n","For this we need:\n","\n"," 1. a random polynomial from the ciphertext space, hence with coefficients modulo $q$, which we'll call $a$;\n"," 2. an *error polynomial*, which is small in the sense that has small coefficients drawn from a discrete Gaussian distribution. We will use this once, then it is discarded. We'll call this $e$.\n","\n","At this point we can define the public key as the pair of polynomials:\n","\n","$$k_p = ([{-(a k_s + e)}]_{\\Phi_n, q} , a)$$\n","\n","where the polynomial arithmetic is done modulo the polynomial modulus and the coefficient arithmetic modulo $q$."]},{"cell_type":"code","execution_count":null,"id":"5d33c497-d3d3-4601-a0dc-ef0b1be437ac","metadata":{"id":"5d33c497-d3d3-4601-a0dc-ef0b1be437ac"},"outputs":[],"source":["coeffs = {}\n","for i in range(0, n):\n"," coeffs[i] = np.random.randint(0, q)\n","\n","a = Poly(coeffs)\n","\n","dist = np.random.default_rng().normal(mu, sigma, 16)\n","coeffs = {}\n","for ind in range(0, n):\n"," coeffs[ind] = round(dist[ind])\n","\n","\n","e = Poly(coeffs)\n","\n","pk_0 = ???\n","pk_0 = divmod(pk_0, polynomial_modulus)[1]\n","pk_0 = mod_on_coefficients(pk_0, q)\n","\n","pk_1 = ???\n","\n","pk = ???\n","\n","print(f\"a: {pr(a)}\\n\")\n","print(f\"e: {pr(e)}\\n\")\n","print(f\"Public key: {pr(pk_0)}, {pr(pk_1)}\\n\")"]},{"cell_type":"markdown","id":"1a9e194d-abfe-415c-b315-1506353add1f","metadata":{"id":"1a9e194d-abfe-415c-b315-1506353add1f"},"source":["It is interesting to note that, in a sense, the secret key is contained in the public key. However, to mask it, it is multiplied with the polynomial $a$ which has coefficients in the range $\\bmod q$ (it is quite \"big\"). \n","\n","However, also $a$ is contained in the public key; a little error $e$ is added to $a*s$ and this makes it computationally hard to retrieve $s$. \n","This is the exact definition of the *Ring Learning with Errors*.\n","\n","To recap:\n","\n","$$ s \\rightarrow \\text{\"small\" polynomial (coefficients in -1, 0, 1)}$$\n","$$ a \\rightarrow \\text{\"big polynomial (coefficients in mod q)}$$\n","$$ e \\rightarrow \\text{\"small noise\" polynomial (coefficients drawn by a Gaussian distribution)}$$\n","\n","$$ k_s$$\n","$$ k_p = \\left ( \\left [ -(a k_s + e) \\right ]_{\\Phi_n, q} , a \\right ) $$"]},{"cell_type":"markdown","id":"35920bdd-2f0b-48eb-81dd-ca6fd3a13f0a","metadata":{"id":"35920bdd-2f0b-48eb-81dd-ca6fd3a13f0a"},"source":["### Encryption\n","Encryption transforms a plaintext - encoded in the form of a polynomial with coefficients modulo $p$ (that, is in the space $R_p = \\mathbb{Z}_p[{x}]/(x^n + 1)$) into a ciphertext.\n","A fresh ciphertext, in BFV, is a pair of polynomials with coefficients modulo $q$ (that is, they are in the space $R_{\\Phi_n, q} = \\mathbb{Z}_{\\Phi_n, q}[{x}]/(x^n + 1)$). "]},{"cell_type":"code","execution_count":null,"id":"dbecf2a5-f2b4-484b-a98a-3fb4dca31a07","metadata":{"id":"dbecf2a5-f2b4-484b-a98a-3fb4dca31a07"},"outputs":[],"source":["m = Poly({0: 1, 1: 1, 2: 1})\n","print(f\"Plain message: {pr(m)}\")"]},{"cell_type":"markdown","id":"43090be6-39fa-4a5a-98ed-7eb8adb4797e","metadata":{"id":"43090be6-39fa-4a5a-98ed-7eb8adb4797e"},"source":["To perform the encryption we need three \"small\" polynomials:\n","\n","1. Two error polynomials (\"small error\" polynomials), extracted from a discrete Gaussian distribution (similarly to the one used in the public key);\n","2. A \"small\" polynomial, $u$, which has coefficients drawn from (-1, 0, 1), similar to $s$."]},{"cell_type":"code","execution_count":null,"id":"e45fba78-b6e2-4ab7-96bb-8e09c868a0c8","metadata":{"id":"e45fba78-b6e2-4ab7-96bb-8e09c868a0c8"},"outputs":[],"source":["dist = np.random.default_rng().normal(mu, sigma, 16)\n","coeffs = {}\n","for ind in range(0, 16):\n"," coeffs[ind] = round(dist[ind])\n","\n","e_1 = Poly(coeffs)\n","\n","dist = np.random.default_rng().normal(mu, sigma, 16)\n","coeffs = {}\n","for ind in range(0, 16):\n"," coeffs[ind] = round(dist[ind])\n","\n","e_2 = Poly(coeffs)\n","\n","coeffs = {}\n","for i in range(0, 16):\n"," coeffs[i] = np.random.randint(-1, 2)\n"," \n","u = Poly(coeffs)\n","\n","print(f\"e_1: {pr(e_1)}\\n\")\n","print(f\"e_2: {pr(e_2)}\\n\")\n","print(f\"u: {pr(u)}\\n\")"]},{"cell_type":"markdown","id":"8061f730-a0df-48a2-87b3-78e7971b089a","metadata":{"id":"8061f730-a0df-48a2-87b3-78e7971b089a"},"source":["We use these polynomials in the encryption phase, then we discard them.\n","The ciphertext is represented by two polynomials calculated as:\n","$$ct = \\left ( \\left [ k_{p_0} u + e_1 + \\left \\lfloor \\frac{q}{p} \\right \\rfloor m \\right ]_{\\Phi_n, q} , \\left [k_{p_1} u + e_2 \\right ]_{\\Phi_n, q} \\right)$$\n","\n","Which can be rewritten, expanding $k_p$:\n","$$ct = \\left ( \\left [ -asu -eu + e_1 + \\left \\lfloor \\frac{q}{p} \\right \\rfloor m \\right ]_{\\Phi_n, q} , \\left [ au + e_2 \\right ]_{\\Phi_n, q} \\right)$$"]},{"cell_type":"markdown","id":"77a7932f-ce12-4266-9350-223bc66a60ec","metadata":{"id":"77a7932f-ce12-4266-9350-223bc66a60ec"},"source":["The initial message had coefficients in the range of $\\bmod p$. However, they have been *scaled* by $\\left \\lfloor \\frac{q}{p} \\right \\rfloor$. This means that, in the ciphertext, the message has coefficients in the range of $\\bmod q$. \n","Morover the role of $u$ become clear: $u$ is a mask which changes at every encryption, and is responsible for the property of confusion typical of HE schemes. The same plaintext message will always have a different encryption.\n","\n","The other terms in the ciphertext *hide* the scaled message, including $-asu$ (which is the biggest one, in terms of coefficients).\n","On the other hand, having $-asu$ and $au$ in the ciphertext is the key for the decryption phase."]},{"cell_type":"code","execution_count":null,"id":"bb7954e0-9626-49b9-b641-33e946c1dba9","metadata":{"id":"bb7954e0-9626-49b9-b641-33e946c1dba9"},"outputs":[],"source":["ct0 = pk[0] * u + e_1 + floor((q/p)) * m\n","ct0 = divmod(ct0, polynomial_modulus)[1]\n","ct0 = mod_on_coefficients(ct0, q)\n","\n","ct1 = pk[1] * u + e_2\n","ct1 = divmod(ct1, polynomial_modulus)[1]\n","ct1 = mod_on_coefficients(ct1, q)\n","\n","print(f\"Ciphertext: {pr(ct0)}, {pr(ct1)}\")"]},{"cell_type":"markdown","id":"c14d7341-7099-449b-90eb-02accc5a4741","metadata":{"id":"c14d7341-7099-449b-90eb-02accc5a4741"},"source":["### Decryption\n","The decryption is relatively simple.\n","\n","In fact, you may have noted that the biggest error term, which hides our message in the ciphertext, is given by $-asu$.\n","However, if we are the entity who crypted the message:\n","\n"," 1. we have $s$;\n"," 2. we have $au$, because it is contained in the second term of the ciphertext.\n","\n","So, we just have to multiply the second term of the ciphertext with $s$, and sum it with the first term of the ciphertext:\n","\n","$$ [{ct_0 + ct_1s}]_{\\Phi_n, q} = \\left [ -asu -eu + e_1 + \\left \\lfloor \\frac{q}{p} \\right \\rfloor m + asu + se_2 \\right ]_{\\Phi_n, q} $$\n","\n","We end up with:\n","\n","$$ \\left [ \\left \\lfloor \\frac{q}{p} \\right \\rfloor m -eu + e_1 + se_2 \\right]_{\\Phi_n, q} $$\n","\n","Inside this polynomial we have the scaled message summed to some noise. If the noise is not too big, we can recover the message.\n","\n","To do that, we just try to make the modulo with the polynomial modulus, than to apply $\\bmod q$ to the coefficients of the resulting polynomial."]},{"cell_type":"code","execution_count":null,"id":"7ac50a01-3251-4dbb-ac27-5a595aa001d6","metadata":{"id":"7ac50a01-3251-4dbb-ac27-5a595aa001d6"},"outputs":[],"source":["decrypt = ???\n","decrypt = divmod(decrypt, polynomial_modulus)[1]\n","decrypt = mod_on_coefficients(decrypt, q)\n","\n","print(f\"Scaled plaintext + errors: {pr(decrypt)}\")"]},{"cell_type":"markdown","id":"37ea3414-f295-4862-98f0-31eeac22b517","metadata":{"id":"37ea3414-f295-4862-98f0-31eeac22b517"},"source":["If we rescale..."]},{"cell_type":"code","execution_count":null,"id":"231e8a41-e93f-485f-ab8f-cd92d0552ab2","metadata":{"id":"231e8a41-e93f-485f-ab8f-cd92d0552ab2"},"outputs":[],"source":["decrypt = ???\n","print(f\"De-scaled plaintext plus errors: {pr(decrypt)}\")"]},{"cell_type":"markdown","id":"a2816ae3-735e-4563-85d2-40405a2b53fb","metadata":{"id":"a2816ae3-735e-4563-85d2-40405a2b53fb"},"source":["Don't forget to round to the nearest integer and make the modulo $\\bmod p$!"]},{"cell_type":"code","execution_count":null,"id":"615f8c7b-319f-4996-acb4-885dc2d8b24b","metadata":{"id":"615f8c7b-319f-4996-acb4-885dc2d8b24b"},"outputs":[],"source":["decrypt = round_on_nearest_integer(decrypt)\n","decrypt = mod_on_coefficients(decrypt, p)\n","\n","print(f\"Plain starting message: {pr(m)}\")\n","print(f\"Final decryption result: {pr(decrypt)}\")"]},{"cell_type":"markdown","id":"10c1e627-7353-4278-aa8a-cb4cc5a0033d","metadata":{"id":"10c1e627-7353-4278-aa8a-cb4cc5a0033d"},"source":["This is exactly our message!\n","\n","If the coefficients are too noisy, they will be rounded to another integer, and we will end up with wrong terms. The amount of room for noise can be adjusted by making the ratio $q/p$ larger or smaller.\n","\n","We are ready to define some other helper functions to automatize the process of encryption and decryption."]},{"cell_type":"code","execution_count":null,"id":"b852ff05-0ce5-49ab-8c36-a7f9873d8567","metadata":{"id":"b852ff05-0ce5-49ab-8c36-a7f9873d8567"},"outputs":[],"source":["def generate_keys(n, q):\n"," \"\"\"\n"," Generate public and secret key, using the parameters\n"," d, q passed as parameters.\n"," \n"," Returns sk, pk.\n"," \"\"\"\n"," polynomial_modulus = Poly({0: 1, n: 1})\n"," coeffs = {}\n"," \n"," # Secret key\n"," for i in range(0, n):\n"," coeffs[i] = np.random.randint(-1, 2)\n","\n"," sk = Poly(coeffs)\n"," \n"," # Public key\n"," coeffs = {}\n"," for i in range(0, n):\n"," coeffs[i] = np.random.randint(0, q)\n","\n"," a = Poly(coeffs)\n","\n"," dist = np.random.default_rng().normal(mu, sigma, 16)\n"," coeffs = {}\n"," for ind in range(0, n):\n"," coeffs[ind] = round(dist[ind])\n","\n"," e = Poly(coeffs)\n","\n"," pk_0 = -((a * sk) + e)\n"," pk_0 = divmod(pk_0, polynomial_modulus)[1]\n"," pk_0 = mod_on_coefficients(pk_0, q)\n","\n"," pk_1 = a\n","\n"," pk = (pk_0, pk_1)\n"," \n"," return sk, pk\n","\n","def encrypt_poly(m, pk, n, p, q):\n"," \"\"\"\n"," Encrypt a polynomial message given the public key.\n"," Returns the encrypted polynomial.\n"," \"\"\"\n"," polynomial_modulus = Poly({0: 1, n: 1})\n","\n"," dist = np.random.default_rng().normal(mu, sigma, 16)\n"," coeffs = {}\n"," for ind in range(0, n):\n"," coeffs[ind] = round(dist[ind])\n","\n"," e_1 = Poly(coeffs)\n","\n"," dist = np.random.default_rng().normal(mu, sigma, 16)\n"," coeffs = {}\n"," for ind in range(0, n):\n"," coeffs[ind] = round(dist[ind])\n","\n"," e_2 = Poly(coeffs)\n","\n"," coeffs = {}\n"," for i in range(0, n):\n"," coeffs[i] = np.random.randint(-1, 2)\n","\n"," u = Poly(coeffs)\n","\n"," ct0 = pk[0] * u + e_1 + floor((q/p)) * m\n"," ct0 = divmod(ct0, polynomial_modulus)[1]\n"," ct0 = mod_on_coefficients(ct0, q)\n"," \n"," ct1 = pk[1] * u + e_2 \n"," ct1 = divmod(ct1, polynomial_modulus)[1]\n"," ct1 = mod_on_coefficients(ct1, q)\n"," \n"," return (ct0, ct1)\n","\n","def decrypt_poly(ct, ks, n, p, q):\n"," \"\"\"\n"," Decrypts an encrypted polynomial, given the secret key.\n"," Returns the decrypted polynomial.\n"," \"\"\"\n"," polynomial_modulus = Poly({0: 1, n: 1})\n"," decrypt = Poly({0: 0})\n"," \n"," for i, term in enumerate(ct):\n"," decrypt += ct[i] * polypow(ks.coef, i, 1000)\n"," \n"," decrypt = divmod(decrypt, polynomial_modulus)[1]\n"," decrypt = mod_on_coefficients(decrypt, q)\n"," \n"," decrypt = decrypt * p/q\n"," decrypt = round_on_nearest_integer(decrypt)\n"," decrypt = mod_on_coefficients(decrypt, p)\n"," \n"," return decrypt "]},{"cell_type":"markdown","id":"49342de7-897a-4fa3-b6cb-4a36238a5e8e","metadata":{"id":"49342de7-897a-4fa3-b6cb-4a36238a5e8e"},"source":["## Homomorphic operations\n","Obviously, we are interested in the special aspect of these HE schemes: the possibility to compute homomorphic additions and multiplications.\n","\n","This means that we can add and multiply numbers while they are still encrypted, without having to decrypt them."]},{"cell_type":"markdown","id":"40b2109a-bbe7-4d04-9b19-38e4d4764680","metadata":{"id":"40b2109a-bbe7-4d04-9b19-38e4d4764680"},"source":["### Homomorphic addition (between ciphertexts)\n","The simplest case is the addition of two ciphertexts. \n","Starting from $ct_1$ and $ct_2$, which are the encryption of messages $m_1$ and $m_2$ respectively, we can compute $ct_3 = ct_1 + ct_2$.\n","The decryption of $ct_3$ is equal to $m_1 + m_2$.\n","\n","Obviously, this assuming the correctness of the processes and suitable encryption parameters."]},{"cell_type":"code","execution_count":null,"id":"fbcf9b16-1861-4e31-92f7-291e0dc5641f","metadata":{"id":"fbcf9b16-1861-4e31-92f7-291e0dc5641f"},"outputs":[],"source":["n = 16\n","p = 7\n","q = 874\n","\n","ks, kp = ??? (n, q)\n","\n","m1 = Poly({0: 1, 1: 1, 2: 1})\n","m2 = Poly({1: 1})\n","print(f\"First message: {pr(m1)}\")\n","print(f\"Second message: {pr(m2)}\")\n","print(f\"Expected sum: {pr(m1+m2)}\")\n","print(f\"-------------------------\")\n","\n","enc_m1 = ??? (m1, kp, n, p, q)\n","enc_m2 = ??? (m2, kp, n, p, q)\n","print(f\"First ciphertext: {pr(enc_m1[0])}, {pr(enc_m1[1])}\\n\")\n","print(f\"Second ciphertext: {pr(enc_m2[0])}, {pr(enc_m2[1])}\\n\")"]},{"cell_type":"markdown","id":"f893ead0-3f4d-4209-8561-36daffbdc616","metadata":{"id":"f893ead0-3f4d-4209-8561-36daffbdc616"},"source":["To perform the addition between the two ciphertexts, it is sufficient to add them element-wise.\n","In the followings $u_1$ and $u_2$, as well as $e_1, \\ldots, e_4$ denote the \"small\" and \"small errors\" polynomials created during the encryption phase.\n","\n","$$ enc\\_m_1 + enc\\_m_2 = \\left ( \\left [k_{p_0}u_1 + e_1 + \\left \\lfloor \\frac{q}{p} \\right \\rfloor m_1 \\right ]_{\\Phi_n, q} , \\left [k_{p_1}u_1 + e_2 \\right]_{\\Phi_n, q} \\right ) + \\left ( \\left [k_{p_0}u_2 + e_3 + \\left \\lfloor \\frac{q}{p} \\right \\rfloor m_2 \\right ]_{\\Phi_n, q} , \\left [ k_{p_1}u_1 + e_4 \\right]_{\\Phi_n, q} \\right ) $$\n","\n","$$ enc\\_m_1 + enc\\_m_2 = \\left ( \\left [k_{p_0}(u_1 + u_2) + e_1 + e_3 + \\left \\lfloor \\frac{q}{p} \\right \\rfloor (m_1 + m_2) \\right ]_{\\Phi_n, q} , \\left [ k_{p_1}(u_1 + u_2) + e_2 + e_4 \\right ]_{\\Phi_n, q} \\right ) $$\n","\n","We are interested in the term $\\left \\lfloor \\frac{q}{p} \\right \\rfloor(m_1 + m_2)$, which is the scaled sum of our messages; however, new noise terms appeared in the ciphertext.\n","\n","Let's try to decrypt this, multiplying the second term of the ciphertext with $s$ and summing it up. Also, expand $k_p$:\n","\n","\n","\n","$$ [{ct_0 + ct_1s}]_{\\Phi_n, q} = \\left [ -e(u_1 + u_2) + e_1 + e_3 + \\left \\lfloor \\frac{q}{p} \\right \\rfloor(m_1 + m_2) + s(e_2 + e_4) \\right ]_{\\Phi_n, q} $$\n","\n","- We have some new noise terms. \n","- the noise polynomials become a problem if one of their coefficients is larger than $\\frac{q}{2p}$, which, in our case, is around $62$.\n","\n","The new noise terms are:\n","\n"," 1. $-e(u_1 + u_2)$: this is the product of a \"small noise\" polynomial with the sum of two \"small\" polynomials (with coefficients $-1, 0, 1$). This can be quite big, because sometimes the sum of $-1, 0, 1$ will be close to $0$, while other times it will become $2, 3, 4...$. This can reach $62$ pretty fast.\n"," 2. $e_1 + e_3$: this is only the sum of two \"small noise\" polynomials. It's less worrying than the first.\n"," 3. $s(e_2 + e_4)$: this is bad similarly to the the first.\n","\n","Can we graphically visualize how bad is this?\n","\n","![image](https://drive.google.com/uc?export=view&id=1ybxLOWVvz4T-0o1BNxYDHaDzqVFXzrqw)\n","\n","- After just one addition, there is a decent probability that a coefficient of the noise polynomials will be higher than $62$, leading to an error in the decryption phase.\n","\n","To have more room, we have to increment the ratio $q/p$.\n","\n","Let's compute an addition."]},{"cell_type":"code","execution_count":null,"id":"b6f74e4b-3533-448f-b24d-024aaef1819c","metadata":{"id":"b6f74e4b-3533-448f-b24d-024aaef1819c"},"outputs":[],"source":["def add_ciphertexts(enc_m1, enc_m2, n, q):\n"," polynomial_modulus = Poly({0: 1, n: 1})\n"," enc_sum_0 = ???\n"," enc_sum_1 = ???\n","\n"," enc_sum_0 = divmod(enc_sum_0, polynomial_modulus)[1]\n"," enc_sum_0 = mod_on_coefficients(enc_sum_0, q)\n","\n"," enc_sum_1 = divmod(enc_sum_1, polynomial_modulus)[1]\n"," enc_sum_1 = mod_on_coefficients(enc_sum_1, q)\n","\n"," enc_sum = (enc_sum_0, enc_sum_1)\n"," return enc_sum\n","\n","enc_sum = add_ciphertexts(enc_m1, enc_m2, n, q)\n","print(f\"Encrypted sum: {pr(enc_sum[0])}, {pr(enc_sum[1])}\\n\")\n","\n","decrypted = decrypt_poly(enc_sum, ks, n, p, q)\n","print(f\"Expected sum: {pr(m1+m2)}\")\n","print(f\"Decrypted result: {pr(decrypted)}\")"]},{"cell_type":"markdown","id":"f6c12a0d-5969-451c-abe3-2a60a4be838f","metadata":{"id":"f6c12a0d-5969-451c-abe3-2a60a4be838f"},"source":["Remember that we are in a ring, so $+6 \\equiv -1$, etc.\n","If the result is not correct, try to encrypt the messages again and re-computing the addition."]},{"cell_type":"markdown","id":"5b510351-9117-435b-9d1a-07843d81da98","metadata":{"id":"5b510351-9117-435b-9d1a-07843d81da98"},"source":["### Homomorphic multiplication (between ciphertexts)\n","The multiplication between ciphertexts is more complicated than the addition and can cause an important increment of the noise in the resulting ciphertext.\n","\n","Key points:\n","1. At some point, we will have a term: $\\left \\lfloor \\frac{q}{p} \\right \\rfloor m_1 * \\left \\lfloor \\frac{q}{p} \\right \\rfloor m_2$ which is $\\frac{q^2}{p^2}m_1m_2$ -> we will multiply the ciphertexts also with $\\frac{p}{q}$ in order to come back to $\\left \\lfloor \\frac{q}{p} \\right \\rfloor m_1m_2$;\n","2. The multiplication between two ciphertexts produces a three-terms ciphertext;\n","3. If we use the encryption parameters that we used in the addition case ($t=7$, $q=874$) we have a problem if a coefficient in the result polynomial will be higher than $\\left \\lfloor \\frac{q}{2t} \\right \\rfloor = 62$.\n","\n","Let's inspect the distribution of the coefficients in one multiplication, for all the noise terms we obtained:\n","\n","![image](https://drive.google.com/uc?export=view&id=10Z7i9SL7QTmjWDxn4gZA08c_a_WI-ZnV)"]},{"cell_type":"markdown","id":"f80c5304-eb07-4cc9-8049-6e749a540647","metadata":{"id":"f80c5304-eb07-4cc9-8049-6e749a540647"},"source":["It is basically sure that this will not work: in the histogram for $-e * u_1 * m_2$ we see that there is a very high probability of coefficients being in the range $[{-500, 500}]$. We have to increment the ratio $\\frac{q}{p}$ to have more room with noise. Let's try with $424242$:"]},{"cell_type":"code","execution_count":null,"id":"9fd3a60d-f43f-4ec8-9c9d-d39db3b916ce","metadata":{"id":"9fd3a60d-f43f-4ec8-9c9d-d39db3b916ce"},"outputs":[],"source":["n = 16\n","p = 7\n","q = 424242\n","\n","ks, kp = generate_keys(n, q)\n","\n","m1 = Poly({0: 1, 1: 1, 2: 3})\n","m2 = Poly({0: -1, 1: -2, 2: 1})\n","print(f\"First message: {pr(m1)}\")\n","print(f\"Second message: {pr(m2)}\")\n","print(f\"Expected product: {pr(m1*m2)}\")\n","print(f\"-------------------------\")\n","\n","enc_m1 = encrypt_poly(m1, kp, n, p, q)\n","enc_m2 = encrypt_poly(m2, kp, n, p, q)\n","print(f\"First ciphertext: {pr(enc_m1[0])}, {pr(enc_m1[1])}\\n\")\n","print(f\"Second ciphertext: {pr(enc_m2[0])}, {pr(enc_m2[1])}\")"]},{"cell_type":"code","execution_count":null,"id":"ae9880f8-ff18-4410-b971-c558f33297d6","metadata":{"id":"ae9880f8-ff18-4410-b971-c558f33297d6"},"outputs":[],"source":["def multiply_ciphertexts(ct1, ct2, n, p, q):\n"," polynomial_modulus = Poly({0: 1, n: 1})\n"," \n"," c0 = (p/q) * ct1[0] * ct2[0]\n"," c0 = divmod(c0, polynomial_modulus)[1]\n"," c0 = mod_on_coefficients(c0, q)\n"," c0 = round_on_nearest_integer(c0)\n"," \n"," c1 = (p/q) * (ct1[0] * ct2[1] + ct1[1] * ct2[0])\n"," c1 = divmod(c1, polynomial_modulus)[1]\n"," c1 = mod_on_coefficients(c1, q)\n"," c1 = round_on_nearest_integer(c1)\n"," \n"," c2 = (p/q) * ct1[1] * ct2[1]\n"," c2 = divmod(c2, polynomial_modulus)[1]\n"," c2 = mod_on_coefficients(c2, q)\n"," c2 = round_on_nearest_integer(c2)\n"," \n"," return (c0, c1, c2) \n","\n","mult = multiply_ciphertexts(enc_m1, enc_m2, n, p, q)\n","decrypted = decrypt_poly(mult, ks, n, p, q)\n","\n","print(f\"Expected product: {pr(m1*m2)}\")\n","print(f\"Decrypted result: {pr(decrypted)}\")"]},{"cell_type":"markdown","id":"9141f6cc-1099-4ce5-87ad-8353aa845c65","metadata":{"id":"9141f6cc-1099-4ce5-87ad-8353aa845c65"},"source":["### Homomorphic operations between ciphertexts and plaintexts\n","An important aspect of schemes like BFV is the possibility to compute additions and multiplications not only between ciphertexts, but also between a ciphertext and a plaintext.\n","\n","This is very important for HE-ML: if a third party wants to run a ML model on encrypted data, it is useless to encrypt the model. Not only, this is actually wasteful because **operations between ciphertexts and plaintexts create much less noise**.\n","\n","They are quite easy with respect to the ones we saw before.\n","Let $ct = (ct_0, ct_1, \\ldots, ct_n)$ be an encrypted polynomial and $pt$ a plain polynomial.\n","\n","$$ ct + pt = (ct_0 + \\left \\lfloor \\frac{q}{p} \\right \\rfloor pt, ct_1, \\ldots, ct_n) $$\n","$$ ct * pt = (ct_0 * pt, ct_1 * pt, \\ldots, ct_n * pt) $$"]},{"cell_type":"code","execution_count":null,"id":"59992e38-6c86-4c03-a3e2-8effa5a52be1","metadata":{"id":"59992e38-6c86-4c03-a3e2-8effa5a52be1"},"outputs":[],"source":["def add_cipher_and_plain(ct, pt, p, q):\n"," ct = list(ct)\n"," ct[0] = ct[0] + floor((q/p))*pt\n"," ct = tuple(ct)\n"," \n"," return ct\n","\n","def multiply_cipher_and_plain(ct, pt):\n"," \n"," new_ct = []\n"," for elem in ct:\n"," new_ct.append(elem * pt)\n"," \n"," return tuple(new_ct)"]},{"cell_type":"code","execution_count":null,"id":"2ef6e031-e43b-4fd3-9562-b969ca9d453e","metadata":{"id":"2ef6e031-e43b-4fd3-9562-b969ca9d453e"},"outputs":[],"source":["n = 16\n","p = 7\n","q = 424242\n","\n","ks, kp = generate_keys(n, q)\n","\n","m1 = Poly({0: 1, 1: 1, 2: 3})\n","m2 = Poly({0: -1, 1: -2, 2: 1})\n","print(f\"First message: {pr(m1)}\")\n","print(f\"Second message: {pr(m2)}\")\n","print(f\"Expected sum: {pr(m1+m2)}\")\n","print(f\"Expected product: {pr(m1*m2)}\")\n","print(f\"-------------------------\")\n","\n","print(f\"Encrypting first message...\")\n","enc_m1 = encrypt_poly(m1, kp, n, p, q)\n","\n","add_result = add_cipher_and_plain(enc_m1, m2, p, q)\n","mult_result = multiply_cipher_and_plain(enc_m1, m2)\n","\n","print(f\"--------------------------\")\n","print(f\"Decrypted sum: {pr(decrypt_poly(add_result, ks, n, p, q))}\")\n","print(f\"Decrypted product: {pr(decrypt_poly(mult_result, ks, n, p, q))}\")"]},{"cell_type":"markdown","id":"87942803-7912-4074-a96c-d589f1b0785d","metadata":{"id":"87942803-7912-4074-a96c-d589f1b0785d"},"source":["In the case of ciphertext-plaintext addition, virtually we are not adding any noise to the ciphertext: in fact, if the plain message is $m_2$, we are adding to the ciphertext $\\left \\lfloor \\frac{q}{p} \\right \\rfloor m_2$ which will be summed to $\\left \\lfloor \\frac{q}{p} \\right \\rfloor m_1$. However, adding a polynomial may lead a coefficient in the ciphertext to go above $t$ or also $\\frac{q}{2t}$.\n","\n","For multiplications, the problem is a more complicated; however, generally, the noise incremenet depends on the \"dimension\" of the plaintext message. This is still an operation which produce much less noise than ciphertext-ciphertext multiplications and do not require a relinearization phase after."]},{"cell_type":"markdown","source":["# Demo 2: Helper libraries for HE"],"metadata":{"id":"5jbxrqsZAtTo"},"id":"5jbxrqsZAtTo"},{"cell_type":"markdown","source":["In this part of the tutorial we will learn how to use an helper Python library for HE, namely [Pyfhel](https://github.com/ibarrond/Pyfhel).\n","Note: we will use the version 2.x of the library for practical reasons, but check out the new version 3!"],"metadata":{"id":"Oeidl9UGYUcX"},"id":"Oeidl9UGYUcX"},{"cell_type":"markdown","id":"Dx8Zfmm3PDMN","metadata":{"id":"Dx8Zfmm3PDMN"},"source":["## Encryption parameters\n","To use BFV, we have to choose two encryption parameters:\n","- $m$: the degree of the cyclotomic polynomial (i.e., the maximum degree of the polynomials);\n","- $p$: the plaintext coefficients modulo.\n","\n","The third parameter, $q$, which is the ciphertext coefficients modulo, is chosen automatically by the library in order to be safe (with a 128 bit level of security---stronger levels of security may be chosen if desired)."]},{"cell_type":"code","execution_count":null,"id":"7c74dbc3-b890-4a87-9e8e-0094b9fc17b4","metadata":{"id":"7c74dbc3-b890-4a87-9e8e-0094b9fc17b4"},"outputs":[],"source":["from Pyfhel import Pyfhel, PyPtxt, PyCtxt\n","\n","HE = ???\n","HE.???\n","HE.???\n","\n","print(HE)"]},{"cell_type":"markdown","source":["Even though the BFV scheme was born for integers, Pyfhel 2.x let the user use a *fractional encoder* which makes it possible to use the BFV with fractional numbers, with fixed precision."],"metadata":{"id":"jGSD5zUVrbd-"},"id":"jGSD5zUVrbd-"},{"cell_type":"code","execution_count":null,"id":"a9f9b90a-f443-4a6b-9fbb-b1fc91c2565d","metadata":{"id":"a9f9b90a-f443-4a6b-9fbb-b1fc91c2565d"},"outputs":[],"source":["a = 127.15717263\n","b = -2.128965182\n","ctxt1 = HE.???\n","ctxt2 = HE.???\n","\n","ctxtSum = ctxt1 + ctxt2\n","ctxtSub = ctxt1 - ctxt2\n","ctxtMul = ctxt1 * ctxt2\n","\n","resSum = HE.???(ctxtSum)\n","resSub = HE.???(ctxtSub) \n","resMul = HE.???(ctxtMul)\n","\n","print(f\"Expected sum: {a+b}, decrypted sum: {resSum}\")\n","print(f\"Expected sub: {a-b}, decrypted sum: {resSub}\")\n","print(f\"Expected mul: {a*b}, decrypted sum: {resMul}\")"]},{"cell_type":"markdown","id":"tYk1ptFwPL2Z","metadata":{"id":"tYk1ptFwPL2Z"},"source":["Noise is added to numbers..."]},{"cell_type":"code","execution_count":null,"id":"6b0293c5-ebf3-4c8b-871f-6f4dd2eb4b89","metadata":{"id":"6b0293c5-ebf3-4c8b-871f-6f4dd2eb4b89"},"outputs":[],"source":["print(f\"Starting Noise Budget: {HE.???(ctxt1)}\")\n","\n","print(f\"sum Noise Budget: {HE.???(ctxtSum)}\")\n","print(f\"prod Noise Budget: {HE.???(ctxtMul)}\")"]},{"cell_type":"code","execution_count":null,"id":"ca445c79-53c2-4507-9438-eb60a5c6302e","metadata":{"id":"ca445c79-53c2-4507-9438-eb60a5c6302e"},"outputs":[],"source":["ptxt1 = HE.???(-2.128965182)\n","\n","print(f\"ctxt-ptxt prod Noise Budget: {HE.???(ctxt1 * ptxt1)}\")"]},{"cell_type":"markdown","source":["Can we use Pyfhel to process encrypted data, with a DL model? Let's start with a simple linear layer...\n","\n","$$ y = Wx^T + b $$\n","\n"],"metadata":{"id":"di86fsAq4PQB"},"id":"di86fsAq4PQB"},{"cell_type":"markdown","source":["Let's try with:\n","$$\n","W = \\begin{bmatrix} \n","1 & 2\\\\ \n","1 & 1\\\\\n","3 & 1\n","\\end{bmatrix}\n","$$\n","\n","$$\n","b = \\begin{bmatrix} \n","0 \\\\\n","1 \\\\\n","2 \n","\\end{bmatrix}\n","$$\n","\n","and\n","\n","$$\n","x = \\begin{bmatrix} \n","1 & 2\\\\ \n","\\end{bmatrix}\n","$$\n","\n","we should obtain:\n","$$\n","y = (\\begin{bmatrix} \n","1 & 2\\\\ \n","1 & 1\\\\\n","3 & 1\n","\\end{bmatrix} * \n","\\begin{bmatrix} \n","1\\\\ \n","2\n","\\end{bmatrix}) + \n","\\begin{bmatrix} \n","0 \\\\ \n","1 \\\\\n","2 \n","\\end{bmatrix}\n","= \n","\\begin{bmatrix} \n","5\\\\\n","4\\\\\n","7\n","\\end{bmatrix}\n","$$"],"metadata":{"id":"UijYS7KV5C0I"},"id":"UijYS7KV5C0I"},{"cell_type":"code","source":["import numpy as np\n","class LinearLayer:\n"," def __init__(self, weights, bias=None):\n"," self.weights = weights\n"," self.bias = bias\n","\n"," def __call__(self, t):\n"," \n"," result = np.array([np.sum(t * row) for row in self.weights])\n","\n"," if self.bias is not None:\n"," result = np.array(result + self.bias)\n","\n"," return result"],"metadata":{"id":"O3KhhkGI4XM5"},"id":"O3KhhkGI4XM5","execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["First, we create weights and biases:"],"metadata":{"id":"WtcJeCTj4tWG"},"id":"WtcJeCTj4tWG"},{"cell_type":"code","source":["W = np.array([[1, 2], [1, 1], [3, 1]])"],"metadata":{"id":"rkm67tge4s23"},"id":"rkm67tge4s23","execution_count":null,"outputs":[]},{"cell_type":"code","source":["b = np.array([0, 1, 2])"],"metadata":{"id":"Q1_08DGX7fDi"},"id":"Q1_08DGX7fDi","execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["Then, the input:"],"metadata":{"id":"d66Byxcr8ERC"},"id":"d66Byxcr8ERC"},{"cell_type":"code","source":["x = np.array([1, 2])"],"metadata":{"id":"nEMv3N107abp"},"id":"nEMv3N107abp","execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["Create a linear layer:"],"metadata":{"id":"XY3-sm0Pryda"},"id":"XY3-sm0Pryda"},{"cell_type":"code","source":["lin_layer = LinearLayer(W, b)"],"metadata":{"id":"S2z_xfei4qFT"},"id":"S2z_xfei4qFT","execution_count":null,"outputs":[]},{"cell_type":"code","source":["result = lin_layer(x)\n","print(result)"],"metadata":{"id":"MoGJTdkw8JYH"},"id":"MoGJTdkw8JYH","execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["What if our x was encrypted?"],"metadata":{"id":"SB3lwJiP8PJK"},"id":"SB3lwJiP8PJK"},{"cell_type":"code","source":["encode = lambda x: HE.encodeInt(x)\n","encrypt = lambda x: HE.encryptInt(x)\n","decrypt = lambda x: HE.decryptInt(x)"],"metadata":{"id":"t-lw40fz8yzP"},"id":"t-lw40fz8yzP","execution_count":null,"outputs":[]},{"cell_type":"code","source":["encrypted_x = np.stack(np.vectorize(encrypt)(x), axis=0)"],"metadata":{"id":"28lmiOWq8Sff"},"id":"28lmiOWq8Sff","execution_count":null,"outputs":[]},{"cell_type":"code","source":["encoded_W = np.stack(np.vectorize(encode)(W), axis=0)\n","encoded_b = np.stack(np.vectorize(encode)(b), axis=0)"],"metadata":{"id":"5DIq9EAS8YS2"},"id":"5DIq9EAS8YS2","execution_count":null,"outputs":[]},{"cell_type":"code","source":["encoded_lin_layer = LinearLayer(encoded_W, encoded_b)"],"metadata":{"id":"YYo8FdJq9YCb"},"id":"YYo8FdJq9YCb","execution_count":null,"outputs":[]},{"cell_type":"code","source":["encrypted_result = encoded_lin_layer(encrypted_x)\n","print(encrypted_result)"],"metadata":{"id":"FgBa2J0H9ln5"},"id":"FgBa2J0H9ln5","execution_count":null,"outputs":[]},{"cell_type":"code","source":["decrypted_result = np.stack(np.vectorize(decrypt)(encrypted_result), axis=0)\n","print(decrypted_result)"],"metadata":{"id":"OPiCfdHi9o7I"},"id":"OPiCfdHi9o7I","execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Designing HE-optimized DL models: a methodology\n","> How can we design machine and deep learning models able to operate on encrypted data with HE-BFV?\n","\n","![image](https://drive.google.com/uc?export=view&id=1d8vBQueEPWdoabEkvoG-q-xZE64CGtjT)"],"metadata":{"id":"Z-L9HLkYdqMZ"},"id":"Z-L9HLkYdqMZ"},{"cell_type":"markdown","source":["## 1. Model approximation\n","Replacing the processing layers of $F(\\cdot)$ that are not compliant with the BFV scheme with those relying only on additions and multiplications.\n","\n","The output of this step is an approximated model $\\varphi(\\cdot)$ comprising processing layers that are HE-compliant.\n","\n","- Maximum pooling -> Average pooling\n","- Layer normalization -> Batch normalization\n","- ReLU/tanh -> Square"],"metadata":{"id":"hMktuqw3hkc2"},"id":"hMktuqw3hkc2"},{"cell_type":"markdown","source":["## 2. Model encoding\n","- Weights have been encoded according to the BFV scheme and the parameters $\\Theta = [n,p,q]$;\n","- Properly setting the parameters $\\Theta$ in a HE processing is still an open research area and generally the selection of the value of these parameters follows a “trial-and-error” approach.\n","\n","Some rules of thumb:\n","- $n \\geq 4096$;\n","- $p \\geq 2^{16}$;\n","- $q$ chosen from known safe values (usually $\\geq 10^{30}$)."],"metadata":{"id":"c_hzng05hnen"},"id":"c_hzng05hnen"},{"cell_type":"markdown","source":["## 3. Model validation\n","Two main purposes:\n","- $\\varphi_\\Theta(\\cdot)$ is evaluated to check if the selected configuration $\\Theta$ provides a sufficient amount of NB in the processing of the ciphertexts;\n","- The loss in accuracy of $\\varphi_\\Theta(\\cdot)$ w.r.t. $F(\\cdot)$ is evaluated.\n","\n","To achieve both goals a (possibly large) set of raw messages (i.e., a validation set) is processed by $\\varphi_\\Theta(\\cdot)$.\n","\n","> If the constraint on the NB is satisfied and the loss in accuracy is below a user-defined threshold (e.g., 1% or 5%): $\\varphi_\\Theta(\\cdot)$ becomes the privacy-preserving version of $F(\\cdot)$ to be considered. "],"metadata":{"id":"yIMELvCthq5X"},"id":"yIMELvCthq5X"},{"cell_type":"markdown","source":["![image](https://drive.google.com/uc?export=view&id=1qfe5oDb7tQU4q-6Zao6Thvruc97AYBHg)\n","\n"],"metadata":{"id":"d5GmhePcmDHC"},"id":"d5GmhePcmDHC"},{"cell_type":"markdown","source":["# Demo 3: PyCrCNN: HE-DL made easy"],"metadata":{"id":"CN3vzqxq99zL"},"id":"CN3vzqxq99zL"},{"cell_type":"markdown","source":["PyCrCNN is a Python library which can automatically process your Torch CNNs on encrypted data, with minimal code."],"metadata":{"id":"PT-VedqujXLa"},"id":"PT-VedqujXLa"},{"cell_type":"code","execution_count":null,"id":"C_hN0ls4Q1j4","metadata":{"id":"C_hN0ls4Q1j4"},"outputs":[],"source":["!git clone https://github.com/AlexMV12/PyCrCNN\n","import os, sys\n","module_path = os.path.abspath(os.path.join('PyCrCNN/'))\n","sys.path.append(module_path)"]},{"cell_type":"code","execution_count":null,"id":"1f2be842-80e1-4c95-b6b2-5724545551e2","metadata":{"id":"1f2be842-80e1-4c95-b6b2-5724545551e2"},"outputs":[],"source":["import time\n","import torch\n","import torchvision\n","import torchvision.transforms as transforms\n","import torch.nn as nn\n","import torch.nn.functional as F\n","import torch.optim as optim\n","\n","import numpy as np\n","\n","device = 'cuda' if torch.cuda.is_available() else 'cpu'"]},{"cell_type":"markdown","id":"1180b17e-7644-4ce8-99a6-c1906758e2eb","metadata":{"id":"1180b17e-7644-4ce8-99a6-c1906758e2eb"},"source":["## Convolutional Neural Networks\n","The simplest model family we can use with HE is Convolutional Neural Networks (CNNs).\n","They are well suited for two reasons:\n","- they are mainly composed of polynomial operations (apart from the activation functions, more on this later...);\n","- the are feed-forward: we can estimate the total number of consecutive multiplications on the inputs."]},{"cell_type":"code","execution_count":null,"id":"8fcf92dc-4929-4adc-b97e-2a1bf946bfeb","metadata":{"id":"8fcf92dc-4929-4adc-b97e-2a1bf946bfeb"},"outputs":[],"source":["transform = transforms.ToTensor()\n","\n","train_set = torchvision.datasets.???(\n"," root = './data',\n"," train=True,\n"," download=True,\n"," transform=transform\n",")\n","\n","test_set = torchvision.datasets.???(\n"," root = './data',\n"," train=False,\n"," download=True,\n"," transform=transform\n",")\n","\n","train_loader = torch.utils.data.DataLoader(\n"," train_set,\n"," batch_size=50,\n"," shuffle=True\n",")\n","\n","test_loader = torch.utils.data.DataLoader(\n"," test_set,\n"," batch_size=50,\n"," shuffle=True\n",")"]},{"cell_type":"code","source":["def get_num_correct(preds, labels):\n"," return preds.argmax(dim=1).eq(labels).sum().item()\n","\n","def train_net(network, epochs, device):\n"," optimizer = optim.Adam(network.parameters(), lr=0.001)\n"," for epoch in range(epochs):\n","\n"," total_loss = 0\n"," total_correct = 0\n","\n"," for batch in train_loader: # Get Batch\n"," images, labels = batch \n"," images, labels = images.to(device), labels.to(device)\n","\n"," preds = network(images) # Pass Batch\n"," loss = F.cross_entropy(preds, labels) # Calculate Loss\n","\n"," optimizer.zero_grad()\n"," loss.backward() # Calculate Gradients\n"," optimizer.step() # Update Weights\n","\n"," total_loss += loss.item()\n"," total_correct += get_num_correct(preds, labels)\n"," \n"," print(f\"Epoch {epoch+1}, loss: {total_loss:.2f}, accuracy: {total_correct} / {len(train_set)}\")\n","\n"," \n","def test_net(network, device):\n"," network.eval()\n"," total_loss = 0\n"," total_correct = 0\n"," \n"," with torch.no_grad():\n"," for batch in test_loader: # Get Batch\n"," images, labels = batch \n"," images, labels = images.to(device), labels.to(device)\n","\n"," preds = network(images) # Pass Batch\n"," loss = F.cross_entropy(preds, labels) # Calculate Loss\n","\n"," total_loss += loss.item()\n"," total_correct += get_num_correct(preds, labels)\n","\n"," accuracy = round(100. * (total_correct / len(test_loader.dataset)), 4)\n","\n"," return total_correct / len(test_loader.dataset)"],"metadata":{"id":"BLidaXOz_Tej"},"id":"BLidaXOz_Tej","execution_count":null,"outputs":[]},{"cell_type":"code","source":["CNN = nn.Sequential(\n"," ???\n"," )"],"metadata":{"id":"vTvl_eDh_VxY"},"id":"vTvl_eDh_VxY","execution_count":null,"outputs":[]},{"cell_type":"code","source":["CNN.to(device)\n","train_net(CNN, 5, device)\n","acc = test_net(CNN, device)"],"metadata":{"id":"SXET9Md4_ZTM"},"id":"SXET9Md4_ZTM","execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(f\"The CNN model obtained an accuracy of {acc}.\")"],"metadata":{"id":"HI_p40beAJAW"},"id":"HI_p40beAJAW","execution_count":null,"outputs":[]},{"cell_type":"markdown","id":"025dfbc9-38d0-41a7-a140-9606bfeff53f","metadata":{"id":"025dfbc9-38d0-41a7-a140-9606bfeff53f"},"source":["## Approximating\n","As we know, there are some operations that cannot be performed homomorphically on encrypted values. Most notably, these operations are division and comparison. It is possible to perform only linear functions.\n","\n","Consequently, in the CNN we used, we can not use `tanh()`. This is because we cannot apply its non-linearities.\n","\n","\n","One of the most common approach is to replace it with a simple polynomial function, for example a square layer (which simply performs $x \\rightarrow x^2$).\n","\n","We define the model with all the non-linearities removed **approximated**. This model can be re-trained, and it will be ready to be used on encrypted values."]},{"cell_type":"code","execution_count":null,"id":"5d4c2478-dabb-487c-99c3-283320a692f2","metadata":{"id":"5d4c2478-dabb-487c-99c3-283320a692f2"},"outputs":[],"source":["class Square(nn.Module):\n"," def __init__(self):\n"," super().__init__()\n","\n"," def forward(self, t):\n"," return torch.pow(t, 2)"]},{"cell_type":"code","source":["CNN_Approx = nn.Sequential(\n"," nn.Conv2d(1, 8, kernel_size=7),\n"," ???\n"," nn.AvgPool2d(kernel_size=4),\n","\n"," nn.Flatten(),\n","\n"," nn.Linear(200, 10),\n"," )"],"metadata":{"id":"VPU3VsVNASzu"},"execution_count":null,"outputs":[],"id":"VPU3VsVNASzu"},{"cell_type":"code","source":["CNN_Approx.to(device)\n","train_net(CNN_Approx, 5, device)\n","acc = test_net(CNN_Approx, device)"],"metadata":{"id":"YbN2M_K9ASzw"},"execution_count":null,"outputs":[],"id":"YbN2M_K9ASzw"},{"cell_type":"code","source":["print(f\"The approximated CNN obtained an accuracy of {acc}.\")"],"metadata":{"id":"qYr4xiG-ASzx"},"execution_count":null,"outputs":[],"id":"qYr4xiG-ASzx"},{"cell_type":"markdown","id":"874fb0fa-2122-46d3-8dcf-dc528959da8c","metadata":{"id":"874fb0fa-2122-46d3-8dcf-dc528959da8c"},"source":["## Encoding\n","Let's use PyCrCNN to automate the processing of an encrypted image.\n"," \n","Let's remember that, in order to be used with the encrypted values, also the weights of the models will have to be **encoded**. This means that each value in the weights of each layer will be encoded in a polynomial."]},{"cell_type":"markdown","id":"a9bfd4c8-79bb-4647-9353-5f76629f1f56","metadata":{"id":"a9bfd4c8-79bb-4647-9353-5f76629f1f56"},"source":["We can now use a function to \"convert\" a PyTorch model to a list of sequential HE-ready-to-be-used layers (`sequential`):"]},{"cell_type":"code","execution_count":null,"id":"886fc91b-7301-4a31-830a-05b326745c98","metadata":{"id":"886fc91b-7301-4a31-830a-05b326745c98"},"outputs":[],"source":["from pycrcnn.he.HE import BFVPyfhel_Fractional\n","from pycrcnn.model.sequential import Sequential"]},{"cell_type":"markdown","id":"06bde28e-7adc-4ede-876a-b5e57b63b0c4","metadata":{"id":"06bde28e-7adc-4ede-876a-b5e57b63b0c4"},"source":["## Encrypted processing\n","\n","Let's list the activities that we will now do:\n"," 1. Create a PyCrCNN BFV HE context, specifiying the encryption parameters `m` (polynomial modulus degree) and `p` (plaintext modulus). Let's remember that `q` will be chosen automatically in order to guarantee a 128-bit RSA equivalent security;\n"," 2. Convert our Torch approximated model to a list of layers able to work on matrices of encrypted values. The weights will be encoded;\n"," 3. Encrypt an image from our testing set;\n"," 4. Verify that the final classification result is correct."]},{"cell_type":"code","execution_count":null,"id":"2d2e537b-1bc0-4dee-8023-4ee75c61a882","metadata":{"id":"2d2e537b-1bc0-4dee-8023-4ee75c61a882"},"outputs":[],"source":["import matplotlib.pyplot as plt\n","\n","images, labels = next(iter(test_loader))\n","\n","sample_image = images[0]\n","sample_label = labels[0]"]},{"cell_type":"code","execution_count":null,"id":"eb5ae8c0-20dd-4456-ac49-145b21a17d6d","metadata":{"id":"eb5ae8c0-20dd-4456-ac49-145b21a17d6d"},"outputs":[],"source":["plt.imshow(sample_image[0], cmap='gray', interpolation='none')"]},{"cell_type":"code","execution_count":null,"id":"7d86b2ef-68ad-4eba-865a-b057efcdb8cc","metadata":{"id":"7d86b2ef-68ad-4eba-865a-b057efcdb8cc"},"outputs":[],"source":["sample_label"]},{"cell_type":"markdown","id":"c3a03f8f-997e-4009-94b4-541f7cc56318","metadata":{"id":"c3a03f8f-997e-4009-94b4-541f7cc56318"},"source":["We will create two PyCrCNN HE contexts: the one we will use to encrypt the image (`HE_Client`), and the one we will use to process the encrypted image (`HE_Server`). We will need to transfer the public key and the relinearization key in order to allow the server to compute some operations on encrypted data."]},{"cell_type":"code","execution_count":null,"id":"c798006e-31d2-4aa1-a270-58e84c3e8a13","metadata":{"id":"c798006e-31d2-4aa1-a270-58e84c3e8a13"},"outputs":[],"source":["encryption_parameters = {\n"," \"m\": ???,\n"," \"p\": ???,\n","}\n","\n","# Context generation\n","HE_Client = BFVPyfhel_Fractional(**encryption_parameters)\n","HE_Client.???\n","HE_Client.???\n","\n","public_key = HE_Client.get_public_key()\n","relin_key = HE_Client.get_relin_key()\n","\n","HE_Server = BFVPyfhel_Fractional(**encryption_parameters)\n","HE_Server.load_public_key(public_key)\n","HE_Server.load_relin_key(relin_key)"]},{"cell_type":"code","source":["encoded_model = Sequential(???, CNN_Approx)"],"metadata":{"id":"RLtmwyA1G_6_"},"id":"RLtmwyA1G_6_","execution_count":null,"outputs":[]},{"cell_type":"code","source":["encrypted_image = ???.encrypt_matrix(sample_image.unsqueeze(0).numpy())"],"metadata":{"id":"WCe1Gg7oG9Vm"},"id":"WCe1Gg7oG9Vm","execution_count":null,"outputs":[]},{"cell_type":"markdown","id":"34a70f52-d87b-489d-b156-2e28333513e3","metadata":{"id":"34a70f52-d87b-489d-b156-2e28333513e3"},"source":["Differences"]},{"cell_type":"code","execution_count":null,"id":"161ac5e4-bd96-46a8-b9fb-ab7a78810c62","metadata":{"id":"161ac5e4-bd96-46a8-b9fb-ab7a78810c62"},"outputs":[],"source":["with torch.no_grad():\n"," expected_output = ???(sample_image.unsqueeze(0))"]},{"cell_type":"code","execution_count":null,"id":"2bace3b5-4e38-4698-963b-768d85e707b1","metadata":{"id":"2bace3b5-4e38-4698-963b-768d85e707b1"},"outputs":[],"source":["start_time = time.time()\n","encrypted_output = ???(encrypted_image, debug=False)\n","\n","requested_time = round(time.time() - start_time, 2)\n","\n","result = HE_Client.decrypt_matrix(encrypted_output)\n","difference = expected_output.numpy() - result\n","\n","print(f\"\\nThe encrypted processing of one image requested {requested_time} seconds.\")\n","print(f\"\\nThe expected result was:\")\n","print(expected_output)\n","\n","print(f\"\\nThe actual result is: \")\n","print(result)\n","\n","print(f\"\\nThe error is:\")\n","print(difference) "]},{"cell_type":"markdown","id":"452f4264-02d5-49c2-9db1-5859d1d94123","metadata":{"id":"452f4264-02d5-49c2-9db1-5859d1d94123"},"source":["In this case we were not able to examine the NB evolution during the computation, because in order to compute the NB we need the secret key. If we use the same PyCrCNN HE context both to encrypt and to process the data, then we will also see the evolution of the NB after each layer."]},{"cell_type":"markdown","source":["# Contacts\n","- Manuel Roveri (manuel.roveri@polimi.it)\n","\n","Manuel Roveri received the Dr. Eng. degree in Computer Science Engineering from the Politecnico di Milano (Italy) in June 2003, the MS in Computer Science from the University of Illinois at Chicago (USA) in December 2003 and the Ph.D. degree in Computer Engineering from the Politecnico di Milano (Italy) in May 2007. He has been Visiting Researcher at Imperial College London (UK) in 2011. Currently, he is an Associate Professor at the Department of Electronics and Information of the Politecnico di Milano (Italy).\n","\n","Current research activity addresses Embedded and Edge Artificial Intelligence, Tiny Machine and Deep Learning, and Learning in nonstationary/evolving environments.\n","\n","- Alessandro Falcetta (alessandro.falcetta@polimi.it)\n","\n","Alessandro Falcetta received the Dr. Eng. degree in Computer Science and Engineering in 2018 and the MS in Computer Science and Engineering in 2020, both from Politecnico di Milano, Italy. Currently, he is a PhD candidate in Information Technology at Dipartimento di Elettronica, Informazione e Bioingegneria (DEIB) at Politecnico di Milano, Italy.\n"],"metadata":{"id":"Hg7lt3BlmZDQ"},"id":"Hg7lt3BlmZDQ"},{"cell_type":"markdown","source":["# References\n","- S. Disabato, A. Falcetta, A. Mongelluzzo, and M. Roveri, “A privacy preserving distributed architecture for deep-learning-as-a-service,” in 2020 International Joint Conference on Neural Networks (IJCNN). IEEE, 2020, pp. 1–8. [GitHub](https://github.com/AlexMV12/PyCrCNN)\n","- A. Falcetta, and M. Roveri, \"Privacy-preserving deep learning with homomorphic encryption: An introduction\", Computational Intelligence Magazine (August 2022). [GitHub](https://github.com/AlexMV12/Introduction-to-BFV-HE-ML)\n","- A.Falcetta, M. Roveri, \"Privacy-preserving time series prediction with temporal convolutional neural networks\", in 2022 International Joint Conference on Neural Networks (IJCNN), [GitHub](https://github.com/AlexMV12/PINPOINT)\n","- J. Fan, and F. Vercauteren, \"Somewhat Practical Fully Homomorphic Encryption\", Cryptology ePrint Archive (2012)\n","- F. Boemer, Y. Lao, R. Cammarota, and C. Wierzynski, “ngraph-he: a graph compiler for deep learning on homomorphically encrypted data,” in Proceedings of the 16th ACM International Conference on Computing Frontiers, 2019, pp. 3–13.\n","- Y. LeCun, B. Boser, J. S. Denker, D. Henderson, R. E. Howard, W. Hubbard, and L. D. Jackel, “Backpropagation applied to handwritten zip code recognition,” Neural computation, vol. 1, no. 4, pp. 541–551, 1989. \n","- B. C. Stahl and D. Wright, “Ethics and privacy in ai and big data: Implementing responsible research and innovation,” IEEE Security & Privacy, vol. 16, no. 3, pp. 26–33, 2018.\n","- E. P. Council of European Union, “Regulation (eu) no 2016/679, article 4(1),” 2016. \n"],"metadata":{"id":"pdPiX7poYVLE"},"id":"pdPiX7poYVLE"},{"cell_type":"markdown","source":["# Other resources\n","\n","\n","Other useful resources on this topic:\n","- [SoK: Privacy-preserving Deep Learning with Homomorphic Encryption](https://arxiv.org/abs/2112.12855)\n","- [Microsoft SEAL](https://github.com/microsoft/SEAL)\n","- [Homomorphic Encryption Standardization](https://homomorphicencryption.org/)\n","\n","**Don't miss our accepted paper here at IJCNN: \"Privacy-preserving time series prediction with temporal convolutional neural networks\" we will present on Friday, July 22nd - 4.00PM!**"],"metadata":{"id":"4gGi9PhUYS-L"},"id":"4gGi9PhUYS-L"},{"cell_type":"code","execution_count":null,"id":"960e28e3-471a-4710-93c8-fc1823a8a259","metadata":{"id":"960e28e3-471a-4710-93c8-fc1823a8a259"},"outputs":[],"source":[""]}],"metadata":{"colab":{"collapsed_sections":[],"name":"Copy of Tutorial_to_fill.ipynb","provenance":[{"file_id":"1WtvumJvvJuFTjV0pxTiivzTWefi14ui5","timestamp":1658096237329}],"toc_visible":true},"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.5"}},"nbformat":4,"nbformat_minor":5} --------------------------------------------------------------------------------