├── configs ├── dwsep_d2_ma.json ├── sdg_d2_ma.json ├── dwclose_d1_ma.json ├── fp_d2.json ├── dwsep_d5_ma.json ├── sdg_d5_ma.json ├── dwsep_d10_ma.json ├── sdg_d10_ma.json ├── fp_d5.json ├── dwsep_d5_ma_second.json ├── dwsep_d10_ma_second.json ├── fp_d10.json ├── fp_d5_ma.json ├── cubicsdg_d10.json ├── cubicsdg_d2.json ├── fp_d10_ma.json ├── cubicsdg_d5_ma.json ├── cubicsdg_d10_ma.json └── dwclose_d1_ma_second.json ├── LICENSE ├── .gitignore ├── utility.py ├── README.md ├── main.py ├── equation.py └── solver.py /configs/dwsep_d2_ma.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "DWSepEigen", 4 | "total_time": 0.2, 5 | "dim": 2, 6 | "num_time_interval": 40, 7 | "initeigen_low": -0.3, 8 | "initeigen_high": -0.3, 9 | "problem_type": "linear", 10 | "eigenpair": "first", 11 | "model_type": "consistent" 12 | }, 13 | "nn_config": { 14 | "trig_order": 5, 15 | "num_hiddens": [60, 60, 60], 16 | "lr_values": [1e-3, 1e-4, 1e-5], 17 | "lr_boundaries": [4000, 7000], 18 | "ma_values": [0.3, 0.7], 19 | "ma_boundaries": [4000], 20 | "num_iterations": 8000, 21 | "batch_size": 256, 22 | "valid_size": 256, 23 | "logging_frequency": 100, 24 | "verbose": true 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /configs/sdg_d2_ma.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "Sdg_d2Eigen", 4 | "total_time": 0.2, 5 | "dim": 2, 6 | "num_time_interval": 40, 7 | "initeigen_low": -1.0, 8 | "initeigen_high": 0.0, 9 | "problem_type": "linear", 10 | "eigenpair": "first", 11 | "model_type": "consistent" 12 | }, 13 | "nn_config": { 14 | "trig_order": 5, 15 | "num_hiddens": [60, 60, 60], 16 | "lr_values": [1e-3, 1e-4, 1e-5], 17 | "lr_boundaries": [4000, 7000], 18 | "ma_values": [0.3, 0.7], 19 | "ma_boundaries": [4000], 20 | "num_iterations": 8000, 21 | "batch_size": 256, 22 | "valid_size": 256, 23 | "logging_frequency": 100, 24 | "verbose": "True" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /configs/dwclose_d1_ma.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "DWClose_d1Eigen", 4 | "total_time": 0.2, 5 | "dim": 1, 6 | "num_time_interval": 160, 7 | "initeigen_low": -2.3, 8 | "initeigen_high": -2.2, 9 | "problem_type": "linear", 10 | "eigenpair": "first", 11 | "model_type": "consistent" 12 | }, 13 | "nn_config": { 14 | "trig_order": 5, 15 | "num_hiddens": [40, 40], 16 | "lr_values": [5e-4, 1e-4, 1e-5], 17 | "lr_boundaries": [2000,4000], 18 | "ma_values": [0.1, 0.2, 0.9], 19 | "ma_boundaries": [2000, 4000], 20 | "num_iterations": 8000, 21 | "batch_size": 512, 22 | "valid_size": 512, 23 | "logging_frequency": 100, 24 | "verbose": true 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /configs/fp_d2.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "FPEigen", 4 | "total_time": 0.2, 5 | "dim": 2, 6 | "num_time_interval": 40, 7 | "initeigen_low": -1.0, 8 | "initeigen_high": 1.0, 9 | "coef":[1.0, 1.0], 10 | "problem_type": "linear", 11 | "eigenpair": "first", 12 | "model_type": "consistent" 13 | }, 14 | "nn_config": { 15 | "trig_order": 5, 16 | "num_hiddens": [30, 30, 30], 17 | "lr_values": [1e-3, 1e-4, 1e-5], 18 | "lr_boundaries": [4000, 5000], 19 | "ma_values": [0, 0], 20 | "ma_boundaries": [0], 21 | "num_iterations": 6000, 22 | "batch_size": 256, 23 | "valid_size": 256, 24 | "logging_frequency": 100, 25 | "verbose": "True" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /configs/dwsep_d5_ma.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "DWSepEigen", 4 | "total_time": 0.2, 5 | "dim": 5, 6 | "num_time_interval": 160, 7 | "initeigen_low": -0.5, 8 | "initeigen_high": -0.5, 9 | "problem_type": "linear", 10 | "eigenpair": "first", 11 | "model_type": "consistent" 12 | }, 13 | "nn_config": { 14 | "trig_order": 5, 15 | "num_hiddens": [60, 60, 60], 16 | "lr_values": [5e-3, 1e-4, 1e-5], 17 | "lr_boundaries": [10000,15000], 18 | "ma_values": [0.1, 0.2, 0.9], 19 | "ma_boundaries": [10000, 15000], 20 | "num_iterations": 20000, 21 | "batch_size": 1024, 22 | "valid_size": 1024, 23 | "logging_frequency": 100, 24 | "verbose": true 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /configs/sdg_d5_ma.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "Sdg_d5Eigen", 4 | "total_time": 0.3, 5 | "dim": 5, 6 | "num_time_interval": 80, 7 | "initeigen_low": -0.2, 8 | "initeigen_high": -0.2, 9 | "problem_type": "linear", 10 | "eigenpair": "first", 11 | "model_type": "consistent" 12 | }, 13 | "nn_config": { 14 | "trig_order": 5, 15 | "num_hiddens": [80, 80, 80], 16 | "lr_values": [1e-4, 5e-5, 1e-5], 17 | "lr_boundaries": [30000, 60000], 18 | "ma_values": [0.2, 0.5, 0.9], 19 | "ma_boundaries": [30000, 60000], 20 | "num_iterations": 80000, 21 | "batch_size": 1024, 22 | "valid_size": 1024, 23 | "logging_frequency": 100, 24 | "verbose": "True" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /configs/dwsep_d10_ma.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "DWSepEigen", 4 | "total_time": 0.2, 5 | "dim": 10, 6 | "num_time_interval": 320, 7 | "initeigen_low": -0.5, 8 | "initeigen_high": -0.5, 9 | "problem_type": "linear", 10 | "eigenpair": "first", 11 | "model_type": "consistent" 12 | }, 13 | "nn_config": { 14 | "trig_order": 5, 15 | "num_hiddens": [200, 200, 200], 16 | "lr_values": [5e-4, 1e-4, 1e-5], 17 | "lr_boundaries": [30000,40000], 18 | "ma_values": [0.1, 0.3, 0.9], 19 | "ma_boundaries": [30000,40000], 20 | "num_iterations": 50000, 21 | "batch_size": 2048, 22 | "valid_size": 2048, 23 | "logging_frequency": 100, 24 | "verbose": true 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /configs/sdg_d10_ma.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "Sdg_d10Eigen", 4 | "total_time": 0.3, 5 | "dim": 10, 6 | "num_time_interval": 120, 7 | "initeigen_low": -0.2, 8 | "initeigen_high": -0.2, 9 | "problem_type": "linear", 10 | "eigenpair": "first", 11 | "model_type": "consistent" 12 | }, 13 | "nn_config": { 14 | "trig_order": 5, 15 | "num_hiddens": [300, 300, 300], 16 | "lr_values": [5e-5, 5e-5, 1e-5], 17 | "lr_boundaries": [40000, 60000], 18 | "ma_values": [0.2, 0.5, 0.9], 19 | "ma_boundaries": [40000, 60000], 20 | "num_iterations": 80000, 21 | "batch_size": 1024, 22 | "valid_size": 1024, 23 | "logging_frequency": 100, 24 | "verbose": "True" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /configs/fp_d5.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "FPEigen", 4 | "total_time": 0.2, 5 | "dim": 5, 6 | "num_time_interval": 80, 7 | "initeigen_low": 0.5, 8 | "initeigen_high": 0.5, 9 | "coef":[1.0, 0.8, 0.6, 0.5, 0.2], 10 | "problem_type": "linear", 11 | "eigenpair": "first", 12 | "model_type": "consistent" 13 | }, 14 | "nn_config": { 15 | "trig_order": 5, 16 | "num_hiddens": [80, 80, 80], 17 | "lr_values": [1e-3, 1e-4, 1e-5], 18 | "lr_boundaries": [4000, 38000], 19 | "ma_values": [0, 0], 20 | "ma_boundaries": [0], 21 | "num_iterations": 40000, 22 | "batch_size": 1024, 23 | "valid_size": 1024, 24 | "logging_frequency": 100, 25 | "verbose": "True" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /configs/dwsep_d5_ma_second.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "DWSepEigen", 4 | "total_time": 0.2, 5 | "dim": 5, 6 | "num_time_interval": 160, 7 | "initeigen_low": 0.16, 8 | "initeigen_high": 0.17, 9 | "problem_type": "linear", 10 | "eigenpair": "second", 11 | "model_type": "consistent", 12 | "pretrain": 10000 13 | }, 14 | "nn_config": { 15 | "trig_order": 5, 16 | "num_hiddens": [60, 60, 60], 17 | "lr_values": [5e-3, 1e-4, 1e-5], 18 | "lr_boundaries": [10000,15000], 19 | "ma_values": [0.1, 0.2, 0.9], 20 | "ma_boundaries": [10000, 15000], 21 | "num_iterations": 40000, 22 | "batch_size": 1024, 23 | "valid_size": 1024, 24 | "logging_frequency": 100, 25 | "verbose": true 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /configs/dwsep_d10_ma_second.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "DWSepEigen", 4 | "total_time": 0.2, 5 | "dim": 10, 6 | "num_time_interval": 320, 7 | "initeigen_low": 0.14, 8 | "initeigen_high": 0.14, 9 | "problem_type": "linear", 10 | "eigenpair": "second", 11 | "model_type": "consistent", 12 | "pretrain": 20000 13 | }, 14 | "nn_config": { 15 | "trig_order": 5, 16 | "num_hiddens": [200, 200, 200], 17 | "lr_values": [5e-4, 1e-4, 1e-5], 18 | "lr_boundaries": [30000,40000], 19 | "ma_values": [0.1, 0.3, 0.9], 20 | "ma_boundaries": [30000,40000], 21 | "num_iterations": 60000, 22 | "batch_size": 2048, 23 | "valid_size": 2048, 24 | "logging_frequency": 100, 25 | "verbose": true 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /configs/fp_d10.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "FPEigen", 4 | "total_time": 0.2, 5 | "dim": 10, 6 | "num_time_interval": 120, 7 | "initeigen_low": 0.3, 8 | "initeigen_high": 0.3, 9 | "coef":[0.1, 0.3, 0.2, 0.5, 0.2, 0.1, 0.3, 0.4, 0.2, 0.2], 10 | "problem_type": "linear", 11 | "eigenpair": "first", 12 | "model_type": "consistent" 13 | }, 14 | "nn_config": { 15 | "trig_order": 5, 16 | "num_hiddens": [300, 300, 300], 17 | "lr_values": [5e-4, 1e-4, 1e-5], 18 | "lr_boundaries": [20000, 25000], 19 | "ma_values": [0, 0], 20 | "ma_boundaries": [0], 21 | "num_iterations": 30000, 22 | "batch_size": 2048, 23 | "valid_size": 2048, 24 | "logging_frequency": 100, 25 | "verbose": "True" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /configs/fp_d5_ma.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "FPEigen", 4 | "total_time": 0.2, 5 | "dim": 5, 6 | "num_time_interval": 80, 7 | "initeigen_low": 0.5, 8 | "initeigen_high": 0.5, 9 | "coef":[1.0, 0.8, 0.6, 0.4, 0.2], 10 | "problem_type": "linear", 11 | "eigenpair": "first", 12 | "model_type": "consistent" 13 | }, 14 | "nn_config": { 15 | "trig_order": 5, 16 | "num_hiddens": [80, 80, 80], 17 | "lr_values": [1e-4, 5e-5, 1e-5], 18 | "lr_boundaries": [30000, 60000], 19 | "ma_values": [0.2, 0.5, 0.9], 20 | "ma_boundaries": [30000, 60000], 21 | "num_iterations": 80000, 22 | "batch_size": 1024, 23 | "valid_size": 1024, 24 | "logging_frequency": 100, 25 | "verbose": "True" 26 | } 27 | } 28 | 29 | 30 | -------------------------------------------------------------------------------- /configs/cubicsdg_d10.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "CubicSdgEigen", 4 | "epsl": 1.0, 5 | "L2mean": 1.0, 6 | "norm_constant": 1.0511402766738, 7 | "total_time": 0.3, 8 | "dim": 10, 9 | "num_time_interval": 200, 10 | "initeigen_low": -3.2, 11 | "initeigen_high": -3.2, 12 | "problem_type": "nonlinear", 13 | "eigenpair": "first", 14 | "model_type": "consistent" 15 | }, 16 | "nn_config": { 17 | "trig_order": 5, 18 | "num_hiddens": [200, 200, 200], 19 | "lr_values": [3e-4, 3e-4], 20 | "lr_boundaries": [80000], 21 | "ma_values": [0, 0], 22 | "ma_boundaries": [80000], 23 | "num_iterations": 90000, 24 | "batch_size": 2048, 25 | "valid_size": 2048, 26 | "logging_frequency": 100, 27 | "verbose": "True" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /configs/cubicsdg_d2.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "CubicSdgEigen", 4 | "epsl": 1.0, 5 | "L2mean": 1, 6 | "norm_constant": 1.266065877752008, 7 | "total_time": 0.2, 8 | "dim": 2, 9 | "num_time_interval": 40, 10 | "initeigen_low": -3.6, 11 | "initeigen_high": -3.5, 12 | "problem_type": "nonlinear", 13 | "eigenpair": "first", 14 | "model_type": "consistent" 15 | }, 16 | "nn_config": { 17 | "trig_order": 3, 18 | "num_hiddens": [30, 30, 30], 19 | "lr_values": [1e-3, 1e-4, 1e-5], 20 | "lr_boundaries": [4000, 5000], 21 | "ma_values": [0, 0], 22 | "ma_boundaries": [0], 23 | "num_iterations": 6000, 24 | "batch_size": 256, 25 | "valid_size": 256, 26 | "logging_frequency": 100, 27 | "verbose": "True" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /configs/fp_d10_ma.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "FPEigen", 4 | "total_time": 0.2, 5 | "dim": 10, 6 | "num_time_interval": 120, 7 | "initeigen_low": 0.2, 8 | "initeigen_high": 0.2, 9 | "coef":[0.1, 0.3, 0.2, 0.5, 0.2, 0.1, 0.3, 0.4, 0.2, 0.2], 10 | "problem_type": "linear", 11 | "eigenpair": "first", 12 | "model_type": "consistent" 13 | }, 14 | "nn_config": { 15 | "trig_order": 5, 16 | "num_hiddens": [300, 300, 300], 17 | "lr_values": [5e-5, 2e-5, 1e-5], 18 | "lr_boundaries": [60000, 80000], 19 | "ma_values": [0.2, 0.5, 0.9], 20 | "ma_boundaries": [60000, 80000], 21 | "num_iterations": 100000, 22 | "batch_size": 1024, 23 | "valid_size": 1024, 24 | "logging_frequency": 100, 25 | "verbose": "True" 26 | } 27 | } 28 | 29 | -------------------------------------------------------------------------------- /configs/cubicsdg_d5_ma.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "CubicSdgEigen", 4 | "epsl": 1.0, 5 | "L2mean": 1, 6 | "norm_constant": 1.104085531496, 7 | "total_time": 0.2, 8 | "dim": 5, 9 | "num_time_interval": 120, 10 | "initeigen_low": -3.3, 11 | "initeigen_high": -3.3, 12 | "problem_type": "nonlinear", 13 | "eigenpair": "first", 14 | "model_type": "consistent" 15 | }, 16 | "nn_config": { 17 | "trig_order": 5, 18 | "num_hiddens": [80, 80, 80], 19 | "lr_values": [5e-5, 2e-5, 1e-5], 20 | "lr_boundaries": [20000, 40000], 21 | "ma_values": [0.2, 0.9, 0.99], 22 | "ma_boundaries": [20000, 40000], 23 | "num_iterations": 60000, 24 | "batch_size": 2048, 25 | "valid_size": 2048, 26 | "logging_frequency": 100, 27 | "verbose": "True" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /configs/cubicsdg_d10_ma.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "CubicSdgEigen", 4 | "epsl": 1.0, 5 | "L2mean": 1.0, 6 | "norm_constant": 1.0511402766738, 7 | "total_time": 0.3, 8 | "dim": 10, 9 | "num_time_interval": 200, 10 | "initeigen_low": -3.2, 11 | "initeigen_high": -3.2, 12 | "problem_type": "nonlinear", 13 | "eigenpair": "first", 14 | "model_type": "consistent" 15 | }, 16 | "nn_config": { 17 | "trig_order": 5, 18 | "num_hiddens": [200, 200, 200], 19 | "lr_values": [5e-5, 2e-5, 1e-5], 20 | "lr_boundaries": [40000, 60000], 21 | "ma_values": [0.1, 0.9, 0.99], 22 | "ma_boundaries": [40000, 60000], 23 | "num_iterations": 80000, 24 | "batch_size": 2048, 25 | "valid_size": 2048, 26 | "logging_frequency": 100, 27 | "verbose": "True" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /configs/dwclose_d1_ma_second.json: -------------------------------------------------------------------------------- 1 | { 2 | "eqn_config": { 3 | "eqn_name": "DWClose_d1Eigen", 4 | "_comment": "results are inaccurate and divese in multiple runs due due to the closedness of eigenvalues", 5 | "total_time": 0.2, 6 | "dim": 1, 7 | "num_time_interval": 160, 8 | "initeigen_low": -2.3, 9 | "initeigen_high": -2.2, 10 | "problem_type": "linear", 11 | "eigenpair": "second", 12 | "model_type": "consistent", 13 | "pretrain": "degenerate" 14 | }, 15 | "nn_config": { 16 | "trig_order": 5, 17 | "num_hiddens": [40, 40], 18 | "lr_values": [5e-4, 1e-4, 1e-5], 19 | "lr_boundaries": [3000,5000], 20 | "ma_values": [0.1, 0.2, 0.9], 21 | "ma_boundaries": [3000, 5000], 22 | "num_iterations": 8000, 23 | "batch_size": 512, 24 | "valid_size": 512, 25 | "logging_frequency": 100, 26 | "verbose": true 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Jiequn Han 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # other folders 2 | ipynb/ 3 | logs/ 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # pyenv 80 | .python-version 81 | 82 | # celery beat schedule file 83 | celerybeat-schedule 84 | 85 | # SageMath parsed files 86 | *.sage.py 87 | 88 | # Environments 89 | .idea/ 90 | .env 91 | .venv 92 | env/ 93 | venv/ 94 | ENV/ 95 | env.bak/ 96 | venv.bak/ 97 | 98 | # Spyder project settings 99 | .spyderproject 100 | .spyproject 101 | 102 | # Rope project settings 103 | .ropeproject 104 | 105 | # mkdocs documentation 106 | /site 107 | 108 | # mypy 109 | .mypy_cache/ 110 | -------------------------------------------------------------------------------- /utility.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | # =================================================================== 4 | # Utility functions for reading config files 5 | # =================================================================== 6 | 7 | 8 | class DictionaryUtility: 9 | """ 10 | Utility methods for dealing with dictionaries. 11 | """ 12 | @staticmethod 13 | def to_object(item): 14 | """ 15 | Convert a dictionary to an object (recursive). 16 | """ 17 | def convert(item): 18 | if isinstance(item, dict): 19 | return type('jo', (), {k: convert(v) for k, v in item.items()}) 20 | if isinstance(item, list): 21 | def yield_convert(item): 22 | for index, value in enumerate(item): 23 | yield convert(value) 24 | return list(yield_convert(item)) 25 | else: 26 | return item 27 | 28 | return convert(item) 29 | 30 | def to_dict(obj): 31 | """ 32 | Convert an object to a dictionary (recursive). 33 | """ 34 | def convert(obj): 35 | if not hasattr(obj, "__dict__"): 36 | return obj 37 | result = {} 38 | for key, val in obj.__dict__.items(): 39 | if key.startswith("_"): 40 | continue 41 | element = [] 42 | if isinstance(val, list): 43 | for item in val: 44 | element.append(convert(item)) 45 | else: 46 | element = convert(val) 47 | result[key] = element 48 | return result 49 | 50 | return convert(obj) 51 | 52 | 53 | def get_config(config_path): 54 | with open(config_path) as json_data_file: 55 | config = json.load(json_data_file) 56 | # convert dict to object recursively for easy call 57 | config = DictionaryUtility.to_object(config) 58 | return config 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Solving high-dimensional eigenvalue problems using deep neural networks: A diffusion Monte Carlo like approach 2 | Accompanying code for [*Solving high-dimensional eigenvalue problems using deep neural networks: A diffusion Monte Carlo like approach*](https://doi.org/10.1016/j.jcp.2020.109792), Jiequn Han, Jianfeng Lu, Mo Zhou, Journal of Computational Physics, 423, 109792 (2020). 3 | 4 | Run the following command to solve the eigenvalue problem directly: 5 | ``` 6 | python main.py --config_path=configs/fp_d2.json 7 | ``` 8 | 9 | **Names of config files**: 10 | "FP" denotes the Fokker-Planck opeartor; 11 | "Sdg" denotes the linear Schrodinger opeartor with a well-behaved potential; 12 | "CubicSdg" denotes the nonlinear Schrodinger operator with a cubic term; 13 | "DWSep" denotes the linear Schrodinger opeartor with a double well potential leading to well separated eigenvalues; 14 | "DWClose" denotes the linear Schrodinger opeartor with a double well potential leading to close eigenvalues. 15 | 16 | If there is "ma" in the name of the config file, it means to use moving average for the normalization factor. 17 | If there is "second" in the name of the config file, it means to solve the second eigenpair. 18 | 19 | | Experiments in the paper | Config names | 20 | |--------------------------------------------------------------|--------------------------------------------------------------| 21 | | Fokker-Planck (Figure 2) | fp_d5_ma.json, fp_d10_ma.json | 22 | | Linear Schrodinger (Figure 3) | sdg_d5_ma.json, sdg_d10_ma.json | 23 | | Nonlinear Schrodinger (Figure 4) | cubicsdg_d5_ma.json, cubicsdg_d10_ma.json | 24 | | Linear Schrodinger with double-well potential (Figure 5) | dwsep_d10_ma.json, dwsep_d10_ma_second.json | 25 | 26 | 27 | **Fileds in config files** 28 | "problem_type": "linear" means to solve linear eigenvalue problems (and similar for "nonlinear"). 29 | "eigenpair": "first" means to solve the first eigenpair (and similar for "second"). 30 | "model_type": "consistent" means to use two neural networks to represent the eigenfunction and its scaled gradient, while "gradient" means to use one neural network for the eigenfunction and call auto-differentiation to get its gradient. 31 | 32 | The code can be run on either Tensorflow 2.0 directly or Tensorflow 1.0 with small modification. For instance, to use Tensorflow 1.12, change the two lines in three .py files 33 | ``` 34 | import tensorflow.compat.v1 as tf 35 | tf.disable_v2_behavior() 36 | ``` 37 | into 38 | ``` 39 | import tensorflow as tf 40 | ``` 41 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import numpy as np 5 | import equation as eqn 6 | from utility import get_config, DictionaryUtility 7 | from solver import FeedForwardModel 8 | 9 | import tensorflow.compat.v1 as tf 10 | tf.disable_v2_behavior() 11 | # import tensorflow as tf 12 | 13 | FLAGS = tf.app.flags.FLAGS 14 | tf.app.flags.DEFINE_string('config_path', './configs/sdg_d2_ma.json', 15 | """The path to load json file.""") 16 | tf.app.flags.DEFINE_string('exp_name', None, 17 | """The name of numerical experiments.""") 18 | tf.app.flags.DEFINE_integer('num_run', 1, 19 | """The number of experiments to repeatedly run for the same problem.""") 20 | tf.app.flags.DEFINE_string('log_dir1', './logs', 21 | """Directory where to write event logs and output array.""") 22 | 23 | 24 | def main(): 25 | config = get_config(FLAGS.config_path) 26 | bsde = getattr(eqn, config.eqn_config.eqn_name)(config.eqn_config) 27 | if FLAGS.exp_name is None: # use config name as exp_name 28 | FLAGS.exp_name = os.path.splitext(os.path.basename(FLAGS.config_path))[0] 29 | if not os.path.exists(FLAGS.log_dir1): 30 | os.mkdir(FLAGS.log_dir1) 31 | path_prefix = os.path.join(FLAGS.log_dir1, FLAGS.exp_name) 32 | with open('{}_config.json'.format(path_prefix), 'w') as outfile: 33 | json.dump(DictionaryUtility.to_dict(config), outfile, indent=2) 34 | logging.basicConfig(level=logging.INFO, 35 | format='%(levelname)-6s %(message)s') 36 | 37 | dim = config.eqn_config.dim 38 | 39 | for idx_run in range(1, FLAGS.num_run+1): 40 | tf.reset_default_graph() 41 | with tf.Session() as sess: 42 | logging.info('Begin to solve %s with run %d' % (FLAGS.exp_name, idx_run)) 43 | model = FeedForwardModel(config, bsde, sess) 44 | if config.eqn_config.eigenpair == 'first': 45 | model.build() 46 | else: 47 | model.build_second() 48 | result = model.train() 49 | training_history = result[0] 50 | # save training history 51 | np.savetxt('{}_log{}.csv'.format(path_prefix, idx_run), 52 | training_history, 53 | fmt=['%d', '%.5e', '%.5e', '%.5e', '%.5e', '%.5e', '%.5e', '%.5e', '%.5e', '%d'], 54 | delimiter=",", 55 | header="step,train_loss, eg_err, egfcn_l2_err, egfcn_infty_err, grad_l2_err, grad_infty_err, consistency_loss, norm_factor, elapsed_time", 56 | comments='') 57 | figure_data = result[1] 58 | if dim == 1: 59 | np.savetxt('{}_hist.csv'.format(path_prefix), figure_data, delimiter=",", 60 | header="x,y_true,y_NN", comments='') 61 | else: 62 | np.savetxt('{}_hist.csv'.format(path_prefix), figure_data, delimiter=",", 63 | header="y_true,y_NN", comments='') 64 | 65 | if __name__ == '__main__': 66 | main() 67 | -------------------------------------------------------------------------------- /equation.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.stats import multivariate_normal as normal 3 | 4 | import tensorflow.compat.v1 as tf 5 | tf.disable_v2_behavior() 6 | # import tensorflow as tf 7 | 8 | class Equation(object): 9 | """Base class for defining PDE related function.""" 10 | def __init__(self, eqn_config): 11 | self.dim = eqn_config.dim 12 | self.total_time = eqn_config.total_time 13 | self.num_time_interval = eqn_config.num_time_interval 14 | self.delta_t = (self.total_time + 0.0) / self.num_time_interval 15 | self.sqrt_delta_t = np.sqrt(self.delta_t) 16 | self.sigma = None 17 | 18 | def sample_hist(self, num_sample): 19 | x_sample = np.random.uniform(0.0, 2*np.pi, size=[num_sample, self.dim]) 20 | return x_sample 21 | 22 | def sample_uniform(self, num_sample): 23 | dw_sample = normal.rvs(size=[num_sample, 24 | self.dim, 25 | self.num_time_interval]) * self.sqrt_delta_t 26 | if self.dim == 1: 27 | dw_sample = np.reshape(dw_sample,[num_sample, 1, self.num_time_interval] ) 28 | x_sample = np.zeros([num_sample, self.dim, self.num_time_interval + 1]) 29 | # uniform X0 30 | x_sample[:, :, 0] = np.random.uniform(0.0, 2*np.pi, size=[num_sample, self.dim]) 31 | for i in range(self.num_time_interval): 32 | x_sample[:, :, i + 1] = x_sample[:, :, i] + self.sigma * dw_sample[:, :, i] 33 | return dw_sample, x_sample 34 | 35 | def f_tf(self, x, y, z): 36 | """Generator function in the PDE.""" 37 | raise NotImplementedError 38 | 39 | def g_tf(self, x): 40 | """Terminal condition of the PDE.""" 41 | raise NotImplementedError 42 | 43 | 44 | class FPEigen(Equation): 45 | # eigenvalue problem for Fokker Planck operator on squares [0, 2pi]^d 46 | # uniform sampling with no drift term 47 | # v(x) = sin(\sum_{i=1}^d c_i cos(x_i)) psi = exp(-v) 48 | def __init__(self, eqn_config): 49 | super(FPEigen, self).__init__(eqn_config) 50 | self.sigma = np.sqrt(2.0) 51 | self.c = eqn_config.coef 52 | self.true_eigen = 0 53 | 54 | def f_tf(self, x, y, z): 55 | temp = tf.reduce_sum(self.c * tf.cos(x), axis=1, keepdims=True) #num_sample x 1 56 | temp1 = tf.cos(x) * tf.cos(temp) 57 | temp2 = self.c * (tf.sin(x) ** 2) * tf.sin(temp) 58 | Laplacian_v = -tf.reduce_sum(self.c * (temp1 + temp2), axis=1, keepdims=True) 59 | gradient_v = self.c * tf.sin(x) * (-tf.cos(temp)) 60 | return tf.reduce_sum(gradient_v * z / self.sigma, axis=1, keepdims=True) + Laplacian_v * y 61 | 62 | def true_y(self, x): 63 | temp = tf.reduce_sum(self.c * tf.cos(x), axis=1, keepdims=True) #num_sample x 1 64 | return tf.exp(-tf.sin(temp)) 65 | 66 | def true_z(self, x): 67 | temp = tf.reduce_sum(self.c * tf.cos(x), axis=1, keepdims=True) #num_sample x 1 68 | return tf.cos(temp) * tf.sin(x) * self.c * tf.exp(-tf.sin(temp)) * self.sigma 69 | 70 | 71 | class Sdg_d10Eigen(Equation): 72 | # Schrodinger V(x)= \sum_{i=1}^d ci*cos(xi) on squares [0, 2pi]^d ci~[0,0.2] 73 | def __init__(self, eqn_config): 74 | super(Sdg_d10Eigen, self).__init__(eqn_config) 75 | self.N = 10 76 | self.sigma = np.sqrt(2.0) 77 | self.ci = [0.162944737278636,0.181158387415124,0.025397363258701,0.182675171227804,0.126471849245082,0.019508080999882,0.055699643773410,0.109376303840997,0.191501367086860,0.192977707039855] 78 | self.coef = [[0.993571793881952,0.992112802401298,0.999838871758632,0.991985354482078,0.996078392055002,0.999904903363006,0.999227344468429,0.997052622938466,0.991226051652424,0.991096131179570], 79 | [-0.160061291194770,-0.177224122143104,-0.0253861093252471,-0.178643778619933,-0.125107059922130,-0.0195029792507090,-0.0555812781968171,-0.108489355137516,-0.186874250374073,-0.188245776894456], 80 | [0.00325007727123395,0.00399794197948296,8.05863952861287e-05,0.00406345319436995,0.00197411443935602,4.75560761524560e-05,0.000386840691309466,0.00148119005866490,0.00445436426915333,0.00452135628602182], 81 | [-2.93797849166571e-05,-4.01667801745879e-05,-1.13700607076658e-07,-4.11655916879119e-05,-1.38587047043284e-05,-5.15393770949078e-08,-1.19684981038921e-06,-8.99463377780295e-06,-4.72979651314309e-05,-4.83779988050594e-05], 82 | [1.49482399351235e-07,2.27167087775217e-07,9.02407255701227e-11,2.34761407474919e-07,5.47463713973189e-08,3.14199988139629e-11,2.08305573451501e-09,3.07325435388975e-08,2.82738262076159e-07,2.91419081794217e-07], 83 | [-4.86883641624589e-10,-8.22539598941590e-10,-4.58385880437081e-14,-8.57145589811237e-10,-1.38431507219819e-10,-1.22591238099021e-14,-2.32034750394283e-12,-6.72113561115208e-11,-1.08212492344907e-09,-1.12393541893816e-09], 84 | [1.10142399609609e-12,2.06867289930651e-12,1.61696832610095e-17,2.17373950410335e-12,2.43099654874719e-13,3.32164395266348e-18,1.79492650923155e-15,1.02081589368955e-13,2.87676001107095e-12,3.01092258164098e-12], 85 | [-1.83072364364542e-15,-3.82285292819827e-15,-4.19063409950640e-21,-4.05062488897179e-15,-3.13659344249419e-16,-6.61232423279210e-22,-1.02011344818844e-18,-1.13912481483233e-16,-5.61948722482390e-15,-5.92687179290964e-15], 86 | [2.32985123369096e-18,5.40924038129660e-18,8.31522997654433e-25,5.77949723343268e-18,3.09855992670163e-19,1.00779299219574e-25,4.43881934470464e-22,9.73238990068554e-20,8.40517382052742e-18,8.93324045857629e-18], 87 | [-2.34284521622012e-21,-6.04791979155140e-21,-1.30365676047138e-28,-6.51597707320250e-21,-2.41860537516298e-22,-1.21362085143571e-29,-1.52609785500994e-25,-6.57005000646960e-23,-9.93391805631913e-21,-1.06393915310827e-20]] 88 | self.coef2 = [[0,0,0,0,0,0,0,0,0,0], 89 | [-0.160061291194770,-0.177224122143104,-0.0253861093252471,-0.178643778619933,-0.125107059922130,-0.0195029792507090,-0.0555812781968171,-0.108489355137516,-0.186874250374073,-0.188245776894456], 90 | [0.00650015454246790,0.00799588395896592,0.000161172790572257,0.00812690638873990,0.00394822887871203,9.51121523049121e-05,0.000773681382618932,0.00296238011732981,0.00890872853830666,0.00904271257204364], 91 | [-8.81393547499713e-05,-0.000120500340523764,-3.41101821229973e-07,-0.000123496775063736,-4.15761141129853e-05,-1.54618131284724e-07,-3.59054943116762e-06,-2.69839013334089e-05,-0.000141893895394293,-0.000145133996415178], 92 | [5.97929597404939e-07,9.08668351100869e-07,3.60962902280491e-10,9.39045629899676e-07,2.18985485589276e-07,1.25679995255852e-10,8.33222293806002e-09,1.22930174155590e-07,1.13095304830463e-06,1.16567632717687e-06], 93 | [-2.43441820812295e-09,-4.11269799470795e-09,-2.29192940218540e-13,-4.28572794905618e-09,-6.92157536099094e-10,-6.12956190495104e-14,-1.16017375197142e-11,-3.36056780557604e-10,-5.41062461724537e-09,-5.61967709469082e-09], 94 | [6.60854397657655e-12,1.24120373958391e-11,9.70180995660572e-17,1.30424370246201e-11,1.45859792924832e-12,1.99298637159809e-17,1.07695590553893e-14,6.12489536213729e-13,1.72605600664257e-11,1.80655354898459e-11], 95 | [-1.28150655055179e-14,-2.67599704973879e-14,-2.93344386965448e-20,-2.83543742228025e-14,-2.19561540974593e-15,-4.62862696295447e-21,-7.14079413731909e-18,-7.97387370382634e-16,-3.93364105737673e-14,-4.14881025503675e-14], 96 | [1.86388098695277e-17,4.32739230503728e-17,6.65218398123546e-24,4.62359778674614e-17,2.47884794136130e-18,8.06234393756592e-25,3.55105547576371e-21,7.78591192054843e-19,6.72413905642194e-17,7.14659236686104e-17], 97 | [-2.10856069459811e-20,-5.44312781239626e-20,-1.17329108442424e-27,-5.86437936588225e-20,-2.17674483764668e-21,-1.09225876629214e-28,-1.37348806950895e-24,-5.91304500582264e-22,-8.94052625068721e-20,-9.57545237797440e-20]] 98 | self.true_eigen = -0.098087448866409 99 | 100 | def f_tf(self, x, y, z): 101 | return -tf.reduce_sum(self.ci * tf.cos(x), axis=1, keepdims=True) *y 102 | 103 | def true_y_np(self,x): 104 | bases_cos = 0 * x 105 | for m in range(self.N): 106 | bases_cos = bases_cos + np.cos(m * x) * self.coef[m] # Broadcasting 107 | return np.prod(bases_cos, axis=1, keepdims=True) 108 | 109 | def true_y(self, x): 110 | bases_cos = 0 * x 111 | for m in range(self.N): 112 | bases_cos = bases_cos + tf.cos(m * x) * self.coef[m] #Broadcasting 113 | return tf.reduce_prod(bases_cos, axis=1, keepdims=True) 114 | 115 | def true_z(self, x): 116 | bases_cos = 0 117 | for m in range(self.N): 118 | bases_cos = bases_cos + tf.cos(m * x) * self.coef[m] #Broadcasting 119 | bases_sin = 0 120 | for m in range(self.N): 121 | bases_sin = bases_sin + tf.sin(m * x) * self.coef2[m] #Broadcasting 122 | y = tf.reduce_prod(bases_cos, axis=1, keepdims=True) 123 | return - y * bases_sin / bases_cos * self.sigma 124 | 125 | 126 | class Sdg_d5Eigen(Equation): 127 | # Schrodinger V(x)= \sum_{i=1}^d ci*cos(xi) on squares [0, 2pi]^d ci~[0,0.2] 128 | def __init__(self, eqn_config): 129 | super(Sdg_d5Eigen, self).__init__(eqn_config) 130 | self.N = 10 131 | self.sigma = np.sqrt(2.0) 132 | self.ci = [0.162944737278636,0.181158387415124,0.025397363258701,0.182675171227804,0.126471849245082] 133 | self.coef = [[0.993571793881952,0.992112802401298,0.999838871758632,0.991985354482078,0.996078392055002], 134 | [-0.160061291194770,-0.177224122143104,-0.0253861093252471,-0.178643778619933,-0.125107059922130], 135 | [0.00325007727123395,0.00399794197948296,8.05863952861287e-05,0.00406345319436995,0.00197411443935602], 136 | [-2.93797849166571e-05,-4.01667801745879e-05,-1.13700607076658e-07,-4.11655916879119e-05,-1.38587047043284e-05], 137 | [1.49482399351235e-07,2.27167087775217e-07,9.02407255701227e-11,2.34761407474919e-07,5.47463713973189e-08], 138 | [-4.86883641624589e-10,-8.22539598941590e-10,-4.58385880437081e-14,-8.57145589811237e-10,-1.38431507219819e-10], 139 | [1.10142399609609e-12,2.06867289930651e-12,1.61696832610095e-17,2.17373950410335e-12,2.43099654874719e-13], 140 | [-1.83072364364542e-15,-3.82285292819827e-15,-4.19063409950640e-21,-4.05062488897179e-15,-3.13659344249419e-16], 141 | [2.32985123369096e-18,5.40924038129660e-18,8.31522997654433e-25,5.77949723343268e-18,3.09855992670163e-19], 142 | [-2.34284521622012e-21,-6.04791979155140e-21,-1.30365676047138e-28,-6.51597707320250e-21,-2.41860537516298e-22]] 143 | self.coef2 = [[0,0,0,0,0], 144 | [-0.160061291194770,-0.177224122143104,-0.0253861093252471,-0.178643778619933,-0.125107059922130], 145 | [0.00650015454246790,0.00799588395896592,0.000161172790572257,0.00812690638873990,0.00394822887871203], 146 | [-8.81393547499713e-05,-0.000120500340523764,-3.41101821229973e-07,-0.000123496775063736,-4.15761141129853e-05], 147 | [5.97929597404939e-07,9.08668351100869e-07,3.60962902280491e-10,9.39045629899676e-07,2.18985485589276e-07], 148 | [-2.43441820812295e-09,-4.11269799470795e-09,-2.29192940218540e-13,-4.28572794905618e-09,-6.92157536099094e-10], 149 | [6.60854397657655e-12,1.24120373958391e-11,9.70180995660572e-17,1.30424370246201e-11,1.45859792924832e-12], 150 | [-1.28150655055179e-14,-2.67599704973879e-14,-2.93344386965448e-20,-2.83543742228025e-14,-2.19561540974593e-15], 151 | [1.86388098695277e-17,4.32739230503728e-17,6.65218398123546e-24,4.62359778674614e-17,2.47884794136130e-18], 152 | [-2.10856069459811e-20,-5.44312781239626e-20,-1.17329108442424e-27,-5.86437936588225e-20,-2.17674483764668e-21]] 153 | self.true_eigen = -0.054018930536326 154 | 155 | def f_tf(self, x, y, z): 156 | return -tf.reduce_sum(self.ci * tf.cos(x), axis=1, keepdims=True) *y 157 | 158 | def true_y_np(self,x): 159 | bases_cos = 0 * x 160 | for m in range(self.N): 161 | bases_cos = bases_cos + np.cos(m * x) * self.coef[m] # Broadcasting 162 | return np.prod(bases_cos, axis=1, keepdims=True) 163 | 164 | def true_y(self, x): 165 | bases_cos = 0 * x 166 | for m in range(self.N): 167 | bases_cos = bases_cos + tf.cos(m * x) * self.coef[m] #Broadcasting 168 | return tf.reduce_prod(bases_cos, axis=1, keepdims=True) 169 | 170 | def true_z(self, x): 171 | bases_cos = 0 172 | for m in range(self.N): 173 | bases_cos = bases_cos + tf.cos(m * x) * self.coef[m] #Broadcasting 174 | bases_sin = 0 175 | for m in range(self.N): 176 | bases_sin = bases_sin + tf.sin(m * x) * self.coef2[m] #Broadcasting 177 | y = tf.reduce_prod(bases_cos, axis=1, keepdims=True) 178 | return - y * bases_sin / bases_cos * self.sigma 179 | 180 | 181 | class Sdg_d2Eigen(Equation): 182 | # Schrodinger V(x)= \sum_{i=1}^d ci*cos(xi) on squares [0, 2pi]^d ci~[0,0.2] 183 | def __init__(self, eqn_config): 184 | super(Sdg_d2Eigen, self).__init__(eqn_config) 185 | self.N = 10 186 | self.sigma = np.sqrt(2.0) 187 | self.ci = [0.814723686393179,0.905791937075619] 188 | self.sup = [1.56400342750522,1.59706136953917] 189 | self.coef = [[0.904929598872363, 0.892479070097153], 190 | [-0.599085866194182, -0.634418280724547], 191 | [0.0573984887007387, 0.0668213136994578], 192 | [-0.00252519085534048, -0.00325082263430659], 193 | [6.32514967687960e-05, 9.02475797129487e-05], 194 | [-1.01983519526066e-06, -1.61448458844806e-06], 195 | [1.14553116486126e-08, 2.01332109031048e-08], 196 | [-9.47170798515555e-11, -1.84883101478958e-10], 197 | [6.00357997713989e-13, 1.30180750716843e-12], 198 | [-3.00925873281827e-15, -7.24995760563704e-15]] 199 | self.coef2 = [[0, 0], 200 | [-0.599085866194182, -0.634418280724547], 201 | [0.114796977401477, 0.133642627398916], 202 | [-0.00757557256602144, -0.00975246790291976], 203 | [0.000253005987075184, 0.000360990318851795], 204 | [-5.09917597630331e-06, -8.07242294224032e-06], 205 | [6.87318698916753e-08, 1.20799265418629e-07], 206 | [-6.63019558960889e-10, -1.29418171035271e-09], 207 | [4.80286398171191e-12, 1.04144600573475e-11], 208 | [-2.70833285953644e-14, -6.52496184507333e-14]] 209 | self.true_eigen = -0.591624518674115 210 | 211 | def f_tf(self, x, y, z): 212 | return -tf.reduce_sum(self.ci * tf.cos(x), axis=1, keepdims=True) *y 213 | 214 | def true_y_np(self,x): 215 | bases_cos = 0 * x 216 | for m in range(self.N): 217 | bases_cos = bases_cos + np.cos(m * x) * self.coef[m] # Broadcasting 218 | return np.prod(bases_cos, axis=1, keepdims=True) 219 | 220 | def true_y(self, x): 221 | bases_cos = 0 * x 222 | for m in range(self.N): 223 | bases_cos = bases_cos + tf.cos(m * x) * self.coef[m] #Broadcasting 224 | return tf.reduce_prod(bases_cos, axis=1, keepdims=True) 225 | 226 | def true_z(self, x): 227 | bases_cos = 0 228 | for m in range(self.N): 229 | bases_cos = bases_cos + tf.cos(m * x) * self.coef[m] #Broadcasting 230 | bases_sin = 0 231 | for m in range(self.N): 232 | bases_sin = bases_sin + tf.sin(m * x) * self.coef2[m] #Broadcasting 233 | y = tf.reduce_prod(bases_cos, axis=1, keepdims=True) 234 | return - y * bases_sin / bases_cos * self.sigma 235 | 236 | 237 | class CubicSdgEigen(Equation): 238 | def __init__(self, eqn_config): 239 | super(CubicSdgEigen, self).__init__(eqn_config) 240 | self.sigma = np.sqrt(2.0) 241 | self.true_eigen = -3.0 242 | self.epsl = eqn_config.epsl 243 | self.dim = eqn_config.dim 244 | # norm_constant makes true_y has unit L2 mean 245 | # sqrt((integral exp(2*cos(x)/dim) from 0 to 2*pi) / 2/pi) ** dim 246 | self.norm_constant = eqn_config.norm_constant 247 | # L2mean is the L2 mean of eigenfunction 248 | self.L2mean = eqn_config.L2mean 249 | 250 | def f_tf(self, x, y, z): 251 | temp = self.epsl / (self.norm_constant**2) * (self.L2mean ** 2) * tf.exp(2 * tf.reduce_mean(tf.cos(x), axis=1, keepdims=True)) \ 252 | - tf.reduce_sum(tf.square(tf.sin(x)/self.dim) - tf.cos(x)/self.dim, axis=1, keepdims=True) 253 | # return -self.epsl * tf.pow(y,3) + (temp + 3.0) * self.true_y(x) 254 | return -self.epsl * tf.pow(y, 3) + (temp + 3.0) * y 255 | 256 | def true_z(self, x): 257 | temp = tf.exp(tf.reduce_mean(tf.cos(x), axis=1, keepdims=True)) 258 | return - tf.sin(x) / self.dim * temp * self.sigma / self.norm_constant * self.L2mean # broadcasting 259 | 260 | def true_y(self, x): 261 | return tf.exp(tf.reduce_mean(tf.cos(x), axis=1, keepdims=True)) / self.norm_constant * self.L2mean 262 | 263 | 264 | class DWClose_d1Eigen(Equation): 265 | #degenerate Schrodinger V(x)= A cos(2x) on [0, 2pi] 266 | def __init__(self, eqn_config): 267 | super(DWClose_d1Eigen, self).__init__(eqn_config) 268 | self.N = 10 269 | self.sigma = np.sqrt(2.0) 270 | self.A = 5 271 | self.true_eigen = -2.15307834 272 | self.second_eigen = -2.07633151 273 | # coef0 is coef for ground eigenfunction: 1, cos2x, cos4x... 274 | self.coef0 = [4.259287663084366793e-01,-3.668232007964972730e-01, 275 | 5.097922425487490944e-02,-3.348739889607468302e-03, 276 | 1.266698872480900102e-04,-3.101300263994905906e-06, 277 | 5.306028830867799572e-08,-6.695172840333706772e-10, 278 | 6.484195313839146743e-12,-4.970641756370646460e-14] 279 | # coef1 is coef for second eigenfunction: sinx, sin3x, sin5x... 280 | self.coef1 = [6.888844529501658709e-01,-1.588103256443261224e-01, 281 | 1.472987240390475835e-02,-7.220376543484285491e-04, 282 | 2.174143337189041368e-05,-4.417561381727552544e-07, 283 | 6.456580422456261813e-09,-7.109053841528895779e-11, 284 | 6.106194946251028526e-13,-4.204648637005190987e-15] 285 | 286 | def f_tf(self, x, y, z): 287 | return -self.A * tf.cos(2*x) *y 288 | 289 | def true_y_np(self,x): 290 | bases_cos = 0 * x 291 | for m in range(self.N): 292 | bases_cos = bases_cos + np.cos(2 * m * x) * self.coef0[m] 293 | return bases_cos 294 | #return np.prod(bases_cos, axis=1, keepdims=True) 295 | 296 | def true_y(self, x): 297 | bases_cos = 0 * x 298 | for m in range(self.N): 299 | bases_cos = bases_cos + tf.cos(2 * m * x) * self.coef0[m] 300 | return bases_cos 301 | #return tf.reduce_prod(bases_cos, axis=1, keepdims=True) 302 | 303 | def second_y(self, x): 304 | bases_sin = 0 * x 305 | for m in range(self.N): 306 | bases_sin = bases_sin + tf.sin((2*m+1) * x) * self.coef1[m] 307 | return bases_sin 308 | 309 | def second_y_approx(self, x): 310 | # you can choose whatever function you like as initialization 311 | # bases_sin = 0 * x 312 | # for m in range(1): 313 | # bases_sin = bases_sin + tf.sin((2*m+1) * x) * self.coef1[m] 314 | return 0.3*self.true_y(x) + 3*self.second_y(x) 315 | 316 | def true_z(self, x): 317 | bases_sin = 0 318 | for m in range(self.N): 319 | bases_sin = bases_sin + 2 * m * tf.sin(2 * m * x) * self.coef0[m] 320 | return - bases_sin * self.sigma 321 | 322 | def second_z(self, x): 323 | bases_cos = 0 * x 324 | for m in range(self.N): 325 | bases_cos = bases_cos + (2*m+1) * tf.cos((2*m+1) *x) * self.coef1[m] 326 | return bases_cos * self.sigma 327 | 328 | def second_z_approx(self, x): 329 | # set as the gradient of second_y_approx 330 | # bases_cos = 0 * x 331 | # for m in range(1): 332 | # bases_cos = bases_cos + (2*m+1) * tf.cos((2*m+1) *x) * self.coef1[m] 333 | return 0.3*self.true_z(x) + 3*self.second_z(x) 334 | 335 | 336 | class DWSepEigen(Equation): 337 | # well-separate Schrodinger example for arbitrary dimensions 338 | def __init__(self, eqn_config): 339 | super(DWSepEigen, self).__init__(eqn_config) 340 | self.N = 10 341 | self.sigma = np.sqrt(2.0) 342 | self.A = np.concatenate(([1.5],0.2*np.ones(self.dim-1))) 343 | self.true_eigen = -0.2658780338622622197 - 0.004994543800531658917 * (self.dim-1) 344 | self.second_eigen = 0.1860163214034415979 - 0.004994543800531658917 * (self.dim-1) 345 | # coef0 is coef for ground eigenfunction in d1: 1, cos2x, cos4x... 346 | self.coef0 = [4.849638936137847245e-01,-1.719216620376243232e-01, 347 | 7.934668280954022823e-03,-1.641332919106110351e-04, 348 | 1.915646686059286880e-06,-1.432980906804100879e-08, 349 | 7.449835308605464979e-11,-2.846866874407665454e-13, 350 | 8.247577849078903519e-16,-1.840772951476521254e-18] 351 | # coef1 is coef for second eigenfunction in d1: sinx, sin3x, sin5x... 352 | self.coef1 = [7.045452301176932108e-01,-6.010586074745370710e-02, 353 | 1.817537370379718794e-03,-2.792944433460181340e-05, 354 | 2.592161556682200467e-07,-1.609229933303384171e-09, 355 | 7.149528211141171171e-12,-2.385168629168197918e-14, 356 | 6.193904135450362291e-17,-1.287490135293999248e-19] 357 | # coef2 is coef for ground eigenfunction in other dim: 1, cos2x, cos4x... 358 | self.coef2 = [4.996884608914199388e-01,-2.495715904542338340e-02, 359 | 1.559362739878917203e-04,-4.330981093820056683e-07, 360 | 6.766640464567428604e-10,-6.766307217324653586e-13, 361 | 4.698663151849139795e-16,-2.397216509273572312e-19, 362 | 9.363952505944370821e-23,3.060820295240397227e-26] 363 | # coef3 is to put coef0 and coef2s together, this is not a list 364 | self.coef3 = np.concatenate([ 365 | np.reshape(self.coef0,[self.N,1]), 366 | np.repeat(np.reshape(self.coef2,[self.N,1]), self.dim-1, axis=1) 367 | ],axis=1 368 | ) 369 | 370 | def f_tf(self, x, y, z): 371 | return - y * tf.reduce_sum(self.A * tf.cos(2*x), axis=1, keepdims=True) 372 | 373 | def true_y(self, x): 374 | bases_cos = x*0 375 | for m in range(self.N): 376 | bases_cos = bases_cos + tf.cos(2*m*x) * self.coef3[m,:] #broadcast 377 | return tf.reduce_prod(bases_cos, axis=1, keepdims=True) 378 | 379 | def second_y(self, x): 380 | bases_cos = x[:,1:self.dim]*0 381 | bases_sin = x[:,0:1]*0 382 | for m in range(self.N): 383 | bases_cos = bases_cos + tf.cos(2*m*x[:,1:self.dim]) * self.coef3[m,1:self.dim] #broadcast 384 | bases_sin = bases_sin + tf.sin((2*m+1) * x[:,0:1]) * self.coef1[m] 385 | return tf.reduce_prod(bases_cos, axis=1, keepdims=True) * bases_sin 386 | 387 | def second_y_approx(self, x): 388 | # you can choose whatever function you like as initialization 389 | return 0.3*self.true_y(x) + 3*self.second_y(x) 390 | 391 | def true_z(self, x): 392 | bases_cos = 0 * x 393 | bases_sin = 0 * x 394 | for m in range(self.N): 395 | bases_cos = bases_cos + tf.cos(2*m*x) * self.coef3[m,:] 396 | bases_sin = bases_sin - 2 * m * tf.sin(2*m*x) * self.coef3[m,:] 397 | y = tf.reduce_prod(bases_cos, axis=1, keepdims=True) 398 | return self.sigma * y * bases_sin / bases_cos 399 | 400 | def second_z(self, x): 401 | y1 = 0 * x[:,0:1] 402 | y2 = 0 * x[:,1:self.dim] 403 | dy1 = 0 * x[:,0:1] 404 | dy2 = 0 * x[:,1:self.dim] 405 | for m in range(self.N): 406 | y1 = y1 + tf.sin((2*m+1) * x[:,0:1]) * self.coef1[m] 407 | y2 = y2 + tf.cos(2 * m * x[:,1:self.dim]) * self.coef3[m,1:self.dim] 408 | dy1 = dy1 + (2*m+1) * tf.cos((2*m+1) * x[:,0:1]) * self.coef1[m] 409 | dy2 = dy2 - 2*m * tf.sin(2*m * x[:,1:self.dim]) * self.coef3[m,1:self.dim] 410 | y = tf.reduce_prod(y2, axis=1, keepdims=True) * y1 411 | yy = tf.concat([y1,y2],1) # expand y 412 | zz = tf.concat([dy1,dy2],1) # expand z 413 | return y * zz / yy 414 | 415 | def second_z_approx(self, x): 416 | # set as the gradient of second_y_approx 417 | return 0.3*self.true_z(x) + 3*self.second_z(x) 418 | -------------------------------------------------------------------------------- /solver.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | import numpy as np 4 | from tensorflow.python.training import moving_averages 5 | 6 | import tensorflow.compat.v1 as tf 7 | tf.disable_v2_behavior() 8 | # import tensorflow as tf 9 | 10 | TF_DTYPE = tf.float64 11 | DELTA_CLIP = 100.0 12 | 13 | 14 | class FeedForwardModel(object): 15 | """The fully connected neural network model.""" 16 | def __init__(self, config, bsde, sess): 17 | self.eqn_config = config.eqn_config 18 | self.nn_config = config.nn_config 19 | self.problem_type = self.eqn_config.problem_type 20 | self.eigenpair = self.eqn_config.eigenpair 21 | self.model_type = self.eqn_config.model_type 22 | self.bsde = bsde 23 | self.sess = sess 24 | self.y_init = None 25 | # make sure consistent with FBSDE equation 26 | self.dim = bsde.dim 27 | self.num_time_interval = bsde.num_time_interval 28 | self.total_time = bsde.total_time 29 | # ops for statistics update of batch normalization 30 | self.extra_train_ops = [] 31 | self.dw = tf.placeholder(TF_DTYPE, [None, self.dim, self.num_time_interval], name='dW') 32 | self.x = tf.placeholder(TF_DTYPE, [None, self.dim, self.num_time_interval + 1], name='X') 33 | self.is_training = tf.placeholder(tf.bool) 34 | self.train_loss, self.eigen_error, self.init_rel_loss, self.NN_consist,self.l2 = None, None, None, None, None 35 | self.init_infty_loss, self.grad_infty_loss = None, None 36 | self.train_ops, self.t_build = None, None 37 | self.train_ops_supervise, self.train_ops_supervise_approx, self.train_ops_fixEigen = None, None, None 38 | self.eigen = tf.get_variable( 39 | 'eigen', shape=[1], dtype=TF_DTYPE, 40 | initializer=tf.random_uniform_initializer(self.eqn_config.initeigen_low, self.eqn_config.initeigen_high), 41 | trainable=True 42 | ) 43 | self.x_hist = tf.placeholder(TF_DTYPE, [None, self.dim], name='x_hist') 44 | self.hist_NN = None 45 | self.hist_true = None 46 | self.y_second = None 47 | self.hist_size = 20000 48 | self.second = False 49 | if self.eigenpair == 'second': 50 | self.second = True 51 | print("Solving the second eigenpair.") 52 | 53 | def train(self): 54 | start_time = time.time() 55 | # to save iteration results 56 | training_history = [] 57 | # for validation 58 | dw_valid, x_valid = self.bsde.sample_uniform(self.nn_config.valid_size) 59 | # can still use batch norm of samples in the validation phase 60 | feed_dict_valid = {self.dw: dw_valid, self.x: x_valid, self.is_training: False} 61 | # initialization 62 | self.sess.run(tf.global_variables_initializer()) 63 | 64 | # begin sgd iteration 65 | for step in range(self.nn_config.num_iterations+1): 66 | if step % self.nn_config.logging_frequency == 0: 67 | train_loss, eigen_error, init_rel_loss, grad_error, NN_consist, l2, init_infty_loss, grad_infty_loss = self.sess.run( 68 | [ 69 | self.train_loss, self.eigen_error, self.init_rel_loss, self.grad_error, 70 | self.NN_consist, self.l2, self.init_infty_loss, self.grad_infty_loss 71 | ], 72 | feed_dict=feed_dict_valid) 73 | elapsed_time = time.time()-start_time+self.t_build 74 | training_history.append( 75 | [ 76 | step, train_loss, eigen_error, init_rel_loss, init_infty_loss, 77 | grad_error, grad_infty_loss, NN_consist, l2, elapsed_time 78 | ] 79 | ) 80 | if self.nn_config.verbose: 81 | logging.info( 82 | "step: %5u, train_loss: %.4e, eg_err: %.4e, egfcn_l2_err: %.4e, egfcn_infty_err: %.4e, grad_l2_err: %.4e " % ( 83 | step, train_loss, eigen_error, init_rel_loss, init_infty_loss, grad_error) + 84 | "grad_infty_err: %.4e, consist_loss: %.4e, norm_fac: %.4e, elapsed time %3u" % ( 85 | grad_infty_loss, NN_consist, l2, elapsed_time)) 86 | dw_train, x_train = self.bsde.sample_uniform(self.nn_config.batch_size) 87 | if self.second: 88 | if self.eqn_config.pretrain == "degenerate": # for dwclose_d1 89 | if step < 1000: # use approximate solution to do supervised learning 90 | self.sess.run(self.train_ops_supervise_approx,feed_dict={self.dw: dw_train, self.x: x_train, self.is_training: True}) 91 | else: # use the default loss function 92 | self.sess.run(self.train_ops,feed_dict={self.dw: dw_train, self.x: x_train, self.is_training: True}) 93 | else: # self.eqn_config.second marks the step to change loss function 94 | if step < self.eqn_config.pretrain: 95 | self.sess.run(self.train_ops_fixEigen,feed_dict={self.dw: dw_train, self.x: x_train, self.is_training: True}) 96 | else: 97 | self.sess.run(self.train_ops,feed_dict={self.dw: dw_train, self.x: x_train, self.is_training: True}) 98 | else: 99 | self.sess.run(self.train_ops,feed_dict={self.dw: dw_train, self.x: x_train, self.is_training: True}) 100 | if step == self.nn_config.num_iterations: 101 | x_hist = self.bsde.sample_hist(self.hist_size) 102 | feed_dict_hist = {self.x_hist: x_hist} 103 | if self.second: 104 | [y_second, y_hist] = self.sess.run([self.y_second, self.hist_NN], feed_dict=feed_dict_hist) 105 | figure_data = np.concatenate([y_second, y_hist], axis=1) 106 | else: 107 | [y_hist_true, y_hist] = self.sess.run([self.hist_true, self.hist_NN], feed_dict=feed_dict_hist) 108 | figure_data = np.concatenate([y_hist_true, y_hist], axis=1) 109 | if self.dim == 1: 110 | figure_data = np.concatenate([x_hist,figure_data], axis=1) 111 | return np.array(training_history, dtype=object), figure_data 112 | 113 | def build(self): 114 | start_time = time.time() 115 | with tf.variable_scope('forward'): 116 | global_step = tf.get_variable('global_step', [], 117 | initializer=tf.constant_initializer(0), 118 | trainable=False, dtype=tf.int32) 119 | decay = tf.train.piecewise_constant( 120 | global_step, self.nn_config.ma_boundaries, 121 | [tf.constant(ma, dtype=TF_DTYPE) for ma in self.nn_config.ma_values]) 122 | x_init = self.x[:, :, 0] 123 | net_y = PeriodNet(self.nn_config.num_hiddens, out_dim=1, 124 | trig_order=self.nn_config.trig_order, name='net_y') 125 | if self.model_type == 'consistent': 126 | net_z = PeriodNet(self.nn_config.num_hiddens, out_dim=self.dim, 127 | trig_order=self.nn_config.trig_order, name='net_z') 128 | 129 | y_init_and_gradient = net_y(x_init,need_grad=True) 130 | y_init = y_init_and_gradient[0] 131 | 132 | yl2_batch = tf.reduce_mean(y_init ** 2) 133 | yl2_ma = tf.get_variable( 134 | 'yl2_ma', [1], TF_DTYPE, 135 | initializer=tf.constant_initializer(100.0, TF_DTYPE), 136 | trainable=False) 137 | yl2 = tf.cond(self.is_training, 138 | lambda: decay * yl2_ma + (1 - decay) * yl2_batch, 139 | lambda: yl2_ma) 140 | true_z = self.bsde.true_z(x_init) 141 | 142 | sign = tf.sign(tf.reduce_sum(y_init)) 143 | 144 | if self.model_type == 'consistent': 145 | z = net_z(x_init, need_grad=False) 146 | else: 147 | z = y_init_and_gradient[1] 148 | z = z / tf.sqrt(yl2) * sign 149 | 150 | normed_true_z = true_z / tf.sqrt(tf.reduce_mean(true_z ** 2)) 151 | error_z = z / tf.sqrt(tf.reduce_mean(z ** 2)) - normed_true_z 152 | y_init = y_init / tf.sqrt(yl2) * sign 153 | 154 | if self.model_type == 'consistent': 155 | x_T = self.x[:, :, -1] 156 | z_T = net_z(x_T, need_grad=False) 157 | yT_and_gradient = net_y(x_T,need_grad=True) 158 | grad_yT = yT_and_gradient[1] 159 | grad_yT = grad_yT * sign / tf.sqrt(yl2) 160 | if self.problem_type == 'nonlinear': 161 | grad_yT = grad_yT * self.bsde.L2mean 162 | NN_consist = z_T - grad_yT 163 | else: 164 | NN_consist = tf.constant(0.0, dtype = TF_DTYPE) 165 | 166 | y = y_init 167 | if self.problem_type == 'nonlinear': 168 | y = y * self.bsde.L2mean 169 | for t in range(0, self.num_time_interval-1): 170 | y = y - self.bsde.delta_t * (self.bsde.f_tf(self.x[:, :, t], y, z) + self.eigen *y) + \ 171 | tf.reduce_sum(z * self.dw[:, :, t], 1, keepdims=True) 172 | if self.problem_type == 'nonlinear': 173 | y = tf.clip_by_value(y, -5, 5, name=None) 174 | if self.model_type == 'consistent': 175 | z = net_z(self.x[:, :, t + 1], need_grad=False, reuse=True) 176 | else: 177 | yz = net_y(self.x[:, :, t + 1], need_grad=True, reuse=True) 178 | z = yz[1] / tf.sqrt(yl2) * sign 179 | # terminal time 180 | y = y - self.bsde.delta_t * (self.bsde.f_tf(self.x[:, :, -2], y, z) + self.eigen *y) + \ 181 | tf.reduce_sum(z * self.dw[:, :, -1], 1, keepdims=True) 182 | y_xT = net_y(self.x[:, :, -1], need_grad=False, reuse=True) 183 | y_xT = y_xT / tf.sqrt(yl2) * sign 184 | if self.problem_type == 'nonlinear': 185 | y_xT = y_xT * self.bsde.L2mean 186 | delta = y - y_xT 187 | 188 | # use linear approximation outside the clipped range 189 | self.train_loss = tf.reduce_mean( 190 | tf.where(tf.abs(delta) < DELTA_CLIP, tf.square(delta), 191 | 2 * DELTA_CLIP * tf.abs(delta) - DELTA_CLIP ** 2)) * 1000 \ 192 | + tf.reduce_mean(tf.square(NN_consist)) * 20\ 193 | + tf.nn.relu(2 - yl2) * 100 194 | if self.problem_type == 'nonlinear': 195 | self.train_loss += tf.nn.relu(2 * self.bsde.L2mean - yl2) * 100 196 | self.extra_train_ops.append( 197 | moving_averages.assign_moving_average(yl2_ma, yl2_batch, decay)) 198 | y_hist = net_y(self.x_hist, need_grad=False, reuse=True) 199 | hist_sign = tf.sign(tf.reduce_sum(y_hist)) 200 | hist_l2 = tf.reduce_mean(y_hist ** 2) 201 | self.hist_NN = y_hist / tf.sqrt(hist_l2) * hist_sign 202 | hist_true_y = self.bsde.true_y(self.x_hist) 203 | self.hist_true = hist_true_y / tf.sqrt(tf.reduce_mean(hist_true_y ** 2)) 204 | true_init = self.bsde.true_y(self.x[:, :, 0]) 205 | true_init = true_init / tf.sqrt(tf.reduce_mean(true_init ** 2)) 206 | error_y = y_init - true_init 207 | self.init_rel_loss = tf.sqrt(tf.reduce_mean(error_y ** 2)) 208 | self.init_infty_loss = tf.reduce_max(tf.abs(error_y)) 209 | self.eigen_error = self.eigen - self.bsde.true_eigen 210 | self.l2 = yl2 211 | self.grad_error = tf.sqrt(tf.reduce_mean(error_z ** 2)) 212 | self.grad_infty_loss = tf.reduce_max(tf.abs(error_z)) 213 | self.NN_consist = tf.sqrt(tf.reduce_mean(NN_consist ** 2)) 214 | 215 | # train operations 216 | learning_rate = tf.train.piecewise_constant(global_step, 217 | self.nn_config.lr_boundaries, 218 | self.nn_config.lr_values) 219 | trainable_variables = tf.trainable_variables() 220 | grads = tf.gradients(self.train_loss, trainable_variables) 221 | optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) 222 | apply_op = optimizer.apply_gradients(zip(grads, trainable_variables), 223 | global_step=global_step, name='train_step') 224 | all_ops = [apply_op] + self.extra_train_ops 225 | self.train_ops = tf.group(*all_ops) 226 | self.t_build = time.time() - start_time 227 | 228 | def build_second(self): 229 | start_time = time.time() 230 | with tf.variable_scope('forward'): 231 | global_step = tf.get_variable('global_step', [], 232 | initializer=tf.constant_initializer(0), 233 | trainable=False, dtype=tf.int32) 234 | decay = tf.train.piecewise_constant( 235 | global_step, self.nn_config.ma_boundaries, 236 | [tf.constant(ma, dtype=TF_DTYPE) for ma in self.nn_config.ma_values]) 237 | x_init = self.x[:, :, 0] 238 | net_y = PeriodNet(self.nn_config.num_hiddens, out_dim=1, 239 | trig_order=self.nn_config.trig_order, name='net_y') 240 | net_z = PeriodNet(self.nn_config.num_hiddens, out_dim=self.dim, 241 | trig_order=self.nn_config.trig_order, name='net_z') 242 | y_init_and_gradient = net_y(x_init,need_grad=True) 243 | y_init = y_init_and_gradient[0] 244 | grad_y = y_init_and_gradient[1] 245 | z = net_z(x_init, need_grad=False) 246 | z_init = z 247 | 248 | yl2_batch = tf.reduce_mean(y_init ** 2) 249 | yl2_ma = tf.get_variable( 250 | 'yl2_ma', [1], TF_DTYPE, 251 | initializer=tf.constant_initializer(100.0, TF_DTYPE), 252 | trainable=False) 253 | yl2 = tf.cond(self.is_training, 254 | lambda: decay * yl2_ma + (1 - decay) * yl2_batch, 255 | lambda: yl2_ma) 256 | true_z = self.bsde.true_z(x_init) 257 | 258 | second_z = self.bsde.second_z(x_init) 259 | normed_second_z = second_z / tf.sqrt(tf.reduce_mean(second_z ** 2)) 260 | error_second_z = z / tf.sqrt(tf.reduce_mean(z ** 2)) - normed_second_z 261 | error_second_z2 = z / tf.sqrt(tf.reduce_mean(z ** 2)) + normed_second_z 262 | 263 | normed_true_z = true_z / tf.sqrt(tf.reduce_mean(true_z ** 2)) 264 | error_z = z / tf.sqrt(tf.reduce_mean(z ** 2)) - normed_true_z 265 | 266 | y_init = y_init / tf.sqrt(yl2) 267 | grad_y = grad_y / tf.sqrt(yl2) 268 | 269 | x_T = self.x[:, :, -1] 270 | z_T = net_z(x_T, need_grad=False) 271 | yT_and_gradient = net_y(x_T,need_grad=True) 272 | grad_yT = yT_and_gradient[1] 273 | grad_yT = grad_yT / tf.sqrt(yl2) 274 | NN_consist = z_T - grad_yT 275 | 276 | y = y_init 277 | for t in range(0, self.num_time_interval-1): 278 | y = y - self.bsde.delta_t * (self.bsde.f_tf(self.x[:, :, t], y, z) + self.eigen *y) + \ 279 | tf.reduce_sum(z * self.dw[:, :, t], 1, keepdims=True) 280 | z = net_z(self.x[:, :, t + 1], need_grad=False, reuse=True) 281 | # terminal time 282 | y = y - self.bsde.delta_t * (self.bsde.f_tf(self.x[:, :, -2], y, z) + self.eigen *y) + \ 283 | tf.reduce_sum(z * self.dw[:, :, -1], 1, keepdims=True) 284 | y_xT = net_y(self.x[:, :, -1], need_grad=False, reuse=True) 285 | y_xT = y_xT / tf.sqrt(yl2) 286 | delta = y - y_xT 287 | 288 | # use linear approximation outside the clipped range 289 | self.train_loss_supervise_approx = \ 290 | tf.reduce_mean((y_init_and_gradient[0] - self.bsde.second_y_approx(x_init))**2) \ 291 | + 0*tf.reduce_mean(NN_consist**2) \ 292 | + tf.reduce_mean((z_init - self.bsde.second_z_approx(x_init))**2) \ 293 | + (self.eigen - self.bsde.second_eigen)**2 294 | self.train_loss_fixEigen = tf.reduce_mean( 295 | tf.where(tf.abs(delta) < DELTA_CLIP, tf.square(delta), 296 | 2 * DELTA_CLIP * tf.abs(delta) - DELTA_CLIP ** 2)) * 1000 \ 297 | + ((self.eigen - self.bsde.second_eigen - 0.1)**2) * 1000000 \ 298 | + tf.reduce_mean(tf.square(NN_consist)) * 20\ 299 | + tf.nn.relu(2 - yl2) * 100 300 | self.train_loss = tf.reduce_mean( 301 | tf.where(tf.abs(delta) < DELTA_CLIP, tf.square(delta), 302 | 2 * DELTA_CLIP * tf.abs(delta) - DELTA_CLIP ** 2)) * 1000 \ 303 | + tf.reduce_mean(tf.square(NN_consist)) * 20\ 304 | + tf.nn.relu(2 - yl2) * 100 305 | self.extra_train_ops.append( 306 | moving_averages.assign_moving_average(yl2_ma, yl2_batch, decay)) 307 | y_hist = net_y(self.x_hist, need_grad=False, reuse=True) 308 | hist_l2 = tf.reduce_mean(y_hist ** 2) 309 | self.hist_NN = y_hist / tf.sqrt(hist_l2) 310 | if not self.second: 311 | hist_sign = tf.sign(tf.reduce_sum(y_hist)) 312 | self.hist_NN = self.hist_NN * hist_sign 313 | hist_true_y = self.bsde.true_y(self.x_hist) 314 | self.hist_true = hist_true_y / tf.sqrt(tf.reduce_mean(hist_true_y ** 2)) 315 | y_second = self.bsde.second_y(self.x_hist) 316 | self.y_second = y_second / tf.sqrt(tf.reduce_mean(y_second ** 2)) 317 | second_init = self.bsde.second_y(self.x[:, :, 0]) 318 | second_init = second_init / tf.sqrt(tf.reduce_mean(second_init ** 2)) 319 | error_y_second = y_init - second_init 320 | self.eigen_error = self.eigen - self.bsde.second_eigen 321 | self.init_rel_loss = tf.sqrt(tf.reduce_mean(error_y_second ** 2)) 322 | self.init_infty_loss = tf.reduce_max(tf.abs(error_y_second)) 323 | self.l2 = yl2 324 | self.grad_error = tf.sqrt(tf.reduce_mean(error_second_z ** 2)) 325 | self.grad_infty_loss = tf.reduce_max(tf.abs(error_second_z)) 326 | self.NN_consist = tf.sqrt(tf.reduce_mean(NN_consist ** 2)) 327 | # the following is to overcome the sign problem, useful when net_y is not wel initialized 328 | error_y_second2 = y_init + second_init 329 | init_rel_loss2 = tf.sqrt(tf.reduce_mean(error_y_second2 ** 2)) 330 | init_infty_loss2 = tf.reduce_max(tf.abs(error_y_second2)) 331 | grad_error2 = tf.sqrt(tf.reduce_mean(error_second_z2 ** 2)) 332 | grad_infty_loss2 = tf.reduce_max(tf.abs(error_second_z2)) 333 | self.init_rel_loss = tf.minimum(self.init_rel_loss, init_rel_loss2) 334 | self.init_infty_loss = tf.minimum(self.init_infty_loss, init_infty_loss2) 335 | self.grad_error = tf.minimum(self.grad_error, grad_error2) 336 | self.grad_infty_loss = tf.minimum(self.grad_infty_loss, grad_infty_loss2) 337 | 338 | # train operations 339 | learning_rate = tf.train.piecewise_constant(global_step, 340 | self.nn_config.lr_boundaries, 341 | self.nn_config.lr_values) 342 | trainable_variables = tf.trainable_variables() 343 | grads = tf.gradients(self.train_loss, trainable_variables) 344 | optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) 345 | apply_op = optimizer.apply_gradients(zip(grads, trainable_variables), 346 | global_step=global_step, name='train_step') 347 | all_ops = [apply_op] + self.extra_train_ops 348 | self.train_ops = tf.group(*all_ops) 349 | 350 | grads_supervise_approx = tf.gradients(self.train_loss_supervise_approx, trainable_variables) 351 | optimizer_supervise_approx = tf.train.AdamOptimizer(learning_rate=learning_rate) 352 | apply_op_supervise_approx = optimizer_supervise_approx.apply_gradients(zip(grads_supervise_approx, trainable_variables), 353 | global_step=global_step, name='train_step') 354 | all_ops_supervise_approx = [apply_op_supervise_approx] + self.extra_train_ops 355 | self.train_ops_supervise_approx = tf.group(*all_ops_supervise_approx) 356 | 357 | grads_fixEigen = tf.gradients(self.train_loss_fixEigen, trainable_variables) 358 | optimizer_fixEigen = tf.train.AdamOptimizer(learning_rate=learning_rate) 359 | apply_op_fixEigen = optimizer_fixEigen.apply_gradients(zip(grads_fixEigen, trainable_variables), 360 | global_step=global_step, name='train_step') 361 | all_ops_fixEigen = [apply_op_fixEigen] + self.extra_train_ops 362 | self.train_ops_fixEigen = tf.group(*all_ops_fixEigen) 363 | 364 | self.t_build = time.time() - start_time 365 | 366 | def build_true(self): 367 | # both true_y and true_z are multiplied by trainable constant 368 | start_time = time.time() 369 | self._init_coef_y = tf.get_variable( 370 | 'init_coef_y', [1], TF_DTYPE, initializer=tf.constant_initializer(1.0, TF_DTYPE), 371 | trainable=False) 372 | self._init_coef_z = tf.get_variable( 373 | 'init_coef_z', [1], TF_DTYPE, initializer=tf.constant_initializer(1.0, TF_DTYPE), 374 | trainable=False) 375 | with tf.variable_scope('forward'): 376 | x_init = self.x[:, :, 0] 377 | y_init = self.bsde.true_y(x_init) * self._init_coef_y 378 | yl2 = tf.reduce_mean(y_init ** 2) 379 | y = y_init 380 | self.z_init = self.bsde.true_z(x_init) * self._init_coef_z 381 | z = self.z_init 382 | for t in range(0, self.num_time_interval - 1): 383 | y = y - self.bsde.delta_t * (self.bsde.f_tf(self.x[:, :, t], y, z) + self.eigen * y) + \ 384 | tf.reduce_sum(z * self.dw[:, :, t], 1, keepdims=True) 385 | z = self.bsde.true_z(self.x[:, :, t + 1]) * self._init_coef_z 386 | # terminal time 387 | y = y - self.bsde.delta_t * (self.bsde.f_tf(self.x[:, :, -2], y, z) + self.eigen * y) + \ 388 | tf.reduce_sum(z * self.dw[:, :, -1], 1, keepdims=True) 389 | y_xT = self.bsde.true_y(self.x[:, :, -1]) 390 | delta = y - y_xT 391 | # use linear approximation outside the clipped range 392 | self.train_loss = tf.reduce_mean( 393 | tf.where(tf.abs(delta) < DELTA_CLIP, tf.square(delta), 394 | 2 * DELTA_CLIP * tf.abs(delta) - DELTA_CLIP ** 2)) * 100 395 | hist_true_y = self.bsde.true_y(self.x_hist) 396 | self.hist_true = hist_true_y / tf.sqrt(tf.reduce_mean(hist_true_y ** 2)) 397 | self.hist_NN = self.hist_true 398 | self.init_rel_loss = self._init_coef_y -1 399 | self.init_infty_loss = self._init_coef_y -1 400 | self.eigen_error = self.eigen - self.bsde.true_eigen 401 | self.l2 = yl2 402 | self.grad_error = self._init_coef_z -1 403 | self.grad_infty_loss = self._init_coef_z -1 404 | self.NN_consist = tf.constant(0) 405 | # train operations 406 | global_step = tf.get_variable('global_step', [], 407 | initializer=tf.constant_initializer(0), 408 | trainable=False, dtype=tf.int32) 409 | learning_rate = tf.train.piecewise_constant(global_step, 410 | self.nn_config.lr_boundaries, 411 | self.nn_config.lr_values) 412 | trainable_variables = tf.trainable_variables() 413 | grads = tf.gradients(self.train_loss, trainable_variables) 414 | optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) 415 | apply_op = optimizer.apply_gradients(zip(grads, trainable_variables), 416 | global_step=global_step, name='train_step') 417 | 418 | all_ops = [apply_op] + self.extra_train_ops 419 | self.train_ops = tf.group(*all_ops) 420 | self.t_build = time.time() - start_time 421 | 422 | 423 | class PeriodNet(object): 424 | def __init__(self, num_hiddens, out_dim, trig_order, name='period_net'): 425 | self.num_hiddens = num_hiddens 426 | self.out_dim = out_dim 427 | self.trig_order = trig_order 428 | self.name = name 429 | 430 | def __call__(self, x, need_grad, reuse=tf.AUTO_REUSE): 431 | with tf.variable_scope(self.name): 432 | trig_bases = [] 433 | for i in range(1, self.trig_order+1): 434 | trig_bases += [tf.sin(i * x), tf.cos(i * x)] 435 | trig_bases = tf.concat(trig_bases, axis=1) 436 | h = trig_bases 437 | for i in range(0, len(self.num_hiddens)): 438 | h = tf.layers.dense(h, self.num_hiddens[i], activation=tf.nn.relu, 439 | name="full%g" % (i + 1), reuse=reuse) 440 | u = tf.layers.dense( 441 | h, self.out_dim, activation=None, name="final_layer", reuse=reuse) 442 | if need_grad: 443 | # the shape of grad is 1 x num_sample x dim 444 | grad = tf.gradients(u, x)[0] 445 | if need_grad: 446 | return u, grad*np.sqrt(2) 447 | else: 448 | return u 449 | --------------------------------------------------------------------------------