├── Examples ├── 1DB │ ├── 1DB.py │ ├── 1DB_hparams.py │ └── burgers.mat ├── 2DH │ ├── 2DH.py │ └── 2DH_hparams.py ├── 3DNS │ ├── 3DNS.py │ └── 3DNS_hparams.py ├── AC │ ├── AC.py │ ├── AC_cosine.mat │ └── AC_hparams.py ├── GS │ ├── GS.py │ └── GS_hparams.py ├── KdV │ ├── KdV.py │ └── KdV_hparams.py ├── LDC │ ├── LDC.py │ └── LDC_hparams.py ├── NLP │ ├── NLP.py │ └── NLP_hparams.py └── NLS │ ├── NLS.mat │ ├── NLS.py │ └── NLS_hparams.py ├── LICENSE ├── README.md ├── modified_minimize.py └── modified_optimize.py /Examples/1DB/1DB.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Dec 11 10:54:50 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import numpy as np 9 | import tensorflow as tf 10 | from tensorflow import keras 11 | from tensorflow.keras.layers import Input, Dense 12 | from tensorflow.keras.models import Model 13 | from tensorflow import convert_to_tensor 14 | from tensorflow.keras.optimizers import Adam 15 | from scipy.optimize import minimize 16 | from scipy.linalg import cholesky,LinAlgError 17 | from scipy import sparse 18 | from scipy.io import loadmat 19 | import json 20 | 21 | def load_hparams(file_config): 22 | with open(file_config, 'r') as file: 23 | config = json.load(file) 24 | return config 25 | 26 | config = load_hparams('config_1DB.json') 27 | 28 | seed = config["seed"]["seed"] #Seed 29 | 30 | #--------------- Architecture hyperparameters ---------------------------------- 31 | neurons = config["architecture_hparams"]["neurons"] #Neurons in every hidden layer 32 | layers = config["architecture_hparams"]["layers"] #Hidden layers 33 | output_dim = config["architecture_hparams"]["output_dim"] #Output dimension 34 | kmax = config["architecture_hparams"]["kmax"] #Maximum wavelength if Fourier inputs are considered 35 | #------------------------------------------------------------------------------- 36 | 37 | #--------------- PDE hyperparameters ------------------------------------------ 38 | nu = config["PDE_hparams"]["nu"] 39 | #------------------------------------------------------------------------------ 40 | 41 | #-------------- Adam hyperparameters ------------------------------------------- 42 | Adam_epochs = config["Adam_hparams"]["Adam_epochs"] #Adam epochs 43 | lr0 = config["Adam_hparams"]["lr0"] #Initial learning rate (We consider an exponential lr schedule) 44 | decay_steps = config["Adam_hparams"]["decay_steps"] #Decay steps 45 | decay_rate = config["Adam_hparams"]["decay_rate"] #Decay rate 46 | b1 = config["Adam_hparams"]["b1"] #beta1 47 | b2 = config["Adam_hparams"]["b2"] #beta2 48 | epsilon= config["Adam_hparams"]["epsilon"] #epsilon 49 | Nprint_adam = config["Adam_hparams"]["Nprint_adam"] #Adam results will be printed and save every Nprint_adam iters 50 | #------------------------------------------------------------------------------ 51 | 52 | #------------ Batch hyperparameters ------------------------------------------- 53 | Nint = config["batch_hparams"]["Nint"] #Number of points at batch 54 | Nchange=config["batch_hparams"]["Nchange"] #Batch is changed every Nchange iterations 55 | k1 = config["batch_hparams"]["k1"] #k hyperparameter (see adaptive_rad function below) 56 | k2 = config["batch_hparams"]["k2"] #c hyperparameter (see adaptive_rad function below) 57 | x0 = config["batch_hparams"]["x0"] #x0 (minimum value of x) 58 | Lx = config["batch_hparams"]["Lx"] #Lx (length in the x direction) 59 | t0 = config["batch_hparams"]["t0"] 60 | tfinal = config["batch_hparams"]["tfinal"] 61 | #------------------------------------------------------------------------------ 62 | 63 | #------------ Quasi-Newton (QN) hyperparameters ------------------------------- 64 | Nbfgs = config["bfgs_hparams"]["BFGS_epochs"] #Number of QN iterations 65 | method = config["bfgs_hparams"]["method"] #Method. See below 66 | method_bfgs = config["bfgs_hparams"]["method_bfgs"] #Quasi-Newton algorithm. See below 67 | use_sqrt = config["bfgs_hparams"]["use_sqrt"] #Use square root of the MSE loss to train 68 | use_log = config["bfgs_hparams"]["use_log"] #Use log of the MSE loss to train 69 | Nprint_bfgs = config["bfgs_hparams"]["Nprint_bfgs"] #QN results will be printed and save every Nprint_adam iters 70 | 71 | #In method, you can choose between: 72 | # -BFGS: Here, we include BFGS and the different self-scaled QN methods. 73 | # To distinguish between these algorithms, we use method_bfgs. See below 74 | # -bfgsr: Personal implementation of the factored BFGS Hessian approximations. 75 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 76 | # Very slow, to be optimized. 77 | # -bfgsz: Personal implementation of the factored inverse BFGS Hessian approximations. 78 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 79 | # Comparable with BFGS in terms of speed. 80 | 81 | #If method=BFGS, the variable "method_bfgs" chooses the different QN methods. 82 | #The options for this are (see the modified Scipy optimize script): 83 | #-BFGS_scipy: The original implementation of BFGS of Scipy 84 | #-BFGS: Equivalent implementation, but faster (avoid repeated calculations in the BFGS formula) 85 | #-SSBFGS_AB: The Self-scaled BFGS formula, where the tauk coefficient is calculated with 86 | # Al-Baali's formula (Formula 11 of "Unveiling the optimization process in PINNs") 87 | #-SSBFGS_OL Same, but tauk is calculated with the original choice of Oren and Luenberger (not recommended) 88 | #-SSBroyden2: Here we use the tauk and phik expressions defined in the paper 89 | # (Formulas 13-23 of "Unveiling the optimization process in PINNs") 90 | #-SSbroyden1: Another possible choice for these parameters (sometimes better, sometimes worse than SSBroyden1) 91 | 92 | #------------------------------------------------------------------------------ 93 | xf = x0 + Lx 94 | tf.keras.backend.set_floatx('float64') 95 | tf.get_logger().setLevel('ERROR') 96 | tf.keras.utils.set_random_seed(seed) 97 | 98 | 99 | class PeriodicC0Layer(keras.layers.Layer): 100 | def __init__(self, **kwargs): 101 | super(PeriodicC0Layer, self).__init__(**kwargs) 102 | 103 | def call(self, inputs): 104 | # Assuming inputs is of shape (batch_size, 2) where inputs[:, 0] is t and inputs[:, 1] is x 105 | t = inputs[:, 0,None] # Extract t 106 | x = inputs[:, 1,None] # Extract x 107 | ks = tf.convert_to_tensor(np.arange(1,kmax+1),dtype=tf.float64) 108 | Xper = 2*np.pi*tf.matmul(x,ks[None,:])/Lx 109 | xcos = tf.math.cos(Xper) 110 | xsin = tf.math.sin(Xper) 111 | Xper = tf.concat([xcos,xsin],axis=1) 112 | Xtot = tf.concat([t,Xper],axis=1) 113 | return Xtot 114 | 115 | def generate_model(layer_dims): 116 | ''' 117 | Parameters 118 | ---------- 119 | layer_dims : TUPLE 120 | DESCRIPTION. 121 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 122 | if i=0, Li corresponds to the input dimension 123 | Returns 124 | ------- 125 | Model 126 | TYPE : TENSORFLOW MODEL 127 | DESCRIPTION. 128 | 129 | ''' 130 | X_input = Input((layer_dims[0],)) 131 | X = PeriodicC0Layer()(X_input) 132 | X = Dense(layer_dims[1],activation="tanh")(X) 133 | if len(layer_dims) > 3: 134 | for i in range(2,len(layer_dims)-1): 135 | X = Dense(layer_dims[i],activation="tanh")(X) 136 | X = Dense(layer_dims[-1],activation=None)(X) 137 | return Model(inputs=X_input,outputs=X) 138 | 139 | def generate_inputs(Nint): 140 | ''' 141 | 142 | Parameters 143 | ---------- 144 | Nint : INTEGER 145 | DESCRIPTION. 146 | Number of training points in a given batch 147 | 148 | Returns 149 | ------- 150 | X: Batch of points (in Tensorflow format) 151 | TYPE : TENSOR 152 | DESCRIPTION. 153 | 154 | ''' 155 | t = (tfinal-t0)*np.random.rand(Nint) + t0 156 | x = Lx*np.random.rand(Nint) + x0 157 | X = np.hstack((t[:,None],x[:,None])) 158 | return convert_to_tensor(X) 159 | 160 | #-----------ADAPTIVE RAD GENERATOR (GIVEN AT WU ET AL., 2023)------------------ 161 | def adaptive_rad(N,Nint,rad_args,Ntest=50000): 162 | ''' 163 | Parameters 164 | ---------- 165 | N : TENSORFLOW MODEL 166 | DESCRIPTION. 167 | PINN model, obtained with generate_model() function 168 | 169 | Nint : INTEGER 170 | DESCRIPTION. 171 | Number of training points in a given batch 172 | 173 | rad_args: TUPLE (k1,k2) 174 | DESCRIPTION. 175 | Adaptive resampling of Wu et al. (2023), formula (2) 176 | k1: k 177 | k2: c 178 | DOI: https://doi.org/10.1016/j.cma.2022.115671 179 | 180 | Ntest: INTEGER 181 | DESCRIPTION. 182 | Number of test points to do the resampling 183 | 184 | Returns 185 | ------- 186 | X: Batch of points (in Tensorflow format) 187 | TYPE 188 | DESCRIPTION. 189 | ''' 190 | Xtest = generate_inputs(Ntest) 191 | k1,k2 = rad_args 192 | Y = tf.math.abs(get_results(N,Xtest)[-1]).numpy() 193 | err_eq = np.power(Y,k1)/np.power(Y,k1).mean() + k2 194 | err_eq_normalized = (err_eq / sum(err_eq)) 195 | X_ids = np.random.choice(a=len(Xtest), size=Nint, replace=False, 196 | p=err_eq_normalized) 197 | return tf.gather(Xtest,X_ids) 198 | 199 | #---------------- PINN OUTPUT ------------------------------------------------- 200 | def output(N,X): 201 | ''' 202 | Parameters 203 | ---------- 204 | N : TENSORFLOW MODEL 205 | PINN model, obtained with generate_model() function 206 | X : TENSOR 207 | Batch of points (in Tensorflow format) 208 | Returns 209 | ------- 210 | u: TENSOR 211 | PINN prediction for u 212 | 213 | ''' 214 | t = X[:,0,None] 215 | x = X[:,1,None] 216 | Nout = N(X) 217 | u = -tf.math.sin(2*np.pi*x/Lx) + t*Nout[:,0,None] 218 | return u 219 | 220 | def get_results(N,X): 221 | ''' 222 | Parameters 223 | ---------- 224 | N : TENSORFLOW MODEL 225 | PINN model, obtained with generate_model() function 226 | X : TENSOR 227 | Batch of points (in Tensorflow format) 228 | 229 | Returns 230 | ------- 231 | u : TENSOR 232 | PINN prediction for u 233 | fu : TENSOR 234 | PDE residuals for u_t = ... 235 | 236 | ''' 237 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt1: 238 | gt1.watch(X) 239 | 240 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt2: 241 | gt2.watch(X) 242 | 243 | # Calculate u,v 244 | u = output(N,X) 245 | 246 | ugrad = gt2.gradient(u, X) 247 | u_t = ugrad[:,0] 248 | u_x = ugrad[:,1] 249 | 250 | u_xx = gt1.gradient(u_x, X)[:,1] 251 | fu = u_t + u[:,0]*u_x - nu*u_xx 252 | return u,fu 253 | 254 | #------------------------------- LOSS FUNCTION -------------------------------- 255 | loss_function = keras.losses.MeanSquaredError() #MSE loss 256 | def loss(fu): 257 | ''' 258 | Parameters 259 | ---------- 260 | fu : TENSOR 261 | PDE residuals for u_t = ... 262 | 263 | Returns 264 | ------- 265 | LOSS 266 | TYPE: FLOAT64 267 | MSE Loss of PDE residuals 268 | 269 | ''' 270 | Ntot = fu.shape[0] 271 | zeros = tf.zeros([Ntot,1],dtype=tf.float64) 272 | return loss_function(fu,zeros) 273 | 274 | #---------------------- Gradients wrt the trainable parameters ---------------- 275 | def grads(N,X): 276 | ''' 277 | Parameters 278 | ---------- 279 | N : TENSORFLOW MODEL 280 | PINN model, obtained with generate_model() function 281 | X : TENSOR 282 | Batch of points (in Tensorflow format) 283 | 284 | Returns 285 | ------- 286 | gradsN : TENSOR 287 | Gradients of loss wrt trainable variables 288 | loss_value: FLOAT64 289 | MSE Loss of PDE residuals 290 | 291 | ''' 292 | with tf.GradientTape() as tape2: 293 | _,fu = get_results(N,X) 294 | loss_value = loss(fu) 295 | gradsN = tape2.gradient(loss_value,N.trainable_variables) 296 | return gradsN,loss_value 297 | 298 | #-------------------------TRAINING STEP --------------------------------------- 299 | @tf.function(jit_compile=True) #Precompile training function to accelerate the process 300 | def training(N,X,optimizer): #Training step function 301 | ''' 302 | Parameters 303 | ---------- 304 | N : TENSORFLOW MODEL 305 | PINN model, obtained with generate_model() function 306 | X : TENSOR 307 | Batch of points (in Tensorflow format) 308 | 309 | Optimizer: TENSORFLOW OPTIMIZER 310 | Tensorflow optimizer (Adam, Nadam,...) 311 | 312 | Returns 313 | ------- 314 | loss_value: FLOAT64 315 | MSE Loss of PDE residuals 316 | ''' 317 | parameter_gradients,loss_value = grads(N,X) 318 | optimizer.apply_gradients(zip(parameter_gradients,N.trainable_variables)) 319 | return loss_value 320 | 321 | rad_args = (k1,k2) 322 | epochs = np.arange(Nprint_adam,Adam_epochs+Nprint_adam,Nprint_adam) 323 | loss_list = np.zeros(len(epochs)) #loss list 324 | X = generate_inputs(Nint) 325 | 326 | layer_dims = [None]*(layers + 2) 327 | layer_dims[0] = X.shape[1] 328 | for i in range(1,len(layer_dims)): 329 | layer_dims[i] = neurons 330 | layer_dims[-1] = output_dim 331 | 332 | N = generate_model(layer_dims) 333 | 334 | lr = tf.keras.optimizers.schedules.ExponentialDecay(lr0,decay_steps,decay_rate) 335 | 336 | optimizer = Adam(lr,b1,b2,epsilon=epsilon) 337 | template = 'Epoch {}, loss: {}' 338 | 339 | #------------------- Adam TRAINING LOOP --------------------------------------- 340 | for i in range(Adam_epochs): 341 | if (i+1)%Nchange == 0: 342 | X = adaptive_rad(N, Nint, rad_args) 343 | #X = random_permutation(X) 344 | if (i+1)%Nprint_adam == 0: 345 | _,fu = get_results(N,X) 346 | loss_value = loss(fu) 347 | print("i=",i+1) 348 | print(template.format(i+1,loss_value)) 349 | loss_list[i//Nprint_adam] = loss_value.numpy() 350 | training(N,X,optimizer) 351 | 352 | np.savetxt("loss_adam_1DB.txt",np.c_[epochs,loss_list]) 353 | initial_weights = np.concatenate([tf.reshape(w, [-1]).numpy() \ 354 | for w in N.weights]) #initial set of trainable variables 355 | layer_dims[0] = 1 + 2*kmax 356 | 357 | def nested_tensor(tparams,layer_dims): 358 | ''' 359 | 360 | Parameters 361 | ---------- 362 | tparams : NUMPY ARRAY 363 | DESCRIPTION: Trainable parameters in Numpy array format 364 | layer_dims : TUPLE 365 | DESCRIPTION: 366 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 367 | if i=0, Li corresponds to the input dimension 368 | 369 | Returns 370 | ------- 371 | temp : LIST 372 | List of tensors (Trainable variables in Tensorflow format) 373 | 374 | ''' 375 | temp = [None]*(2*len(layer_dims)-2) 376 | index = 0 377 | for i in range(len(temp)): 378 | if i%2==0: 379 | temp[i] = np.reshape(tparams[index:index+layer_dims[i//2]*\ 380 | layer_dims[i//2 +1]],(layer_dims[i//2], 381 | layer_dims[i//2 +1])) 382 | index+=layer_dims[i//2]*layer_dims[i//2 +1] 383 | else: 384 | temp[i] = tparams[index:index+layer_dims[i-i//2]] 385 | index+=layer_dims[i-i//2] 386 | return temp 387 | 388 | @tf.function(jit_compile=True) 389 | def loss_and_gradient_TF(N,X,use_sqrt,use_log): 390 | ''' 391 | Parameters 392 | ---------- 393 | N : TENSORFLOW MODEL 394 | PINN model, obtained with generate_model() function 395 | X : TENSOR 396 | Batch of points (in Tensorflow format) 397 | use_sqrt: BOOL 398 | If the square root of the MSE residuals is used for training 399 | use_log: BOOL 400 | If the logarithm of the MSE residuals is used for training 401 | 402 | Returns 403 | ------- 404 | loss_value : FLOAT64 405 | LOSS USED FOR TRAINING 406 | gradsN: TENSOR 407 | Gradients wrt trainable variables 408 | 409 | ''' 410 | with tf.GradientTape() as tape: 411 | _,fu = get_results(N,X) 412 | if use_sqrt: 413 | loss_value = tf.math.sqrt(loss(fu)) 414 | elif use_log: 415 | loss_value = tf.math.log(loss(fu)) 416 | else: 417 | loss_value = loss(fu) 418 | gradsN = tape.gradient(loss_value,N.trainable_variables) 419 | return loss_value,gradsN 420 | 421 | def loss_and_gradient(weights,N,X,layer_dims,use_sqrt,use_log): 422 | ''' 423 | Parameters 424 | ---------- 425 | weights : NUMPY ARRAY 426 | DESCRIPTION: Trainable parameters in Numpy array format 427 | N : TENSORFLOW MODEL 428 | PINN model 429 | X : TENSOR 430 | Batch of training params 431 | layer_dims : TUPLE 432 | DESCRIPTION: 433 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 434 | if i=0, Li corresponds to the input dimension 435 | use_sqrt : BOOL 436 | DESCRIPTION. If the square root of the MSE residuals is used for training 437 | use_log : BOOL 438 | DESCRIPTION. If the log of the MSE residuals is used for training 439 | 440 | Returns 441 | ------- 442 | loss_value : FLOAT64 (NUMPY) 443 | LOSS USED FOR TRAINING 444 | grads_flat : ARRAY OF FLOAT64 (NUMPY) 445 | Gradients wrt trainable variableS 446 | ''' 447 | resh_weights = nested_tensor(weights,layer_dims) 448 | N.set_weights(resh_weights) 449 | loss_value,grads = loss_and_gradient_TF(N,X,use_sqrt,use_log) 450 | grads_flat = np.concatenate([tf.reshape(g, [-1]).numpy() for g in grads]) 451 | return loss_value.numpy(), grads_flat 452 | 453 | data = loadmat("burgers.mat") 454 | u_ref = data["usol"] 455 | t_star = data["t"].flatten() 456 | x_star = data["x"].flatten() 457 | x_star,t_star = np.meshgrid(x_star,t_star) 458 | 459 | X_star = tf.convert_to_tensor(np.hstack((t_star.flatten()[:,None], 460 | x_star.flatten()[:,None]))) 461 | 462 | epochs_bfgs = np.arange(0,Nbfgs+Nprint_bfgs,Nprint_bfgs) #iterations bfgs list 463 | epochs_bfgs+=Adam_epochs 464 | lossbfgs = np.zeros(len(epochs_bfgs)) #loss bfgs list 465 | error_list = np.zeros(len(lossbfgs)) 466 | 467 | cont=0 468 | def callback(*,intermediate_result): 469 | global N,cont,lossbfgs,X_star,x_star,Nprint_bfgs,u_ref 470 | if (cont+1)%Nprint_bfgs == 0 or cont == 0: 471 | if use_sqrt: 472 | loss_value = np.power(intermediate_result.fun,2) 473 | elif use_log: 474 | loss_value = np.exp(intermediate_result.fun) 475 | else: 476 | loss_value = intermediate_result.fun 477 | lossbfgs[(cont+1)//Nprint_bfgs] = loss_value 478 | 479 | utest = output(N,X_star).numpy().reshape(x_star.shape) 480 | error = np.linalg.norm(u_ref-utest)/np.linalg.norm(utest) 481 | error_list[(cont+1)//Nprint_bfgs] = error 482 | print(loss_value,error,cont+1) 483 | cont+=1 484 | 485 | if method == "BFGS": 486 | method_bfgs = method_bfgs 487 | initial_scale=False 488 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 489 | H0 = H0.numpy() 490 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 491 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 492 | 493 | elif method == "bfgsr": 494 | R0 = sparse.csr_matrix(np.eye(len(initial_weights))) 495 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 496 | 497 | elif method == "bfgsz": 498 | Z0 = tf.eye(len(initial_weights),dtype=tf.float64) 499 | Z0 = Z0.numpy() 500 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 501 | 502 | #------------------------- BFGS TRAINING -------------------------------------- 503 | while cont < Nbfgs: #Training loop 504 | result = minimize(loss_and_gradient,initial_weights, args = (N,X,layer_dims,use_sqrt,use_log), 505 | method=method,jac=True, options=options, 506 | tol=0,callback=callback) 507 | initial_weights = result.x 508 | 509 | if method=="BFGS": 510 | H0 = result.hess_inv 511 | H0 = (H0 + np.transpose(H0))/2 512 | try: 513 | cholesky(H0) 514 | except LinAlgError: 515 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 516 | H0 = H0.numpy() 517 | 518 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 519 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 520 | 521 | elif method=="bfgsr": 522 | R0 = result.r_inv 523 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 524 | 525 | elif method == "bfgsz": 526 | Z0 = result.Z 527 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 528 | 529 | X = adaptive_rad(N, Nint, rad_args) 530 | 531 | if use_sqrt: 532 | fname_loss = f"1DB_loss_{method}_{method_bfgs}_sqrt.txt" 533 | fname_error = f"1DB_error_{method}_{method_bfgs}_sqrt.txt" 534 | elif use_log: 535 | fname_loss = f"1DB_loss_{method}_{method_bfgs}_log.txt" 536 | fname_error = f"1DB_error_{method}_{method_bfgs}_log.txt" 537 | else: 538 | fname_loss = f"1DB_loss_{method}_{method_bfgs}.txt" 539 | fname_error = f"1DB_error_{method}_{method_bfgs}.txt" 540 | 541 | np.savetxt(fname_loss,np.c_[epochs_bfgs,lossbfgs]) 542 | np.savetxt(fname_error,np.c_[epochs_bfgs,error_list]) -------------------------------------------------------------------------------- /Examples/1DB/1DB_hparams.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Dec 11 11:21:27 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import json 9 | import numpy as np 10 | 11 | def hyperparameter_configuration(): 12 | 13 | seed={"seed":2} 14 | architecture_hparams={"neurons":20, 15 | "layers":3, 16 | "output_dim":1, 17 | "kmax":1 18 | } 19 | 20 | PDE_hparams={"nu":0.01/np.pi} 21 | 22 | Adam_hparams={"Adam_epochs":5000, 23 | "lr0":5e-3, 24 | "decay_steps":1000, 25 | "decay_rate":0.98, 26 | "b1":0.99, 27 | "b2":0.999, 28 | "epsilon":1e-20, 29 | "Nprint_adam":100 30 | } 31 | 32 | batch_hparams={"Nint":10000, 33 | "Nchange":500, 34 | "k1":1., 35 | "k2":1., 36 | "x0":-1., 37 | "t0":0., 38 | "Lx":2., 39 | "tfinal":1. 40 | } 41 | 42 | bfgs_hparams={"BFGS_epochs":10000, 43 | "method":"BFGS", 44 | "method_bfgs":"BFGS", 45 | "use_sqrt":False, 46 | "use_log":False, 47 | "Nprint_bfgs":100} 48 | 49 | hparams={ 50 | "seed":seed, 51 | "architecture_hparams":architecture_hparams, 52 | "PDE_hparams":PDE_hparams, 53 | "Adam_hparams":Adam_hparams, 54 | "batch_hparams":batch_hparams, 55 | "bfgs_hparams":bfgs_hparams} 56 | 57 | # Guarda los hiperparámetros en un archivo JSON 58 | with open('config_1DB.json', 'w') as params: 59 | json.dump(hparams, params, indent=4) 60 | 61 | print("Hiperparámetros guardados en 'config_1DB.json'.") 62 | 63 | hyperparameter_configuration() -------------------------------------------------------------------------------- /Examples/1DB/burgers.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jorgeurban/self_scaled_algorithms_pinns/a4e7bc1bd00b89880ab9361712b371327645f82c/Examples/1DB/burgers.mat -------------------------------------------------------------------------------- /Examples/2DH/2DH.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Thu Jul 25 17:04:08 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import numpy as np 9 | import tensorflow as tf 10 | from tensorflow import keras 11 | from tensorflow.keras.layers import Input, Dense 12 | from tensorflow.keras.models import Model 13 | from tensorflow import convert_to_tensor 14 | from tensorflow.keras.optimizers import Adam 15 | from scipy.optimize import minimize 16 | from scipy.linalg import cholesky,LinAlgError 17 | from scipy import sparse 18 | import json 19 | 20 | def load_hparams(file_config): 21 | with open(file_config, 'r') as file: 22 | config = json.load(file) 23 | return config 24 | 25 | 26 | config = load_hparams('config_2DH.json') 27 | 28 | seed = config["seed"]["seed"] #Seed 29 | 30 | #--------------- Architecture hyperparameters ---------------------------------- 31 | neurons = config["architecture_hparams"]["neurons"] #Neurons in every hidden layer 32 | layers = config["architecture_hparams"]["layers"] #Hidden layers 33 | output_dim = config["architecture_hparams"]["output_dim"] #Output dimension 34 | kmax = config["architecture_hparams"]["kmax"] #Maximum wavelength if Fourier inputs are considered 35 | #------------------------------------------------------------------------------- 36 | 37 | #--------------- PDE parameters --------------------------------------- 38 | a1 = config["PDE_hparams"]["a1"] #Source parameter a1 39 | a2 = config["PDE_hparams"]["a2"] #Source parameter a2 40 | k = config["PDE_hparams"]["k"] #Source parameter k 41 | #------------------------------------------------------------------------------ 42 | 43 | #-------------- Adam hyperparameters ------------------------------------------- 44 | Adam_epochs = config["Adam_hparams"]["Adam_epochs"] #Adam epochs 45 | lr0 = config["Adam_hparams"]["lr0"] #Initial learning rate (We consider an exponential lr schedule) 46 | decay_steps = config["Adam_hparams"]["decay_steps"] #Decay steps 47 | decay_rate = config["Adam_hparams"]["decay_rate"] #Decay rate 48 | b1 = config["Adam_hparams"]["b1"] #beta1 49 | b2 = config["Adam_hparams"]["b2"] #beta2 50 | epsilon= config["Adam_hparams"]["epsilon"] #epsilon 51 | Nprint_adam = config["Adam_hparams"]["Nprint_adam"] #Adam results will be printed and save every Nprint_adam iters 52 | #------------------------------------------------------------------------------ 53 | 54 | #------------ Batch hyperparameters ------------------------------------------- 55 | Nint = config["batch_hparams"]["Nint"] #Number of points at batch 56 | Nchange=config["batch_hparams"]["Nchange"] #Batch is changed every Nchange iterations 57 | k1 = config["batch_hparams"]["k1"] #k hyperparameter (see adaptive_rad function below) 58 | k2 = config["batch_hparams"]["k2"] #c hyperparameter (see adaptive_rad function below) 59 | x0 = config["batch_hparams"]["x0"] #x0 (minimum value of x) 60 | y0 = config["batch_hparams"]["y0"] #y0 (minimum value of y) 61 | Lx = config["batch_hparams"]["Lx"] #Lx (length in the x direction) 62 | Ly = config["batch_hparams"]["Ly"] #Ly (length in the y direction) 63 | #------------------------------------------------------------------------------ 64 | 65 | #------------ Test hyperparameters -------------------------------------- 66 | Nx = config["test_hparams"]["Nx"] #Number of grid points for test set in x direction 67 | Ny = config["test_hparams"]["Ny"] #Number of grid points for test set in y direction 68 | 69 | #------------ Quasi-Newton (QN) hyperparameters ------------------------------- 70 | Nbfgs = config["bfgs_hparams"]["BFGS_epochs"] #Number of QN iterations 71 | method = config["bfgs_hparams"]["method"] #Method. See below 72 | method_bfgs = config["bfgs_hparams"]["method_bfgs"] #Quasi-Newton algorithm. See below 73 | use_sqrt = config["bfgs_hparams"]["use_sqrt"] #Use square root of the MSE loss to train 74 | use_log = config["bfgs_hparams"]["use_log"] #Use log of the MSE loss to train 75 | Nprint_bfgs = config["bfgs_hparams"]["Nprint_bfgs"] #QN results will be printed and save every Nprint_adam iters 76 | 77 | #In method, you can choose between: 78 | # -BFGS: Here, we include BFGS and the different self-scaled QN methods. 79 | # To distinguish between these algorithms, we use method_bfgs. See below 80 | # -bfgsr: Personal implementation of the factored BFGS Hessian approximations. 81 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 82 | # Very slow, to be optimized. 83 | # -bfgsz: Personal implementation of the factored inverse BFGS Hessian approximations. 84 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 85 | # Comparable with BFGS in terms of speed. 86 | 87 | #If method=BFGS, the variable "method_bfgs" chooses the different QN methods. 88 | #The options for this are (see the modified Scipy optimize script): 89 | #-BFGS_scipy: The original implementation of BFGS of Scipy 90 | #-BFGS: Equivalent implementation, but faster (avoid repeated calculations in the BFGS formula) 91 | #-SSBFGS_AB: The Self-scaled BFGS formula, where the tauk coefficient is calculated with 92 | # Al-Baali's formula (Formula 11 of "Unveiling the optimization process in PINNs") 93 | #-SSBFGS_OL Same, but tauk is calculated with the original choice of Oren and Luenberger (not recommended) 94 | #-SSBroyden2: Here we use the tauk and phik expressions defined in the paper 95 | # (Formulas 13-23 of "Unveiling the optimization process in PINNs") 96 | #-SSbroyden1: Another possible choice for these parameters (sometimes better, sometimes worse than SSBroyden1) 97 | 98 | #------------------------------------------------------------------------------ 99 | 100 | xf = x0 + Lx 101 | yf = y0 + Ly 102 | 103 | tf.keras.backend.set_floatx('float64') 104 | tf.get_logger().setLevel('ERROR') 105 | tf.keras.utils.set_random_seed(seed) 106 | 107 | class PeriodicC0Layer(keras.layers.Layer): #Implementation of the Fourier layer 108 | def __init__(self, **kwargs): 109 | super(PeriodicC0Layer, self).__init__(**kwargs) 110 | 111 | def call(self, inputs): 112 | x = inputs[:, 0,None] 113 | y = inputs[:, 1,None] 114 | ks = tf.convert_to_tensor(np.arange(1,kmax+1),dtype=tf.float64) 115 | xper = 2*np.pi*tf.matmul(x,ks[None,:])/Lx 116 | yper = 2*np.pi*tf.matmul(y,ks[None,:])/Ly 117 | xcos = tf.math.cos(xper) 118 | xsin = tf.math.sin(xper) 119 | ycos = tf.math.cos(yper) 120 | ysin = tf.math.sin(yper) 121 | Xper = tf.concat([tf.concat([xcos,xsin],axis=1),tf.concat([ycos,ysin],axis=1)],axis=1) 122 | return Xper 123 | 124 | def generate_model(layer_dims): 125 | ''' 126 | Parameters 127 | ---------- 128 | layer_dims : TUPLE 129 | DESCRIPTION. 130 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 131 | if i=0, Li corresponds to the input dimension 132 | Returns 133 | ------- 134 | Model 135 | TYPE : TENSORFLOW MODEL 136 | DESCRIPTION. 137 | 138 | ''' 139 | X_input = Input((layer_dims[0],)) 140 | X = PeriodicC0Layer()(X_input) 141 | X = Dense(layer_dims[1],activation="tanh")(X) 142 | if len(layer_dims) > 3: 143 | for i in range(2,len(layer_dims)-1): 144 | X = Dense(layer_dims[i],activation="tanh")(X) 145 | X = Dense(layer_dims[-1],activation=None)(X) 146 | return Model(inputs=X_input,outputs=X) 147 | 148 | def generate_inputs(Nint): 149 | ''' 150 | 151 | Parameters 152 | ---------- 153 | Nint : INTEGER 154 | DESCRIPTION. 155 | Number of training points in a given batch 156 | 157 | Returns 158 | ------- 159 | X: Batch of points (in Tensorflow format) 160 | TYPE : TENSOR 161 | DESCRIPTION. 162 | 163 | ''' 164 | y = (yf-y0)*np.random.rand(Nint) + y0 165 | x = (xf-x0)*np.random.rand(Nint) + x0 166 | X = np.hstack((x[:,None],y[:,None])) 167 | return convert_to_tensor(X) 168 | 169 | def adaptive_rad(N,Nint,rad_args,Ntest=100000): 170 | ''' 171 | Parameters 172 | ---------- 173 | N : TENSORFLOW MODEL 174 | DESCRIPTION. 175 | PINN model, obtained with generate_model() function 176 | 177 | Nint : INTEGER 178 | DESCRIPTION. 179 | Number of training points in a given batch 180 | 181 | rad_args: TUPLE (k1,k2) 182 | DESCRIPTION. 183 | Adaptive resampling of Wu et al. (2023), formula (2) 184 | k1: k 185 | k2: c 186 | DOI: https://doi.org/10.1016/j.cma.2022.115671 187 | 188 | Ntest: INTEGER 189 | DESCRIPTION. 190 | Number of test points to do the resampling 191 | 192 | Returns 193 | ------- 194 | X: Batch of points (in Tensorflow format) 195 | TYPE 196 | DESCRIPTION. 197 | ''' 198 | Xtest = generate_inputs(Ntest) 199 | k1,k2 = rad_args 200 | Y = tf.math.abs(get_results(N,Xtest)[-1]).numpy() 201 | err_eq = np.power(Y,k1)/np.power(Y,k1).mean() + k2 202 | err_eq_normalized = (err_eq / sum(err_eq)) 203 | X_ids = np.random.choice(a=len(Xtest), size=Nint, replace=False, 204 | p=err_eq_normalized) 205 | return tf.gather(Xtest,X_ids) 206 | 207 | def output(N,X): 208 | ''' 209 | Parameters 210 | ---------- 211 | N : TENSORFLOW MODEL 212 | PINN model, obtained with generate_model() function 213 | X : TENSOR 214 | Batch of points (in Tensorflow format) 215 | Returns 216 | ------- 217 | u: TENSOR 218 | PINN prediction. Fourier 219 | 220 | ''' 221 | Nout = N(X) 222 | u = Nout[:,0,None] 223 | return u 224 | 225 | 226 | def get_results(N,X): 227 | ''' 228 | Parameters 229 | ---------- 230 | N : TENSORFLOW MODEL 231 | PINN model, obtained with generate_model() function 232 | X : TENSOR 233 | Batch of points (in Tensorflow format) 234 | 235 | Returns 236 | ------- 237 | u : TENSOR 238 | PINN prediction 239 | fu : TENSOR 240 | PDE residuals 241 | 242 | ''' 243 | x =X[:,0] 244 | y = X[:,1] 245 | 246 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt1: 247 | gt1.watch(X) 248 | 249 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt2: 250 | gt2.watch(X) 251 | 252 | # Calculate u,v 253 | u = output(N,X) 254 | 255 | ugrad = gt2.gradient(u, X) 256 | u_x = ugrad[:,0] 257 | u_y = ugrad[:,1] 258 | 259 | u_xx = gt1.gradient(u_x, X)[:,0] 260 | u_yy = gt1.gradient(u_y,X)[:,1] 261 | 262 | q = -(a1*np.pi)**2*tf.math.sin(a1*np.pi*x)*tf.math.sin(a2*np.pi*y)-\ 263 | (a2*np.pi)**2*tf.math.sin(a1*np.pi*x)*tf.math.sin(a2*np.pi*y)+\ 264 | k**2*tf.math.sin(a1*np.pi*x)*tf.math.sin(a2*np.pi*y) 265 | 266 | fu = u_xx + u_yy + k**2*u[:,0] -q 267 | 268 | return u,fu 269 | 270 | loss_function = keras.losses.MeanSquaredError() #MSE loss 271 | def loss(fu): 272 | ''' 273 | Parameters 274 | ---------- 275 | fu : TENSOR 276 | PDE residuals 277 | 278 | Returns 279 | ------- 280 | LOSS 281 | TYPE: FLOAT64 282 | MSE Loss of PDE residuals 283 | 284 | ''' 285 | Ntot = fu.shape[0] 286 | zeros = tf.zeros([Ntot,1],dtype=tf.float64) 287 | return loss_function(fu,zeros) 288 | 289 | def grads(N,X): 290 | ''' 291 | Parameters 292 | ---------- 293 | N : TENSORFLOW MODEL 294 | PINN model, obtained with generate_model() function 295 | X : TENSOR 296 | Batch of points (in Tensorflow format) 297 | 298 | Returns 299 | ------- 300 | gradsN : TENSOR 301 | Gradients of loss wrt trainable variables 302 | loss_value: FLOAT64 303 | MSE Loss of PDE residuals 304 | 305 | ''' 306 | with tf.GradientTape() as tape2: 307 | _,fu = get_results(N,X) 308 | loss_value = loss(fu) 309 | gradsN = tape2.gradient(loss_value,N.trainable_variables) 310 | return gradsN,loss_value 311 | 312 | @tf.function #Precompile training function to accelerate the process 313 | def training(N,X,optimizer): 314 | ''' 315 | Parameters 316 | ---------- 317 | N : TENSORFLOW MODEL 318 | PINN model, obtained with generate_model() function 319 | X : TENSOR 320 | Batch of points (in Tensorflow format) 321 | 322 | Optimizer: TENSORFLOW OPTIMIZER 323 | Tensorflow optimizer (Adam, Nadam,...) 324 | 325 | Returns 326 | ------- 327 | loss_value: FLOAT64 328 | MSE Loss of PDE residuals 329 | ''' 330 | parameter_gradients,loss_value = grads(N,X) 331 | optimizer.apply_gradients(zip(parameter_gradients,N.trainable_variables)) 332 | return loss_value 333 | 334 | 335 | rad_args = (k1,k2) 336 | epochs = np.arange(Nprint_adam,Adam_epochs+Nprint_adam,Nprint_adam) 337 | loss_list = np.zeros(len(epochs)) #loss list 338 | X = generate_inputs(Nint) 339 | 340 | layer_dims = [None]*(layers + 2) 341 | layer_dims[0] = X.shape[1] 342 | for i in range(1,len(layer_dims)): 343 | layer_dims[i] = neurons 344 | layer_dims[-1] = output_dim 345 | 346 | N = generate_model(layer_dims) 347 | 348 | lr = tf.keras.optimizers.schedules.ExponentialDecay(lr0,decay_steps,decay_rate) 349 | 350 | optimizer = Adam(lr,b1,b2,epsilon=epsilon) 351 | template = 'Epoch {}, loss: {}' 352 | 353 | 354 | #------------------- Adam TRAINING LOOP --------------------------------------- 355 | for i in range(Adam_epochs): 356 | if (i+1)%Nchange == 0: 357 | X = adaptive_rad(N, Nint, rad_args) 358 | #X = random_permutation(X) 359 | if (i+1)%Nprint_adam == 0: 360 | _,fu = get_results(N,X) 361 | loss_value = loss(fu) 362 | print("i=",i+1) 363 | print(template.format(i+1,loss_value)) 364 | loss_list[i//Nprint_adam] = loss_value.numpy() 365 | 366 | training(N,X,optimizer) 367 | 368 | np.savetxt(f"loss_adam_2DH_{a1}_{a2}_{k}.txt",np.c_[epochs,loss_list]) 369 | initial_weights = np.concatenate([tf.reshape(w, [-1]).numpy() \ 370 | for w in N.weights]) #initial set of trainable variables 371 | layer_dims[0] = 2*kmax 372 | 373 | def nested_tensor(tparams,layer_dims): 374 | ''' 375 | 376 | Parameters 377 | ---------- 378 | tparams : NUMPY ARRAY 379 | DESCRIPTION: Trainable parameters in Numpy array format 380 | layer_dims : TUPLE 381 | DESCRIPTION: 382 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 383 | if i=0, Li corresponds to the input dimension 384 | 385 | Returns 386 | ------- 387 | temp : LIST 388 | List of tensors (Trainable variables in Tensorflow format) 389 | 390 | ''' 391 | temp = [None]*(2*len(layer_dims)-2) 392 | index = 0 393 | for i in range(len(temp)): 394 | if i%2==0: 395 | temp[i] = np.reshape(tparams[index:index+layer_dims[i//2]*\ 396 | layer_dims[i//2 +1]],(layer_dims[i//2], 397 | layer_dims[i//2 +1])) 398 | index+=layer_dims[i//2]*layer_dims[i//2 +1] 399 | else: 400 | temp[i] = tparams[index:index+layer_dims[i-i//2]] 401 | index+=layer_dims[i-i//2] 402 | return temp 403 | 404 | 405 | @tf.function 406 | def loss_and_gradient_TF(N,X,use_sqrt,use_log): 407 | ''' 408 | Parameters 409 | ---------- 410 | N : TENSORFLOW MODEL 411 | PINN model, obtained with generate_model() function 412 | X : TENSOR 413 | Batch of points (in Tensorflow format) 414 | use_sqrt: BOOL 415 | If the square root of the MSE residuals is used for training 416 | use_log: BOOL 417 | If the logarithm of the MSE residuals is used for training 418 | 419 | Returns 420 | ------- 421 | loss_value : FLOAT64 422 | LOSS USED FOR TRAINING 423 | gradsN: TENSOR 424 | Gradients wrt trainable variables 425 | 426 | ''' 427 | with tf.GradientTape() as tape: 428 | fu = get_results(N,X)[-1] 429 | if use_sqrt: 430 | loss_value = tf.math.sqrt(loss(fu)) 431 | elif use_log: 432 | loss_value = tf.math.log(loss(fu)) 433 | else: 434 | loss_value = loss(fu) 435 | gradsN = tape.gradient(loss_value,N.trainable_variables) 436 | return loss_value,gradsN 437 | 438 | #LOSS AND GRADIENT IN NUMPY FORMAT 439 | def loss_and_gradient(weights,N,X,use_sqrt,use_log): 440 | ''' 441 | Parameters 442 | ---------- 443 | weights : NUMPY ARRAY 444 | DESCRIPTION: Trainable parameters in Numpy array format 445 | N : TENSORFLOW MODEL 446 | PINN model 447 | X : TENSOR 448 | Batch of training params 449 | layer_dims : TUPLE 450 | DESCRIPTION: 451 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 452 | if i=0, Li corresponds to the input dimension 453 | use_sqrt : BOOL 454 | DESCRIPTION. If the square root of the MSE residuals is used for training 455 | use_log : BOOL 456 | DESCRIPTION. If the log of the MSE residuals is used for training 457 | 458 | Returns 459 | ------- 460 | loss_value : FLOAT64 (NUMPY) 461 | LOSS USED FOR TRAINING 462 | grads_flat : ARRAY OF FLOAT64 (NUMPY) 463 | Gradients wrt trainable variableS 464 | ''' 465 | resh_weights = nested_tensor(weights,layer_dims) 466 | N.set_weights(resh_weights) 467 | loss_value,grads = loss_and_gradient_TF(N,X,use_sqrt,use_log) 468 | grads_flat = np.concatenate([tf.reshape(g, [-1]).numpy() for g in grads]) 469 | return loss_value.numpy(), grads_flat 470 | 471 | 472 | epochs_bfgs = np.arange(0,Nbfgs+Nprint_bfgs,Nprint_bfgs) #iterations bfgs list 473 | epochs_bfgs+=Adam_epochs 474 | lossbfgs = np.zeros(len(epochs_bfgs)) #loss bfgs list 475 | validation_list = np.zeros(len(epochs_bfgs)) 476 | error_list = np.zeros(len(epochs_bfgs)) #loss bfgs list 477 | 478 | 479 | def generate_test(Nx,Ny): 480 | x = np.linspace(x0,xf,Nx) 481 | y = np.linspace(y0,yf,Ny) 482 | x,y = np.meshgrid(x,y) 483 | X = np.hstack((x.flatten()[:,None],y.flatten()[:,None])) 484 | return X,x,y 485 | 486 | Xtest,x,y = generate_test(Nx,Ny) 487 | uexact = tf.math.sin(a1*np.pi*Xtest[:,0,None])*tf.math.sin(a2*np.pi*Xtest[:,1,None]) 488 | Xtest = convert_to_tensor(Xtest) 489 | uexact = uexact.numpy().reshape(x.shape) 490 | 491 | cont=0 492 | def callback(*,intermediate_result): 493 | global N,cont,lossbfgs,Nprint_bfgs,x,Xtest,uexact,error_list 494 | if (cont+1)%Nprint_bfgs == 0 or cont == 0: 495 | if use_sqrt: 496 | loss_value = np.power(intermediate_result.fun,2) 497 | elif use_log: 498 | loss_value = np.exp(intermediate_result.fun) 499 | else: 500 | loss_value = intermediate_result.fun 501 | lossbfgs[(cont+1)//Nprint_bfgs] = loss_value 502 | 503 | utest = output(N, Xtest).numpy().reshape(x.shape) 504 | error = np.linalg.norm(utest-uexact)/np.linalg.norm(uexact) 505 | error_list[(cont+1)//Nprint_bfgs] = error 506 | 507 | _,fuval = get_results(N,Xtest) 508 | validation_value = loss(fuval).numpy() 509 | validation_list[(cont+1)//Nprint_bfgs] = validation_value 510 | 511 | print(loss_value,error,validation_value,cont+1) 512 | cont+=1 513 | 514 | if method == "BFGS": 515 | method_bfgs = method_bfgs 516 | initial_scale=False 517 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 518 | H0 = H0.numpy() 519 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 520 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 521 | 522 | elif method == "bfgsr": 523 | R0 = sparse.csr_matrix(np.eye(len(initial_weights))) 524 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 525 | 526 | elif method == "bfgsz": 527 | Z0 = tf.eye(len(initial_weights),dtype=tf.float64) 528 | Z0 = Z0.numpy() 529 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 530 | 531 | 532 | #------------------------- BFGS TRAINING -------------------------------------- 533 | while cont < Nbfgs: #Training loop 534 | result = minimize(loss_and_gradient,initial_weights, args = (N,X,layer_dims,use_sqrt,use_log), 535 | method=method,jac=True, options=options, 536 | tol=0,callback=callback) 537 | initial_weights = result.x 538 | 539 | if method=="BFGS": 540 | H0 = result.hess_inv 541 | H0 = (H0 + np.transpose(H0))/2 542 | try: 543 | cholesky(H0) 544 | except LinAlgError: 545 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 546 | H0 = H0.numpy() 547 | 548 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 549 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 550 | 551 | elif method=="bfgsr": 552 | R0 = result.r_inv 553 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 554 | 555 | elif method == "bfgsz": 556 | Z0 = result.Z 557 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 558 | 559 | X = adaptive_rad(N, Nint, rad_args) 560 | 561 | 562 | if use_sqrt: 563 | fname_loss = f"2DH_{a1}_{a2}_{k}_loss_{method}_{method_bfgs}_sqrt.txt" 564 | fname_error = f"2DH_{a1}_{a2}_{k}_error_{method}_{method_bfgs}_sqrt.txt" 565 | fname_val = f"2DH_{a1}_{a2}_{k}_validation_{method}_{method_bfgs}_sqrt.txt" 566 | elif use_log: 567 | fname_loss = f"2DH_{a1}_{a2}_{k}_loss_{method}_{method_bfgs}_log.txt" 568 | fname_error = f"2DH_{a1}_{a2}_{k}_error_{method}_{method_bfgs}_log.txt" 569 | fname_val = f"2DH_{a1}_{a2}_{k}_validation_{method}_{method_bfgs}_log.txt" 570 | else: 571 | fname_loss = f"2DH_{a1}_{a2}_{k}_loss_{method}_{method_bfgs}.txt" 572 | fname_error = f"2DH_{a1}_{a2}_{k}_error_{method}_{method_bfgs}.txt" 573 | fname_val = f"2DH_{a1}_{a2}_{k}_validation_{method}_{method_bfgs}.txt" 574 | 575 | np.savetxt(fname_loss,np.c_[epochs_bfgs,lossbfgs]) 576 | np.savetxt(fname_error,np.c_[epochs_bfgs,error_list]) 577 | np.savetxt(fname_val,np.c_[epochs_bfgs,validation_list]) 578 | 579 | 580 | -------------------------------------------------------------------------------- /Examples/2DH/2DH_hparams.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Dec 10 18:56:13 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import json 9 | 10 | def hyperparameter_configuration(): 11 | 12 | seed={"seed":2} 13 | architecture_hparams={"neurons":30, 14 | "layers":3, 15 | "output_dim":1, 16 | "kmax":1 17 | } 18 | PDE_hparams={"a1":6., 19 | "a2":6., 20 | "k":1. 21 | } 22 | 23 | Adam_hparams={"Adam_epochs":5000, 24 | "lr0":5e-3, 25 | "decay_steps":1000, 26 | "decay_rate":0.98, 27 | "b1":0.99, 28 | "b2":0.999, 29 | "epsilon":1e-20, 30 | "Nprint_adam":100 31 | } 32 | 33 | batch_hparams={"Nint":15000, 34 | "Nchange":500, 35 | "k1":1., 36 | "k2":0., 37 | "x0":-1., 38 | "y0":-1., 39 | "Lx":2., 40 | "Ly":2. 41 | } 42 | 43 | bfgs_hparams={"BFGS_epochs":50000, 44 | "method":"BFGS", 45 | "method_bfgs":"SSBroyden2", 46 | "use_sqrt":False, 47 | "use_log":False, 48 | "Nprint_bfgs":100} 49 | 50 | test_hparams={"Nx":100, 51 | "Ny":100} 52 | 53 | hparams={ 54 | "seed":seed, 55 | "architecture_hparams":architecture_hparams, 56 | "PDE_hparams":PDE_hparams, 57 | "Adam_hparams":Adam_hparams, 58 | "batch_hparams":batch_hparams, 59 | "test_hparams":test_hparams, 60 | "bfgs_hparams":bfgs_hparams} 61 | 62 | # Guarda los hiperparámetros en un archivo JSON 63 | with open('config_2DH.json', 'w') as params: 64 | json.dump(hparams, params, indent=4) 65 | 66 | print("Hiperparámetros guardados en 'config_2DH.json'.") 67 | 68 | hyperparameter_configuration() -------------------------------------------------------------------------------- /Examples/3DNS/3DNS.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Dec 11 12:49:43 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import numpy as np 9 | import tensorflow as tf 10 | from tensorflow import keras 11 | from tensorflow.keras.layers import Input, Dense 12 | from tensorflow.keras.models import Model 13 | from tensorflow import convert_to_tensor 14 | from tensorflow.keras.optimizers import Adam 15 | from scipy.optimize import minimize 16 | from scipy.linalg import cholesky,LinAlgError 17 | from scipy import sparse 18 | import json 19 | 20 | def load_hparams(file_config): 21 | with open(file_config, 'r') as file: 22 | config = json.load(file) 23 | return config 24 | 25 | config = load_hparams('config_3DNS.json') 26 | 27 | seed = config["seed"]["seed"] #Seed 28 | 29 | #--------------- Architecture hyperparameters ---------------------------------- 30 | neurons = config["architecture_hparams"]["neurons"] #Neurons in every hidden layer 31 | layers = config["architecture_hparams"]["layers"] #Hidden layers 32 | output_dim = config["architecture_hparams"]["output_dim"] #Output dimension 33 | #------------------------------------------------------------------------------- 34 | 35 | #--------------- PDE hyperparameters ------------------------------------------ 36 | Re = config["PDE_hparams"]["Re"] 37 | #------------------------------------------------------------------------------ 38 | 39 | #-------------- Adam hyperparameters ------------------------------------------- 40 | Adam_epochs = config["Adam_hparams"]["Adam_epochs"] #Adam epochs 41 | lr0 = config["Adam_hparams"]["lr0"] #Initial learning rate (We consider an exponential lr schedule) 42 | decay_steps = config["Adam_hparams"]["decay_steps"] #Decay steps 43 | decay_rate = config["Adam_hparams"]["decay_rate"] #Decay rate 44 | b1 = config["Adam_hparams"]["b1"] #beta1 45 | b2 = config["Adam_hparams"]["b2"] #beta2 46 | epsilon= config["Adam_hparams"]["epsilon"] #epsilon 47 | Nprint_adam = config["Adam_hparams"]["Nprint_adam"] #Adam results will be printed and save every Nprint_adam iters 48 | #------------------------------------------------------------------------------ 49 | 50 | #------------ Batch hyperparameters ------------------------------------------- 51 | Nint = config["batch_hparams"]["Nint"] #Number of points at batch 52 | Nchange=config["batch_hparams"]["Nchange"] #Batch is changed every Nchange iterations 53 | k1 = config["batch_hparams"]["k1"] #k hyperparameter (see adaptive_rad function below) 54 | k2 = config["batch_hparams"]["k2"] #c hyperparameter (see adaptive_rad function below) 55 | x0 = config["batch_hparams"]["x0"] #x0 (minimum value of x) 56 | Lx = config["batch_hparams"]["Lx"] #Lx (length in the x direction) 57 | y0 = config["batch_hparams"]["y0"] #x0 (minimum value of x) 58 | Ly = config["batch_hparams"]["Ly"] #Lx (length in the x direction) 59 | z0 = config["batch_hparams"]["z0"] #x0 (minimum value of x) 60 | Lz = config["batch_hparams"]["Lz"] #Lx (length in the x direction) 61 | t0 = config["batch_hparams"]["t0"] 62 | tfinal = config["batch_hparams"]["tfinal"] 63 | #------------------------------------------------------------------------------ 64 | 65 | #------------ Quasi-Newton (QN) hyperparameters ------------------------------- 66 | Nbfgs = config["bfgs_hparams"]["BFGS_epochs"] #Number of QN iterations 67 | method = config["bfgs_hparams"]["method"] #Method. See below 68 | method_bfgs = config["bfgs_hparams"]["method_bfgs"] #Quasi-Newton algorithm. See below 69 | use_sqrt = config["bfgs_hparams"]["use_sqrt"] #Use square root of the MSE loss to train 70 | use_log = config["bfgs_hparams"]["use_log"] #Use log of the MSE loss to train 71 | Nprint_bfgs = config["bfgs_hparams"]["Nprint_bfgs"] #QN results will be printed and save every Nprint_adam iters 72 | 73 | #In method, you can choose between: 74 | # -BFGS: Here, we include BFGS and the different self-scaled QN methods. 75 | # To distinguish between these algorithms, we use method_bfgs. See below 76 | # -bfgsr: Personal implementation of the factored BFGS Hessian approximations. 77 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 78 | # Very slow, to be optimized. 79 | # -bfgsz: Personal implementation of the factored inverse BFGS Hessian approximations. 80 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 81 | # Comparable with BFGS in terms of speed. 82 | 83 | #If method=BFGS, the variable "method_bfgs" chooses the different QN methods. 84 | #The options for this are (see the modified Scipy optimize script): 85 | #-BFGS_scipy: The original implementation of BFGS of Scipy 86 | #-BFGS: Equivalent implementation, but faster (avoid repeated calculations in the BFGS formula) 87 | #-SSBFGS_AB: The Self-scaled BFGS formula, where the tauk coefficient is calculated with 88 | # Al-Baali's formula (Formula 11 of "Unveiling the optimization process in PINNs") 89 | #-SSBFGS_OL Same, but tauk is calculated with the original choice of Oren and Luenberger (not recommended) 90 | #-SSBroyden2: Here we use the tauk and phik expressions defined in the paper 91 | # (Formulas 13-23 of "Unveiling the optimization process in PINNs") 92 | #-SSbroyden1: Another possible choice for these parameters (sometimes better, sometimes worse than SSBroyden1) 93 | 94 | #------------------------------------------------------------------------------ 95 | 96 | xf = x0 + Lx 97 | yf = y0 + Ly 98 | zf = z0 + Lz 99 | tf.keras.backend.set_floatx('float64') 100 | tf.get_logger().setLevel('ERROR') 101 | tf.keras.utils.set_random_seed(seed) 102 | 103 | def generate_model(layer_dims): 104 | X_input = Input((layer_dims[0],)) 105 | X = Dense(layer_dims[1],activation="tanh")(X_input) 106 | if len(layer_dims) > 3: 107 | for i in range(2,len(layer_dims)-1): 108 | X = Dense(layer_dims[i],activation="tanh")(X) 109 | X = Dense(layer_dims[-1],activation=None)(X) 110 | return Model(inputs=X_input,outputs=X) 111 | 112 | def generate_inputs(Nint,x0,y0,z0,Lx,Ly,Lz): 113 | t = np.random.rand(Nint) 114 | x = Lx*np.random.rand(Nint) + x0 115 | y = Ly*np.random.rand(Nint) + y0 116 | z = Lz*np.random.rand(Nint) + z0 117 | Xcoord = np.hstack((x[:,None],np.hstack((y[:,None],z[:,None])))) 118 | X = np.hstack((t[:,None],Xcoord)) 119 | return convert_to_tensor(X) 120 | 121 | def adaptive_rad(N,Nint,rad_args,x0,y0,z0,Lx,Ly,Lz,Ntest=50000): 122 | Xtest = generate_inputs(Ntest,x0,y0,z0,Lx,Ly,Lz) 123 | k1,k2 = rad_args 124 | Y = tf.math.abs(PDEs(N,Xtest,x0,y0,z0,Lx,Ly,Lz)[-1]).numpy() 125 | err_eq = np.power(Y,k1)/np.power(Y,k1).mean() + k2 126 | err_eq_normalized = (err_eq / sum(err_eq)) 127 | X_ids = np.random.choice(a=len(Xtest), size=Nint, replace=False, 128 | p=err_eq_normalized) 129 | return tf.gather(Xtest,X_ids) 130 | 131 | def velocityt(x,y,z,t): 132 | sinu,cosu = tf.math.sin(y+z),tf.math.cos(x+y) 133 | sinv,cosv = tf.math.sin(x+z),tf.math.cos(z+y) 134 | sinw,cosw = tf.math.sin(y+x),tf.math.cos(x+z) 135 | u = -tf.math.exp(x-t)*sinu - cosu*tf.math.exp(z-t) 136 | v = -tf.math.exp(y-t)*sinv - cosv*tf.math.exp(x-t) 137 | w = -tf.math.exp(z-t)*sinw - cosw*tf.math.exp(y-t) 138 | return u,v,w 139 | 140 | def pressure(X): 141 | t = X[:,0,None] 142 | x = X[:,1,None] 143 | y = X[:,2,None] 144 | z = X[:,3,None] 145 | sinu,cosu = tf.math.sin(y+z),tf.math.cos(x+y) 146 | sinv,cosv = tf.math.sin(x+z),tf.math.cos(z+y) 147 | sinw,cosw = tf.math.sin(y+x),tf.math.cos(x+z) 148 | return -tf.math.exp(-2*t)*(tf.math.exp(2*x)+tf.math.exp(2*y) + tf.math.exp(2*z) + 2*sinw*cosw*\ 149 | tf.math.exp(y+z) +2*sinu*cosu*tf.math.exp(x+z) + 2*sinv*cosv*tf.math.exp(x+y))/2 150 | 151 | def boundary_conditions(t,x,y,z,x0,y0,z0,Lx,Ly,Lz): 152 | 153 | xi_x = (x-x0)/Lx 154 | xi_y = (y-y0)/Lx 155 | xi_z = (z-z0)/Lz 156 | 157 | zero = tf.constant(0.,dtype=tf.float64) 158 | 159 | ux0,vx0,wx0 = velocityt(x0,y,z,t) 160 | u0x0,v0x0,w0x0 = velocityt(x0,y,z,zero) 161 | ux0pl,vx0pl,wx0pl = velocityt(x0+Lx,y,z,t) 162 | u0x0pl,v0x0pl,w0x0pl = velocityt(x0+Lx,y,z,zero) 163 | 164 | uy0,vy0,wy0 = velocityt(x,y0,z,t) 165 | u0y0,v0y0,w0y0 = velocityt(x,y0,z,zero) 166 | uy0pl,vy0pl,wy0pl = velocityt(x,y0+Ly,z,t) 167 | u0y0pl,v0y0pl,w0y0pl = velocityt(x,y0+Ly,z,zero) 168 | 169 | uz0,vz0,wz0 = velocityt(x,y,z0,t) 170 | u0z0,v0z0,w0z0 = velocityt(x,y,z0,zero) 171 | uz0pl,vz0pl,wz0pl = velocityt(x,y,z0+Lz,t) 172 | u0z0pl,v0z0pl,w0z0pl = velocityt(x,y,z0+Lz,zero) 173 | 174 | fou,fov,fow = (ux0-u0x0,vx0-v0x0,wx0-w0x0) 175 | f1u,f1v,f1w = (ux0pl-u0x0pl,vx0pl-v0x0pl,wx0pl-w0x0pl) 176 | 177 | gou,gov,gow = (uy0-u0y0,vy0-v0y0,wy0-w0y0) 178 | g1u,g1v,g1w = (uy0pl-u0y0pl,vy0pl-v0y0pl,wy0pl-w0y0pl) 179 | 180 | hou,hov,how = (uz0-u0z0,vz0-v0z0,wz0-w0z0) 181 | h1u,h1v,h1w = (uz0pl-u0z0pl,vz0pl-v0z0pl,wz0pl-w0z0pl) 182 | 183 | 184 | ux0y0,vx0y0,wx0y0 = velocityt(x0,y0,z,t) 185 | ux0y0pl,vx0y0pl,wx0y0pl = velocityt(x0,y0+Ly,z,t) 186 | ux0ply0,vx0ply0,wx0ply0 = velocityt(x0+Lx,y0,z,t) 187 | ux0ply0pl,vx0ply0pl,wx0ply0pl = velocityt(x0+Lx,y0+Ly,z,t) 188 | 189 | u0x0y0,v0x0y0,w0x0y0 = velocityt(x0,y0,z,zero) 190 | u0x0y0pl,v0x0y0pl,w0x0y0pl = velocityt(x0,y0+Ly,z,zero) 191 | u0x0ply0,v0x0ply0,w0x0ply0 = velocityt(x0+Lx,y0,z,zero) 192 | u0x0ply0pl,v0x0ply0pl,w0x0ply0pl = velocityt(x0+Lx,y0+Ly,z,zero) 193 | 194 | G0u = gou - (1-xi_x)*(ux0y0-u0x0y0) - xi_x*(ux0ply0-u0x0ply0) 195 | G1u = g1u - (1-xi_x)*(ux0y0pl-u0x0y0pl) - xi_x*(ux0ply0pl-u0x0ply0pl) 196 | 197 | G0v = gov - (1-xi_x)*(vx0y0-v0x0y0) - xi_x*(vx0ply0-v0x0ply0) 198 | G1v = g1v - (1-xi_x)*(vx0y0pl-v0x0y0pl) - xi_x*(vx0ply0pl-v0x0ply0pl) 199 | 200 | G0w = gow - (1-xi_x)*(wx0y0-w0x0y0) - xi_x*(wx0ply0-w0x0ply0) 201 | G1w = g1w - (1-xi_x)*(wx0y0pl-w0x0y0pl) - xi_x*(wx0ply0pl-w0x0ply0pl) 202 | 203 | ux0z0,vx0z0,wx0z0 = velocityt(x0,y,z0,t) 204 | ux0z0pl,vx0z0pl,wx0z0pl = velocityt(x0,y,z0+Lz,t) 205 | u0x0z0,v0x0z0,w0x0z0 = velocityt(x0,y,z0,0.) 206 | u0x0z0pl,v0x0z0pl,w0x0z0pl = velocityt(x0,y,z0+Lz,0.) 207 | 208 | ux0plz0,vx0plz0,wx0plz0 = velocityt(x0+Lx,y,z0,t) 209 | ux0plz0pl,vx0plz0pl,wx0plz0pl = velocityt(x0+Lx,y,z0+Lz,t) 210 | u0x0plz0,v0x0plz0,w0x0plz0 = velocityt(x0+Lx,y,z0,0.) 211 | u0x0plz0pl,v0x0plz0pl,w0x0plz0pl = velocityt(x0+Lx,y,z0+Lz,0.) 212 | 213 | uy0z0,vy0z0,wy0z0 = velocityt(x,y0,z0,t) 214 | uy0z0pl,vy0z0pl,wy0z0pl = velocityt(x,y0,z0+Lz,t) 215 | u0y0z0,v0y0z0,w0y0z0 = velocityt(x,y0,z0,0.) 216 | u0y0z0pl,v0y0z0pl,w0y0z0pl = velocityt(x,y0,z0+Lz,0.) 217 | 218 | ux0y0z0,vx0y0z0,wx0y0z0 = velocityt(x0,y0,z0,t) 219 | ux0y0z0pl,vx0y0z0pl,wx0y0z0pl = velocityt(x0,y0,z0+Lz,t) 220 | u0x0y0z0,v0x0y0z0,w0x0y0z0 = velocityt(x0,y0,z0,0.) 221 | u0x0y0z0pl,v0x0y0z0pl,w0x0y0z0pl = velocityt(x0,y0,z0+Lz,0.) 222 | 223 | ux0ply0z0,vx0ply0z0,wx0ply0z0 = velocityt(x0+Lx,y0,z0,t) 224 | ux0ply0z0pl,vx0ply0z0pl,wx0ply0z0pl = velocityt(x0+Lx,y0,z0+Lz,t) 225 | u0x0ply0z0,v0x0ply0z0,w0x0ply0z0 = velocityt(x0+Lx,y0,z0,0.) 226 | u0x0ply0z0pl,v0x0ply0z0pl,w0x0ply0z0pl = velocityt(x0+Lx,y0,z0+Lz,0.) 227 | 228 | uy0plz0,vy0plz0,wy0plz0 = velocityt(x,y0+Ly,z0,t) 229 | uy0plz0pl,vy0plz0pl,wy0plz0pl = velocityt(x,y0+Ly,z0+Lz,t) 230 | u0y0plz0,v0y0plz0,w0y0plz0 = velocityt(x,y0+Ly,z0,0.) 231 | u0y0plz0pl,v0y0plz0pl,w0y0plz0pl = velocityt(x,y0+Ly,z0+Lz,0.) 232 | 233 | ux0y0plz0,vx0y0plz0,wx0y0plz0 = velocityt(x0,y0+Ly,z0,t) 234 | ux0y0plz0pl,vx0y0plz0pl,wx0y0plz0pl = velocityt(x0,y0+Ly,z0+Lz,t) 235 | u0x0y0plz0,v0x0y0plz0,w0x0y0plz0 = velocityt(x0,y0+Ly,z0,0.) 236 | u0x0y0plz0pl,v0x0y0plz0pl,w0x0y0plz0pl = velocityt(x0,y0+Ly,z0+Lz,0.) 237 | 238 | ux0ply0plz0,vx0ply0plz0,wx0ply0plz0 = velocityt(x0+Lx,y0+Ly,z0,t) 239 | ux0ply0plz0pl,vx0ply0plz0pl,wx0ply0plz0pl = velocityt(x0+Lx,y0+Ly,z0+Lz,t) 240 | u0x0ply0plz0,v0x0ply0plz0,w0x0ply0plz0 = velocityt(x0+Lx,y0+Ly,z0,0.) 241 | u0x0ply0plz0pl,v0x0ply0plz0pl,w0x0ply0plz0pl = velocityt(x0+Lx,y0+Ly,z0+Lz,0.) 242 | 243 | H0u = hou - (1-xi_x)*(ux0z0-u0x0z0) - xi_x*(ux0plz0-u0x0plz0) - (1-xi_y)*\ 244 | (uy0z0-u0y0z0 - (1-xi_x)*(ux0y0z0-u0x0y0z0) - xi_x*(ux0ply0z0-u0x0ply0z0))-\ 245 | xi_y*(uy0plz0-u0y0plz0 - (1-xi_x)*(ux0y0plz0-u0x0y0plz0)-xi_x*\ 246 | (ux0ply0plz0-u0x0ply0plz0)) 247 | 248 | H1u = h1u - (1-xi_x)*(ux0z0pl-u0x0z0pl) - xi_x*(ux0plz0pl-u0x0plz0pl) - (1-xi_y)*\ 249 | (uy0z0pl-u0y0z0pl - (1-xi_x)*(ux0y0z0pl-u0x0y0z0pl) - xi_x*(ux0ply0z0pl-u0x0ply0z0pl))-\ 250 | xi_y*(uy0plz0pl-u0y0plz0pl - (1-xi_x)*(ux0y0plz0pl-u0x0y0plz0pl)-xi_x*\ 251 | (ux0ply0plz0pl-u0x0ply0plz0pl)) 252 | 253 | H0v = hov - (1-xi_x)*(vx0z0-v0x0z0) - xi_x*(vx0plz0-v0x0plz0) - (1-xi_y)*\ 254 | (vy0z0-v0y0z0 - (1-xi_x)*(vx0y0z0-v0x0y0z0) - xi_x*(vx0ply0z0-v0x0ply0z0))-\ 255 | xi_y*(vy0plz0-v0y0plz0 - (1-xi_x)*(vx0y0plz0-v0x0y0plz0)-xi_x*\ 256 | (vx0ply0plz0-v0x0ply0plz0)) 257 | 258 | H1v = h1v - (1-xi_x)*(vx0z0pl-v0x0z0pl) - xi_x*(vx0plz0pl-v0x0plz0pl) - (1-xi_y)*\ 259 | (vy0z0pl-v0y0z0pl - (1-xi_x)*(vx0y0z0pl-v0x0y0z0pl) - xi_x*(vx0ply0z0pl-v0x0ply0z0pl))-\ 260 | xi_y*(vy0plz0pl-v0y0plz0pl - (1-xi_x)*(vx0y0plz0pl-v0x0y0plz0pl)-xi_x*\ 261 | (vx0ply0plz0pl-v0x0ply0plz0pl)) 262 | 263 | H0w = how - (1-xi_x)*(wx0z0-w0x0z0) - xi_x*(wx0plz0-w0x0plz0) - (1-xi_y)*\ 264 | (wy0z0-w0y0z0 - (1-xi_x)*(wx0y0z0-w0x0y0z0) - xi_x*(wx0ply0z0-w0x0ply0z0))-\ 265 | xi_y*(wy0plz0-w0y0plz0 - (1-xi_x)*(wx0y0plz0-w0x0y0plz0)-xi_x*\ 266 | (wx0ply0plz0-w0x0ply0plz0)) 267 | 268 | H1w = h1w - (1-xi_x)*(wx0z0pl-w0x0z0pl) - xi_x*(wx0plz0pl-w0x0plz0pl) - (1-xi_y)*\ 269 | (wy0z0pl-w0y0z0pl - (1-xi_x)*(wx0y0z0pl-w0x0y0z0pl) - xi_x*(wx0ply0z0pl-w0x0ply0z0pl))-\ 270 | xi_y*(wy0plz0pl-w0y0plz0pl - (1-xi_x)*(wx0y0plz0pl-w0x0y0plz0pl)-xi_x*\ 271 | (wx0ply0plz0pl-w0x0ply0plz0pl)) 272 | 273 | Au = (1-xi_x)*fou + xi_x*f1u + (1-xi_y)*G0u + xi_y*G1u + (1-xi_z)*H0u + xi_z*H1u 274 | Av = (1-xi_x)*fov + xi_x*f1v + (1-xi_y)*G0v + xi_y*G1v + (1-xi_z)*H0v + xi_z*H1v 275 | Aw = (1-xi_x)*fow + xi_x*f1w + (1-xi_y)*G0w + xi_y*G1w + (1-xi_z)*H0w + xi_z*H1w 276 | return Au,Av,Aw 277 | 278 | 279 | def output(N,X,x0,y0,z0,Lx,Ly,Lz): 280 | t = X[:,0,None] 281 | x = X[:,1,None] 282 | y = X[:,2,None] 283 | z = X[:,3,None] 284 | Au,Av,Aw = boundary_conditions(t,x,y,z,x0,y0,z0,Lx,Ly,Lz) 285 | zero = tf.constant(0.,dtype=tf.float64) 286 | u0,v0,w0 = velocityt(x, y, z, zero) 287 | Nout = N(X) 288 | h = t*(x-x0)*(x-x0-Lx)*(y-y0)*(y-y0-Ly)*(z-z0)*(z-z0-Lz) 289 | 290 | X0 = tf.concat([tf.zeros([len(t),1],dtype=tf.float64), 291 | tf.concat([tf.zeros([len(t),1],dtype=tf.float64), 292 | tf.zeros([len(t),1],dtype=tf.float64)],axis=1)],axis=1) 293 | X0 = tf.concat([t,X0],axis=1) 294 | P0 = pressure(X0) 295 | 296 | u = u0 + Au + h*Nout[:,0,None] 297 | v = v0 + Av + h*Nout[:,1,None] 298 | w = w0 + Aw + h*Nout[:,2,None] 299 | p = Nout[:,3,None] - N(X0)[:,3,None] + P0 300 | return u,v,w,p 301 | 302 | def PDEs(N,X,x0,y0,z0,Lx,Ly,Lz): 303 | 304 | with tf.GradientTape(persistent=True, 305 | watch_accessed_variables=False) as gt1: 306 | gt1.watch(X) 307 | 308 | with tf.GradientTape(persistent=True, 309 | watch_accessed_variables=False) as gt2: 310 | gt2.watch(X) 311 | 312 | u,v,w,p = output(N,X,x0,y0,z0,Lx,Ly,Lz) 313 | 314 | ugrad = gt2.gradient(u,X) 315 | vgrad = gt2.gradient(v,X) 316 | wgrad = gt2.gradient(w,X) 317 | pgrad = gt2.gradient(p,X) 318 | 319 | u_t = ugrad[:,0] 320 | v_t = vgrad[:,0] 321 | w_t = wgrad[:,0] 322 | 323 | u_x = ugrad[:,1] 324 | v_x = vgrad[:,1] 325 | w_x = wgrad[:,1] 326 | 327 | u_y = ugrad[:,2] 328 | v_y = vgrad[:,2] 329 | w_y = wgrad[:,2] 330 | 331 | u_z = ugrad[:,3] 332 | v_z = vgrad[:,3] 333 | w_z = wgrad[:,3] 334 | 335 | p_x = pgrad[:,1] 336 | p_y = pgrad[:,2] 337 | p_z = pgrad[:,3] 338 | 339 | u_xx = gt1.gradient(u_x,X)[:,1] 340 | u_yy = gt1.gradient(u_y,X)[:,2] 341 | u_zz = gt1.gradient(u_z,X)[:,3] 342 | 343 | v_xx = gt1.gradient(v_x,X)[:,1] 344 | v_yy = gt1.gradient(v_y,X)[:,2] 345 | v_zz = gt1.gradient(v_z,X)[:,3] 346 | 347 | w_xx = gt1.gradient(w_x,X)[:,1] 348 | w_yy = gt1.gradient(w_y,X)[:,2] 349 | w_zz = gt1.gradient(w_z,X)[:,3] 350 | 351 | equ = u_t + u[:,0]*u_x + v[:,0]*u_y + w[:,0]*u_z - (1/Re)*(u_xx + u_yy + u_zz) + p_x 352 | eqv = v_t + u[:,0]*v_x + v[:,0]*v_y + w[:,0]*v_z - (1/Re)*(v_xx + v_yy + v_zz) + p_y 353 | eqw = w_t + u[:,0]*w_x + v[:,0]*w_y + w[:,0]*w_z - (1/Re)*(w_xx + w_yy + w_zz) + p_z 354 | eq_inc = u_x + v_y + w_z 355 | 356 | return equ,eqv,eqw,eq_inc 357 | 358 | loss_function = keras.losses.MeanSquaredError() 359 | def loss(equ,eqv,eqw,eq_inc): 360 | zeros = tf.zeros([len(equ),1],dtype=tf.float64) 361 | return loss_function(equ,zeros) + loss_function(eqv,zeros) + loss_function(eqw,zeros) + \ 362 | loss_function(eq_inc,zeros) 363 | 364 | def grads(N,X,x0,y0,z0,Lx,Ly,Lz): #Gradients wrt the trainable parameters 365 | with tf.GradientTape() as tape2: 366 | equ,eqv,eqw,eq_inc = PDEs(N,X,x0,y0,z0,Lx,Ly,Lz) 367 | loss_value = loss(equ,eqv,eqw,eq_inc) 368 | gradsN = tape2.gradient(loss_value,N.trainable_variables) 369 | return gradsN,loss_value 370 | 371 | @tf.function(jit_compile=True) 372 | def training(N,X,x0,y0,z0,Lx,Ly,Lz,optimizer): #Training step function 373 | parameter_gradients,loss_value = grads(N,X,x0,y0,z0,Lx,Ly,Lz) 374 | optimizer.apply_gradients(zip(parameter_gradients,N.trainable_variables)) 375 | return loss_value 376 | 377 | x0 = tf.constant(x0,dtype=tf.float64) 378 | y0 = tf.constant(y0,dtype=tf.float64) 379 | z0 = tf.constant(z0,dtype=tf.float64) 380 | Lx = tf.constant(Lx,dtype=tf.float64) 381 | Ly = tf.constant(Ly,dtype=tf.float64) 382 | Lz = tf.constant(Lz,dtype=tf.float64) 383 | 384 | rad_args = (k1,k2) #If random uniform, select k1 = 0 385 | epochs = np.arange(Nprint_adam,Adam_epochs+Nprint_adam,Nprint_adam) 386 | loss_list = np.zeros(len(epochs)) #loss list 387 | X = generate_inputs(Nint,x0,y0,z0,Lx,Ly,Lz) 388 | layer_dims = [None]*(layers + 2) 389 | layer_dims[0] = X.shape[1] 390 | for i in range(1,len(layer_dims)): 391 | layer_dims[i] = neurons 392 | layer_dims[-1] = output_dim 393 | 394 | N = generate_model(layer_dims) 395 | lr = tf.keras.optimizers.schedules.ExponentialDecay(lr0,decay_steps,decay_rate) 396 | 397 | optimizer = Adam(lr,b1,b2,epsilon=epsilon) 398 | template = 'Epoch {}, loss: {}' 399 | 400 | for i in range(Adam_epochs): 401 | if (i+1)%Nchange == 0: 402 | X = adaptive_rad(N, Nint, rad_args,x0,y0,z0,Lx,Ly,Lz) 403 | #X = random_permutation(X) 404 | if (i+1)%Nprint_adam == 0: 405 | equ,eqv,eqw,eq_inc = PDEs(N,X,x0,y0,z0,Lx,Ly,Lz) 406 | loss_value = loss(equ,eqv,eqw,eq_inc) 407 | print("i=",i+1) 408 | print(template.format(i+1,loss_value)) 409 | loss_list[i//Nprint_adam] = loss_value.numpy() 410 | 411 | training(N,X,x0,y0,z0,Lx,Ly,Lz,optimizer) 412 | 413 | np.savetxt("loss_adam_3DNS.txt",np.c_[epochs,loss_list]) 414 | initial_weights = np.concatenate([tf.reshape(w, [-1]).numpy() \ 415 | for w in N.weights]) #initial set of trainable variables 416 | 417 | def nested_tensor(tparams,layer_dims): 418 | ''' 419 | 420 | Parameters 421 | ---------- 422 | tparams : NUMPY ARRAY 423 | DESCRIPTION: Trainable parameters in Numpy array format 424 | layer_dims : TUPLE 425 | DESCRIPTION: 426 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 427 | if i=0, Li corresponds to the input dimension 428 | 429 | Returns 430 | ------- 431 | temp : LIST 432 | List of tensors (Trainable variables in Tensorflow format) 433 | 434 | ''' 435 | temp = [None]*(2*len(layer_dims)-2) 436 | index = 0 437 | for i in range(len(temp)): 438 | if i%2==0: 439 | temp[i] = np.reshape(tparams[index:index+layer_dims[i//2]*\ 440 | layer_dims[i//2 +1]],(layer_dims[i//2], 441 | layer_dims[i//2 +1])) 442 | index+=layer_dims[i//2]*layer_dims[i//2 +1] 443 | else: 444 | temp[i] = tparams[index:index+layer_dims[i-i//2]] 445 | index+=layer_dims[i-i//2] 446 | return temp 447 | 448 | @tf.function(jit_compile=True) 449 | def loss_and_gradient_TF(N,X,use_sqrt,use_log): #Gradients wrt the trainable parameters 450 | with tf.GradientTape() as tape: 451 | equ,eqv,eqw,eq_inc = PDEs(N,X,x0,y0,z0,Lx,Ly,Lz) 452 | if use_sqrt: 453 | loss_value = tf.math.sqrt(loss(equ,eqv,eqw,eq_inc)) 454 | elif use_log: 455 | loss_value = tf.math.log(loss(equ,eqv,eqw,eq_inc)) 456 | else: 457 | loss_value = loss(equ,eqv,eqw,eq_inc) 458 | gradsN = tape.gradient(loss_value,N.trainable_variables) 459 | return loss_value,gradsN 460 | 461 | def loss_and_gradient(weights,N,X,layer_dims,use_sqrt,use_log): 462 | resh_weights = nested_tensor(weights,layer_dims) 463 | N.set_weights(resh_weights) 464 | loss_value,grads = loss_and_gradient_TF(N,X,use_sqrt,use_log) 465 | grads_flat = np.concatenate([tf.reshape(g, [-1]).numpy() for g in grads]) 466 | return loss_value.numpy(), grads_flat 467 | 468 | def generate_slice(Nx,Ny,zval=0.5,tval=1.): 469 | x = np.linspace(x0,xf,Nx) 470 | y = np.linspace(y0,yf,Ny) 471 | y,x = np.meshgrid(y,x) 472 | z = zval*np.ones(Nx*Ny) 473 | t = tval*np.ones(Nx*Ny) 474 | Xcoord = np.hstack((x.flatten()[:,None],np.hstack((y.flatten()[:,None],z[:,None])))) 475 | X = np.hstack((t[:,None],Xcoord)) 476 | return convert_to_tensor(X),x,y 477 | 478 | Nx = 300 479 | Ny = 300 480 | Xstar,x,y = generate_slice(Nx, Ny) 481 | ustar,vstar,wstar = velocityt(Xstar[:,1], Xstar[:,2], Xstar[:,3], Xstar[:,0]) 482 | ustar = ustar.numpy().reshape(x.shape) 483 | vstar = vstar.numpy().reshape(x.shape) 484 | wstar = wstar.numpy().reshape(x.shape) 485 | umod_star = np.sqrt(ustar**2 + vstar**2 + wstar**2) 486 | 487 | epochs_bfgs = np.arange(0,Nbfgs+Nprint_bfgs,Nprint_bfgs) #iterations bfgs list 488 | epochs_bfgs+=Adam_epochs 489 | lossbfgs = np.zeros(len(epochs_bfgs)) #loss bfgs list 490 | error_list = np.zeros(len(lossbfgs)) 491 | 492 | cont=0 493 | def callback(*,intermediate_result): #Callback function, to obtain the loss at every iteration 494 | global N,cont,lossbfgs,Xstar,umod_star,error_list 495 | if (cont+1)%Nprint_bfgs == 0 or cont == 0: 496 | if use_sqrt: 497 | loss_value = np.power(intermediate_result.fun,2) 498 | elif use_log: 499 | loss_value = np.exp(intermediate_result.fun) 500 | else: 501 | loss_value = intermediate_result.fun 502 | lossbfgs[(cont+1)//Nprint_bfgs] = loss_value 503 | 504 | utest,vtest,wtest = output(N,Xstar, x0, y0, z0, Lx, Ly, Lz)[:-1] 505 | utest = utest.numpy().reshape(umod_star.shape) 506 | vtest = vtest.numpy().reshape(umod_star.shape) 507 | wtest = wtest.numpy().reshape(umod_star.shape) 508 | umod_test = np.sqrt(utest**2 + vtest**2 + wtest**2) 509 | error = np.linalg.norm(umod_star-umod_test)/np.linalg.norm(umod_star) 510 | error_list = np.append(error_list,error) 511 | #Bk = intermediate_result.hess_inv 512 | #maxlamb = np.append(maxlamb,np.max(np.linalg.eig(Bk)[0])) 513 | #minlamb = np.append(minlamb,np.min(np.linalg.eig(Bk)[0])) 514 | print(cont+1,error,loss_value) 515 | cont+=1 516 | 517 | if method == "BFGS": 518 | method_bfgs = method_bfgs 519 | initial_scale=False 520 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 521 | H0 = H0.numpy() 522 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 523 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 524 | 525 | elif method == "bfgsr": 526 | R0 = sparse.csr_matrix(np.eye(len(initial_weights))) 527 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 528 | 529 | elif method == "bfgsz": 530 | Z0 = tf.eye(len(initial_weights),dtype=tf.float64) 531 | Z0 = Z0.numpy() 532 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 533 | 534 | #------------------------- BFGS TRAINING -------------------------------------- 535 | while cont < Nbfgs: #Training loop 536 | result = minimize(loss_and_gradient,initial_weights, args = (N,X,layer_dims,use_sqrt,use_log), 537 | method=method,jac=True, options=options, 538 | tol=0,callback=callback) 539 | initial_weights = result.x 540 | 541 | if method=="BFGS": 542 | H0 = result.hess_inv 543 | H0 = (H0 + np.transpose(H0))/2 544 | try: 545 | cholesky(H0) 546 | except LinAlgError: 547 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 548 | H0 = H0.numpy() 549 | 550 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 551 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 552 | 553 | elif method=="bfgsr": 554 | R0 = result.r_inv 555 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 556 | 557 | elif method == "bfgsz": 558 | Z0 = result.Z 559 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 560 | 561 | X = adaptive_rad(N, Nint, rad_args,x0,y0,z0,Lx,Ly,Lz) 562 | 563 | if use_sqrt: 564 | fname_loss = f"3DNS_loss_{method}_{method_bfgs}_sqrt.txt" 565 | fname_error = f"3DNS_error_{method}_{method_bfgs}_sqrt.txt" 566 | elif use_log: 567 | fname_loss = f"3DNS_loss_{method}_{method_bfgs}_log.txt" 568 | fname_error = f"3DNS_error_{method}_{method_bfgs}_log.txt" 569 | else: 570 | fname_loss = f"3DNS_loss_{method}_{method_bfgs}.txt" 571 | fname_error = f"3DNS_error_{method}_{method_bfgs}.txt" 572 | 573 | np.savetxt(fname_loss,np.c_[epochs_bfgs,lossbfgs]) 574 | np.savetxt(fname_error,np.c_[epochs_bfgs,error_list]) 575 | 576 | -------------------------------------------------------------------------------- /Examples/3DNS/3DNS_hparams.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Dec 11 13:05:23 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import json 9 | import numpy as np 10 | 11 | def hyperparameter_configuration(): 12 | 13 | seed={"seed":2} 14 | architecture_hparams={"neurons":40, 15 | "layers":2, 16 | "output_dim":4 17 | } 18 | 19 | PDE_hparams={"Re":1.} 20 | 21 | Adam_hparams={"Adam_epochs":10000, 22 | "lr0":5e-3, 23 | "decay_steps":1000, 24 | "decay_rate":0.98, 25 | "b1":0.99, 26 | "b2":0.999, 27 | "epsilon":1e-20, 28 | "Nprint_adam":100 29 | } 30 | 31 | batch_hparams={"Nint":8000, 32 | "Nchange":500, 33 | "k1":1., 34 | "k2":1., 35 | "x0":-1., 36 | "Lx":2., 37 | "y0":-1., 38 | "Ly":2., 39 | "z0":-1., 40 | "Lz":2., 41 | "t0":0., 42 | "tfinal":1. 43 | } 44 | 45 | bfgs_hparams={"BFGS_epochs":20000, 46 | "method":"BFGS", 47 | "method_bfgs":"BFGS", 48 | "use_sqrt":False, 49 | "use_log":False, 50 | "Nprint_bfgs":100} 51 | 52 | hparams={ 53 | "seed":seed, 54 | "architecture_hparams":architecture_hparams, 55 | "PDE_hparams":PDE_hparams, 56 | "Adam_hparams":Adam_hparams, 57 | "batch_hparams":batch_hparams, 58 | "bfgs_hparams":bfgs_hparams} 59 | 60 | # Guarda los hiperparámetros en un archivo JSON 61 | with open('config_3DNS.json', 'w') as params: 62 | json.dump(hparams, params, indent=4) 63 | 64 | print("Hiperparámetros guardados en 'config_3DNS.json'.") 65 | 66 | hyperparameter_configuration() -------------------------------------------------------------------------------- /Examples/AC/AC.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Dec 11 12:03:24 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import numpy as np 9 | import tensorflow as tf 10 | from tensorflow import keras 11 | from tensorflow.keras.layers import Input, Dense 12 | from tensorflow.keras.models import Model 13 | from tensorflow import convert_to_tensor 14 | from tensorflow.keras.optimizers import Adam 15 | from scipy.optimize import minimize 16 | from scipy.linalg import cholesky,LinAlgError 17 | from scipy import sparse 18 | from scipy.io import loadmat 19 | import json 20 | 21 | def load_hparams(file_config): 22 | with open(file_config, 'r') as file: 23 | config = json.load(file) 24 | return config 25 | 26 | config = load_hparams('config_AC.json') 27 | 28 | seed = config["seed"]["seed"] #Seed 29 | 30 | #--------------- Architecture hyperparameters ---------------------------------- 31 | neurons = config["architecture_hparams"]["neurons"] #Neurons in every hidden layer 32 | layers = config["architecture_hparams"]["layers"] #Hidden layers 33 | output_dim = config["architecture_hparams"]["output_dim"] #Output dimension 34 | kmax = config["architecture_hparams"]["kmax"] #Maximum wavelength if Fourier inputs are considered 35 | #------------------------------------------------------------------------------- 36 | 37 | #--------------- PDE hyperparameters ------------------------------------------ 38 | k = config["PDE_hparams"]["k"] 39 | eps = config["PDE_hparams"]["eps"] 40 | #------------------------------------------------------------------------------ 41 | 42 | #-------------- Adam hyperparameters ------------------------------------------- 43 | Adam_epochs = config["Adam_hparams"]["Adam_epochs"] #Adam epochs 44 | lr0 = config["Adam_hparams"]["lr0"] #Initial learning rate (We consider an exponential lr schedule) 45 | decay_steps = config["Adam_hparams"]["decay_steps"] #Decay steps 46 | decay_rate = config["Adam_hparams"]["decay_rate"] #Decay rate 47 | b1 = config["Adam_hparams"]["b1"] #beta1 48 | b2 = config["Adam_hparams"]["b2"] #beta2 49 | epsilon= config["Adam_hparams"]["epsilon"] #epsilon 50 | Nprint_adam = config["Adam_hparams"]["Nprint_adam"] #Adam results will be printed and save every Nprint_adam iters 51 | #------------------------------------------------------------------------------ 52 | 53 | #------------ Batch hyperparameters ------------------------------------------- 54 | Nint = config["batch_hparams"]["Nint"] #Number of points at batch 55 | Nchange=config["batch_hparams"]["Nchange"] #Batch is changed every Nchange iterations 56 | k1 = config["batch_hparams"]["k1"] #k hyperparameter (see adaptive_rad function below) 57 | k2 = config["batch_hparams"]["k2"] #c hyperparameter (see adaptive_rad function below) 58 | x0 = config["batch_hparams"]["x0"] #x0 (minimum value of x) 59 | Lx = config["batch_hparams"]["Lx"] #Lx (length in the x direction) 60 | t0 = config["batch_hparams"]["t0"] 61 | tfinal = config["batch_hparams"]["tfinal"] 62 | #------------------------------------------------------------------------------ 63 | 64 | #------------ Quasi-Newton (QN) hyperparameters ------------------------------- 65 | Nbfgs = config["bfgs_hparams"]["BFGS_epochs"] #Number of QN iterations 66 | method = config["bfgs_hparams"]["method"] #Method. See below 67 | method_bfgs = config["bfgs_hparams"]["method_bfgs"] #Quasi-Newton algorithm. See below 68 | use_sqrt = config["bfgs_hparams"]["use_sqrt"] #Use square root of the MSE loss to train 69 | use_log = config["bfgs_hparams"]["use_log"] #Use log of the MSE loss to train 70 | Nprint_bfgs = config["bfgs_hparams"]["Nprint_bfgs"] #QN results will be printed and save every Nprint_adam iters 71 | 72 | #In method, you can choose between: 73 | # -BFGS: Here, we include BFGS and the different self-scaled QN methods. 74 | # To distinguish between these algorithms, we use method_bfgs. See below 75 | # -bfgsr: Personal implementation of the factored BFGS Hessian approximations. 76 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 77 | # Very slow, to be optimized. 78 | # -bfgsz: Personal implementation of the factored inverse BFGS Hessian approximations. 79 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 80 | # Comparable with BFGS in terms of speed. 81 | 82 | #If method=BFGS, the variable "method_bfgs" chooses the different QN methods. 83 | #The options for this are (see the modified Scipy optimize script): 84 | #-BFGS_scipy: The original implementation of BFGS of Scipy 85 | #-BFGS: Equivalent implementation, but faster (avoid repeated calculations in the BFGS formula) 86 | #-SSBFGS_AB: The Self-scaled BFGS formula, where the tauk coefficient is calculated with 87 | # Al-Baali's formula (Formula 11 of "Unveiling the optimization process in PINNs") 88 | #-SSBFGS_OL Same, but tauk is calculated with the original choice of Oren and Luenberger (not recommended) 89 | #-SSBroyden2: Here we use the tauk and phik expressions defined in the paper 90 | # (Formulas 13-23 of "Unveiling the optimization process in PINNs") 91 | #-SSbroyden1: Another possible choice for these parameters (sometimes better, sometimes worse than SSBroyden1) 92 | 93 | #------------------------------------------------------------------------------ 94 | 95 | xf = x0 + Lx 96 | tf.keras.backend.set_floatx('float64') 97 | tf.get_logger().setLevel('ERROR') 98 | tf.keras.utils.set_random_seed(seed) 99 | 100 | 101 | class PeriodicC0Layer(keras.layers.Layer): 102 | def __init__(self, **kwargs): 103 | super(PeriodicC0Layer, self).__init__(**kwargs) 104 | 105 | def call(self, inputs): 106 | # Assuming inputs is of shape (batch_size, 2) where inputs[:, 0] is t and inputs[:, 1] is x 107 | t = inputs[:, 0,None] # Extract t 108 | x = inputs[:, 1,None] # Extract x 109 | ks = tf.convert_to_tensor(np.arange(1,kmax+1),dtype=tf.float64) 110 | Xper = 2*np.pi*tf.matmul(x,ks[None,:])/Lx 111 | xcos = tf.math.cos(Xper) 112 | xsin = tf.math.sin(Xper) 113 | Xper = tf.concat([xcos,xsin],axis=1) 114 | Xtot = tf.concat([t,Xper],axis=1) 115 | return Xtot 116 | 117 | def generate_model(layer_dims): 118 | ''' 119 | Parameters 120 | ---------- 121 | layer_dims : TUPLE 122 | DESCRIPTION. 123 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 124 | if i=0, Li corresponds to the input dimension 125 | Returns 126 | ------- 127 | Model 128 | TYPE : TENSORFLOW MODEL 129 | DESCRIPTION. 130 | 131 | ''' 132 | X_input = Input((layer_dims[0],)) 133 | X = PeriodicC0Layer()(X_input) 134 | X = Dense(layer_dims[1],activation="tanh")(X) 135 | if len(layer_dims) > 3: 136 | for i in range(2,len(layer_dims)-1): 137 | X = Dense(layer_dims[i],activation="tanh")(X) 138 | X = Dense(layer_dims[-1],activation=None)(X) 139 | return Model(inputs=X_input,outputs=X) 140 | 141 | def generate_inputs(Nint): 142 | ''' 143 | 144 | Parameters 145 | ---------- 146 | Nint : INTEGER 147 | DESCRIPTION. 148 | Number of training points in a given batch 149 | 150 | Returns 151 | ------- 152 | X: Batch of points (in Tensorflow format) 153 | TYPE : TENSOR 154 | DESCRIPTION. 155 | 156 | ''' 157 | t = (tfinal-t0)*np.random.rand(Nint) + t0 158 | x = Lx*np.random.rand(Nint) + x0 159 | X = np.hstack((t[:,None],x[:,None])) 160 | return convert_to_tensor(X) 161 | 162 | #-----------ADAPTIVE RAD GENERATOR (GIVEN AT WU ET AL., 2023)------------------ 163 | def adaptive_rad(N,Nint,rad_args,k,eps,Ntest=100000): 164 | ''' 165 | Parameters 166 | ---------- 167 | N : TENSORFLOW MODEL 168 | DESCRIPTION. 169 | PINN model, obtained with generate_model() function 170 | 171 | Nint : INTEGER 172 | DESCRIPTION. 173 | Number of training points in a given batch 174 | 175 | rad_args: TUPLE (k1,k2) 176 | DESCRIPTION. 177 | Adaptive resampling of Wu et al. (2023), formula (2) 178 | k1: k 179 | k2: c 180 | DOI: https://doi.org/10.1016/j.cma.2022.115671 181 | 182 | Ntest: INTEGER 183 | DESCRIPTION. 184 | Number of test points to do the resampling 185 | 186 | Returns 187 | ------- 188 | X: Batch of points (in Tensorflow format) 189 | TYPE 190 | DESCRIPTION. 191 | ''' 192 | Xtest = generate_inputs(Ntest) 193 | k1,k2 = rad_args 194 | Y = tf.math.abs(get_results(N,Xtest,k,eps)[-1]).numpy() 195 | err_eq = np.power(Y,k1)/np.power(Y,k1).mean() + k2 196 | err_eq_normalized = (err_eq / sum(err_eq)) 197 | X_ids = np.random.choice(a=len(Xtest), size=Nint, replace=False, 198 | p=err_eq_normalized) 199 | return tf.gather(Xtest,X_ids) 200 | 201 | 202 | #---------------- PINN OUTPUT ------------------------------------------------- 203 | def output(N,X): 204 | ''' 205 | Parameters 206 | ---------- 207 | N : TENSORFLOW MODEL 208 | PINN model, obtained with generate_model() function 209 | X : TENSOR 210 | Batch of points (in Tensorflow format) 211 | Returns 212 | ------- 213 | u: TENSOR 214 | PINN prediction for u 215 | 216 | ''' 217 | Nout = N(X) 218 | return Nout[:,0,None] 219 | 220 | #-------------------- PDEs ---------------------------------------------------- 221 | def get_results(N,X,k,eps): 222 | ''' 223 | 224 | Parameters 225 | ---------- 226 | N : TENSORFLOW MODEL 227 | PINN model, obtained with generate_model() function 228 | X : TENSOR 229 | Batch of points (in Tensorflow format) 230 | 231 | k : TYPE 232 | Coefficient of the nonlinear part 233 | eps : TYPE 234 | Coefficient that multiplies uxx 235 | 236 | Returns 237 | ------- 238 | u : TENSOR 239 | PINN prediction for u 240 | fu : TENSOR 241 | PDE residuals for u_t = ... 242 | 243 | ''' 244 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt1: 245 | gt1.watch(X) 246 | 247 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt2: 248 | gt2.watch(X) 249 | 250 | # Calculate u,v 251 | u = output(N,X) 252 | 253 | ugrad = gt2.gradient(u, X) 254 | u_t = ugrad[:,0] 255 | u_x = ugrad[:,1] 256 | 257 | u_xx = gt1.gradient(u_x, X)[:,1] 258 | fu = u_t - eps*u_xx +k*(u[:,0]**3-u[:,0]) 259 | return u,fu 260 | 261 | #------------------------------- LOSS FUNCTION -------------------------------- 262 | loss_function = keras.losses.MeanSquaredError() #MSE loss 263 | lam0 = 10. 264 | def loss(fu,u0,u0pinn): 265 | ''' 266 | 267 | 268 | Parameters 269 | ---------- 270 | fu : TENSOR 271 | PDE residuals for u_t = ... 272 | u0 : TENSOR 273 | Initial condition. 274 | u0pinn : TENSOR 275 | PINN prediction of initial condition 276 | 277 | Returns 278 | ------- 279 | TENSOR 280 | Total loss function. The term regarding the initial condition is controlled by lam0 281 | 282 | ''' 283 | Ntot = fu.shape[0] 284 | zeros = tf.zeros([Ntot,1],dtype=tf.float64) 285 | loss_init = loss_function(u0,u0pinn) 286 | return loss_function(fu,zeros) + lam0*loss_init 287 | 288 | #---------------------- Gradients wrt the trainable parameters ---------------- 289 | def grads(N,X,u0,X0,k,eps): 290 | ''' 291 | Parameters 292 | ---------- 293 | N : TENSORFLOW MODEL 294 | PINN model, obtained with generate_model() function 295 | X : TENSOR 296 | Batch of points (in Tensorflow format) 297 | u0 : TENSOR 298 | Initial condition. 299 | X0 : TENSOR 300 | Batch of points at t=0 (in Tensorflow format) 301 | k : TYPE 302 | Coefficient of the nonlinear part 303 | eps : TYPE 304 | Coefficient that multiplies uxx 305 | 306 | Returns 307 | ------- 308 | gradsN : TENSOR 309 | Gradients of loss wrt trainable variables 310 | loss_value: FLOAT64 311 | MSE Loss of PDE residuals 312 | 313 | ''' 314 | with tf.GradientTape() as tape2: 315 | _,fu = get_results(N,X,k,eps) 316 | u0pinn = output(N, X0) 317 | loss_value = loss(fu,u0,u0pinn) 318 | gradsN = tape2.gradient(loss_value,N.trainable_variables) 319 | return gradsN,loss_value 320 | 321 | #-------------------------TRAINING STEP --------------------------------------- 322 | @tf.function #Precompile training function to accelerate the process 323 | def training(N,X,u0,X0,k,eps,optimizer): #Training step function 324 | ''' 325 | Parameters 326 | ---------- 327 | N : TENSORFLOW MODEL 328 | PINN model, obtained with generate_model() function 329 | X : TENSOR 330 | Batch of points (in Tensorflow format) 331 | u0 : TENSOR 332 | Initial condition. 333 | X0 : TENSOR 334 | Batch of points at t=0 (in Tensorflow format) 335 | k : TYPE 336 | Coefficient of the nonlinear part 337 | eps : TYPE 338 | Coefficient that multiplies uxx 339 | 340 | Optimizer: TENSORFLOW OPTIMIZER 341 | Tensorflow optimizer (Adam, Nadam,...) 342 | 343 | Returns 344 | ------- 345 | loss_value: FLOAT64 346 | MSE Loss of PDE residuals 347 | ''' 348 | parameter_gradients,loss_value = grads(N,X,u0,X0,k,eps) 349 | optimizer.apply_gradients(zip(parameter_gradients,N.trainable_variables)) 350 | return loss_value 351 | 352 | rad_args = (k1,k2) 353 | epochs = np.arange(Nprint_adam,Adam_epochs+Nprint_adam,Nprint_adam) 354 | loss_list = np.zeros(len(epochs)) #loss list 355 | X = generate_inputs(Nint) 356 | 357 | layer_dims = [None]*(layers + 2) 358 | layer_dims[0] = X.shape[1] 359 | for i in range(1,len(layer_dims)): 360 | layer_dims[i] = neurons 361 | layer_dims[-1] = output_dim 362 | 363 | N = generate_model(layer_dims) 364 | 365 | lr = tf.keras.optimizers.schedules.ExponentialDecay(lr0,decay_steps,decay_rate) 366 | 367 | optimizer = Adam(lr,b1,b2,epsilon=epsilon) 368 | template = 'Epoch {}, loss: {}' 369 | 370 | data = loadmat("AC_cosine.mat") 371 | unum = data["uu"] 372 | u0 = convert_to_tensor(unum[:,0,None]) 373 | X0 = data["x"][0] 374 | T0 = np.zeros(len(X0)) 375 | X0 = convert_to_tensor(np.hstack((T0[:,None],X0[:,None]))) 376 | 377 | #START HERE WITH Adam CONSIDERING A LOWER VALUE OF K, AND THEN MOVE TO THE ORIGINAL PROBLEM. 378 | k=1. 379 | eps=1e-4 380 | 381 | #------------------- Adam TRAINING LOOP --------------------------------------- 382 | for i in range(Adam_epochs): 383 | if (i+1)%Nchange == 0: 384 | X = adaptive_rad(N,Nint, rad_args,k,eps) 385 | if (i+1)%Nprint_adam == 0: 386 | _,fu = get_results(N,X,k,eps) 387 | u0pinn = output(N, X0) 388 | loss_value = loss(fu,u0,u0pinn) 389 | print("i=",i+1) 390 | print(template.format(i+1,loss_value)) 391 | loss_list[i//Nprint_adam] = loss_value.numpy() 392 | 393 | training(N,X,u0,X0,k,eps,optimizer) 394 | 395 | #RETURN TO THE ORIGINAL PROBLEM 396 | k = config["PDE_hparams"]["k"] 397 | eps = config["PDE_hparams"]["eps"] 398 | 399 | np.savetxt("loss_adam_AC.txt",np.c_[epochs,loss_list]) 400 | initial_weights = np.concatenate([tf.reshape(w, [-1]).numpy() \ 401 | for w in N.weights]) #initial set of trainable variables 402 | layer_dims[0] = 1 + 2*kmax 403 | 404 | def nested_tensor(tparams,layer_dims): 405 | ''' 406 | 407 | Parameters 408 | ---------- 409 | tparams : NUMPY ARRAY 410 | DESCRIPTION: Trainable parameters in Numpy array format 411 | layer_dims : TUPLE 412 | DESCRIPTION: 413 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 414 | if i=0, Li corresponds to the input dimension 415 | 416 | Returns 417 | ------- 418 | temp : LIST 419 | List of tensors (Trainable variables in Tensorflow format) 420 | 421 | ''' 422 | temp = [None]*(2*len(layer_dims)-2) 423 | index = 0 424 | for i in range(len(temp)): 425 | if i%2==0: 426 | temp[i] = np.reshape(tparams[index:index+layer_dims[i//2]*\ 427 | layer_dims[i//2 +1]],(layer_dims[i//2], 428 | layer_dims[i//2 +1])) 429 | index+=layer_dims[i//2]*layer_dims[i//2 +1] 430 | else: 431 | temp[i] = tparams[index:index+layer_dims[i-i//2]] 432 | index+=layer_dims[i-i//2] 433 | return temp 434 | 435 | @tf.function 436 | def loss_and_gradient_TF(N,X,u0,X0,k,eps,use_sqrt,use_log): 437 | with tf.GradientTape() as tape: 438 | _,fu = get_results(N,X,k,eps) 439 | u0pinn = output(N, X0) 440 | if use_sqrt: 441 | loss_value = tf.math.sqrt(loss(fu,u0,u0pinn)) 442 | elif use_log: 443 | loss_value = tf.math.log(loss(fu,u0,u0pinn)) 444 | else: 445 | loss_value = loss(fu,u0,u0pinn) 446 | gradsN = tape.gradient(loss_value,N.trainable_variables) 447 | return loss_value,gradsN 448 | 449 | #LOSS AND GRADIENT IN NUMPY FORMAT 450 | def loss_and_gradient(weights,N,X,u0,X0,k,eps,layer_dims,use_sqrt,use_log): 451 | resh_weights = nested_tensor(weights,layer_dims) 452 | N.set_weights(resh_weights) 453 | loss_value,grads = loss_and_gradient_TF(N,X,u0,X0,k,eps,use_sqrt,use_log) 454 | grads_flat = np.concatenate([tf.reshape(g, [-1]).numpy() for g in grads]) 455 | return loss_value.numpy(), grads_flat 456 | 457 | data = loadmat("AC_cosine.mat") 458 | u_ref = data["uu"] 459 | t_star = data["tt"].flatten() 460 | x_star = data["x"].flatten() 461 | x_star,t_star = np.meshgrid(x_star,t_star) 462 | 463 | epochs_bfgs = np.arange(0,Nbfgs+100,100) #iterations bfgs list 464 | epochs_bfgs+=Adam_epochs 465 | lossbfgs = np.zeros(len(epochs_bfgs)) #loss bfgs list 466 | 467 | X_star = tf.convert_to_tensor(np.hstack((t_star.flatten()[:,None],x_star.flatten()[:,None]))) 468 | error_list = np.zeros(len(epochs_bfgs)) 469 | cont=0 470 | def callback(*,intermediate_result): 471 | global N,cont,lossbfgs,Nprint_bfgs,u_ref,X_star,x_star,\ 472 | error_list 473 | if (cont+1)%Nprint_bfgs == 0 or cont == 0: 474 | 475 | if use_sqrt: 476 | loss_value = np.power(intermediate_result.fun,2) 477 | elif use_log: 478 | loss_value = np.exp(intermediate_result.fun) 479 | else: 480 | loss_value = intermediate_result.fun 481 | lossbfgs[(cont+1)//Nprint_bfgs] = loss_value 482 | 483 | utest = output(N,X_star).numpy().reshape(x_star.shape) 484 | erroru = np.linalg.norm(u_ref.T-utest)/np.linalg.norm(utest) 485 | error_list[(cont+1)//Nprint_bfgs] = erroru 486 | print(loss_value,cont+1,erroru) 487 | cont+=1 488 | 489 | if method == "BFGS": 490 | method_bfgs = method_bfgs 491 | initial_scale=False 492 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 493 | H0 = H0.numpy() 494 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 495 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 496 | 497 | elif method == "bfgsr": 498 | R0 = sparse.csr_matrix(np.eye(len(initial_weights))) 499 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 500 | 501 | elif method == "bfgsz": 502 | Z0 = tf.eye(len(initial_weights),dtype=tf.float64) 503 | Z0 = Z0.numpy() 504 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 505 | 506 | #------------------------- BFGS TRAINING -------------------------------------- 507 | while cont < Nbfgs: #Training loop 508 | result = minimize(loss_and_gradient,initial_weights, 509 | args = (N,X,u0,X0,k,eps,layer_dims,use_sqrt,use_log), 510 | method=method,jac=True, options=options, 511 | tol=0,callback=callback) 512 | 513 | initial_weights = result.x 514 | 515 | if method=="BFGS": 516 | H0 = result.hess_inv 517 | H0 = (H0 + np.transpose(H0))/2 518 | try: 519 | cholesky(H0) 520 | except LinAlgError: 521 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 522 | H0 = H0.numpy() 523 | 524 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 525 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 526 | 527 | elif method=="bfgsr": 528 | R0 = result.r_inv 529 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 530 | 531 | elif method == "bfgsz": 532 | Z0 = result.Z 533 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 534 | 535 | X = adaptive_rad(N,Nint, rad_args,k,eps) 536 | 537 | if use_sqrt: 538 | fname_loss = f"AC_loss_{method}_{method_bfgs}_sqrt.txt" 539 | fname_error = f"AC_error_{method}_{method_bfgs}_sqrt.txt" 540 | elif use_log: 541 | fname_loss = f"AC_loss_{method}_{method_bfgs}_log.txt" 542 | fname_error = f"AC_error_{method}_{method_bfgs}_log.txt" 543 | else: 544 | fname_loss = f"AC_loss_{method}_{method_bfgs}.txt" 545 | fname_error = f"AC_error_{method}_{method_bfgs}.txt" 546 | 547 | np.savetxt(fname_loss,np.c_[epochs_bfgs,lossbfgs]) 548 | np.savetxt(fname_error,np.c_[epochs_bfgs,error_list]) -------------------------------------------------------------------------------- /Examples/AC/AC_cosine.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jorgeurban/self_scaled_algorithms_pinns/a4e7bc1bd00b89880ab9361712b371327645f82c/Examples/AC/AC_cosine.mat -------------------------------------------------------------------------------- /Examples/AC/AC_hparams.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Dec 11 12:30:45 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import json 9 | import numpy as np 10 | 11 | def hyperparameter_configuration(): 12 | 13 | seed={"seed":5} 14 | architecture_hparams={"neurons":30, 15 | "layers":3, 16 | "output_dim":1, 17 | "kmax":1 18 | } 19 | 20 | PDE_hparams={"k":5., 21 | "eps":1e-4} 22 | 23 | Adam_hparams={"Adam_epochs":5000, 24 | "lr0":5e-3, 25 | "decay_steps":1000, 26 | "decay_rate":0.98, 27 | "b1":0.99, 28 | "b2":0.999, 29 | "epsilon":1e-20, 30 | "Nprint_adam":100 31 | } 32 | 33 | batch_hparams={"Nint":15000, 34 | "Nchange":500, 35 | "k1":1., 36 | "k2":0., 37 | "x0":-1., 38 | "t0":0., 39 | "Lx":2., 40 | "tfinal":1. 41 | } 42 | 43 | bfgs_hparams={"BFGS_epochs":20000, 44 | "method":"BFGS", 45 | "method_bfgs":"SSBroyden2", 46 | "use_sqrt":False, 47 | "use_log":False, 48 | "Nprint_bfgs":100} 49 | 50 | hparams={ 51 | "seed":seed, 52 | "architecture_hparams":architecture_hparams, 53 | "PDE_hparams":PDE_hparams, 54 | "PDE_hparams":PDE_hparams, 55 | "Adam_hparams":Adam_hparams, 56 | "batch_hparams":batch_hparams, 57 | "bfgs_hparams":bfgs_hparams} 58 | 59 | # Guarda los hiperparámetros en un archivo JSON 60 | with open('config_AC.json', 'w') as params: 61 | json.dump(hparams, params, indent=4) 62 | 63 | print("Hiperparámetros guardados en 'config_AC.json'.") 64 | 65 | hyperparameter_configuration() -------------------------------------------------------------------------------- /Examples/GS/GS.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Mon Sep 18 12:57:10 2023 4 | 5 | @author: Usuario 6 | """ 7 | 8 | import numpy as np 9 | import tensorflow as tf 10 | from tensorflow import keras 11 | from tensorflow.keras.layers import Input, Dense 12 | from tensorflow.keras.models import Model 13 | from tensorflow import convert_to_tensor 14 | from tensorflow.keras.optimizers import Adam 15 | from scipy.optimize import minimize 16 | from scipy.linalg import cholesky,LinAlgError 17 | from scipy import sparse 18 | import json 19 | 20 | def load_hparams(file_config): 21 | with open(file_config, 'r') as file: 22 | config = json.load(file) 23 | return config 24 | 25 | config = load_hparams('config.json') 26 | 27 | seed = config["seed"]["seed"] #Seed 28 | 29 | #--------------- Architecture hyperparameters ---------------------------------- 30 | neurons = config["architecture_hparams"]["neurons"] #Neurons in every hidden layer 31 | layers = config["architecture_hparams"]["layers"] #Hidden layers 32 | output_dim = config["architecture_hparams"]["output_dim"] #Output dimension 33 | #------------------------------------------------------------------------------- 34 | 35 | #--------------- Source parameters T(P) --------------------------------------- 36 | sigma = config["PDE_hparams"]["sigma"] #Source parameter sigma 37 | s = config["PDE_hparams"]["s"] #Source parameter s 38 | Pc = config["PDE_hparams"]["Pc"] #Source parameter Pc 39 | #------------------------------------------------------------------------------ 40 | 41 | #-------------- Adam hyperparameters ------------------------------------------- 42 | Adam_epochs = config["Adam_hparams"]["Adam_epochs"] #Adam epochs 43 | lr0 = config["Adam_hparams"]["lr0"] #Initial learning rate (We consider an exponential lr schedule) 44 | decay_steps = config["Adam_hparams"]["decay_steps"] #Decay steps 45 | decay_rate = config["Adam_hparams"]["decay_rate"] #Decay rate 46 | b1 = config["Adam_hparams"]["b1"] #beta1 47 | b2 = config["Adam_hparams"]["b2"] #beta2 48 | epsilon= config["Adam_hparams"]["epsilon"] #epsilon 49 | Nprint_adam = config["Adam_hparams"]["Nprint_adam"] #Adam results will be printed and save every Nprint_adam iters 50 | #------------------------------------------------------------------------------ 51 | 52 | #------------ Batch hyperparameters ------------------------------------------- 53 | Nint = config["batch_hparams"]["Nint"] #Number of points at batch 54 | Nchange=config["batch_hparams"]["Nchange"] #Batch is changed every Nchange iterations 55 | k1 = config["batch_hparams"]["k1"] #k hyperparameter (see adaptive_rad function below) 56 | k2 = config["batch_hparams"]["k2"] #c hyperparameter (see adaptive_rad function below) 57 | #------------------------------------------------------------------------------ 58 | 59 | #------------ Quasi-Newton (QN) hyperparameters ------------------------------- 60 | Nbfgs = config["bfgs_hparams"]["BFGS_epochs"] #Number of QN iterations 61 | method = config["bfgs_hparams"]["method"] #Method. See below 62 | method_bfgs = config["bfgs_hparams"]["method_bfgs"] #Quasi-Newton algorithm. See below 63 | use_sqrt = config["bfgs_hparams"]["use_sqrt"] #Use square root of the MSE loss to train 64 | use_log = config["bfgs_hparams"]["use_log"] #Use log of the MSE loss to train 65 | Nprint_bfgs = config["bfgs_hparams"]["Nprint_bfgs"] #QN results will be printed and save every Nprint_adam iters 66 | 67 | #In method, you can choose between: 68 | # -BFGS: Here, we include BFGS and the different self-scaled QN methods. 69 | # To distinguish between these algorithms, we use method_bfgs. See below 70 | # -bfgsr: Personal implementation of the factored BFGS Hessian approximations. 71 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 72 | # Very slow, to be optimized. 73 | # -bfgsz: Personal implementation of the factored inverse BFGS Hessian approximations. 74 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 75 | # Comparable with BFGS in terms of speed. 76 | 77 | #If method=BFGS, the variable "method_bfgs" chooses the different QN methods. 78 | #The options for this are (see the modified Scipy optimize script): 79 | #-BFGS_scipy: The original implementation of BFGS of Scipy 80 | #-BFGS: Equivalent implementation, but faster (avoid repeated calculations in the BFGS formula) 81 | #-SSBFGS_AB: The Self-scaled BFGS formula, where the tauk coefficient is calculated with 82 | # Al-Baali's formula (Formula 11 of "Unveiling the optimization process in PINNs") 83 | #-SSBFGS_OL Same, but tauk is calculated with the original choice of Oren and Luenberger (not recommended) 84 | #-SSBroyden2: Here we use the tauk and phik expressions defined in the paper 85 | # (Formulas 13-23 of "Unveiling the optimization process in PINNs") 86 | #-SSbroyden1: Another possible choice for these parameters (sometimes better, sometimes worse than SSBroyden1) 87 | 88 | #------------------------------------------------------------------------------ 89 | 90 | tf.keras.backend.set_floatx('float64') 91 | tf.get_logger().setLevel('ERROR') 92 | tf.keras.utils.set_random_seed(seed) 93 | 94 | def generate_model(layer_dims): 95 | ''' 96 | Parameters 97 | ---------- 98 | layer_dims : TUPLE 99 | DESCRIPTION. 100 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 101 | if i=0, Li corresponds to the input dimension 102 | Returns 103 | ------- 104 | Model 105 | TYPE : TENSORFLOW MODEL 106 | DESCRIPTION. 107 | 108 | ''' 109 | X_input = Input((layer_dims[0],)) 110 | X = Dense(layer_dims[1],activation="tanh")(X_input) 111 | if len(layer_dims) > 3: 112 | for i in range(2,len(layer_dims)-1): 113 | X = Dense(layer_dims[i],activation="tanh")(X) 114 | X = Dense(layer_dims[-1],activation=None)(X) 115 | return Model(inputs=X_input,outputs=X) 116 | 117 | def generate_inputs(Nint): 118 | ''' 119 | 120 | Parameters 121 | ---------- 122 | Nint : INTEGER 123 | DESCRIPTION. 124 | Number of training points in a given batch 125 | 126 | Returns 127 | ------- 128 | X: Batch of points (in Tensorflow format) 129 | TYPE : TENSOR 130 | DESCRIPTION. 131 | 132 | ''' 133 | q = np.random.rand(Nint) 134 | theta = np.pi*np.random.rand(Nint) 135 | mu = np.cos(theta) 136 | X = np.hstack((q[:,None],mu[:,None])) 137 | return convert_to_tensor(X) 138 | 139 | def adaptive_rad(N,Nint,rad_args,Ntest=50000): 140 | ''' 141 | Parameters 142 | ---------- 143 | N : TENSORFLOW MODEL 144 | DESCRIPTION. 145 | PINN model, obtained with generate_model() function 146 | 147 | Nint : INTEGER 148 | DESCRIPTION. 149 | Number of training points in a given batch 150 | 151 | rad_args: TUPLE (k1,k2) 152 | DESCRIPTION. 153 | Adaptive resampling of Wu et al. (2023), formula (2) 154 | k1: k 155 | k2: c 156 | DOI: https://doi.org/10.1016/j.cma.2022.115671 157 | 158 | Ntest: INTEGER 159 | DESCRIPTION. 160 | Number of test points to do the resampling 161 | 162 | Returns 163 | ------- 164 | X: Batch of points (in Tensorflow format) 165 | TYPE 166 | DESCRIPTION. 167 | ''' 168 | Xtest = generate_inputs(Ntest) 169 | k1,k2 = rad_args 170 | Y = tf.math.abs(get_results(N,Xtest)[-1]).numpy() 171 | err_eq = np.power(Y,k1)/np.power(Y,k1).mean() + k2 172 | err_eq_normalized = (err_eq / sum(err_eq)) 173 | X_ids = np.random.choice(a=len(Xtest), size=Nint, replace=False, 174 | p=err_eq_normalized) 175 | return tf.gather(Xtest,X_ids) 176 | 177 | def P_t(N,X,n=7): 178 | ''' 179 | Parameters 180 | ---------- 181 | N : TENSORFLOW MODEL 182 | PINN model, obtained with generate_model() function 183 | X : TENSOR 184 | Batch of points (in Tensorflow format) 185 | n: Additional optional power at the function f in hard-enforcement 186 | P = q^n*(1-mu^2)*Pmu + q(1-q)(1-mu^2)N(q,mu) 187 | which controls the weight of the boundary condition at surface 188 | Returns 189 | ------- 190 | P: TENSOR 191 | PINN prediction 192 | 193 | ''' 194 | q = X[:,0,None] 195 | mu = X[:,1,None] 196 | Pmu = 1+3*mu/2+(15*mu**2 -3)/6 + (140*mu**3 - 60*mu)/32 + \ 197 | (315*mu**4 -210*mu**2 +15)/40 + (1386*mu**5 -1260*mu**3 +210*mu)/96 + \ 198 | (3003*mu**6 -3465*mu**4 + 945*mu**2 -35 )/112 + \ 199 | (51480*mu**7 - 72072*mu**5 +27720*mu**3 - 2520*mu)/1024 200 | return q*(1-mu**2)*(q**n*Pmu + (1-q)*N(X)) 201 | 202 | 203 | def get_results(N,X): 204 | ''' 205 | Parameters 206 | ---------- 207 | N : TENSORFLOW MODEL 208 | PINN model, obtained with generate_model() function 209 | X : TENSOR 210 | Batch of points (in Tensorflow format) 211 | 212 | Returns 213 | ------- 214 | P : TENSOR 215 | PINN prediction 216 | gs : TENSOR 217 | PDE residuals 218 | 219 | ''' 220 | q = X[:,0] 221 | mu = X[:,1] 222 | 223 | with tf.GradientTape(persistent=True, 224 | watch_accessed_variables=False) as gt1: 225 | gt1.watch(X) 226 | 227 | with tf.GradientTape(persistent=True, 228 | watch_accessed_variables=False) as gt2: 229 | gt2.watch(X) 230 | 231 | # Calculate P 232 | P = P_t (N,X) 233 | 234 | # Calculate 1st derivatives 235 | Pgrad = gt2.gradient(P, X) 236 | P_q = Pgrad[:,0] 237 | P_mu = Pgrad[:,1] 238 | 239 | # Calculate 2nd derivatives 240 | P_qq = gt1.gradient(P_q, X)[:,0] 241 | P_mumu = gt1.gradient(P_mu, X)[:,1] 242 | TTp = s**2*sigma*tf.math.abs(P[:,0]-Pc)**(2*sigma-1)*tf.experimental.numpy.heaviside(P[:,0]-Pc,0) 243 | gs = q*P_qq + 2*P_q + (1-mu**2)*P_mumu/q + TTp/q**3 244 | 245 | #print(P_q[:,None]) 246 | return P,gs 247 | 248 | loss_function = keras.losses.MeanSquaredError() 249 | def loss(GS): 250 | ''' 251 | Parameters 252 | ---------- 253 | GS : TENSOR 254 | PDE residuals 255 | 256 | Returns 257 | ------- 258 | LOSS 259 | TYPE: FLOAT64 260 | MSE Loss of PDE residuals 261 | 262 | ''' 263 | Ntot = GS.shape[0] 264 | zeros = tf.zeros([Ntot,1],dtype=tf.float64) 265 | return loss_function(GS,zeros) 266 | 267 | def grads(N,X): 268 | ''' 269 | Parameters 270 | ---------- 271 | N : TENSORFLOW MODEL 272 | PINN model, obtained with generate_model() function 273 | X : TENSOR 274 | Batch of points (in Tensorflow format) 275 | 276 | Returns 277 | ------- 278 | gradsN : TENSOR 279 | Gradients of loss wrt trainable variables 280 | P : TENSOR 281 | PINN prediction 282 | gs : TENSOR 283 | PDE residuals 284 | loss_value: FLOAT64 285 | MSE Loss of PDE residuals 286 | 287 | ''' 288 | with tf.GradientTape() as tape2: 289 | P,GS = get_results(N,X) 290 | loss_value = loss(GS) 291 | gradsN = tape2.gradient(loss_value,N.trainable_variables) 292 | return gradsN,P,GS,loss_value 293 | 294 | @tf.function(jit_compile=True) #Precompile training function to accelerate the process 295 | def training(N,X,optimizer): #ADAM Training step function 296 | ''' 297 | Parameters 298 | ---------- 299 | N : TENSORFLOW MODEL 300 | PINN model, obtained with generate_model() function 301 | X : TENSOR 302 | Batch of points (in Tensorflow format) 303 | 304 | Optimizer: TENSORFLOW OPTIMIZER 305 | Tensorflow optimizer (Adam, Nadam,...) 306 | 307 | Returns 308 | ------- 309 | P : TENSOR 310 | PINN prediction 311 | GS : TENSOR 312 | PDE residuals 313 | loss_value: FLOAT64 314 | MSE Loss of PDE residuals 315 | ''' 316 | parameter_gradients,P,GS,loss_value = grads(N,X) 317 | optimizer.apply_gradients(zip(parameter_gradients,N.trainable_variables)) 318 | return P,GS,loss_value 319 | 320 | rad_args = (k1,k2) 321 | loss_list = np.array([]) 322 | X = generate_inputs(Nint) 323 | 324 | layer_dims = [None]*(layers + 2) 325 | layer_dims[0] = X.shape[1] 326 | for i in range(1,len(layer_dims)): 327 | layer_dims[i] = neurons 328 | layer_dims[-1] = output_dim 329 | 330 | N = generate_model(layer_dims) 331 | 332 | lr = tf.keras.optimizers.schedules.ExponentialDecay(lr0,decay_steps,decay_rate) 333 | 334 | optimizer = Adam(lr,b1,b2,epsilon=epsilon) 335 | template = 'Epoch {}, loss: {}' 336 | 337 | for i in range(Adam_epochs): 338 | if (i+1)%Nchange == 0: 339 | X = adaptive_rad(N, Nint, rad_args) 340 | if (i+1)%Nprint_adam == 0: 341 | GS = get_results(N,X)[-1] 342 | loss_value = loss(GS) 343 | print("i=",i+1) 344 | print(template.format(i+1,loss_value)) 345 | loss_list = np.append(loss_list,loss_value.numpy()) 346 | 347 | training(N,X,optimizer) 348 | 349 | epochs = np.arange(Nprint_adam,Adam_epochs+Nprint_adam,Nprint_adam) 350 | np.savetxt("loss_adam_GS.txt",np.c_[epochs,loss_list]) 351 | initial_weights = np.concatenate([tf.reshape(w, [-1]).numpy() \ 352 | for w in N.weights]) 353 | lossbfgs = np.array([]) 354 | 355 | def nested_tensor(tparams,layer_dims): 356 | ''' 357 | 358 | Parameters 359 | ---------- 360 | tparams : NUMPY ARRAY 361 | DESCRIPTION: Trainable parameters in Numpy array format 362 | layer_dims : TUPLE 363 | DESCRIPTION: 364 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 365 | if i=0, Li corresponds to the input dimension 366 | 367 | Returns 368 | ------- 369 | temp : LIST 370 | List of tensors (Trainable variables in Tensorflow format) 371 | 372 | ''' 373 | temp = [None]*(2*len(layer_dims)-2) 374 | index = 0 375 | for i in range(len(temp)): 376 | if i%2==0: 377 | temp[i] = np.reshape(tparams[index:index+layer_dims[i//2]*\ 378 | layer_dims[i//2 +1]],(layer_dims[i//2], 379 | layer_dims[i//2 +1])) 380 | index+=layer_dims[i//2]*layer_dims[i//2 +1] 381 | else: 382 | temp[i] = tparams[index:index+layer_dims[i-i//2]] 383 | index+=layer_dims[i-i//2] 384 | return temp 385 | 386 | power = 1. 387 | @tf.function(jit_compile=True) 388 | def loss_and_gradient_TF(N,X,use_sqrt,use_log): #Gradients wrt the trainable parameters 389 | ''' 390 | Parameters 391 | ---------- 392 | N : TENSORFLOW MODEL 393 | PINN model, obtained with generate_model() function 394 | X : TENSOR 395 | Batch of points (in Tensorflow format) 396 | use_sqrt: BOOL 397 | If the square root of the MSE residuals is used for training 398 | use_log: BOOL 399 | If the logarithm of the MSE residuals is used for training 400 | 401 | Returns 402 | ------- 403 | loss_value : FLOAT64 404 | LOSS USED FOR TRAINING 405 | gradsN: TENSOR 406 | Gradients wrt trainable variables 407 | 408 | ''' 409 | with tf.GradientTape() as tape: 410 | GS = get_results(N,X)[-1] 411 | if use_sqrt: 412 | loss_value = tf.math.sqrt(loss(GS)) 413 | elif use_log: 414 | loss_value = tf.math.log(loss(GS)) 415 | else: 416 | loss_value = loss(GS) 417 | gradsN = tape.gradient(loss_value,N.trainable_variables) 418 | return loss_value,gradsN 419 | 420 | def loss_and_gradient(weights,N,X,layer_dims,use_sqrt,use_log): 421 | ''' 422 | Parameters 423 | ---------- 424 | weights : NUMPY ARRAY 425 | DESCRIPTION: Trainable parameters in Numpy array format 426 | N : TENSORFLOW MODEL 427 | PINN model 428 | X : TENSOR 429 | Batch of training params 430 | layer_dims : TUPLE 431 | DESCRIPTION: 432 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 433 | if i=0, Li corresponds to the input dimension 434 | use_sqrt : BOOL 435 | DESCRIPTION. If the square root of the MSE residuals is used for training 436 | use_log : BOOL 437 | DESCRIPTION. If the log of the MSE residuals is used for training 438 | 439 | Returns 440 | ------- 441 | loss_value : FLOAT64 (NUMPY) 442 | LOSS USED FOR TRAINING 443 | grads_flat : ARRAY OF FLOAT64 (NUMPY) 444 | Gradients wrt trainable variableS 445 | ''' 446 | resh_weights = nested_tensor(weights,layer_dims) 447 | N.set_weights(resh_weights) 448 | loss_value,grads = loss_and_gradient_TF(N,X,use_sqrt,use_log) 449 | grads_flat = np.concatenate([tf.reshape(g, [-1]).numpy() for g in grads]) 450 | return loss_value.numpy(), grads_flat 451 | 452 | 453 | cont=0 454 | def callback(*,intermediate_result): #Callback function, to obtain the loss at every iteration 455 | global N,cont,lossbfgs,use_sqrt,use_log,Nprint_bfgs 456 | if (cont+1)%Nprint_bfgs == 0 or cont == 0: 457 | if use_sqrt: 458 | loss_value = np.power(intermediate_result.fun,2) 459 | elif use_log: 460 | loss_value = np.exp(intermediate_result.fun) 461 | else: 462 | loss_value = intermediate_result.fun 463 | lossbfgs = np.append(lossbfgs,loss_value) 464 | print(cont+1,loss_value) 465 | cont+=1 466 | 467 | if method == "BFGS": 468 | method_bfgs = method_bfgs 469 | initial_scale=False 470 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 471 | H0 = H0.numpy() 472 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 473 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 474 | 475 | elif method == "bfgsr": 476 | R0 = sparse.csr_matrix(np.eye(len(initial_weights))) 477 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 478 | 479 | elif method == "bfgsz": 480 | Z0 = tf.eye(len(initial_weights),dtype=tf.float64) 481 | Z0 = Z0.numpy() 482 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 483 | 484 | while cont < Nbfgs: #Training BFGS loop 485 | result = minimize(loss_and_gradient,initial_weights, args = (N,X,layer_dims,use_sqrt,use_log), 486 | method=method,jac=True, options=options, 487 | tol=0,callback=callback) 488 | initial_weights = result.x 489 | 490 | if method=="BFGS": 491 | H0 = result.hess_inv 492 | H0 = (H0 + np.transpose(H0))/2 493 | try: 494 | cholesky(H0) 495 | except LinAlgError: 496 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 497 | H0 = H0.numpy() 498 | 499 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 500 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 501 | 502 | elif method=="bfgsr": 503 | R0 = result.r_inv 504 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 505 | 506 | elif method == "bfgsz": 507 | Z0 = result.Z 508 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 509 | 510 | X = adaptive_rad(N, Nint, rad_args) 511 | 512 | epochs_bfgs = np.arange(0,Nbfgs+Nprint_bfgs,Nprint_bfgs) 513 | epochs_bfgs+=Adam_epochs 514 | if use_sqrt: 515 | fname = f"GS_loss_{method}_{method_bfgs}_sqrt.txt" 516 | elif use_log: 517 | fname = f"GS_loss_{method}_{method_bfgs}_log.txt" 518 | else: 519 | fname = f"GS_loss_{method}_{method_bfgs}.txt" 520 | 521 | np.savetxt(fname,np.c_[epochs_bfgs,lossbfgs]) 522 | 523 | -------------------------------------------------------------------------------- /Examples/GS/GS_hparams.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Dec 10 13:39:58 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import json 9 | 10 | def hyperparameter_configuration(): 11 | 12 | seed={"seed":2} 13 | architecture_hparams={"neurons":30, 14 | "layers":2, 15 | "output_dim":1, 16 | } 17 | PDE_hparams={"sigma":2., 18 | "s":1.5, 19 | "Pc":0.4 20 | } 21 | 22 | Adam_hparams={"Adam_epochs":5000, 23 | "lr0":1e-2, 24 | "decay_steps":2000, 25 | "decay_rate":0.98, 26 | "b1":0.99, 27 | "b2":0.999, 28 | "epsilon":1e-20, 29 | "Nprint_adam":100 30 | } 31 | batch_hparams={"Nint":8000, 32 | "Nchange":500, 33 | "k1":0., 34 | "k2":1., 35 | } 36 | 37 | bfgs_hparams={"BFGS_epochs":20000, 38 | "method":"bfgsz", 39 | "method_bfgs":"BFGS", 40 | "use_sqrt":False, 41 | "use_log":True, 42 | "Nprint_bfgs":100} 43 | 44 | hparams={ 45 | "seed":seed, 46 | "architecture_hparams":architecture_hparams, 47 | "PDE_hparams":PDE_hparams, 48 | "Adam_hparams":Adam_hparams, 49 | "batch_hparams":batch_hparams, 50 | "bfgs_hparams":bfgs_hparams} 51 | 52 | # Guarda los hiperparámetros en un archivo JSON 53 | with open('config.json', 'w') as params: 54 | json.dump(hparams, params, indent=4) 55 | 56 | print("Hiperparámetros guardados en 'config.json'.") 57 | 58 | hyperparameter_configuration() -------------------------------------------------------------------------------- /Examples/KdV/KdV.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Feb 27 18:46:56 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import numpy as np 9 | import tensorflow as tf 10 | from tensorflow import keras 11 | from tensorflow.keras.layers import Input, Dense 12 | from tensorflow.keras.models import Model 13 | from tensorflow import convert_to_tensor 14 | from tensorflow.keras.optimizers import Adam 15 | from scipy.optimize import minimize 16 | from scipy.linalg import cholesky,LinAlgError 17 | from scipy import sparse 18 | import json 19 | 20 | 21 | 22 | def load_hparams(file_config): 23 | with open(file_config, 'r') as file: 24 | config = json.load(file) 25 | return config 26 | 27 | config = load_hparams('config_KdV.json') 28 | 29 | seed = config["seed"]["seed"] #Seed 30 | 31 | #--------------- Architecture hyperparameters ---------------------------------- 32 | neurons = config["architecture_hparams"]["neurons"] #Neurons in every hidden layer 33 | layers = config["architecture_hparams"]["layers"] #Hidden layers 34 | output_dim = config["architecture_hparams"]["output_dim"] #Output dimension 35 | #------------------------------------------------------------------------------- 36 | 37 | #--------------- PDE parameters --------------------------------------- 38 | c1 = config["PDE_hparams"]["c1"] #Velocity soliton 1 39 | c2 = config["PDE_hparams"]["c2"] #Velocity soliton 2 40 | x1 = config["PDE_hparams"]["x1"] #Initial position soliton 1 41 | x2 = config["PDE_hparams"]["x2"] #Initial position soliton 2 42 | #------------------------------------------------------------------------------ 43 | 44 | #-------------- Adam hyperparameters ------------------------------------------- 45 | Adam_epochs = config["Adam_hparams"]["Adam_epochs"] #Adam epochs 46 | lr0 = config["Adam_hparams"]["lr0"] #Initial learning rate (We consider an exponential lr schedule) 47 | decay_steps = config["Adam_hparams"]["decay_steps"] #Decay steps 48 | decay_rate = config["Adam_hparams"]["decay_rate"] #Decay rate 49 | b1 = config["Adam_hparams"]["b1"] #beta1 50 | b2 = config["Adam_hparams"]["b2"] #beta2 51 | epsilon= config["Adam_hparams"]["epsilon"] #epsilon 52 | Nprint_adam = config["Adam_hparams"]["Nprint_adam"] #Adam results will be printed and save every Nprint_adam iters 53 | #------------------------------------------------------------------------------ 54 | 55 | #------------ Batch hyperparameters ------------------------------------------- 56 | Nint = config["batch_hparams"]["Nint"] #Number of points at batch 57 | Nb = config["batch_hparams"]["Nb"] #Number of points at boundary 58 | Nchange=config["batch_hparams"]["Nchange"] #Batch is changed every Nchange iterations 59 | k1 = config["batch_hparams"]["k1"] #k hyperparameter (see adaptive_rad function below) 60 | k2 = config["batch_hparams"]["k2"] #c hyperparameter (see adaptive_rad function below) 61 | x0 = config["batch_hparams"]["x0"] #x0 (minimum value of x) 62 | Lx = config["batch_hparams"]["Lx"] #Lx (length in the x direction) 63 | t0 = config["batch_hparams"]["t0"] 64 | tfinal = config["batch_hparams"]["tfinal"] 65 | #------------------------------------------------------------------------------ 66 | 67 | #------------ Test hyperparameters -------------------------------------- 68 | Nx = config["test_hparams"]["Nx"] #Number of grid points for test set in x direction 69 | Nt = config["test_hparams"]["Nt"] #Number of grid points for test set in t direction 70 | 71 | #------------ Quasi-Newton (QN) hyperparameters ------------------------------- 72 | Nbfgs = config["bfgs_hparams"]["BFGS_epochs"] #Number of QN iterations 73 | method = config["bfgs_hparams"]["method"] #Method. See below 74 | method_bfgs = config["bfgs_hparams"]["method_bfgs"] #Quasi-Newton algorithm. See below 75 | use_sqrt = config["bfgs_hparams"]["use_sqrt"] #Use square root of the MSE loss to train 76 | use_log = config["bfgs_hparams"]["use_log"] #Use log of the MSE loss to train 77 | Nprint_bfgs = config["bfgs_hparams"]["Nprint_bfgs"] #QN results will be printed and save every Nprint_adam iters 78 | 79 | #In method, you can choose between: 80 | # -BFGS: Here, we include BFGS and the different self-scaled QN methods. 81 | # To distinguish between these algorithms, we use method_bfgs. See below 82 | # -bfgsr: Personal implementation of the factored BFGS Hessian approximations. 83 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 84 | # Very slow, to be optimized. 85 | # -bfgsz: Personal implementation of the factored inverse BFGS Hessian approximations. 86 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 87 | # Comparable with BFGS in terms of speed. 88 | 89 | #If method=BFGS, the variable "method_bfgs" chooses the different QN methods. 90 | #The options for this are (see the modified Scipy optimize script): 91 | #-BFGS_scipy: The original implementation of BFGS of Scipy 92 | #-BFGS: Equivalent implementation, but faster (avoid repeated calculations in the BFGS formula) 93 | #-SSBFGS_AB: The Self-scaled BFGS formula, where the tauk coefficient is calculated with 94 | # Al-Baali's formula (Formula 11 of "Unveiling the optimization process in PINNs") 95 | #-SSBFGS_OL Same, but tauk is calculated with the original choice of Oren and Luenberger (not recommended) 96 | #-SSBroyden2: Here we use the tauk and phik expressions defined in the paper 97 | # (Formulas 13-23 of "Unveiling the optimization process in PINNs") 98 | #-SSbroyden1: Another possible choice for these parameters (sometimes better, sometimes worse than SSBroyden1) 99 | 100 | 101 | #------------------------------------------------------------------------------ 102 | xf = x0 + Lx 103 | tf.keras.backend.set_floatx('float64') 104 | tf.get_logger().setLevel('ERROR') 105 | tf.keras.utils.set_random_seed(seed) 106 | 107 | def generate_model(layer_dims): 108 | ''' 109 | Parameters 110 | ---------- 111 | layer_dims : TUPLE 112 | DESCRIPTION. 113 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 114 | if i=0, Li corresponds to the input dimension 115 | Returns 116 | ------- 117 | Model 118 | TYPE : TENSORFLOW MODEL 119 | DESCRIPTION. 120 | 121 | ''' 122 | X_input = Input((layer_dims[0],)) 123 | X = Dense(layer_dims[1],activation="tanh")(X_input) 124 | if len(layer_dims) > 3: 125 | for i in range(2,len(layer_dims)-1): 126 | X = Dense(layer_dims[i],activation="tanh")(X) 127 | X = Dense(layer_dims[-1],activation=None)(X) 128 | return Model(inputs=X_input,outputs=X) 129 | 130 | def generate_inputs(Nint,Nb): 131 | ''' 132 | 133 | Parameters 134 | ---------- 135 | Nint : INTEGER 136 | DESCRIPTION. 137 | Number of training points in a given batch 138 | 139 | Returns 140 | ------- 141 | X: Batch of points (in Tensorflow format) 142 | TYPE : TENSOR 143 | DESCRIPTION. 144 | 145 | ''' 146 | t = (tfinal-t0)*np.random.rand(Nint) + t0 147 | x = (xf-x0)*np.random.rand(Nint) + x0 148 | X = np.hstack((t[:,None],x[:,None])) 149 | tb = (tfinal-t0)*np.random.rand(Nb) + t0 150 | xb = np.ones(Nb)*xf 151 | Xb = np.hstack((tb[:,None],xb[:,None])) 152 | return convert_to_tensor(X), convert_to_tensor(Xb) 153 | 154 | #-----------ADAPTIVE RAD GENERATOR (GIVEN AT WU ET AL., 2023)------------------ 155 | def adaptive_rad(N,Nint,rad_args,Ntest=50000): 156 | ''' 157 | Parameters 158 | ---------- 159 | N : TENSORFLOW MODEL 160 | DESCRIPTION. 161 | PINN model, obtained with generate_model() function 162 | 163 | Nint : INTEGER 164 | DESCRIPTION. 165 | Number of training points in a given batch 166 | 167 | rad_args: TUPLE (k1,k2) 168 | DESCRIPTION. 169 | Adaptive resampling of Wu et al. (2023), formula (2) 170 | k1: k 171 | k2: c 172 | DOI: https://doi.org/10.1016/j.cma.2022.115671 173 | 174 | Ntest: INTEGER 175 | DESCRIPTION. 176 | Number of test points to do the resampling 177 | 178 | Returns 179 | ------- 180 | X: Batch of points (in Tensorflow format) 181 | TYPE 182 | DESCRIPTION. 183 | ''' 184 | Xtest = generate_inputs(Ntest,1)[0] 185 | k1,k2 = rad_args 186 | Y = tf.math.abs(get_results(N,Xtest)[-1]).numpy() 187 | err_eq = np.power(Y,k1)/np.power(Y,k1).mean() + k2 188 | err_eq_normalized = (err_eq / sum(err_eq)) 189 | X_ids = np.random.choice(a=len(Xtest), size=Nint, replace=False, 190 | p=err_eq_normalized) 191 | return tf.gather(Xtest,X_ids) 192 | 193 | 194 | def usol(x,t): 195 | xi1 = x-c1*t-x1 196 | xi2 = x-c2*t-x2 197 | num = 2*(c1-c2)*(c1*tf.math.cosh(np.sqrt(c2)*xi2/2)**2 + c2*tf.math.sinh(np.sqrt(c1)*xi1/2)**2) 198 | den = ((np.sqrt(c1)-np.sqrt(c2))*tf.math.cosh((xi1*np.sqrt(c1)+xi2*np.sqrt(c2))/2)+\ 199 | (np.sqrt(c1)+np.sqrt(c2))*tf.math.cosh((np.sqrt(c1)*xi1-np.sqrt(c2)*xi2)/2))**2 200 | return num/den 201 | 202 | def uteor(X): 203 | t = X[:,0,None] 204 | x = X[:,1,None] 205 | xi1 = x-c1*t-x1 206 | xi2 = x-c2*t-x2 207 | num = 2*(c1-c2)*(c1*tf.math.cosh(np.sqrt(c2)*xi2/2)**2 + c2*tf.math.sinh(np.sqrt(c1)*xi1/2)**2) 208 | den = ((np.sqrt(c1)-np.sqrt(c2))*tf.math.cosh((xi1*np.sqrt(c1)+xi2*np.sqrt(c2))/2)+\ 209 | (np.sqrt(c1)+np.sqrt(c2))*tf.math.cosh((np.sqrt(c1)*xi1-np.sqrt(c2)*xi2)/2))**2 210 | return num/den 211 | 212 | def uteor_x(X): 213 | with tf.GradientTape(persistent=True) as tape: 214 | tape.watch(X) 215 | ut = uteor(X) 216 | gradu = tape.gradient(ut,X) 217 | ut_x = gradu[:,1] 218 | return ut_x 219 | 220 | def u0(x): 221 | return usol(x,0) 222 | 223 | def g1(t): 224 | return usol(x0,t) 225 | 226 | def g2(t): 227 | return usol(xf,t) 228 | 229 | def r(x,t): 230 | zero = tf.constant(0.,tf.float64) 231 | return ((x-x0)*(g2(t)-g2(zero)) + (xf-x)*(g1(t)-g1(zero)))/(xf-x0) 232 | 233 | def output(X): 234 | t = X[:,0,None] 235 | x = X[:,1,None] 236 | Xnorm = tf.concat([t/tfinal,x/(xf-x0)],axis=1) 237 | u = u0(x) + r(x,t) + t*(x-x0)*(x-xf)*N(Xnorm)/(xf-x0) 238 | return u 239 | 240 | def get_results(N,X): 241 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt1: 242 | gt1.watch(X) 243 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt2: 244 | gt2.watch(X) 245 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt3: 246 | gt3.watch(X) 247 | u = output(X) 248 | 249 | ugrad = gt3.gradient(u, X) 250 | u_t = ugrad[:,0] 251 | u_x = ugrad[:,1] 252 | 253 | u_xx = gt2.gradient(u_x, X)[:,1] 254 | 255 | u_xxx = gt1.gradient(u_xx,X)[:,1] 256 | 257 | fu = u_t + 6*u[:,0]*u_x + u_xxx 258 | return u,fu 259 | 260 | def u_xpinn(X): 261 | with tf.GradientTape(persistent=True) as tape: 262 | tape.watch(X) 263 | utpinn = output(X) 264 | gradupinn = tape.gradient(utpinn,X) 265 | ut_xpinn = gradupinn[:,1] 266 | return ut_xpinn 267 | 268 | loss_function = keras.losses.MeanSquaredError() 269 | def loss(fu,uxf,uxfpinn): 270 | Ntot = fu.shape[0] 271 | zeros = tf.zeros([Ntot,1],dtype=tf.float64) 272 | return loss_function(fu,zeros) + 5*loss_function(uxf,uxfpinn) 273 | 274 | def grads(N,X,Xb,uxf): #Gradients wrt the trainable parameters 275 | with tf.GradientTape() as tape2: 276 | _,fu = get_results(N,X) 277 | uxfpinn = u_xpinn(Xb) 278 | loss_value = loss(fu,uxf,uxfpinn) 279 | gradsN = tape2.gradient(loss_value,N.trainable_variables) 280 | return gradsN,loss_value 281 | 282 | @tf.function(jit_compile=True) #Precompile training function to accelerate the process 283 | def training(N,X,Xb,uxf,optimizer): #Training step function 284 | parameter_gradients,loss_value = grads(N,X,Xb,uxf) 285 | optimizer.apply_gradients(zip(parameter_gradients,N.trainable_variables)) 286 | return loss_value 287 | 288 | 289 | rad_args = (k1,k2) 290 | epochs = np.arange(Nprint_adam,Adam_epochs+Nprint_adam,Nprint_adam) 291 | loss_list = np.zeros(len(epochs)) #loss list 292 | X,Xb = generate_inputs(Nint,Nb) 293 | uxf = uteor_x(Xb) 294 | 295 | layer_dims = [None]*(layers + 2) 296 | layer_dims[0] = X.shape[1] 297 | for i in range(1,len(layer_dims)): 298 | layer_dims[i] = neurons 299 | layer_dims[-1] = output_dim 300 | 301 | N = generate_model(layer_dims) 302 | 303 | lr = tf.keras.optimizers.schedules.ExponentialDecay(lr0,decay_steps,decay_rate) 304 | 305 | optimizer = Adam(lr,b1,b2,epsilon=epsilon) 306 | template = 'Epoch {}, loss: {}' 307 | 308 | for i in range(Adam_epochs): 309 | if (i+1)%Nchange == 0: 310 | X = adaptive_rad(N, Nint, rad_args) 311 | Xb = generate_inputs(Nint, Nb)[-1] 312 | uxf = uteor_x(Xb) 313 | #X = random_permutation(X) 314 | if (i+1)%Nprint_adam == 0: 315 | _,fu = get_results(N,X) 316 | uxfpinn = u_xpinn(Xb) 317 | loss_value = loss(fu,uxf,uxfpinn) 318 | print("i=",i+1) 319 | print(template.format(i+1,loss_value)) 320 | loss_list[i//Nprint_adam] = loss_value.numpy() 321 | 322 | training(N,X,Xb,uxf,optimizer) 323 | 324 | 325 | np.savetxt("loss_adam_NLS.txt",np.c_[epochs,loss_list]) 326 | initial_weights = np.concatenate([tf.reshape(w, [-1]).numpy() \ 327 | for w in N.weights]) #initial set of trainable variables 328 | 329 | 330 | def nested_tensor(tparams,layer_dims): 331 | ''' 332 | 333 | Parameters 334 | ---------- 335 | tparams : NUMPY ARRAY 336 | DESCRIPTION: Trainable parameters in Numpy array format 337 | layer_dims : TUPLE 338 | DESCRIPTION: 339 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 340 | if i=0, Li corresponds to the input dimension 341 | 342 | Returns 343 | ------- 344 | temp : LIST 345 | List of tensors (Trainable variables in Tensorflow format) 346 | 347 | ''' 348 | temp = [None]*(2*len(layer_dims)-2) 349 | index = 0 350 | for i in range(len(temp)): 351 | if i%2==0: 352 | temp[i] = np.reshape(tparams[index:index+layer_dims[i//2]*\ 353 | layer_dims[i//2 +1]],(layer_dims[i//2], 354 | layer_dims[i//2 +1])) 355 | index+=layer_dims[i//2]*layer_dims[i//2 +1] 356 | else: 357 | temp[i] = tparams[index:index+layer_dims[i-i//2]] 358 | index+=layer_dims[i-i//2] 359 | return temp 360 | 361 | 362 | @tf.function(jit_compile=True) 363 | def loss_and_gradient_TF(N,X,Xb,uxf,use_sqrt,use_log): #Gradients wrt the trainable parameters 364 | ''' 365 | Parameters 366 | ---------- 367 | weights : NUMPY ARRAY 368 | DESCRIPTION: Trainable parameters in Numpy array format 369 | N : TENSORFLOW MODEL 370 | PINN model 371 | X : TENSOR 372 | Batch of training params 373 | Xb: TENSOR 374 | Batch of boundary inputs 375 | uxf: TENSOR 376 | First derivative at boundary 377 | use_sqrt : BOOL 378 | DESCRIPTION. If the square root of the MSE residuals is used for training 379 | use_log : BOOL 380 | DESCRIPTION. If the log of the MSE residuals is used for training 381 | 382 | Returns 383 | ------- 384 | loss_value : LOSS (FLOAT64) 385 | LOSS USED FOR TRAINING 386 | gradsN : ARRAY OF FLOAT64 (TENSORFLOW) 387 | Gradients wrt trainable variableS 388 | ''' 389 | with tf.GradientTape() as tape: 390 | _,fu = get_results(N,X) 391 | uxfpinn = u_xpinn(Xb) 392 | if use_sqrt: 393 | loss_value = tf.math.sqrt(loss(fu,uxf,uxfpinn)) 394 | elif use_log: 395 | loss_value = tf.math.log(loss(fu,uxf,uxfpinn)) 396 | else: 397 | loss_value = loss(fu,uxf,uxfpinn) 398 | gradsN = tape.gradient(loss_value,N.trainable_variables) 399 | return loss_value,gradsN 400 | 401 | def loss_and_gradient(weights,N,X,Xb,uxf,use_sqrt,use_log,layer_dims): 402 | ''' 403 | Parameters 404 | ---------- 405 | weights : NUMPY ARRAY 406 | DESCRIPTION: Trainable parameters in Numpy array format 407 | N : TENSORFLOW MODEL 408 | PINN model 409 | X : TENSOR 410 | Batch of training params 411 | Xb: TENSOR 412 | Batch of boundary inputs 413 | uxf: TENSOR 414 | First derivative at boundary 415 | use_sqrt : BOOL 416 | DESCRIPTION. If the square root of the MSE residuals is used for training 417 | use_log : BOOL 418 | DESCRIPTION. If the log of the MSE residuals is used for training 419 | layer_dims : TUPLE 420 | DESCRIPTION: 421 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 422 | if i=0, Li corresponds to the input dimension 423 | 424 | Returns 425 | ------- 426 | loss_value : LOSS (FLOAT64) 427 | LOSS USED FOR TRAINING 428 | gradsN : ARRAY OF FLOAT64 (TENSORFLOW) 429 | Gradients wrt trainable variableS 430 | ''' 431 | resh_weights = nested_tensor(weights,layer_dims) 432 | N.set_weights(resh_weights) 433 | loss_value,grads = loss_and_gradient_TF(N,X,Xb,uxf,use_sqrt,use_log) 434 | grads_flat = np.concatenate([tf.reshape(g, [-1]).numpy() for g in grads]) 435 | return loss_value.numpy(), grads_flat 436 | 437 | 438 | def generate_test(Nt,Nx): 439 | t = np.linspace(t0,tfinal,Nt) 440 | x = np.linspace(x0,xf,Nx) 441 | t,x = np.meshgrid(t,x) 442 | X = np.hstack((t.flatten()[:,None],x.flatten()[:,None])) 443 | return convert_to_tensor(X),t,x 444 | 445 | 446 | Xtest,t,x = generate_test(Nt,Nx) 447 | 448 | epochs_bfgs = np.arange(0,Nbfgs+Nprint_bfgs,Nprint_bfgs) #iterations bfgs list 449 | epochs_bfgs+=Adam_epochs 450 | lossbfgs = np.zeros(len(epochs_bfgs)) #loss bfgs list 451 | error_list = np.zeros(len(lossbfgs)) 452 | 453 | cont=0 454 | def callback(*,intermediate_result): #Callback function, to obtain the loss at every iteration 455 | global N,cont,lossbfgs,Xtest,t,x,error_list,Nprint_bfgs 456 | if (cont+1)%Nprint_bfgs == 0 or cont == 0: 457 | if use_sqrt: 458 | loss_value = np.power(intermediate_result.fun,2) 459 | elif use_log: 460 | loss_value = np.exp(intermediate_result.fun) 461 | else: 462 | loss_value = intermediate_result.fun 463 | 464 | lossbfgs[(cont+1)//Nprint_bfgs] = loss_value 465 | 466 | utest = output(Xtest).numpy().reshape(t.shape) 467 | uteor = usol(x,t) 468 | error = np.linalg.norm(utest-uteor)/np.linalg.norm(uteor) 469 | error_list[(cont+1)//Nprint_bfgs] = error 470 | print(cont+1,loss_value) 471 | cont+=1 472 | 473 | 474 | if method == "BFGS": 475 | method_bfgs = method_bfgs 476 | initial_scale=False 477 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 478 | H0 = H0.numpy() 479 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 480 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 481 | 482 | elif method == "bfgsr": 483 | R0 = sparse.csr_matrix(np.eye(len(initial_weights))) 484 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 485 | 486 | elif method == "bfgsz": 487 | Z0 = tf.eye(len(initial_weights),dtype=tf.float64) 488 | Z0 = Z0.numpy() 489 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 490 | 491 | 492 | while cont < Nbfgs: #Training loop 493 | print(cont) 494 | result = minimize(loss_and_gradient,initial_weights, 495 | args = (N,X,Xb,uxf,use_sqrt,use_log,layer_dims), 496 | method=method,jac=True, options=options, 497 | tol=0,callback=callback) 498 | initial_weights = result.x 499 | 500 | 501 | if method=="BFGS": 502 | H0 = result.hess_inv 503 | H0 = (H0 + np.transpose(H0))/2 504 | try: 505 | cholesky(H0) 506 | except LinAlgError: 507 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 508 | H0 = H0.numpy() 509 | 510 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 511 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 512 | 513 | elif method=="bfgsr": 514 | R0 = result.r_inv 515 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 516 | 517 | elif method == "bfgsz": 518 | Z0 = result.Z 519 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 520 | 521 | 522 | X = adaptive_rad(N, Nint, rad_args) 523 | Xb = generate_inputs(Nint, Nb)[-1] 524 | uxf = uteor_x(Xb) 525 | 526 | if use_sqrt: 527 | fname_loss = f"KdV_loss_{method}_{method_bfgs}_sqrt.txt" 528 | fname_error = f"KdV_error_{method}_{method_bfgs}_sqrt.txt" 529 | elif use_log: 530 | fname_loss = f"KdV_loss_{method}_{method_bfgs}_log.txt" 531 | fname_error = f"KdV_error_{method}_{method_bfgs}_log.txt" 532 | else: 533 | fname_loss = f"KdV_loss_{method}_{method_bfgs}.txt" 534 | fname_error = f"KdV_error_{method}_{method_bfgs}.txt" 535 | 536 | np.savetxt(fname_loss,np.c_[epochs_bfgs,lossbfgs]) 537 | np.savetxt(fname_error,np.c_[epochs_bfgs,error_list]) -------------------------------------------------------------------------------- /Examples/KdV/KdV_hparams.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Dec 11 09:24:04 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import json 9 | import numpy as np 10 | 11 | def hyperparameter_configuration(): 12 | 13 | seed={"seed":2} 14 | architecture_hparams={"neurons":30, 15 | "layers":3, 16 | "output_dim":1 17 | } 18 | 19 | PDE_hparams={"c1":6., 20 | "c2":2., 21 | "x1":-2., 22 | "x2":2. 23 | } 24 | 25 | Adam_hparams={"Adam_epochs":10000, 26 | "lr0":1e-2, 27 | "decay_steps":1000, 28 | "decay_rate":0.98, 29 | "b1":0.99, 30 | "b2":0.999, 31 | "epsilon":1e-20, 32 | "Nprint_adam":100 33 | } 34 | 35 | batch_hparams={"Nint":15000, 36 | "Nb":1000, 37 | "Nchange":500, 38 | "k1":0., 39 | "k2":1., 40 | "x0":0., 41 | "t0":0., 42 | "Lx":20., 43 | "tfinal":5. 44 | } 45 | 46 | test_hparams={"Nt":600, 47 | "Nx":600} 48 | 49 | bfgs_hparams={"BFGS_epochs":20000, 50 | "method":"BFGS", 51 | "method_bfgs":"BFGS", 52 | "use_sqrt":False, 53 | "use_log":False, 54 | "Nprint_bfgs":100} 55 | 56 | hparams={ 57 | "seed":seed, 58 | "architecture_hparams":architecture_hparams, 59 | "PDE_hparams":PDE_hparams, 60 | "Adam_hparams":Adam_hparams, 61 | "batch_hparams":batch_hparams, 62 | "test_hparams":test_hparams, 63 | "bfgs_hparams":bfgs_hparams} 64 | 65 | # Guarda los hiperparámetros en un archivo JSON 66 | with open('config_KdV.json', 'w') as params: 67 | json.dump(hparams, params, indent=4) 68 | 69 | print("Hiperparámetros guardados en 'config_KdV.json'.") 70 | 71 | hyperparameter_configuration() -------------------------------------------------------------------------------- /Examples/LDC/LDC.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Dec 11 13:20:18 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import numpy as np 9 | import tensorflow as tf 10 | from tensorflow import keras 11 | from tensorflow.keras.layers import Input, Dense 12 | from tensorflow.keras.models import Model 13 | from tensorflow import convert_to_tensor 14 | from tensorflow.keras.optimizers import Adam 15 | from scipy.optimize import minimize 16 | from scipy.linalg import cholesky,LinAlgError 17 | from scipy import sparse 18 | import json 19 | 20 | def load_hparams(file_config): 21 | with open(file_config, 'r') as file: 22 | config = json.load(file) 23 | return config 24 | 25 | config = load_hparams('config_LDC.json') 26 | 27 | seed = config["seed"]["seed"] #Seed 28 | 29 | #--------------- Architecture hyperparameters ---------------------------------- 30 | neurons = config["architecture_hparams"]["neurons"] #Neurons in every hidden layer 31 | layers = config["architecture_hparams"]["layers"] #Hidden layers 32 | output_dim = config["architecture_hparams"]["output_dim"] #Output dimension 33 | #------------------------------------------------------------------------------- 34 | 35 | #--------------- PDE hyperparameters ------------------------------------------ 36 | Re = config["PDE_hparams"]["Re"] 37 | #------------------------------------------------------------------------------ 38 | 39 | #-------------- Adam hyperparameters ------------------------------------------- 40 | Adam_epochs = config["Adam_hparams"]["Adam_epochs"] #Adam epochs 41 | lr0 = config["Adam_hparams"]["lr0"] #Initial learning rate (We consider an exponential lr schedule) 42 | decay_steps = config["Adam_hparams"]["decay_steps"] #Decay steps 43 | decay_rate = config["Adam_hparams"]["decay_rate"] #Decay rate 44 | b1 = config["Adam_hparams"]["b1"] #beta1 45 | b2 = config["Adam_hparams"]["b2"] #beta2 46 | epsilon= config["Adam_hparams"]["epsilon"] #epsilon 47 | Nprint_adam = config["Adam_hparams"]["Nprint_adam"] #Adam results will be printed and save every Nprint_adam iters 48 | #------------------------------------------------------------------------------ 49 | 50 | #------------ Batch hyperparameters ------------------------------------------- 51 | Nint = config["batch_hparams"]["Nint"] #Number of points at batch 52 | Nbound = config["batch_hparams"]["Nbound"] #Number of points at the boundary 53 | Ncorner = config["batch_hparams"]["Ncorner"] #Number of points at the corners of top edge. Included at the interior points batch 54 | Nchange=config["batch_hparams"]["Nchange"] #Batch is changed every Nchange iterations 55 | k1 = config["batch_hparams"]["k1"] #k hyperparameter (see adaptive_rad function below) 56 | k2 = config["batch_hparams"]["k2"] #c hyperparameter (see adaptive_rad function below) 57 | x0 = config["batch_hparams"]["x0"] #x0 (minimum value of x) 58 | Lx = config["batch_hparams"]["Lx"] #Lx (length in the x direction) 59 | y0 = config["batch_hparams"]["y0"] #x0 (minimum value of x) 60 | Ly = config["batch_hparams"]["Ly"] #Lx (length in the x direction) 61 | #------------------------------------------------------------------------------ 62 | 63 | #------------ Quasi-Newton (QN) hyperparameters ------------------------------- 64 | Nbfgs = config["bfgs_hparams"]["BFGS_epochs"] #Number of QN iterations 65 | method = config["bfgs_hparams"]["method"] #Method. See below 66 | method_bfgs = config["bfgs_hparams"]["method_bfgs"] #Quasi-Newton algorithm. See below 67 | use_sqrt = config["bfgs_hparams"]["use_sqrt"] #Use square root of the MSE loss to train 68 | use_log = config["bfgs_hparams"]["use_log"] #Use log of the MSE loss to train 69 | Nprint_bfgs = config["bfgs_hparams"]["Nprint_bfgs"] #QN results will be printed and save every Nprint_adam iters 70 | 71 | #In method, you can choose between: 72 | # -BFGS: Here, we include BFGS and the different self-scaled QN methods. 73 | # To distinguish between these algorithms, we use method_bfgs. See below 74 | # -bfgsr: Personal implementation of the factored BFGS Hessian approximations. 75 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 76 | # Very slow, to be optimized. 77 | # -bfgsz: Personal implementation of the factored inverse BFGS Hessian approximations. 78 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 79 | # Comparable with BFGS in terms of speed. 80 | 81 | #If method=BFGS, the variable "method_bfgs" chooses the different QN methods. 82 | #The options for this are (see the modified Scipy optimize script): 83 | #-BFGS_scipy: The original implementation of BFGS of Scipy 84 | #-BFGS: Equivalent implementation, but faster (avoid repeated calculations in the BFGS formula) 85 | #-SSBFGS_AB: The Self-scaled BFGS formula, where the tauk coefficient is calculated with 86 | # Al-Baali's formula (Formula 11 of "Unveiling the optimization process in PINNs") 87 | #-SSBFGS_OL Same, but tauk is calculated with the original choice of Oren and Luenberger (not recommended) 88 | #-SSBroyden2: Here we use the tauk and phik expressions defined in the paper 89 | # (Formulas 13-23 of "Unveiling the optimization process in PINNs") 90 | #-SSbroyden1: Another possible choice for these parameters (sometimes better, sometimes worse than SSBroyden1) 91 | 92 | #------------------------------------------------------------------------------ 93 | 94 | xf = x0 + Lx 95 | yf = y0 + Ly 96 | tf.keras.backend.set_floatx('float64') 97 | tf.get_logger().setLevel('ERROR') 98 | tf.keras.utils.set_random_seed(seed) 99 | 100 | def generate_model(layer_dims): 101 | ''' 102 | 103 | 104 | Parameters 105 | ---------- 106 | layer_dims : LIST OR TUPLE 107 | DESCRIPTION. 108 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 109 | if i=0, Li corresponds to the input dimension 110 | Returns 111 | ------- 112 | Model 113 | TYPE : TENSORFLOW MODEL 114 | DESCRIPTION. 115 | 116 | ''' 117 | X_input = Input((layer_dims[0],)) 118 | X = Dense(layer_dims[1],activation="tanh")(X_input) 119 | if len(layer_dims) > 3: 120 | for i in range(2,len(layer_dims)-1): 121 | X = Dense(layer_dims[i],activation="tanh")(X) 122 | X = Dense(layer_dims[-1],activation=None)(X) 123 | return Model(inputs=X_input,outputs=X) 124 | 125 | def u0(x,C0=50): 126 | ''' 127 | 128 | 129 | Parameters 130 | ---------- 131 | x : TENSOR 132 | x coordinate in [0,1] 133 | C0 : FLOAT 134 | COEFFICIENT TO SMOOTH THE BOUNDARY CONDITION AT TOP EDGE. The default is 50. 135 | 136 | Returns 137 | ------- 138 | TYPE 139 | DESCRIPTION. 140 | 141 | ''' 142 | return 1 - tf.math.cosh(C0*(x-0.5))/np.cosh(0.5*C0) 143 | 144 | def generate_inputs(Nint): 145 | ''' 146 | 147 | 148 | Parameters 149 | ---------- 150 | Nint : INTEGER 151 | Number of interior points 152 | 153 | Returns 154 | ------- 155 | TENSOR 156 | Interior points 157 | 158 | ''' 159 | y = Lx*np.random.rand(Nint) + y0 160 | x = Ly*np.random.rand(Nint) + x0 161 | X = np.hstack((x[:,None],y[:,None])) 162 | 163 | return convert_to_tensor(X) 164 | 165 | def boundary_inputs(Nb): 166 | ''' 167 | 168 | 169 | Parameters 170 | ---------- 171 | Nb : INTEGER 172 | 173 | 174 | Returns 175 | ------- 176 | TENSORS 177 | xy_bnd Points at the boundary 178 | uv_bnd Values for u and v at the boundary 179 | 180 | ''' 181 | xy_ub = np.random.rand(Nb//2, 2) # top-bottom boundaries 182 | xy_ub[:,1] = np.round(xy_ub[:,1]) # y-position is 0 or 1 183 | xy_lr = np.random.rand(Nb//2, 2) # left-right boundaries 184 | xy_lr[:,0] = np.round(xy_lr[:,0]) # x-position is 0 or 1 185 | xy_bnd = np.random.permutation(np.concatenate([xy_ub, xy_lr])) 186 | uv_bnd = np.zeros((Nb, 2)) 187 | uv_bnd[:,0] = u0(xy_bnd[:,0])*np.floor(xy_bnd[:,1]) 188 | return convert_to_tensor(xy_bnd),convert_to_tensor(uv_bnd) 189 | 190 | def corner_points(Ncorner,alpha=0.99): 191 | ''' 192 | Parameters 193 | ---------- 194 | Ncorner : INT 195 | Number of points at the corners of the top edge 196 | alpha : TYPE, optional 197 | Corners are defined as (x,y) in [0,1-alpha] x [alpha,1] (left corner) 198 | (x,y) in [alpha,1] x [alpha,1] (right corner) 199 | Returns 200 | ------- 201 | TENSOR 202 | (x,y) points at corners 203 | 204 | ''' 205 | xl = np.random.uniform(low=0,high=(1-alpha),size=Ncorner//2) 206 | xr = np.random.uniform(low=alpha,high=1,size=Ncorner//2) 207 | x = np.concatenate((xl,xr)) 208 | yl = np.random.uniform(low=alpha,high=1,size=Ncorner//2) 209 | yr = np.random.uniform(low=alpha,high=1,size=Ncorner//2) 210 | y = np.concatenate((yl,yr)) 211 | X = np.hstack((x[:,None],y[:,None])) 212 | return convert_to_tensor(X) 213 | 214 | #-----------ADAPTIVE RAD GENERATOR (GIVEN AT WU ET AL., 2023)------------------ 215 | def adaptive_rad(N,Nint,rad_args,Re,Ntest=50000): 216 | ''' 217 | 218 | Parameters 219 | ---------- 220 | N : TENSORFLOW MODEL 221 | PINN MODEL 222 | Nint : INTEGER 223 | NUMBER OF INTERIOR POINTS AT BATCH 224 | rad_args: TUPLE (k1,k2) 225 | DESCRIPTION. 226 | Adaptive resampling of Wu et al. (2023), formula (2) 227 | k1: k 228 | k2: c 229 | DOI: https://doi.org/10.1016/j.cma.2022.115671 230 | 231 | Ntest: INTEGER 232 | DESCRIPTION. 233 | Number of test points to do the resampling 234 | 235 | 236 | Returns 237 | ------- 238 | TYPE 239 | DESCRIPTION. 240 | 241 | ''' 242 | Xtest = generate_inputs(Ntest) 243 | k1,k2 = rad_args 244 | fu,fv = get_results(N,Xtest,Re)[-2:] 245 | Y = tf.math.sqrt(fu**2 + fv**2).numpy() 246 | err_eq = np.power(Y,k1)/np.power(Y,k1).mean() + k2 247 | err_eq_normalized = (err_eq / sum(err_eq)) 248 | X_ids = np.random.choice(a=len(Xtest), size=Nint, replace=False, 249 | p=err_eq_normalized) 250 | return tf.gather(Xtest,X_ids) 251 | 252 | 253 | def output(N,X): 254 | ''' 255 | 256 | 257 | Parameters 258 | ---------- 259 | N : TENSORFLOW MODEL 260 | PINN MODEL 261 | X : TENSOR 262 | INTERIOR POINTS 263 | 264 | Returns 265 | ------- 266 | u : TENSOR 267 | VELOCITY U 268 | v : TENSOR 269 | VELOCITY V 270 | p : TENSOR 271 | PRESSURE P 272 | 273 | ''' 274 | with tf.GradientTape() as tape: 275 | tape.watch(X) 276 | Nout = N(X) 277 | psi = Nout[:,0,None] 278 | psigrad = tape.gradient(psi,X) 279 | psi_x = psigrad[:,0,None] 280 | psi_y = psigrad[:,1,None] 281 | u = psi_y 282 | v = -psi_x 283 | p = Nout[:,1,None] - N(tf.constant([[0.,0.]],dtype=tf.float64))[:,1,None] 284 | return u,v,p 285 | 286 | def get_results(N,X,Re): 287 | ''' 288 | Parameters 289 | ---------- 290 | N : TENSORFLOW MODEL 291 | PINN MODEL 292 | X : TENSOR 293 | INTERIOR POINTS 294 | Re : FLOAT 295 | REYNOLDS NUMBER 296 | 297 | Returns 298 | ------- 299 | u : TENSOR 300 | VELOCITY U 301 | v : TENSOR 302 | VELOCITY V 303 | fu : TENSOR 304 | RESIDUALS FOR u_t = ... 305 | fv : TENSOR 306 | RESIDUALS for v_t = ... 307 | 308 | ''' 309 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt1: 310 | gt1.watch(X) 311 | 312 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt2: 313 | gt2.watch(X) 314 | 315 | # Calculate u,v 316 | u,v,p = output(N,X) 317 | 318 | ugrad = gt2.gradient(u, X) 319 | vgrad = gt2.gradient(v, X) 320 | pgrad = gt2.gradient(p, X) 321 | 322 | u_x = ugrad[:,0] 323 | u_y = ugrad[:,1] 324 | v_x = vgrad[:,0] 325 | v_y = vgrad[:,1] 326 | p_x = pgrad[:,0] 327 | p_y = pgrad[:,1] 328 | 329 | u_xx = gt1.gradient(u_x,X)[:,0] 330 | u_yy = gt1.gradient(u_y,X)[:,1] 331 | 332 | v_xx = gt1.gradient(v_x,X)[:,0] 333 | v_yy = gt1.gradient(v_y,X)[:,1] 334 | 335 | nu=1/Re 336 | fu = u[:,0]*u_x + v[:,0]*u_y + p_x - nu*(u_xx + u_yy) 337 | fv = u[:,0]*v_x + v[:,0]*v_y + p_y - nu*(v_xx + v_yy) 338 | return u,v,fu,fv 339 | 340 | loss_function = keras.losses.MeanSquaredError() 341 | lam = 10 342 | def loss(fu,fv,uv,uv_pinn): 343 | ''' 344 | Parameters 345 | ---------- 346 | fu : TENSOR 347 | RESIDUALS FOR u_t = ... 348 | fv : TENSOR 349 | RESIDUALS for v_t = ... 350 | uv : TENSOR 351 | u and v at boundary 352 | uv_pinn : TENSOR 353 | PINN prediction for u and v 354 | 355 | Returns 356 | ------- 357 | loss_val : FLOAT 358 | LOSS VALUE 359 | 360 | ''' 361 | Ntot = fu.shape[0] 362 | zeros = tf.zeros([Ntot,1],dtype=tf.float64) 363 | loss_val = loss_function(fu,zeros) + loss_function(fv,zeros) + lam*loss_function(uv,uv_pinn) 364 | return loss_val 365 | 366 | def grads(N,X,Xb,uv,Re): 367 | ''' 368 | Parameters 369 | ---------- 370 | N : TENSORFLOW MODEL 371 | PINN MODEL 372 | X : TENSOR 373 | INTERIOR POINTS 374 | Xb : TENSOR 375 | BOUNDARY POINTS 376 | uv : TENSOR 377 | u and v at boundary 378 | Re : FLOAT 379 | REYNOLDS NUMBER 380 | 381 | Returns 382 | ------- 383 | gradsN : LIST OF TENSORS 384 | GRADIENT OF LOSS WRT TRAINABLE VARIABLES 385 | loss_value : FLOAT64 386 | LOSS VALUE 387 | 388 | ''' 389 | with tf.GradientTape() as tape2: 390 | _,_,fu,fv = get_results(N,X,Re) 391 | uv_pinn = output(N, Xb)[0:2] 392 | uv_pinn = tf.concat([uv_pinn[0],uv_pinn[1]],axis=1) 393 | loss_value = loss(fu,fv,uv,uv_pinn) 394 | gradsN = tape2.gradient(loss_value,N.trainable_variables, 395 | unconnected_gradients=tf.UnconnectedGradients.ZERO) 396 | return gradsN,loss_value 397 | 398 | @tf.function(jit_compile=True) #Precompile training function to accelerate the process 399 | def training(N,X,Xb,uv,Re,optimizer): #Training step function 400 | ''' 401 | Parameters 402 | ---------- 403 | N : TENSORFLOW MODEL 404 | PINN MODEL 405 | X : TENSOR 406 | INTERIOR POINTS 407 | Xb : TENSOR 408 | BOUNDARY POINTS 409 | uv : TENSOR 410 | u and v at boundary 411 | Re : FLOAT 412 | REYNOLDS NUMBER 413 | optimizer: TENSORLFOW OPTIMIZER 414 | ADAM (OR GRADIENT DESCENT BASED) OPTIMIZER 415 | Returns 416 | ------- 417 | loss_value : FLOAT64 418 | LOSS VALUE 419 | ''' 420 | 421 | parameter_gradients,loss_value = grads(N,X,Xb,uv,Re) 422 | optimizer.apply_gradients(zip(parameter_gradients,N.trainable_variables)) 423 | return loss_value 424 | 425 | rad_args = (k1,k2) #If random uniform, select k1 = 0 426 | epochs = np.arange(Nprint_adam,Adam_epochs+Nprint_adam,Nprint_adam) 427 | loss_list = np.zeros(len(epochs)) #loss list 428 | 429 | X = generate_inputs(Nint) #Initial collocation points 430 | Xcorner = corner_points(Ncorner) 431 | X = tf.concat([X,Xcorner],axis=0) 432 | Xb,uv = boundary_inputs(Nbound) 433 | layer_dims = [None]*(layers + 2) 434 | layer_dims[0] = X.shape[1] 435 | for i in range(1,len(layer_dims)): 436 | layer_dims[i] = neurons 437 | layer_dims[-1] = output_dim 438 | 439 | N = generate_model(layer_dims) 440 | 441 | lr = tf.keras.optimizers.schedules.ExponentialDecay(lr0,decay_steps,decay_rate) 442 | 443 | optimizer = Adam(lr,b1,b2,epsilon=epsilon) 444 | template = 'Epoch {}, loss: {}' 445 | 446 | for i in range(Adam_epochs): 447 | if (i+1)%Nchange == 0: 448 | X = adaptive_rad(N, Nint, rad_args, Re) 449 | Xb,uv = boundary_inputs(Nbound) 450 | Xcorner = corner_points(Ncorner) 451 | X = tf.concat([X,Xcorner],axis=0) 452 | #X = random_permutation(X) 453 | if (i+1)%Nprint_adam == 0: 454 | _,_,fu,fv = get_results(N,X,Re) 455 | uv_pinn = output(N, Xb)[0:2] 456 | uv_pinn = tf.concat([uv_pinn[0],uv_pinn[1]],axis=1) 457 | loss_value = loss(fu,fv,uv,uv_pinn) 458 | print("i=",i+1) 459 | print(template.format(i+1,loss_value)) 460 | loss_list[i//Nprint_adam] = loss_value.numpy() 461 | 462 | training(N,X,Xb,uv,Re,optimizer) 463 | 464 | np.savetxt("loss_adam_LDC.txt",np.c_[epochs,loss_list]) 465 | initial_weights = np.concatenate([tf.reshape(w, [-1]).numpy() \ 466 | for w in N.weights]) #initial set of trainable variables 467 | 468 | def nested_tensor(tparams,layer_dims): 469 | ''' 470 | 471 | Parameters 472 | ---------- 473 | tparams : NUMPY ARRAY 474 | DESCRIPTION: Trainable parameters in Numpy array format 475 | layer_dims : TUPLE 476 | DESCRIPTION: 477 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 478 | if i=0, Li corresponds to the input dimension 479 | 480 | Returns 481 | ------- 482 | temp : LIST 483 | List of tensors (Trainable variables in Tensorflow format) 484 | 485 | ''' 486 | temp = [None]*(2*len(layer_dims)-2) 487 | index = 0 488 | for i in range(len(temp)): 489 | if i%2==0: 490 | temp[i] = np.reshape(tparams[index:index+layer_dims[i//2]*\ 491 | layer_dims[i//2 +1]],(layer_dims[i//2], 492 | layer_dims[i//2 +1])) 493 | index+=layer_dims[i//2]*layer_dims[i//2 +1] 494 | else: 495 | temp[i] = tparams[index:index+layer_dims[i-i//2]] 496 | index+=layer_dims[i-i//2] 497 | return temp 498 | 499 | 500 | @tf.function(jit_compile=True) 501 | def loss_and_gradient_TF(N,X,Xb,uv,Re,use_sqrt,use_log): 502 | ''' 503 | Parameters 504 | ---------- 505 | Parameters 506 | ---------- 507 | N : TENSORFLOW MODEL 508 | PINN MODEL 509 | X : TENSOR 510 | INTERIOR POINTS 511 | Xb : TENSOR 512 | BOUNDARY POINTS 513 | uv : TENSOR 514 | u and v at boundary 515 | Re : FLOAT 516 | REYNOLDS NUMBER 517 | use_sqrt: BOOL 518 | If True, the square root of the loss is used 519 | use_log: BOOL 520 | If True, the logarithm of the loss is used 521 | 522 | Returns 523 | ------- 524 | gradsN : LIST OF TENSORS 525 | GRADIENT OF LOSS WRT TRAINABLE VARIABLES 526 | loss_value : FLOAT64 527 | LOSS VALUE 528 | 529 | ''' 530 | with tf.GradientTape() as tape: 531 | _,_,fu,fv = get_results(N,X,Re) 532 | uv_pinn = output(N, Xb)[0:2] 533 | uv_pinn = tf.concat([uv_pinn[0],uv_pinn[1]],axis=1) 534 | if use_sqrt: 535 | loss_value = tf.math.sqrt(loss(fu,fv,uv,uv_pinn)) 536 | elif use_log: 537 | loss_value = tf.math.log(loss(fu,fv,uv,uv_pinn)) 538 | else: 539 | loss_value = loss(fu,fv,uv,uv_pinn) 540 | 541 | gradsN = tape.gradient(loss_value,N.trainable_variables, 542 | unconnected_gradients=tf.UnconnectedGradients.ZERO) 543 | return loss_value,gradsN 544 | 545 | def loss_and_gradient(weights,N,X,Xb,uv,Re,use_sqrt,use_log,layer_dims): 546 | ''' 547 | 548 | 549 | Parameters 550 | ---------- 551 | weights : NUMPY ARRAY 552 | TRAINABLE VARIABLES 553 | N : TENSORFLOW MODEL 554 | PINN MODEL 555 | X : TENSOR 556 | INTERIOR POINTS 557 | Xb : TENSOR 558 | BOUNDARY POINTS 559 | uv : TENSOR 560 | u and v at boundary 561 | Re : FLOAT 562 | REYNOLDS NUMBER 563 | use_sqrt: BOOL 564 | If True, the square root of the loss is used 565 | use_log: BOOL 566 | If True, the logarithm of the loss is used 567 | layer_dims : LIST OR TUPLE 568 | DESCRIPTION. 569 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 570 | if i=0, Li corresponds to the input dimension 571 | 572 | Returns 573 | ------- 574 | loss_value : FLOAT 575 | LOSS VALUE 576 | grads_flat : ARRAY 577 | GRADIENT OF LOSS WRT TRAINABLE VARIABLES 578 | 579 | ''' 580 | resh_weights = nested_tensor(weights,layer_dims) 581 | N.set_weights(resh_weights) 582 | loss_value,grads = loss_and_gradient_TF(N,X,Xb,uv,Re,use_sqrt,use_log) 583 | grads_flat = np.concatenate([tf.reshape(g, [-1]).numpy() for g in grads]) 584 | return loss_value.numpy(), grads_flat 585 | 586 | epochs_bfgs = np.arange(0,Nbfgs+Nprint_bfgs,Nprint_bfgs) #iterations bfgs list 587 | epochs_bfgs+=Adam_epochs 588 | lossbfgs = np.zeros(len(epochs_bfgs)) #loss bfgs list 589 | validationbfgs = np.zeros(len(epochs_bfgs)) 590 | 591 | Nval = 100000 592 | Nbval = 10000 593 | Xval = generate_inputs(Nval) 594 | Xbval,uvval = boundary_inputs(Nbval) 595 | 596 | cont=0 597 | def callback(*,intermediate_result): 598 | global N,cont,lossbfgs,Nprint_bfgs,Xval,Xbval,uvval 599 | if (cont+1)%Nprint_bfgs == 0 or cont == 0: 600 | if use_sqrt: 601 | loss_value = np.power(intermediate_result.fun,2) 602 | elif use_log: 603 | loss_value = np.exp(intermediate_result.fun) 604 | else: 605 | loss_value = intermediate_result.fun 606 | lossbfgs[(cont+1)//Nprint_bfgs] = loss_value 607 | _,_,fuval,fvval = get_results(N,Xval,Re) 608 | uv_pinn_val = output(N, Xbval)[0:2] 609 | uv_pinn_val = tf.concat([uv_pinn_val[0],uv_pinn_val[1]],axis=1) 610 | validation_value = loss(fuval,fvval,uvval,uv_pinn_val) 611 | validationbfgs[(cont+1)//Nprint_bfgs] = validation_value 612 | print(loss_value,validation_value.numpy(),cont+1) 613 | cont+=1 614 | 615 | if method == "BFGS": 616 | method_bfgs = method_bfgs 617 | initial_scale=False 618 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 619 | H0 = H0.numpy() 620 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 621 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 622 | 623 | elif method == "bfgsr": 624 | R0 = sparse.csr_matrix(np.eye(len(initial_weights))) 625 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 626 | 627 | elif method == "bfgsz": 628 | Z0 = tf.eye(len(initial_weights),dtype=tf.float64) 629 | Z0 = Z0.numpy() 630 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 631 | 632 | 633 | #------------------------- BFGS TRAINING -------------------------------------- 634 | while cont < Nbfgs: #Training loop 635 | result = minimize(loss_and_gradient,initial_weights, 636 | args = (N,X,Xb,uv,Re,use_sqrt,use_log,layer_dims), 637 | method=method,jac=True, options=options, 638 | tol=0,callback=callback) 639 | 640 | initial_weights = result.x 641 | 642 | if method=="BFGS": 643 | H0 = result.hess_inv 644 | H0 = (H0 + np.transpose(H0))/2 645 | try: 646 | cholesky(H0) 647 | except LinAlgError: 648 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 649 | H0 = H0.numpy() 650 | 651 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 652 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 653 | 654 | elif method=="bfgsr": 655 | R0 = result.r_inv 656 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 657 | 658 | elif method == "bfgsz": 659 | Z0 = result.Z 660 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 661 | 662 | X = adaptive_rad(N, Nint, rad_args, Re) 663 | Xb,uv = boundary_inputs(Nbound) 664 | Xcorner = corner_points(5000) 665 | X = tf.concat([X,Xcorner],axis=0) 666 | 667 | if use_sqrt: 668 | fname_loss = f"LDC_loss_{method}_{method_bfgs}_sqrt.txt" 669 | fname_error = f"LDC_validation_{method}_{method_bfgs}_sqrt.txt" 670 | elif use_log: 671 | fname_loss = f"LDC_loss_{method}_{method_bfgs}_log.txt" 672 | fname_error = f"LDC_validation_{method}_{method_bfgs}_log.txt" 673 | else: 674 | fname_loss = f"LDC_loss_{method}_{method_bfgs}.txt" 675 | fname_error = f"LDC_validation_{method}_{method_bfgs}.txt" 676 | 677 | np.savetxt(fname_loss,np.c_[epochs_bfgs,lossbfgs]) 678 | np.savetxt(fname_error,np.c_[epochs_bfgs,validationbfgs]) 679 | 680 | -------------------------------------------------------------------------------- /Examples/LDC/LDC_hparams.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Wed Dec 11 13:48:47 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import json 9 | import numpy as np 10 | 11 | def hyperparameter_configuration(): 12 | 13 | seed={"seed":1} 14 | architecture_hparams={"neurons":20, 15 | "layers":6, 16 | "output_dim":2 17 | } 18 | 19 | PDE_hparams={"Re":1000.} 20 | 21 | Adam_hparams={"Adam_epochs":5000, 22 | "lr0":5e-3, 23 | "decay_steps":1000, 24 | "decay_rate":0.98, 25 | "b1":0.99, 26 | "b2":0.999, 27 | "epsilon":1e-20, 28 | "Nprint_adam":100 29 | } 30 | 31 | batch_hparams={"Nint":15000, 32 | "Nbound":5000, 33 | "Ncorner":5000, 34 | "Nchange":500, 35 | "k1":1., 36 | "k2":0., 37 | "x0":0., 38 | "Lx":1., 39 | "y0":0., 40 | "Ly":1. 41 | } 42 | 43 | bfgs_hparams={"BFGS_epochs":20000, 44 | "method":"BFGS", 45 | "method_bfgs":"SSBroyden2", 46 | "use_sqrt":False, 47 | "use_log":False, 48 | "Nprint_bfgs":100} 49 | 50 | hparams={ 51 | "seed":seed, 52 | "architecture_hparams":architecture_hparams, 53 | "PDE_hparams":PDE_hparams, 54 | "Adam_hparams":Adam_hparams, 55 | "batch_hparams":batch_hparams, 56 | "bfgs_hparams":bfgs_hparams} 57 | 58 | # Guarda los hiperparámetros en un archivo JSON 59 | with open('config_LDC.json', 'w') as params: 60 | json.dump(hparams, params, indent=4) 61 | 62 | print("Hiperparámetros guardados en 'config_LDC.json'.") 63 | 64 | hyperparameter_configuration() -------------------------------------------------------------------------------- /Examples/NLP/NLP.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Dec 10 20:21:50 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import numpy as np 9 | import tensorflow as tf 10 | from tensorflow import keras 11 | from tensorflow.keras.layers import Input, Dense 12 | from tensorflow.keras.models import Model 13 | from tensorflow import convert_to_tensor 14 | from tensorflow.keras.optimizers import Adam 15 | from scipy.optimize import minimize 16 | from scipy.linalg import cholesky,LinAlgError 17 | from scipy import sparse 18 | import json 19 | 20 | def load_hparams(file_config): 21 | with open(file_config, 'r') as file: 22 | config = json.load(file) 23 | return config 24 | 25 | config = load_hparams('config_NLP.json') 26 | 27 | seed = config["seed"]["seed"] #Seed 28 | 29 | #--------------- Architecture hyperparameters ---------------------------------- 30 | neurons = config["architecture_hparams"]["neurons"] #Neurons in every hidden layer 31 | layers = config["architecture_hparams"]["layers"] #Hidden layers 32 | output_dim = config["architecture_hparams"]["output_dim"] #Output dimension 33 | #------------------------------------------------------------------------------- 34 | 35 | #--------------- PDE parameters --------------------------------------- 36 | k = config["PDE_hparams"]["k"] #Source parameter k 37 | #------------------------------------------------------------------------------ 38 | 39 | #-------------- Adam hyperparameters ------------------------------------------- 40 | Adam_epochs = config["Adam_hparams"]["Adam_epochs"] #Adam epochs 41 | lr0 = config["Adam_hparams"]["lr0"] #Initial learning rate (We consider an exponential lr schedule) 42 | decay_steps = config["Adam_hparams"]["decay_steps"] #Decay steps 43 | decay_rate = config["Adam_hparams"]["decay_rate"] #Decay rate 44 | b1 = config["Adam_hparams"]["b1"] #beta1 45 | b2 = config["Adam_hparams"]["b2"] #beta2 46 | epsilon= config["Adam_hparams"]["epsilon"] #epsilon 47 | Nprint_adam = config["Adam_hparams"]["Nprint_adam"] #Adam results will be printed and save every Nprint_adam iters 48 | #------------------------------------------------------------------------------ 49 | 50 | #------------ Batch hyperparameters ------------------------------------------- 51 | Nint = config["batch_hparams"]["Nint"] #Number of points at batch 52 | Nchange=config["batch_hparams"]["Nchange"] #Batch is changed every Nchange iterations 53 | k1 = config["batch_hparams"]["k1"] #k hyperparameter (see adaptive_rad function below) 54 | k2 = config["batch_hparams"]["k2"] #c hyperparameter (see adaptive_rad function below) 55 | x0 = config["batch_hparams"]["x0"] #x0 (minimum value of x) 56 | y0 = config["batch_hparams"]["y0"] #y0 (minimum value of y) 57 | Lx = config["batch_hparams"]["Lx"] #Lx (length in the x direction) 58 | Ly = config["batch_hparams"]["Ly"] #Ly (length in the y direction) 59 | #------------------------------------------------------------------------------ 60 | 61 | #------------ Test hyperparameters -------------------------------------- 62 | Nx = config["test_hparams"]["Nx"] #Number of grid points for test set in x direction 63 | Ny = config["test_hparams"]["Ny"] #Number of grid points for test set in y direction 64 | 65 | #------------ Quasi-Newton (QN) hyperparameters ------------------------------- 66 | Nbfgs = config["bfgs_hparams"]["BFGS_epochs"] #Number of QN iterations 67 | method = config["bfgs_hparams"]["method"] #Method. See below 68 | method_bfgs = config["bfgs_hparams"]["method_bfgs"] #Quasi-Newton algorithm. See below 69 | use_sqrt = config["bfgs_hparams"]["use_sqrt"] #Use square root of the MSE loss to train 70 | use_log = config["bfgs_hparams"]["use_log"] #Use log of the MSE loss to train 71 | Nprint_bfgs = config["bfgs_hparams"]["Nprint_bfgs"] #QN results will be printed and save every Nprint_adam iters 72 | 73 | #In method, you can choose between: 74 | # -BFGS: Here, we include BFGS and the different self-scaled QN methods. 75 | # To distinguish between these algorithms, we use method_bfgs. See below 76 | # -bfgsr: Personal implementation of the factored BFGS Hessian approximations. 77 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 78 | # Very slow, to be optimized. 79 | # -bfgsz: Personal implementation of the factored inverse BFGS Hessian approximations. 80 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 81 | # Comparable with BFGS in terms of speed. 82 | 83 | #If method=BFGS, the variable "method_bfgs" chooses the different QN methods. 84 | #The options for this are (see the modified Scipy optimize script): 85 | #-BFGS_scipy: The original implementation of BFGS of Scipy 86 | #-BFGS: Equivalent implementation, but faster (avoid repeated calculations in the BFGS formula) 87 | #-SSBFGS_AB: The Self-scaled BFGS formula, where the tauk coefficient is calculated with 88 | # Al-Baali's formula (Formula 11 of "Unveiling the optimization process in PINNs") 89 | #-SSBFGS_OL Same, but tauk is calculated with the original choice of Oren and Luenberger (not recommended) 90 | #-SSBroyden2: Here we use the tauk and phik expressions defined in the paper 91 | # (Formulas 13-23 of "Unveiling the optimization process in PINNs") 92 | #-SSbroyden1: Another possible choice for these parameters (sometimes better, sometimes worse than SSBroyden1) 93 | 94 | #------------------------------------------------------------------------------ 95 | 96 | xf = x0 + Lx 97 | yf = y0 + Ly 98 | 99 | tf.keras.backend.set_floatx('float64') 100 | tf.get_logger().setLevel('ERROR') 101 | tf.keras.utils.set_random_seed(seed) 102 | 103 | def generate_model(layer_dims): 104 | ''' 105 | Parameters 106 | ---------- 107 | layer_dims : TUPLE 108 | DESCRIPTION. 109 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 110 | if i=0, Li corresponds to the input dimension 111 | Returns 112 | ------- 113 | Model 114 | TYPE : TENSORFLOW MODEL 115 | DESCRIPTION. 116 | 117 | ''' 118 | X_input = Input((layer_dims[0],)) 119 | X = Dense(layer_dims[1],activation="tanh")(X_input) 120 | if len(layer_dims) > 3: 121 | for i in range(2,len(layer_dims)-1): 122 | X = Dense(layer_dims[i],activation="tanh")(X) 123 | X = Dense(layer_dims[-1],activation=None)(X) 124 | return Model(inputs=X_input,outputs=X) 125 | 126 | 127 | def generate_inputs(Nint): 128 | ''' 129 | 130 | Parameters 131 | ---------- 132 | Nint : INTEGER 133 | DESCRIPTION. 134 | Number of training points in a given batch 135 | 136 | Returns 137 | ------- 138 | X: Batch of points (in Tensorflow format) 139 | TYPE : TENSOR 140 | DESCRIPTION. 141 | 142 | ''' 143 | y = (yf-y0)*np.random.rand(Nint) + y0 144 | x = (xf-x0)*np.random.rand(Nint) + x0 145 | X = np.hstack((x[:,None],y[:,None])) 146 | return convert_to_tensor(X) 147 | 148 | def adaptive_rad(N,Nint,rad_args,Ntest=100000): 149 | ''' 150 | Parameters 151 | ---------- 152 | N : TENSORFLOW MODEL 153 | DESCRIPTION. 154 | PINN model, obtained with generate_model() function 155 | 156 | Nint : INTEGER 157 | DESCRIPTION. 158 | Number of training points in a given batch 159 | 160 | rad_args: TUPLE (k1,k2) 161 | DESCRIPTION. 162 | Adaptive resampling of Wu et al. (2023), formula (2) 163 | k1: k 164 | k2: c 165 | DOI: https://doi.org/10.1016/j.cma.2022.115671 166 | 167 | Ntest: INTEGER 168 | DESCRIPTION. 169 | Number of test points to do the resampling 170 | 171 | Returns 172 | ------- 173 | X: Batch of points (in Tensorflow format) 174 | TYPE 175 | DESCRIPTION. 176 | ''' 177 | Xtest = generate_inputs(Ntest) 178 | k1,k2 = rad_args 179 | Y = tf.math.abs(get_results(N,Xtest)[-1]).numpy() 180 | err_eq = np.power(Y,k1)/np.power(Y,k1).mean() + k2 181 | err_eq_normalized = (err_eq / sum(err_eq)) 182 | X_ids = np.random.choice(a=len(Xtest), size=Nint, replace=False, 183 | p=err_eq_normalized) 184 | return tf.gather(Xtest,X_ids) 185 | 186 | 187 | def output(N,X): 188 | ''' 189 | Parameters 190 | ---------- 191 | N : TENSORFLOW MODEL 192 | PINN model, obtained with generate_model() function 193 | X : TENSOR 194 | Batch of points (in Tensorflow format) 195 | Returns 196 | ------- 197 | u: TENSOR 198 | PINN prediction. Fourier 199 | 200 | ''' 201 | x = X[:,0,None] 202 | y = X[:,1,None] 203 | A = 1 + (1-y*(1-np.cos(k*np.pi)))*tf.math.sin(k*np.pi*x) 204 | u = A + x*y*(1-x)*(1-y)*N(X) 205 | return u 206 | 207 | def usol(x,y): 208 | ''' 209 | 210 | Parameters 211 | ---------- 212 | x : array for x 213 | y : array for y 214 | 215 | Returns 216 | ------- 217 | u: analytical solution 218 | 219 | ''' 220 | return 1+np.sin(k*np.pi*x)*np.cos(k*np.pi*y) 221 | 222 | def get_results(N,X): 223 | ''' 224 | Parameters 225 | ---------- 226 | N : TENSORFLOW MODEL 227 | PINN model, obtained with generate_model() function 228 | X : TENSOR 229 | Batch of points (in Tensorflow format) 230 | 231 | Returns 232 | ------- 233 | u : TENSOR 234 | PINN prediction 235 | fu : TENSOR 236 | PDE residuals 237 | 238 | ''' 239 | x = X[:,0] 240 | y = X[:,1] 241 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt1: 242 | gt1.watch(X) 243 | 244 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt2: 245 | gt2.watch(X) 246 | 247 | # Calculate u,v 248 | u = output(N,X) 249 | 250 | ugrad = gt2.gradient(u, X) 251 | u_x = ugrad[:,0] 252 | u_y = ugrad[:,1] 253 | 254 | u_xx = gt1.gradient(u_x, X)[:,0] 255 | u_yy = gt1.gradient(u_y, X)[:,1] 256 | 257 | f = -2*k**2*np.pi**2*tf.math.sin(k*np.pi*x)*tf.math.cos(k*np.pi*y)-\ 258 | tf.math.exp(1+tf.math.sin(k*np.pi*x)*tf.math.cos(k*np.pi*y)) 259 | fu = u_xx + u_yy - tf.math.exp(u[:,0]) - f 260 | return u,fu 261 | 262 | loss_function = keras.losses.MeanSquaredError() 263 | def loss(fu): 264 | ''' 265 | Parameters 266 | ---------- 267 | fu : TENSOR 268 | PDE residuals 269 | 270 | Returns 271 | ------- 272 | LOSS 273 | TYPE: FLOAT64 274 | MSE Loss of PDE residuals 275 | 276 | ''' 277 | Ntot = fu.shape[0] 278 | zeros = tf.zeros([Ntot,1],dtype=tf.float64) 279 | return loss_function(fu,zeros) 280 | 281 | def grads(N,X): #Gradients wrt the trainable parameters 282 | ''' 283 | Parameters 284 | ---------- 285 | N : TENSORFLOW MODEL 286 | PINN model, obtained with generate_model() function 287 | X : TENSOR 288 | Batch of points (in Tensorflow format) 289 | 290 | Returns 291 | ------- 292 | gradsN : TENSOR 293 | Gradients of loss wrt trainable variables 294 | loss_value: FLOAT64 295 | MSE Loss of PDE residuals 296 | 297 | ''' 298 | with tf.GradientTape() as tape2: 299 | _,fu = get_results(N,X) 300 | loss_value = loss(fu) 301 | gradsN = tape2.gradient(loss_value,N.trainable_variables) 302 | return gradsN,loss_value 303 | 304 | @tf.function(jit_compile=True) #Precompile training function to accelerate the process 305 | def training(N,X,optimizer): #Training step function 306 | ''' 307 | Parameters 308 | ---------- 309 | N : TENSORFLOW MODEL 310 | PINN model, obtained with generate_model() function 311 | X : TENSOR 312 | Batch of points (in Tensorflow format) 313 | 314 | Optimizer: TENSORFLOW OPTIMIZER 315 | Tensorflow optimizer (Adam, Nadam,...) 316 | 317 | Returns 318 | ------- 319 | loss_value: FLOAT64 320 | MSE Loss of PDE residuals 321 | ''' 322 | parameter_gradients,loss_value = grads(N,X) 323 | optimizer.apply_gradients(zip(parameter_gradients,N.trainable_variables)) 324 | return loss_value 325 | 326 | rad_args = (k1,k2) 327 | epochs = np.arange(Nprint_adam,Adam_epochs+Nprint_adam,Nprint_adam) 328 | loss_list = np.zeros(len(epochs)) #loss list 329 | X = generate_inputs(Nint) 330 | 331 | layer_dims = [None]*(layers + 2) 332 | layer_dims[0] = X.shape[1] 333 | for i in range(1,len(layer_dims)): 334 | layer_dims[i] = neurons 335 | layer_dims[-1] = output_dim 336 | 337 | N = generate_model(layer_dims) 338 | 339 | lr = tf.keras.optimizers.schedules.ExponentialDecay(lr0,decay_steps,decay_rate) 340 | 341 | optimizer = Adam(lr,b1,b2,epsilon=epsilon) 342 | template = 'Epoch {}, loss: {}' 343 | 344 | for i in range(Adam_epochs): 345 | 346 | if (i+1)%Nchange == 0: 347 | X = adaptive_rad(N, Nint, rad_args) 348 | #X = random_permutation(X) 349 | if (i+1)%Nprint_adam == 0: 350 | _,fu = get_results(N,X) 351 | loss_value = loss(fu) 352 | print("i=",i+1) 353 | print(template.format(i+1,loss_value)) 354 | loss_list[i//Nprint_adam] = loss_value.numpy() 355 | 356 | training(N,X,optimizer) 357 | 358 | np.savetxt(f"loss_adam_NLP_{k}.txt",np.c_[epochs,loss_list]) 359 | initial_weights = np.concatenate([tf.reshape(w, [-1]).numpy() \ 360 | for w in N.weights]) #initial set of trainable variables 361 | 362 | def nested_tensor(tparams,layer_dims): 363 | ''' 364 | 365 | Parameters 366 | ---------- 367 | tparams : NUMPY ARRAY 368 | DESCRIPTION: Trainable parameters in Numpy array format 369 | layer_dims : TUPLE 370 | DESCRIPTION: 371 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 372 | if i=0, Li corresponds to the input dimension 373 | 374 | Returns 375 | ------- 376 | temp : LIST 377 | List of tensors (Trainable variables in Tensorflow format) 378 | 379 | ''' 380 | temp = [None]*(2*len(layer_dims)-2) 381 | index = 0 382 | for i in range(len(temp)): 383 | if i%2==0: 384 | temp[i] = np.reshape(tparams[index:index+layer_dims[i//2]*\ 385 | layer_dims[i//2 +1]],(layer_dims[i//2], 386 | layer_dims[i//2 +1])) 387 | index+=layer_dims[i//2]*layer_dims[i//2 +1] 388 | else: 389 | temp[i] = tparams[index:index+layer_dims[i-i//2]] 390 | index+=layer_dims[i-i//2] 391 | return temp 392 | 393 | 394 | @tf.function 395 | def loss_and_gradient_TF(N,X,use_sqrt,use_log): 396 | ''' 397 | Parameters 398 | ---------- 399 | N : TENSORFLOW MODEL 400 | PINN model, obtained with generate_model() function 401 | X : TENSOR 402 | Batch of points (in Tensorflow format) 403 | use_sqrt: BOOL 404 | If the square root of the MSE residuals is used for training 405 | use_log: BOOL 406 | If the logarithm of the MSE residuals is used for training 407 | 408 | Returns 409 | ------- 410 | loss_value : FLOAT64 411 | LOSS USED FOR TRAINING 412 | gradsN: TENSOR 413 | Gradients wrt trainable variables 414 | 415 | ''' 416 | with tf.GradientTape() as tape: 417 | fu = get_results(N,X)[-1] 418 | if use_sqrt: 419 | loss_value = tf.math.sqrt(loss(fu)) 420 | elif use_log: 421 | loss_value = tf.math.log(loss(fu)) 422 | else: 423 | loss_value = loss(fu) 424 | gradsN = tape.gradient(loss_value,N.trainable_variables) 425 | return loss_value,gradsN 426 | 427 | #LOSS AND GRADIENT IN NUMPY FORMAT 428 | def loss_and_gradient(weights,N,X,layer_dims,use_sqrt,use_log): 429 | ''' 430 | Parameters 431 | ---------- 432 | weights : NUMPY ARRAY 433 | DESCRIPTION: Trainable parameters in Numpy array format 434 | N : TENSORFLOW MODEL 435 | PINN model 436 | X : TENSOR 437 | Batch of training params 438 | layer_dims : TUPLE 439 | DESCRIPTION: 440 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 441 | if i=0, Li corresponds to the input dimension 442 | use_sqrt : BOOL 443 | DESCRIPTION. If the square root of the MSE residuals is used for training 444 | use_log : BOOL 445 | DESCRIPTION. If the log of the MSE residuals is used for training 446 | 447 | Returns 448 | ------- 449 | loss_value : FLOAT64 (NUMPY) 450 | LOSS USED FOR TRAINING 451 | grads_flat : ARRAY OF FLOAT64 (NUMPY) 452 | Gradients wrt trainable variableS 453 | ''' 454 | resh_weights = nested_tensor(weights,layer_dims) 455 | N.set_weights(resh_weights) 456 | loss_value,grads = loss_and_gradient_TF(N,X,use_sqrt,use_log) 457 | grads_flat = np.concatenate([tf.reshape(g, [-1]).numpy() for g in grads]) 458 | return loss_value.numpy(), grads_flat 459 | 460 | 461 | epochs_bfgs = np.arange(0,Nbfgs+Nprint_bfgs,Nprint_bfgs) #iterations bfgs list 462 | epochs_bfgs+=Adam_epochs 463 | lossbfgs = np.zeros(len(epochs_bfgs)) #loss bfgs list 464 | validation_list = np.zeros(len(epochs_bfgs)) 465 | error_list = np.zeros(len(epochs_bfgs)) #loss bfgs list 466 | 467 | def generate_test(Ny,Nx): 468 | y = np.linspace(y0,yf,Ny) 469 | x = np.linspace(x0,xf,Nx) 470 | x,y = np.meshgrid(x,y) 471 | X = np.hstack((x.flatten()[:,None],y.flatten()[:,None])) 472 | return convert_to_tensor(X),x,y 473 | 474 | 475 | Xtest,x,y = generate_test(Nx,Ny) 476 | pteor = usol(x, y) 477 | cont=0 478 | def callback(*,intermediate_result): #Callback function, to obtain the loss at every iteration 479 | global N,cont,lossbfgs,Xtest,x,error_list,pteor,Nprint_bfgs 480 | ''' 481 | if cont%Nsave == 0: 482 | N.save(fname.format(cont+Nepochs)) 483 | ''' 484 | if (cont+1)%Nprint_bfgs == 0 or cont == 0: 485 | if use_sqrt: 486 | loss_value = np.power(intermediate_result.fun,2) 487 | elif use_log: 488 | loss_value = np.exp(intermediate_result.fun) 489 | else: 490 | loss_value = intermediate_result.fun 491 | lossbfgs[(cont+1)//Nprint_bfgs] = loss_value 492 | 493 | ptest = output(N,Xtest).numpy().reshape(x.shape) 494 | error = np.linalg.norm(ptest-pteor)/np.linalg.norm(pteor) 495 | error_list[(cont+1)//Nprint_bfgs] = error 496 | #Bk = intermediate_result.hess_inv 497 | #maxlamb = np.append(maxlamb,np.max(np.linalg.eig(Bk)[0])) 498 | #minlamb = np.append(minlamb,np.min(np.linalg.eig(Bk)[0])) 499 | print(cont+1,loss_value,error) 500 | cont+=1 501 | 502 | if method == "BFGS": 503 | method_bfgs = method_bfgs 504 | initial_scale=False 505 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 506 | H0 = H0.numpy() 507 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 508 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 509 | 510 | elif method == "bfgsr": 511 | R0 = sparse.csr_matrix(np.eye(len(initial_weights))) 512 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 513 | 514 | elif method == "bfgsz": 515 | Z0 = tf.eye(len(initial_weights),dtype=tf.float64) 516 | Z0 = Z0.numpy() 517 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 518 | 519 | #------------------------- BFGS TRAINING -------------------------------------- 520 | while cont < Nbfgs: #Training loop 521 | result = minimize(loss_and_gradient,initial_weights, args = (N,X,layer_dims,use_sqrt,use_log), 522 | method=method,jac=True, options=options, 523 | tol=0,callback=callback) 524 | initial_weights = result.x 525 | 526 | if method=="BFGS": 527 | H0 = result.hess_inv 528 | H0 = (H0 + np.transpose(H0))/2 529 | try: 530 | cholesky(H0) 531 | except LinAlgError: 532 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 533 | H0 = H0.numpy() 534 | 535 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 536 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 537 | 538 | elif method=="bfgsr": 539 | R0 = result.r_inv 540 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 541 | 542 | elif method == "bfgsz": 543 | Z0 = result.Z 544 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 545 | 546 | X = adaptive_rad(N, Nint, rad_args) 547 | 548 | 549 | if use_sqrt: 550 | fname_loss = f"NLP_{k}_loss_{method}_{method_bfgs}_sqrt.txt" 551 | fname_error = f"NLP_{k}_error_{method}_{method_bfgs}_sqrt.txt" 552 | elif use_log: 553 | fname_loss = f"NLP_{k}_loss_{method}_{method_bfgs}_log.txt" 554 | fname_error = f"NLP_{k}_error_{method}_{method_bfgs}_log.txt" 555 | else: 556 | fname_loss = f"NLP_{k}_loss_{method}_{method_bfgs}.txt" 557 | fname_error = f"NLP_{k}_error_{method}_{method_bfgs}.txt" 558 | 559 | np.savetxt(fname_loss,np.c_[epochs_bfgs,lossbfgs]) 560 | np.savetxt(fname_error,np.c_[epochs_bfgs,error_list]) 561 | -------------------------------------------------------------------------------- /Examples/NLP/NLP_hparams.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Dec 10 20:53:00 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import json 9 | 10 | def hyperparameter_configuration(): 11 | 12 | seed={"seed":2} 13 | architecture_hparams={"neurons":30, 14 | "layers":2, 15 | "output_dim":1 16 | } 17 | PDE_hparams={"k":4. 18 | } 19 | 20 | Adam_hparams={"Adam_epochs":10000, 21 | "lr0":5e-3, 22 | "decay_steps":1000, 23 | "decay_rate":0.98, 24 | "b1":0.99, 25 | "b2":0.999, 26 | "epsilon":1e-20, 27 | "Nprint_adam":100 28 | } 29 | 30 | batch_hparams={"Nint":8000, 31 | "Nchange":500, 32 | "k1":0., 33 | "k2":1., 34 | "x0":0., 35 | "y0":0., 36 | "Lx":1., 37 | "Ly":1. 38 | } 39 | 40 | bfgs_hparams={"BFGS_epochs":20000, 41 | "method":"BFGS", 42 | "method_bfgs":"BFGS", 43 | "use_sqrt":False, 44 | "use_log":True, 45 | "Nprint_bfgs":100} 46 | 47 | test_hparams={"Nx":1000, 48 | "Ny":1000} 49 | 50 | hparams={ 51 | "seed":seed, 52 | "architecture_hparams":architecture_hparams, 53 | "PDE_hparams":PDE_hparams, 54 | "Adam_hparams":Adam_hparams, 55 | "batch_hparams":batch_hparams, 56 | "test_hparams":test_hparams, 57 | "bfgs_hparams":bfgs_hparams} 58 | 59 | # Guarda los hiperparámetros en un archivo JSON 60 | with open('config_NLP.json', 'w') as params: 61 | json.dump(hparams, params, indent=4) 62 | 63 | print("Hiperparámetros guardados en 'config_NLP.json'.") 64 | 65 | hyperparameter_configuration() -------------------------------------------------------------------------------- /Examples/NLS/NLS.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jorgeurban/self_scaled_algorithms_pinns/a4e7bc1bd00b89880ab9361712b371327645f82c/Examples/NLS/NLS.mat -------------------------------------------------------------------------------- /Examples/NLS/NLS.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Dec 10 21:02:18 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import numpy as np 9 | import tensorflow as tf 10 | from tensorflow import keras 11 | from tensorflow.keras.layers import Input, Dense 12 | from tensorflow.keras.models import Model 13 | from tensorflow import convert_to_tensor 14 | from tensorflow.keras.optimizers import Adam 15 | from scipy.optimize import minimize 16 | from scipy.linalg import cholesky,LinAlgError 17 | from scipy import sparse 18 | from scipy.io import loadmat 19 | import json 20 | 21 | def load_hparams(file_config): 22 | with open(file_config, 'r') as file: 23 | config = json.load(file) 24 | return config 25 | 26 | config = load_hparams('config_NLS.json') 27 | 28 | seed = config["seed"]["seed"] #Seed 29 | 30 | #--------------- Architecture hyperparameters ---------------------------------- 31 | neurons = config["architecture_hparams"]["neurons"] #Neurons in every hidden layer 32 | layers = config["architecture_hparams"]["layers"] #Hidden layers 33 | output_dim = config["architecture_hparams"]["output_dim"] #Output dimension 34 | kmax = config["architecture_hparams"]["kmax"] #Maximum wavelength if Fourier inputs are considered 35 | #------------------------------------------------------------------------------- 36 | 37 | #-------------- Adam hyperparameters ------------------------------------------- 38 | Adam_epochs = config["Adam_hparams"]["Adam_epochs"] #Adam epochs 39 | lr0 = config["Adam_hparams"]["lr0"] #Initial learning rate (We consider an exponential lr schedule) 40 | decay_steps = config["Adam_hparams"]["decay_steps"] #Decay steps 41 | decay_rate = config["Adam_hparams"]["decay_rate"] #Decay rate 42 | b1 = config["Adam_hparams"]["b1"] #beta1 43 | b2 = config["Adam_hparams"]["b2"] #beta2 44 | epsilon= config["Adam_hparams"]["epsilon"] #epsilon 45 | Nprint_adam = config["Adam_hparams"]["Nprint_adam"] #Adam results will be printed and save every Nprint_adam iters 46 | #------------------------------------------------------------------------------ 47 | 48 | #------------ Batch hyperparameters ------------------------------------------- 49 | Nint = config["batch_hparams"]["Nint"] #Number of points at batch 50 | Nchange=config["batch_hparams"]["Nchange"] #Batch is changed every Nchange iterations 51 | k1 = config["batch_hparams"]["k1"] #k hyperparameter (see adaptive_rad function below) 52 | k2 = config["batch_hparams"]["k2"] #c hyperparameter (see adaptive_rad function below) 53 | x0 = config["batch_hparams"]["x0"] #x0 (minimum value of x) 54 | Lx = config["batch_hparams"]["Lx"] #Lx (length in the x direction) 55 | t0 = config["batch_hparams"]["t0"] 56 | tfinal = config["batch_hparams"]["tfinal"] 57 | #------------------------------------------------------------------------------ 58 | 59 | #------------ Quasi-Newton (QN) hyperparameters ------------------------------- 60 | Nbfgs = config["bfgs_hparams"]["BFGS_epochs"] #Number of QN iterations 61 | method = config["bfgs_hparams"]["method"] #Method. See below 62 | method_bfgs = config["bfgs_hparams"]["method_bfgs"] #Quasi-Newton algorithm. See below 63 | use_sqrt = config["bfgs_hparams"]["use_sqrt"] #Use square root of the MSE loss to train 64 | use_log = config["bfgs_hparams"]["use_log"] #Use log of the MSE loss to train 65 | Nprint_bfgs = config["bfgs_hparams"]["Nprint_bfgs"] #QN results will be printed and save every Nprint_adam iters 66 | 67 | #In method, you can choose between: 68 | # -BFGS: Here, we include BFGS and the different self-scaled QN methods. 69 | # To distinguish between these algorithms, we use method_bfgs. See below 70 | # -bfgsr: Personal implementation of the factored BFGS Hessian approximations. 71 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 72 | # Very slow, to be optimized. 73 | # -bfgsz: Personal implementation of the factored inverse BFGS Hessian approximations. 74 | # See https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf for details 75 | # Comparable with BFGS in terms of speed. 76 | 77 | #If method=BFGS, the variable "method_bfgs" chooses the different QN methods. 78 | #The options for this are (see the modified Scipy optimize script): 79 | #-BFGS_scipy: The original implementation of BFGS of Scipy 80 | #-BFGS: Equivalent implementation, but faster (avoid repeated calculations in the BFGS formula) 81 | #-SSBFGS_AB: The Self-scaled BFGS formula, where the tauk coefficient is calculated with 82 | # Al-Baali's formula (Formula 11 of "Unveiling the optimization process in PINNs") 83 | #-SSBFGS_OL Same, but tauk is calculated with the original choice of Oren and Luenberger (not recommended) 84 | #-SSBroyden2: Here we use the tauk and phik expressions defined in the paper 85 | # (Formulas 13-23 of "Unveiling the optimization process in PINNs") 86 | #-SSbroyden1: Another possible choice for these parameters (sometimes better, sometimes worse than SSBroyden1) 87 | 88 | #------------------------------------------------------------------------------ 89 | xf = x0 + Lx 90 | tf.keras.backend.set_floatx('float64') 91 | tf.get_logger().setLevel('ERROR') 92 | tf.keras.utils.set_random_seed(seed) 93 | 94 | class PeriodicC0Layer(keras.layers.Layer): 95 | def __init__(self, **kwargs): 96 | super(PeriodicC0Layer, self).__init__(**kwargs) 97 | 98 | def call(self, inputs): 99 | # Assuming inputs is of shape (batch_size, 2) where inputs[:, 0] is t and inputs[:, 1] is x 100 | t = inputs[:, 0,None] # Extract t 101 | x = inputs[:, 1,None] # Extract x 102 | ks = tf.convert_to_tensor(np.arange(1,kmax+1),dtype=tf.float64) 103 | Xper = 2*np.pi*tf.matmul(x,ks[None,:])/Lx 104 | xcos = tf.math.cos(Xper) 105 | xsin = tf.math.sin(Xper) 106 | Xper = tf.concat([xcos,xsin],axis=1) 107 | Xtot = tf.concat([t,Xper],axis=1) 108 | return Xtot 109 | 110 | def generate_model(layer_dims): 111 | ''' 112 | Parameters 113 | ---------- 114 | layer_dims : TUPLE 115 | DESCRIPTION. 116 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 117 | if i=0, Li corresponds to the input dimension 118 | Returns 119 | ------- 120 | Model 121 | TYPE : TENSORFLOW MODEL 122 | DESCRIPTION. 123 | 124 | ''' 125 | X_input = Input((layer_dims[0],)) 126 | X = PeriodicC0Layer()(X_input) 127 | X = Dense(layer_dims[1],activation="tanh")(X) 128 | if len(layer_dims) > 3: 129 | for i in range(2,len(layer_dims)-1): 130 | X = Dense(layer_dims[i],activation="tanh")(X) 131 | X = Dense(layer_dims[-1],activation=None)(X) 132 | return Model(inputs=X_input,outputs=X) 133 | 134 | def generate_inputs(Nint): 135 | ''' 136 | 137 | Parameters 138 | ---------- 139 | Nint : INTEGER 140 | DESCRIPTION. 141 | Number of training points in a given batch 142 | 143 | Returns 144 | ------- 145 | X: Batch of points (in Tensorflow format) 146 | TYPE : TENSOR 147 | DESCRIPTION. 148 | 149 | ''' 150 | t = (tfinal-t0)*np.random.rand(Nint) + t0 151 | x = Lx*np.random.rand(Nint) + x0 152 | X = np.hstack((t[:,None],x[:,None])) 153 | return convert_to_tensor(X) 154 | 155 | #-----------ADAPTIVE RAD GENERATOR (GIVEN AT WU ET AL., 2023)------------------ 156 | def adaptive_rad(N,Nint,rad_args,Ntest=50000): 157 | ''' 158 | Parameters 159 | ---------- 160 | N : TENSORFLOW MODEL 161 | DESCRIPTION. 162 | PINN model, obtained with generate_model() function 163 | 164 | Nint : INTEGER 165 | DESCRIPTION. 166 | Number of training points in a given batch 167 | 168 | rad_args: TUPLE (k1,k2) 169 | DESCRIPTION. 170 | Adaptive resampling of Wu et al. (2023), formula (2) 171 | k1: k 172 | k2: c 173 | DOI: https://doi.org/10.1016/j.cma.2022.115671 174 | 175 | Ntest: INTEGER 176 | DESCRIPTION. 177 | Number of test points to do the resampling 178 | 179 | Returns 180 | ------- 181 | X: Batch of points (in Tensorflow format) 182 | TYPE 183 | DESCRIPTION. 184 | ''' 185 | Xtest = generate_inputs(Ntest) 186 | k1,k2 = rad_args 187 | Y = tf.math.abs(get_results(N,Xtest)[-1]).numpy() 188 | err_eq = np.power(Y,k1)/np.power(Y,k1).mean() + k2 189 | err_eq_normalized = (err_eq / sum(err_eq)) 190 | X_ids = np.random.choice(a=len(Xtest), size=Nint, replace=False, 191 | p=err_eq_normalized) 192 | return tf.gather(Xtest,X_ids) 193 | 194 | #---------------- PINN OUTPUT ------------------------------------------------- 195 | def output(N,X): 196 | ''' 197 | Parameters 198 | ---------- 199 | N : TENSORFLOW MODEL 200 | PINN model, obtained with generate_model() function 201 | X : TENSOR 202 | Batch of points (in Tensorflow format) 203 | Returns 204 | ------- 205 | u: TENSOR 206 | PINN prediction for u 207 | v: TENSOR 208 | PINN prediction for v 209 | 210 | ''' 211 | t = X[:,0,None] 212 | x = X[:,1,None] 213 | Nout = N(X) 214 | u = 2/tf.math.cosh(x) + t*Nout[:,0,None] 215 | v = t*Nout[:,1,None] 216 | return u,v 217 | 218 | #-------------------- PDEs ---------------------------------------------------- 219 | def get_results(N,X): 220 | ''' 221 | Parameters 222 | ---------- 223 | N : TENSORFLOW MODEL 224 | PINN model, obtained with generate_model() function 225 | X : TENSOR 226 | Batch of points (in Tensorflow format) 227 | 228 | Returns 229 | ------- 230 | u : TENSOR 231 | PINN prediction for u 232 | fu : TENSOR 233 | PDE residuals for u_t = ... 234 | 235 | v : TENSOR 236 | PINN prediction for v 237 | fv : TENSOR 238 | PDE residuals for v_t = ... 239 | 240 | ''' 241 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt1: 242 | gt1.watch(X) 243 | 244 | with tf.GradientTape(persistent=True, watch_accessed_variables=False) as gt2: 245 | gt2.watch(X) 246 | 247 | # Calculate u,v 248 | u,v = output(N,X) 249 | 250 | ugrad = gt2.gradient(u, X) 251 | vgrad = gt2.gradient(v, X) 252 | u_t = ugrad[:,0] 253 | u_x = ugrad[:,1] 254 | v_t = vgrad[:,0] 255 | v_x = vgrad[:,1] 256 | 257 | u_xx = gt1.gradient(u_x, X)[:,1] 258 | v_xx = gt1.gradient(v_x, X)[:,1] 259 | hmod = u**2 + v**2 260 | fu = u_t + v_xx/2 + v[:,0]*hmod[:,0] 261 | fv = v_t - u_xx/2 - u[:,0]*hmod[:,0] 262 | return u,v,fu,fv 263 | 264 | 265 | #------------------------------- LOSS FUNCTION -------------------------------- 266 | loss_function = keras.losses.MeanSquaredError() #MSE loss 267 | def loss(fu,fv): 268 | ''' 269 | Parameters 270 | ---------- 271 | fu : TENSOR 272 | PDE residuals for u_t = ... 273 | 274 | fv : TENSOR 275 | PDE residuals for v_t = ... 276 | Returns 277 | ------- 278 | LOSS 279 | TYPE: FLOAT64 280 | MSE Loss of PDE residuals 281 | 282 | ''' 283 | Ntot = fu.shape[0] 284 | zeros = tf.zeros([Ntot,1],dtype=tf.float64) 285 | return loss_function(fu,zeros) + loss_function(fv,zeros) 286 | 287 | #---------------------- Gradients wrt the trainable parameters ---------------- 288 | def grads(N,X): 289 | ''' 290 | Parameters 291 | ---------- 292 | N : TENSORFLOW MODEL 293 | PINN model, obtained with generate_model() function 294 | X : TENSOR 295 | Batch of points (in Tensorflow format) 296 | 297 | Returns 298 | ------- 299 | gradsN : TENSOR 300 | Gradients of loss wrt trainable variables 301 | loss_value: FLOAT64 302 | MSE Loss of PDE residuals 303 | 304 | ''' 305 | with tf.GradientTape() as tape2: 306 | _,_,fu,fv = get_results(N,X) 307 | loss_value = loss(fu,fv) 308 | gradsN = tape2.gradient(loss_value,N.trainable_variables) 309 | return gradsN,loss_value 310 | 311 | #-------------------------TRAINING STEP --------------------------------------- 312 | @tf.function(jit_compile=True) #Precompile training function to accelerate the process 313 | def training(N,X,optimizer): #Training step function 314 | ''' 315 | Parameters 316 | ---------- 317 | N : TENSORFLOW MODEL 318 | PINN model, obtained with generate_model() function 319 | X : TENSOR 320 | Batch of points (in Tensorflow format) 321 | 322 | Optimizer: TENSORFLOW OPTIMIZER 323 | Tensorflow optimizer (Adam, Nadam,...) 324 | 325 | Returns 326 | ------- 327 | loss_value: FLOAT64 328 | MSE Loss of PDE residuals 329 | ''' 330 | parameter_gradients,loss_value = grads(N,X) 331 | optimizer.apply_gradients(zip(parameter_gradients,N.trainable_variables)) 332 | return loss_value 333 | 334 | rad_args = (k1,k2) 335 | epochs = np.arange(Nprint_adam,Adam_epochs+Nprint_adam,Nprint_adam) 336 | loss_list = np.zeros(len(epochs)) #loss list 337 | X = generate_inputs(Nint) 338 | 339 | layer_dims = [None]*(layers + 2) 340 | layer_dims[0] = X.shape[1] 341 | for i in range(1,len(layer_dims)): 342 | layer_dims[i] = neurons 343 | layer_dims[-1] = output_dim 344 | 345 | N = generate_model(layer_dims) 346 | 347 | lr = tf.keras.optimizers.schedules.ExponentialDecay(lr0,decay_steps,decay_rate) 348 | 349 | optimizer = Adam(lr,b1,b2,epsilon=epsilon) 350 | template = 'Epoch {}, loss: {}' 351 | 352 | #------------------- Adam TRAINING LOOP --------------------------------------- 353 | for i in range(Adam_epochs): 354 | if (i+1)%Nchange == 0: 355 | X = adaptive_rad(N, Nint, rad_args) 356 | #X = random_permutation(X) 357 | if (i+1)%Nprint_adam == 0: 358 | _,_,fu,fv = get_results(N,X) 359 | loss_value = loss(fu,fv) 360 | print("i=",i+1) 361 | print(template.format(i+1,loss_value)) 362 | loss_list[i//Nprint_adam] = loss_value.numpy() 363 | training(N,X,optimizer) 364 | 365 | np.savetxt("loss_adam_NLS.txt",np.c_[epochs,loss_list]) 366 | initial_weights = np.concatenate([tf.reshape(w, [-1]).numpy() \ 367 | for w in N.weights]) #initial set of trainable variables 368 | layer_dims[0] = 1 + 2*kmax 369 | 370 | def nested_tensor(tparams,layer_dims): 371 | ''' 372 | 373 | Parameters 374 | ---------- 375 | tparams : NUMPY ARRAY 376 | DESCRIPTION: Trainable parameters in Numpy array format 377 | layer_dims : TUPLE 378 | DESCRIPTION: 379 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 380 | if i=0, Li corresponds to the input dimension 381 | 382 | Returns 383 | ------- 384 | temp : LIST 385 | List of tensors (Trainable variables in Tensorflow format) 386 | 387 | ''' 388 | temp = [None]*(2*len(layer_dims)-2) 389 | index = 0 390 | for i in range(len(temp)): 391 | if i%2==0: 392 | temp[i] = np.reshape(tparams[index:index+layer_dims[i//2]*\ 393 | layer_dims[i//2 +1]],(layer_dims[i//2], 394 | layer_dims[i//2 +1])) 395 | index+=layer_dims[i//2]*layer_dims[i//2 +1] 396 | else: 397 | temp[i] = tparams[index:index+layer_dims[i-i//2]] 398 | index+=layer_dims[i-i//2] 399 | return temp 400 | 401 | @tf.function 402 | def loss_and_gradient_TF(N,X,use_sqrt,use_log): 403 | ''' 404 | Parameters 405 | ---------- 406 | N : TENSORFLOW MODEL 407 | PINN model, obtained with generate_model() function 408 | X : TENSOR 409 | Batch of points (in Tensorflow format) 410 | use_sqrt: BOOL 411 | If the square root of the MSE residuals is used for training 412 | use_log: BOOL 413 | If the logarithm of the MSE residuals is used for training 414 | 415 | Returns 416 | ------- 417 | loss_value : FLOAT64 418 | LOSS USED FOR TRAINING 419 | gradsN: TENSOR 420 | Gradients wrt trainable variables 421 | 422 | ''' 423 | with tf.GradientTape() as tape: 424 | _,_,fu,fv = get_results(N,X) 425 | if use_sqrt: 426 | loss_value = tf.math.sqrt(loss(fu,fv)) 427 | elif use_log: 428 | loss_value = tf.math.log(loss(fu,fv)) 429 | else: 430 | loss_value = loss(fu,fv) 431 | gradsN = tape.gradient(loss_value,N.trainable_variables) 432 | return loss_value,gradsN 433 | 434 | #LOSS AND GRADIENT IN NUMPY FORMAT 435 | def loss_and_gradient(weights,N,X,layer_dims,use_sqrt,use_log): 436 | ''' 437 | Parameters 438 | ---------- 439 | weights : NUMPY ARRAY 440 | DESCRIPTION: Trainable parameters in Numpy array format 441 | N : TENSORFLOW MODEL 442 | PINN model 443 | X : TENSOR 444 | Batch of training params 445 | layer_dims : TUPLE 446 | DESCRIPTION: 447 | (L0,L1,...,Ln), where Li is the number of neurons at ith layer 448 | if i=0, Li corresponds to the input dimension 449 | use_sqrt : BOOL 450 | DESCRIPTION. If the square root of the MSE residuals is used for training 451 | use_log : BOOL 452 | DESCRIPTION. If the log of the MSE residuals is used for training 453 | 454 | Returns 455 | ------- 456 | loss_value : FLOAT64 (NUMPY) 457 | LOSS USED FOR TRAINING 458 | grads_flat : ARRAY OF FLOAT64 (NUMPY) 459 | Gradients wrt trainable variableS 460 | ''' 461 | resh_weights = nested_tensor(weights,layer_dims) 462 | N.set_weights(resh_weights) 463 | loss_value,grads = loss_and_gradient_TF(N,X,use_sqrt,use_log) 464 | grads_flat = np.concatenate([tf.reshape(g, [-1]).numpy() for g in grads]) 465 | return loss_value.numpy(), grads_flat 466 | 467 | 468 | data_sol = loadmat('NLS.mat') 469 | t = data_sol['t'].flatten()[:,None] 470 | x = data_sol['x'].flatten()[:,None] 471 | ustar = np.real(data_sol['usol']).T 472 | vstar = np.imag(data_sol['usol']).T 473 | psistar = np.sqrt(ustar**2 + vstar**2) 474 | 475 | x, t = np.meshgrid(x,t) 476 | 477 | tflat = t.flatten()[:,None] 478 | xflat = x.flatten()[:,None] 479 | Xstar = np.hstack((tflat, xflat)) 480 | epochs_bfgs = np.arange(0,Nbfgs+Nprint_bfgs,Nprint_bfgs) #iterations bfgs list 481 | epochs_bfgs+=Adam_epochs 482 | lossbfgs = np.zeros(len(epochs_bfgs)) #loss bfgs list 483 | error_list = np.zeros(len(lossbfgs)) 484 | 485 | cont=0 486 | def callback(*,intermediate_result): 487 | global N,cont,lossbfgs,psistar,Xstar,x,Nprint_bfgs 488 | if (cont+1)%Nprint_bfgs == 0 or cont == 0: 489 | if use_sqrt: 490 | loss_value = np.power(intermediate_result.fun,2) 491 | elif use_log: 492 | loss_value = np.exp(intermediate_result.fun) 493 | else: 494 | loss_value = intermediate_result.fun 495 | lossbfgs[(cont+1)//Nprint_bfgs] = loss_value 496 | 497 | utest,vtest = output(N, Xstar) 498 | utest = utest.numpy().reshape(x.shape) 499 | vtest = vtest.numpy().reshape(x.shape) 500 | psitest = np.sqrt(utest**2 + vtest**2) 501 | error = np.linalg.norm(psitest-psistar)/np.linalg.norm(psistar) 502 | error_list[(cont+1)//Nprint_bfgs] = error 503 | 504 | print(loss_value,error,cont+1) 505 | cont+=1 506 | 507 | if method == "BFGS": 508 | method_bfgs = method_bfgs 509 | initial_scale=False 510 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 511 | H0 = H0.numpy() 512 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 513 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 514 | 515 | elif method == "bfgsr": 516 | R0 = sparse.csr_matrix(np.eye(len(initial_weights))) 517 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 518 | 519 | elif method == "bfgsz": 520 | Z0 = tf.eye(len(initial_weights),dtype=tf.float64) 521 | Z0 = Z0.numpy() 522 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 523 | 524 | #------------------------- BFGS TRAINING -------------------------------------- 525 | while cont < Nbfgs: #Training loop 526 | result = minimize(loss_and_gradient,initial_weights, args = (N,X,layer_dims,use_sqrt,use_log), 527 | method=method,jac=True, options=options, 528 | tol=0,callback=callback) 529 | initial_weights = result.x 530 | 531 | if method=="BFGS": 532 | H0 = result.hess_inv 533 | H0 = (H0 + np.transpose(H0))/2 534 | try: 535 | cholesky(H0) 536 | except LinAlgError: 537 | H0 = tf.eye(len(initial_weights),dtype=tf.float64) 538 | H0 = H0.numpy() 539 | 540 | options={'maxiter':Nchange, 'gtol': 0, "hess_inv0":H0, 541 | "method_bfgs":method_bfgs, "initial_scale":initial_scale} 542 | 543 | elif method=="bfgsr": 544 | R0 = result.r_inv 545 | options={"maxiter":Nchange,"gtol":0, "r_inv0":R0} 546 | 547 | elif method == "bfgsz": 548 | Z0 = result.Z 549 | options={"maxiter":Nchange,"gtol":0, "Z0":Z0} 550 | 551 | X = adaptive_rad(N, Nint, rad_args) 552 | 553 | 554 | if use_sqrt: 555 | fname_loss = f"NLS_loss_{method}_{method_bfgs}_sqrt.txt" 556 | fname_error = f"NLS_error_{method}_{method_bfgs}_sqrt.txt" 557 | elif use_log: 558 | fname_loss = f"NLS_loss_{method}_{method_bfgs}_log.txt" 559 | fname_error = f"NLS_error_{method}_{method_bfgs}_log.txt" 560 | else: 561 | fname_loss = f"NLS_loss_{method}_{method_bfgs}.txt" 562 | fname_error = f"NLS_error_{method}_{method_bfgs}.txt" 563 | 564 | np.savetxt(fname_loss,np.c_[epochs_bfgs,lossbfgs]) 565 | np.savetxt(fname_error,np.c_[epochs_bfgs,error_list]) 566 | 567 | -------------------------------------------------------------------------------- /Examples/NLS/NLS_hparams.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on Tue Dec 10 21:32:10 2024 4 | 5 | @author: USUARIO 6 | """ 7 | 8 | import json 9 | import numpy as np 10 | 11 | def hyperparameter_configuration(): 12 | 13 | seed={"seed":2} 14 | architecture_hparams={"neurons":40, 15 | "layers":2, 16 | "output_dim":2, 17 | "kmax":1 18 | } 19 | 20 | Adam_hparams={"Adam_epochs":10000, 21 | "lr0":5e-3, 22 | "decay_steps":1000, 23 | "decay_rate":0.98, 24 | "b1":0.99, 25 | "b2":0.999, 26 | "epsilon":1e-20, 27 | "Nprint_adam":100 28 | } 29 | 30 | batch_hparams={"Nint":10000, 31 | "Nchange":500, 32 | "k1":1., 33 | "k2":1., 34 | "x0":-15., 35 | "t0":0., 36 | "Lx":30., 37 | "tfinal":np.pi/2 38 | } 39 | 40 | bfgs_hparams={"BFGS_epochs":20000, 41 | "method":"BFGS", 42 | "method_bfgs":"SSBroyden2", 43 | "use_sqrt":False, 44 | "use_log":False, 45 | "Nprint_bfgs":100} 46 | 47 | hparams={ 48 | "seed":seed, 49 | "architecture_hparams":architecture_hparams, 50 | "Adam_hparams":Adam_hparams, 51 | "batch_hparams":batch_hparams, 52 | "bfgs_hparams":bfgs_hparams} 53 | 54 | # Guarda los hiperparámetros en un archivo JSON 55 | with open('config_NLS.json', 'w') as params: 56 | json.dump(hparams, params, indent=4) 57 | 58 | print("Hiperparámetros guardados en 'config_NLS.json'.") 59 | 60 | hyperparameter_configuration() -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2024, Jorge F. Urbán. University of Alicante 4 | 5 | This project contains modified versions of the scripts 6 | "_optimize.py" and "_minimize.py", which are 7 | within "scipy.optimize". While it is based on the original 8 | work of SciPy, this project is NOT officially maintained or 9 | endorsed by the original developers of SciPy. 10 | The use of the name 'SciPy' or any related names should not be 11 | interpreted as endorsement, sponsorship, or support by the SciPy developers. 12 | 13 | Redistribution and use in source and binary forms, with or without 14 | modification, are permitted provided that the following conditions are met: 15 | 16 | 1. Redistributions of source code must retain the above copyright notice, this 17 | list of conditions and the following disclaimer. 18 | 19 | 2. Redistributions in binary form must reproduce the above copyright notice, 20 | this list of conditions and the following disclaimer in the documentation 21 | and/or other materials provided with the distribution. 22 | 23 | 3. Neither the name of the copyright holder nor the names of its 24 | contributors may be used to endorse or promote products derived from 25 | this software without specific prior written permission. 26 | 27 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 28 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 30 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 31 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 33 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 34 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 35 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37 | 38 | Copyright (c) 2001-2002 Enthought, Inc. 2003-2024, SciPy Developers. 39 | All rights reserved. 40 | 41 | This project contains modified versions of the scripts 42 | "_optimize.py" and "_minimize.py". These files contain 43 | original SciPy code. 44 | 45 | Redistribution and use in source and binary forms, with or without 46 | modification, are permitted provided that the following conditions 47 | are met: 48 | 49 | 1. Redistributions of source code must retain the above copyright 50 | notice, this list of conditions and the following disclaimer. 51 | 52 | 2. Redistributions in binary form must reproduce the above 53 | copyright notice, this list of conditions and the following 54 | disclaimer in the documentation and/or other materials provided 55 | with the distribution. 56 | 57 | 3. Neither the name of the copyright holder nor the names of its 58 | contributors may be used to endorse or promote products derived 59 | from this software without specific prior written permission. 60 | 61 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 62 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 63 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 64 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 65 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 66 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 67 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 68 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 69 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 70 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 71 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 72 | 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Quasi-Newton optimization algorithms to enhance PINNs 2 | 3 | This repository includes the implementation of the Self-scaled Quasi-Newton algorithms suggested in: 4 | * [Unveiling the optimization process of Physics Informed Neural Networks: How accurate and competitive can PINNs be?](https://www.sciencedirect.com/science/article/pii/S0021999124009045) 5 | 6 | together with numerous scripts corresponding to the examples discussed in this work. These scripts contain numerous comments, in order to facilitate their use and modification. Other optimization algorithms, as well as other examples, will also be included in the future. 7 | 8 | # Requirements 9 | * The Machine learning frameworks considered here are [Tensorflow](https://www.tensorflow.org/?hl=es-419) and [Keras](https://keras.io/). The specific versions used in this work are 2.10.1 for Tensorflow, and 2.10.0 for Keras. 10 | * [Numpy](https://numpy.org/) and [Scipy](https://scipy.org/) are also needed. This work has used 1.24.1 for Numpy, and 1.12.0 for Scipy. 11 | 12 | # How to use 13 | 14 | * To use the codes associated with each example, please download the `modified_minimize.py` and `modified_optimize.py` files and replace the `_minimize.py` and `_optimize.py` scripts with these two files, respectively. The `_minimize.py` and `_optimize.py` scripts can be found in a folder called `optimize`, which is located within the SciPy folder. 15 | * The examples referenced in [our article](https://www.sciencedirect.com/science/article/pii/S0021999124009045) are within the folder `Examples` and saved in different subfolders. These folders have been named according to the problem they refer to. 16 | * Each of these folders contains two `.py` files: 17 | - **Main file**: Same name as the folder. This is the file used for training. 18 | - **Hyperparameter file**: Same name as the main file, but followed by `_hparams`. In this file, you can choose: 19 | + **Architecture hyperparameters**: Hidden layers, neurons at every hidden layer, and output neurons. 20 | + **PDE parameters (if any)** 21 | + **Adam hyperparameters**: All the hyperparameters related with Adam optimization. See the Main file for details. 22 | + **Batch hyperparameters**: Number of points, number of iterations per batch, and adaptive resampling hyperparameters. We have incorporated here the RAD algorithm introduced in [A comprehensive study of non-adaptive and residual-based adaptive sampling for physics-informed neural networks](https://www.sciencedirect.com/science/article/abs/pii/S0045782522006260). 23 | + **Quasi-Newton hyperparameters**: All the hyperparameters related with Quasi-Newton optimization. See the **Main file** for details. 24 | * Run first the **hyperparameter file**, to generate a `.json` file with all the hyperparameters. Next, run the **main file**. 25 | * Within the different options for the **Quasi-Newton hyperparameters**, we can select the different Self-scaled Quasi-Newton algorithms, as well as other additional algorithms. In order to choose between them we have two different variables: `method` and `method_bfgs`: 26 | - In `method`, you can choose between: 27 | + `BFGS`: Here, we include BFGS and the different self-scaled Quasi-Newton methods. To distinguish between the different Quasi-Newton algorithms, we use `method_bfgs` (see below). 28 | + `bfgsr`: Personal implementation of the factored BFGS Hessian approximations. See [On Recent Developments in BFGS Methods for Unconstrained Optimization](https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf) for details. 29 | + `bfgsz`: Personal implementation of the factored inverse BFGS Hessian approximations. See [On Recent Developments in BFGS Methods for Unconstrained Optimization](https://ccom.ucsd.edu/reports/UCSD-CCoM-22-01.pdf) for details. 30 | - If `method=BFGS`, the variable `method_bfgs` chooses different Quasi-Newton methods. We have implemented: 31 | + `BFGS_scipy`: The original implementation of BFGS of Scipy. 32 | + `BFGS`: Equivalent implementation, but faster. 33 | + `SSBFGS_AB`: The Self-scaled BFGS formula, where the tauk coefficient is introduced originally in [Numerical Experience with a Class of Self-Scaling Quasi-Newton Algorithms](https://link.springer.com/article/10.1023/A:1022608410710) (see also expression 11 of [our article](https://www.sciencedirect.com/science/article/pii/S0021999124009045)). 34 | + `SSBFGS_OL` Same, but tauk is calculated with the original choice of [Self-Scaling Variable Metric (SSVM) Algorithms](https://pubsonline.informs.org/doi/10.1287/mnsc.20.5.845). 35 | + `SSBroyden2`: Here we use the tauk and phik expressions originally introduced in [A Wide Interval for Efficient Self-Scaling Quasi-Newton Algorithms](https://optimization-online.org/2003/08/699/) 36 | (Formulas 13-23 of [our article](https://www.sciencedirect.com/science/article/pii/S0021999124009045)) 37 | + `SSBroyden1`: Another possible choice for these parameters introduced in [A Wide Interval for Efficient Self-Scaling Quasi-Newton Algorithms](https://optimization-online.org/2003/08/699/) (sometimes better, sometimes worse than `SSBroyden2`). 38 | * Finally, you can also choose to train against the square root or the logarithm of the loss function during the Quasi-Newton optimization. To do that, choose `use_sqrt = True` or `use_log = True` respectively in the Hyperparameter file. 39 | 40 | # IMPORTANT 41 | This repository contains modified versions of two scripts from the ‘optimize’ package within the Scipy library [scipy.optimize](https://docs.scipy.org/doc/scipy/reference/optimize.html). Specifically, the scripts named ‘_optimize.py’ and ‘_minimize.py’ have been modified. The original codes can be found in [scipy](https://github.com/scipy/scipy/tree/main/scipy/optimize). 42 | 43 | This project is NOT officially maintained or endorsed by the original developers of SciPy. The use of the name 'SciPy' or any related names should not be interpreted as endorsement, sponsorship, or support by the SciPy developers. 44 | 45 | # License 46 | This project uses the BSD 3-clause license. This work uses modified scripts of Scipy (see `LICENSE.txt`). 47 | 48 | # Citation 49 | ```bibtex 50 | @article{UrbanStefanouPons2025, 51 | title = {Unveiling the optimization process of physics informed neural networks: How accurate and competitive can PINNs be?}, 52 | journal = {Journal of Computational Physics}, 53 | volume = {523}, 54 | pages = {113656}, 55 | year = {2025}, 56 | issn = {0021-9991}, 57 | doi = {https://doi.org/10.1016/j.jcp.2024.113656}, 58 | url = {https://www.sciencedirect.com/science/article/pii/S0021999124009045}, 59 | author = {Jorge F. Urbán and Petros Stefanou and José A. Pons}, 60 | keywords = {Physics-informed neural networks, Optimization algorithms, Non-linear PDEs}, 61 | abstract = {This study investigates the potential accuracy boundaries of physics-informed neural networks, contrasting their approach with previous similar works and traditional numerical methods. We find that selecting improved optimization algorithms significantly enhances the accuracy of the results. Simple modifications to the loss function may also improve precision, offering an additional avenue for enhancement. Despite optimization algorithms having a greater impact on convergence than adjustments to the loss function, practical considerations often favor tweaking the latter due to ease of implementation. On a global scale, the integration of an enhanced optimizer and a marginally adjusted loss function enables a reduction in the loss function by several orders of magnitude across diverse physical problems. Consequently, our results obtained using compact networks (typically comprising 2 or 3 layers of 20-30 neurons) achieve accuracies comparable to finite difference schemes employing thousands of grid points. This study encourages the continued advancement of PINNs and associated optimization techniques for broader applications across various fields.} 62 | } 63 | --------------------------------------------------------------------------------