dfNGP/configs.py at main · kimbente/dfNGP · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
################
### SIM DATA ###
################

# directories (alphabetic order)
dfGP_SIM_RESULTS_DIR = "results_sim/dfGP"
dfGPcm_SIM_RESULTS_DIR = "results_sim/dfGPcm"
dfNGP_SIM_RESULTS_DIR = "results_sim/dfNGP"
dfNN_SIM_RESULTS_DIR = "results_sim/dfNN"
GP_SIM_RESULTS_DIR = "results_sim/GP"
PINN_SIM_RESULTS_DIR = "results_sim/PINN"

# learning rates (alphabetic order)
# NOTE: df is always smallcap.
# NOTE: All lr are the same for SIM
dfGP_SIM_LEARNING_RATE = 0.005
dfGPcm_SIM_LEARNING_RATE = 0.005
dfNGP_SIM_LEARNING_RATE = 0.005 # lr x 0.02 for NN mean function params
dfNN_SIM_LEARNING_RATE = 0.005
GP_SIM_LEARNING_RATE = 0.005
PINN_SIM_LEARNING_RATE = 0.005

# SIM-specific hyperparameters
# test grid resolution
N_SIDE = 20

#################
### REAL DATA ###
#################

# directories (alphabetic order)
dfGP_REAL_RESULTS_DIR = "results_real/dfGP"
dfGPcm_REAL_RESULTS_DIR = "results_real/dfGPcm"
dfNGP_REAL_RESULTS_DIR = "results_real/dfNGP"
dfNN_REAL_RESULTS_DIR = "results_real/dfNN"
GP_REAL_RESULTS_DIR = "results_real/GP"
PINN_REAL_RESULTS_DIR = "results_real/PINN"

# learning rates (alphabetic order)
dfGP_REAL_LEARNING_RATE = 0.005
dfGPcm_REAL_LEARNING_RATE = 0.005
dfNGP_REAL_LEARNING_RATE = 0.005 # lr x 0.2 for NN mean function params
dfNN_REAL_LEARNING_RATE = 0.005
GP_REAL_LEARNING_RATE = 0.005
# NOTE: PINN requires slightly lower lr for smooth descent on real train
PINN_REAL_LEARNING_RATE = 0.001

# infer at higher resolution grid across the domain
# NOTE: for visualisations only, not evaluations
N_SIDE_INFERENCE = 60

################################
### TRAINING HYPERPARAMETERS ###
################################

# Toggle emission tracking with codecarbon on or off
# TRACK_EMISSIONS_BOOL = False
TRACK_EMISSIONS_BOOL = True

# Define how often to print training progress
PRINT_FREQUENCY = 50

NUM_RUNS = 8
# NUM_RUNS = 3
MAX_NUM_EPOCHS = 2000

PATIENCE = 100 # Stop after {PATIENCE} epochs with no improvement
GP_PATIENCE = 50 # NOTE: GP convergence is more smooth so less patience is needed

# WEIGHT_DECAY is L2 regularisation; `decay` because it pulls weights towards 0
# Only for NN parameters, not for GP parameters
WEIGHT_DECAY = 1e-4 # i.e. 0.0001

BATCH_SIZE = 32

# PINN HYPERPARAM (SIM & REAL)
W_PINN_DIV_WEIGHT = 0.3

# FOR SIM
# Noise parameter for training: independent Gaussian noise to perturb inputs
# NOTE: This corresponds to a true noise variance of 0.0004 (contained in initialisation range)
# We do scale by the var of each experiment however (which is slighly <1 or >1)
STD_GAUSSIAN_NOISE = 0.02

##########################################
### DEFAULT/SIM (df)GP HYPERPARAMETERS ###
##########################################
# order: lengthscale, outputscale variance, noise variance

# HYPERPARAMETER 1: Range for lengthscale parameter (l)
# NOTE: This corresponds to a l^2 range of (0.09, 0.49) (domain is [0, 1])
L_RANGE = (0.3, 0.7)

# HYPERPARAMETER 2: Range for outputscale variance parameter (sigma_f^2)
# NOTE: This corresponds to a sigma_f range of (0.64, ~1.22)
OUTPUTSCALE_VAR_RANGE = (0.8, 1.5)

# NOTE: For residual models (i.e. models with non-zero mean function), we use a different range for the outputscale variance, acknowledging that the residuals are smaller than the original data.
OUTPUTSCALE_VAR_RESIDUAL_MODEL_RANGE = (0.1, 0.6)

# For regular GP only, we scale for each task
# NOTE: The multitask GP is parameterised via a covariance factor F, which is used to construct the covariance matrix B together with a TASK Variance D.
# B = (FF^T + D), where D is a diagonal matrix and F is the covar_factor
TASK_COVAR_FACTOR_RANGE = (-0.2, 0.5)

# Define initialisation ranges FOR GP MODELs
# HYPERPARAMETER 3: Range for noise variance parameter (sigma_n^2)
# NOTE: This corresponds to a sigma_n range of (0.01, 0.07)
NOISE_VAR_RANGE = (0.0001, 0.0049)

###################################
### REAL (df)GP HYPERPARAMETERS ###
###################################

# NOTE: Initialise with higher noise variance range for real data
# REAL_NOISE_VAR_RANGE = (0.005, 0.03)
REAL_NOISE_VAR_RANGE = (0.01, 0.025) # try
REAL_OUTPUTSCALE_VAR_RANGE = (0.1, 0.4)