
%matplotlib inline
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import os, datetime, warnings
print('Last updated on {}'.format(datetime.datetime.now().ctime()))

Last updated on Sun Feb  5 13:22:09 2023


import sns_settings
sns.set_context('paper')
pd.options.display.max_columns = 50
warnings.filterwarnings('ignore')


df_bottle_dic = pd.read_csv('data/bottle_data_DIC_prepared.csv', parse_dates=['Date'],
                            index_col=0, na_values=['<undefined>',-9999.])
df_bottle_dic['log_Chl'] = np.log(df_bottle_dic.Chl)
df_bottle_dic['log_KD490'] = np.log(df_bottle_dic.KD490)


features = ['Depth', 'Temperature', 'Salinity', 'pCO2_atm', 'ADT', 'SST_hires', 'log_KD490']
target = ['DIC']
varlist = features + target
fg = sns.pairplot(df_bottle_dic, vars=varlist, hue='Season', plot_kws={'alpha':0.7}, diag_kind='hist')


data = df_bottle_dic[varlist]
corr_mat = data.corr()
fig, ax = plt.subplots(figsize=(7,7))
_ = sns.heatmap(corr_mat, ax=ax, cmap='vlag', center=0, square=True, annot=True, annot_kws={'fontsize':9})
_ = ax.set_title('Correlation')


from sklearn.model_selection import train_test_split, cross_val_score

data = df_bottle_dic[features + target + ['Season']].dropna()

X = data[features].values
y = data[target].values

X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=data.Season.values, random_state=77)
X.shape, X_train.shape, X_test.shape, y_train.shape, y_test.shape

((4350, 7), (3262, 7), (1088, 7), (3262, 1), (1088, 1))


import tensorflow as tf
from tensorflow import keras

keras.utils.set_random_seed(42) # make things reproducible
n_hidden = 256 # number of nodes in hidden layers
alpha=0.01

model = keras.models.Sequential([
    keras.layers.BatchNormalization(),
    keras.layers.Dense(n_hidden, input_shape=X_train.shape[1:]),
    keras.layers.LeakyReLU(alpha=alpha),
    keras.layers.BatchNormalization(),
    keras.layers.Dense(n_hidden),
    keras.layers.LeakyReLU(alpha=alpha),
    keras.layers.BatchNormalization(),
    keras.layers.Dense(y_train.shape[1])
])

early_stopping_cb = keras.callbacks.EarlyStopping(patience=20, restore_best_weights=True)
model.compile(loss='mean_squared_error', optimizer=keras.optimizers.Adam())
history = model.fit(X_train, y_train, epochs=700, verbose=2, validation_split=0.2, callbacks=[early_stopping_cb])

Epoch 1/700

2023-02-05 13:23:00.252410: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2 AVX AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.

82/82 - 1s - loss: 4295560.5000 - val_loss: 4132539.7500 - 1s/epoch - 15ms/step
Epoch 2/700
82/82 - 0s - loss: 4261573.0000 - val_loss: 4032585.2500 - 227ms/epoch - 3ms/step
Epoch 3/700
82/82 - 0s - loss: 4195029.5000 - val_loss: 3983974.0000 - 234ms/epoch - 3ms/step
Epoch 4/700
82/82 - 0s - loss: 4091591.2500 - val_loss: 3899459.0000 - 228ms/epoch - 3ms/step
Epoch 5/700
82/82 - 0s - loss: 3953473.2500 - val_loss: 3790599.0000 - 218ms/epoch - 3ms/step
Epoch 6/700
82/82 - 0s - loss: 3784571.0000 - val_loss: 3638522.5000 - 212ms/epoch - 3ms/step
Epoch 7/700
82/82 - 0s - loss: 3589534.0000 - val_loss: 3446617.5000 - 213ms/epoch - 3ms/step
Epoch 8/700
82/82 - 0s - loss: 3372989.2500 - val_loss: 3237511.5000 - 218ms/epoch - 3ms/step
Epoch 9/700
82/82 - 0s - loss: 3139982.7500 - val_loss: 3006159.5000 - 213ms/epoch - 3ms/step
Epoch 10/700
82/82 - 0s - loss: 2895169.7500 - val_loss: 2762621.2500 - 213ms/epoch - 3ms/step
Epoch 11/700
82/82 - 0s - loss: 2643437.0000 - val_loss: 2503281.7500 - 215ms/epoch - 3ms/step
Epoch 12/700
82/82 - 0s - loss: 2389205.5000 - val_loss: 2243648.0000 - 226ms/epoch - 3ms/step
Epoch 13/700
82/82 - 0s - loss: 2136912.2500 - val_loss: 2007204.6250 - 221ms/epoch - 3ms/step
Epoch 14/700
82/82 - 0s - loss: 1890236.2500 - val_loss: 1759047.7500 - 228ms/epoch - 3ms/step
Epoch 15/700
82/82 - 0s - loss: 1653044.5000 - val_loss: 1527760.0000 - 226ms/epoch - 3ms/step
Epoch 16/700
82/82 - 0s - loss: 1428074.6250 - val_loss: 1303412.0000 - 222ms/epoch - 3ms/step
Epoch 17/700
82/82 - 0s - loss: 1218087.5000 - val_loss: 1109421.6250 - 222ms/epoch - 3ms/step
Epoch 18/700
82/82 - 0s - loss: 1024863.0000 - val_loss: 928347.6250 - 229ms/epoch - 3ms/step
Epoch 19/700
82/82 - 0s - loss: 850001.0625 - val_loss: 758912.8125 - 215ms/epoch - 3ms/step
Epoch 20/700
82/82 - 0s - loss: 694251.1250 - val_loss: 610982.6875 - 226ms/epoch - 3ms/step
Epoch 21/700
82/82 - 0s - loss: 557734.0625 - val_loss: 489008.0625 - 223ms/epoch - 3ms/step
Epoch 22/700
82/82 - 0s - loss: 440363.3750 - val_loss: 382267.9375 - 218ms/epoch - 3ms/step
Epoch 23/700
82/82 - 0s - loss: 341472.2188 - val_loss: 292852.7812 - 225ms/epoch - 3ms/step
Epoch 24/700
82/82 - 0s - loss: 259549.8750 - val_loss: 222554.6562 - 213ms/epoch - 3ms/step
Epoch 25/700
82/82 - 0s - loss: 193339.5312 - val_loss: 159651.6406 - 220ms/epoch - 3ms/step
Epoch 26/700
82/82 - 0s - loss: 140959.2969 - val_loss: 115116.3281 - 211ms/epoch - 3ms/step
Epoch 27/700
82/82 - 0s - loss: 100500.3750 - val_loss: 80076.5312 - 215ms/epoch - 3ms/step
Epoch 28/700
82/82 - 0s - loss: 69961.7422 - val_loss: 54898.4883 - 213ms/epoch - 3ms/step
Epoch 29/700
82/82 - 0s - loss: 47687.4648 - val_loss: 36964.5977 - 225ms/epoch - 3ms/step
Epoch 30/700
82/82 - 0s - loss: 31720.3457 - val_loss: 24061.6074 - 218ms/epoch - 3ms/step
Epoch 31/700
82/82 - 0s - loss: 20651.9043 - val_loss: 15115.0645 - 219ms/epoch - 3ms/step
Epoch 32/700
82/82 - 0s - loss: 13172.2236 - val_loss: 9261.3828 - 213ms/epoch - 3ms/step
Epoch 33/700
82/82 - 0s - loss: 8262.4893 - val_loss: 5487.0923 - 219ms/epoch - 3ms/step
Epoch 34/700
82/82 - 0s - loss: 5144.9814 - val_loss: 3618.4624 - 218ms/epoch - 3ms/step
Epoch 35/700
82/82 - 0s - loss: 3244.5405 - val_loss: 1845.8739 - 215ms/epoch - 3ms/step
Epoch 36/700
82/82 - 0s - loss: 2064.8386 - val_loss: 1413.0588 - 223ms/epoch - 3ms/step
Epoch 37/700
82/82 - 0s - loss: 1473.8643 - val_loss: 774.7010 - 218ms/epoch - 3ms/step
Epoch 38/700
82/82 - 0s - loss: 1052.0219 - val_loss: 601.1273 - 218ms/epoch - 3ms/step
Epoch 39/700
82/82 - 0s - loss: 818.3077 - val_loss: 630.7568 - 225ms/epoch - 3ms/step
Epoch 40/700
82/82 - 0s - loss: 740.6975 - val_loss: 453.7334 - 218ms/epoch - 3ms/step
Epoch 41/700
82/82 - 0s - loss: 639.7831 - val_loss: 483.4555 - 219ms/epoch - 3ms/step
Epoch 42/700
82/82 - 0s - loss: 654.0411 - val_loss: 621.3655 - 217ms/epoch - 3ms/step
Epoch 43/700
82/82 - 0s - loss: 663.7580 - val_loss: 656.1754 - 215ms/epoch - 3ms/step
Epoch 44/700
82/82 - 0s - loss: 630.9948 - val_loss: 456.1325 - 216ms/epoch - 3ms/step
Epoch 45/700
82/82 - 0s - loss: 586.7437 - val_loss: 474.2964 - 208ms/epoch - 3ms/step
Epoch 46/700
82/82 - 0s - loss: 621.9514 - val_loss: 510.2854 - 213ms/epoch - 3ms/step
Epoch 47/700
82/82 - 0s - loss: 568.8397 - val_loss: 552.4397 - 216ms/epoch - 3ms/step
Epoch 48/700
82/82 - 0s - loss: 598.1824 - val_loss: 531.1625 - 218ms/epoch - 3ms/step
Epoch 49/700
82/82 - 0s - loss: 648.2249 - val_loss: 529.2685 - 216ms/epoch - 3ms/step
Epoch 50/700
82/82 - 0s - loss: 610.4193 - val_loss: 671.7320 - 210ms/epoch - 3ms/step
Epoch 51/700
82/82 - 0s - loss: 674.0334 - val_loss: 498.7827 - 208ms/epoch - 3ms/step
Epoch 52/700
82/82 - 0s - loss: 586.7018 - val_loss: 486.5792 - 211ms/epoch - 3ms/step
Epoch 53/700
82/82 - 0s - loss: 597.3868 - val_loss: 506.1737 - 216ms/epoch - 3ms/step
Epoch 54/700
82/82 - 0s - loss: 603.5836 - val_loss: 489.9623 - 216ms/epoch - 3ms/step
Epoch 55/700
82/82 - 0s - loss: 640.6675 - val_loss: 546.9764 - 215ms/epoch - 3ms/step
Epoch 56/700
82/82 - 0s - loss: 641.6469 - val_loss: 470.7936 - 219ms/epoch - 3ms/step
Epoch 57/700
82/82 - 0s - loss: 583.0955 - val_loss: 421.6942 - 217ms/epoch - 3ms/step
Epoch 58/700
82/82 - 0s - loss: 560.4546 - val_loss: 528.5685 - 217ms/epoch - 3ms/step
Epoch 59/700
82/82 - 0s - loss: 631.6149 - val_loss: 478.0450 - 219ms/epoch - 3ms/step
Epoch 60/700
82/82 - 0s - loss: 544.5762 - val_loss: 509.6238 - 222ms/epoch - 3ms/step
Epoch 61/700
82/82 - 0s - loss: 585.2369 - val_loss: 496.2092 - 222ms/epoch - 3ms/step
Epoch 62/700
82/82 - 0s - loss: 622.8492 - val_loss: 514.6617 - 225ms/epoch - 3ms/step
Epoch 63/700
82/82 - 0s - loss: 585.7975 - val_loss: 558.8933 - 226ms/epoch - 3ms/step
Epoch 64/700
82/82 - 0s - loss: 603.0429 - val_loss: 555.9790 - 230ms/epoch - 3ms/step
Epoch 65/700
82/82 - 0s - loss: 581.4849 - val_loss: 489.4061 - 230ms/epoch - 3ms/step
Epoch 66/700
82/82 - 0s - loss: 539.1697 - val_loss: 486.3523 - 227ms/epoch - 3ms/step
Epoch 67/700
82/82 - 0s - loss: 562.6075 - val_loss: 579.6304 - 228ms/epoch - 3ms/step
Epoch 68/700
82/82 - 0s - loss: 614.6166 - val_loss: 505.0974 - 228ms/epoch - 3ms/step
Epoch 69/700
82/82 - 0s - loss: 567.3275 - val_loss: 513.2411 - 231ms/epoch - 3ms/step
Epoch 70/700
82/82 - 0s - loss: 680.1465 - val_loss: 565.9537 - 224ms/epoch - 3ms/step
Epoch 71/700
82/82 - 0s - loss: 601.1392 - val_loss: 420.8592 - 227ms/epoch - 3ms/step
Epoch 72/700
82/82 - 0s - loss: 626.3896 - val_loss: 444.7740 - 228ms/epoch - 3ms/step
Epoch 73/700
82/82 - 0s - loss: 635.1450 - val_loss: 489.1148 - 212ms/epoch - 3ms/step
Epoch 74/700
82/82 - 0s - loss: 569.4388 - val_loss: 522.0839 - 221ms/epoch - 3ms/step
Epoch 75/700
82/82 - 0s - loss: 559.9667 - val_loss: 602.7838 - 223ms/epoch - 3ms/step
Epoch 76/700
82/82 - 0s - loss: 567.8832 - val_loss: 467.2472 - 220ms/epoch - 3ms/step
Epoch 77/700
82/82 - 0s - loss: 572.0048 - val_loss: 496.6600 - 213ms/epoch - 3ms/step
Epoch 78/700
82/82 - 0s - loss: 594.4030 - val_loss: 525.7438 - 212ms/epoch - 3ms/step
Epoch 79/700
82/82 - 0s - loss: 614.0696 - val_loss: 438.4948 - 213ms/epoch - 3ms/step
Epoch 80/700
82/82 - 0s - loss: 512.9919 - val_loss: 567.3928 - 220ms/epoch - 3ms/step
Epoch 81/700
82/82 - 0s - loss: 665.2734 - val_loss: 530.2236 - 215ms/epoch - 3ms/step
Epoch 82/700
82/82 - 0s - loss: 581.5056 - val_loss: 493.5941 - 222ms/epoch - 3ms/step
Epoch 83/700
82/82 - 0s - loss: 657.0535 - val_loss: 599.4510 - 215ms/epoch - 3ms/step
Epoch 84/700
82/82 - 0s - loss: 580.4384 - val_loss: 539.7863 - 222ms/epoch - 3ms/step
Epoch 85/700
82/82 - 0s - loss: 519.9036 - val_loss: 554.7277 - 213ms/epoch - 3ms/step
Epoch 86/700
82/82 - 0s - loss: 533.4694 - val_loss: 546.8605 - 214ms/epoch - 3ms/step
Epoch 87/700
82/82 - 0s - loss: 580.8466 - val_loss: 609.5256 - 211ms/epoch - 3ms/step
Epoch 88/700
82/82 - 0s - loss: 633.6235 - val_loss: 524.9554 - 218ms/epoch - 3ms/step
Epoch 89/700
82/82 - 0s - loss: 606.5544 - val_loss: 449.8918 - 216ms/epoch - 3ms/step
Epoch 90/700
82/82 - 0s - loss: 648.5673 - val_loss: 448.7724 - 218ms/epoch - 3ms/step
Epoch 91/700
82/82 - 0s - loss: 583.4109 - val_loss: 452.3459 - 214ms/epoch - 3ms/step


model.save('models/nn_regression_dic_noO2.h5')


df_history = pd.DataFrame(history.history)
df_history.index.name = 'epoch'
df_history = df_history.reset_index()
df_history.to_csv('results/nn_regression_history_dic_noO2.csv')

fig, ax = plt.subplots(figsize=(6, 6))
_ = sns.lineplot(x=df_history.epoch-0.5, y='loss', data=df_history, ax=ax, label='training set')
_ = sns.lineplot(x=df_history.epoch, y='val_loss', data=df_history, ax=ax, label='validation set')
_ = ax.set(ylabel = 'MSE')
# _ = ax.set(yscale='log')


from sklearn.metrics import r2_score

print('MSE on training set = {:.2f}'.format(model.evaluate(X_train, y_train, verbose=0)))
print('MSE on test set     = {:.2f}\n'.format(model.evaluate(X_test, y_test, verbose=0)))

y_pred_train = model.predict(X_train)
y_pred_test = model.predict(X_test)

print('R squared on training set = {:.3f}'.format(r2_score(y_train, y_pred_train)))
print('R squared on test set     = {:.3f}'.format(r2_score(y_test, y_pred_test)))

MSE on training set = 308.82
MSE on test set     = 365.76

R squared on training set = 0.952
R squared on test set     = 0.944


fig, ax = plt.subplots(figsize=(6,6))
_ = sns.scatterplot(x=y_test.ravel(), y=y_pred_test.ravel(), ax=ax)
_ = ax.set(xlabel='observed DIC', ylabel='predicted DIC', title='Test dataset')
_ = ax.axis('equal')


# save test set features, target & predictions
df_test = pd.DataFrame(np.c_[X_test, y_test, y_pred_test], columns = features + ['DIC observed', 'DIC predicted'])
df_test['DIC residuals'] = df_test['DIC observed'] - df_test['DIC predicted']
df_test.to_csv('results/bottle_data_test_dic_noO2.csv')


from sklearn.model_selection import KFold

kf = KFold(n_splits=5, shuffle=True, random_state=42)
score_vals = [] # store score values

nn_reg = keras.models.Sequential([
    keras.layers.BatchNormalization(),
    keras.layers.Dense(n_hidden, input_shape=X_train.shape[1:]),
    keras.layers.LeakyReLU(alpha=alpha),
    keras.layers.BatchNormalization(),
    keras.layers.Dense(n_hidden),
    keras.layers.LeakyReLU(alpha=alpha),
    keras.layers.BatchNormalization(),
    keras.layers.Dense(y_train.shape[1])
])
nn_reg.compile(loss='mean_squared_error', optimizer=keras.optimizers.Adam())

for k, (train_idx, test_idx) in enumerate(kf.split(X_train)):
    X_tr, X_te = X_train[train_idx], X_train[test_idx]
    y_tr, y_te = y_train[train_idx], y_train[test_idx]
    history_cv = nn_reg.fit(X_tr, y_tr, epochs=700, verbose=0, validation_split=0.2, callbacks=[early_stopping_cb])
    y_pred = nn_reg.predict(X_te)
    score = r2_score(y_te, y_pred)
    score_vals.append(score)
    print('Fold {} test set R squared: {:.3f}'.format(k+1, score))

scores = np.array(score_vals)
print('\nBest R squared:  {:.3f}'.format(scores.max()))
print('Worst R squared: {:.3f}'.format(scores.min()))
print('Mean R squared:  {:.3f}'.format(scores.mean()))

Fold 1 test set R squared: 0.933
Fold 2 test set R squared: 0.948
Fold 3 test set R squared: 0.951
Fold 4 test set R squared: 0.931
Fold 5 test set R squared: 0.950

Best R squared:  0.951
Worst R squared: 0.931
Mean R squared:  0.943

Neural network model to predict DIC¶

Load DIC bottle data¶

Select input features and target variable¶

Split data into training and test sets¶

Train Neural Network regression¶

Save trained model¶

Learning curve¶

MSE & $R^2$¶

K-fold cross-validation¶