
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import shap, os, datetime, warnings
print('Last updated on {}'.format(datetime.datetime.now().ctime()))

# import sns_settings
sns.set_theme(context='paper', style='ticks')
pd.options.display.max_columns = 100
warnings.filterwarnings('ignore')

Last updated on Fri Jun 23 08:57:17 2023

/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/utils/_clustering.py:35: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def _pt_shuffle_rec(i, indexes, index_mask, partition_tree, M, pos):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/utils/_clustering.py:54: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def delta_minimization_order(all_masks, max_swap_size=100, num_passes=2):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/utils/_clustering.py:63: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def _reverse_window(order, start, length):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/utils/_clustering.py:69: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def _reverse_window_score_gain(masks, order, start, length):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/utils/_clustering.py:77: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def _mask_delta_score(m1, m2):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/links.py:5: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def identity(x):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/links.py:10: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def _identity_inverse(x):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/links.py:15: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def logit(x):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/links.py:20: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def _logit_inverse(x):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/utils/_masked_model.py:363: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def _build_fixed_single_output(averaged_outs, last_outs, outputs, batch_positions, varying_rows, num_varying_rows, link, linearizing_weights):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/utils/_masked_model.py:385: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def _build_fixed_multi_output(averaged_outs, last_outs, outputs, batch_positions, varying_rows, num_varying_rows, link, linearizing_weights):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/utils/_masked_model.py:428: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def _init_masks(cluster_matrix, M, indices_row_pos, indptr):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/utils/_masked_model.py:439: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def _rec_fill_masks(cluster_matrix, indices_row_pos, indptr, indices, M, ind):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/maskers/_tabular.py:186: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def _single_delta_mask(dind, masked_inputs, last_mask, data, x, noop_code):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/maskers/_tabular.py:197: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def _delta_masking(masks, x, curr_delta_inds, varying_rows_out,
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/maskers/_image.py:175: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def _jit_build_partition_tree(xmin, xmax, ymin, ymax, zmin, zmax, total_ywidth, total_zwidth, M, clustering, q):
/home/ivan/mambaforge/envs/py310/lib/python3.10/site-packages/shap/explainers/_partition.py:676: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
  def lower_credit(i, value, M, values, clustering):
The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.
The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.


# df_dic_test = pd.read_csv('results/bottle_data_test_dic_all_vars.csv', index_col=0)
# df_dic_test = df_dic_test.rename(columns={'DIC observed':'DIC'})

# features = ['Depth', 'Temperature', 'Salinity', 'Oxygen', 'pCO2_atm', 'ADT', 'SST_hires', 'log_KD490']
# target = ['DIC']

# X_test = df_dic_test[features]
# X_test.head()


df_bottle_dic = pd.read_csv('data/bottle_data_DIC_prepared.csv', parse_dates=['Date'],
                            index_col=0, na_values=['<undefined>',-9999.])
df_bottle_dic = df_bottle_dic.loc[df_bottle_dic.Oxygen_flag.isin([2, 6])]
df_bottle_dic = df_bottle_dic.loc[df_bottle_dic.Oxygen.notnull()]
df_bottle_dic['log_Chl'] = np.log(df_bottle_dic.Chl)
df_bottle_dic['log_KD490'] = np.log(df_bottle_dic.KD490)

features = ['Depth', 'Temperature', 'Salinity', 'Oxygen', 'pCO2_atm', 'ADT', 'SST_hires', 'log_KD490']

X_dic = df_bottle_dic[features].dropna()


df_bottle_ta = pd.read_csv('data/bottle_data_TA_prepared.csv', parse_dates=['Date'],
                           index_col=0, na_values=['<undefined>',-9999.])
df_bottle_ta = df_bottle_ta.loc[df_bottle_ta.Oxygen_flag.isin([2, 6])]
df_bottle_ta = df_bottle_ta.loc[df_bottle_ta.Oxygen.notnull()]
df_bottle_ta['log_Chl'] = np.log(df_bottle_ta.Chl)
df_bottle_ta['log_KD490'] = np.log(df_bottle_ta.KD490)

X_ta = df_bottle_ta[features].dropna()


import tensorflow as tf
from tensorflow import keras

model_dic = keras.models.load_model('models/nn_regression_dic_all_vars.h5')
model_ta = keras.models.load_model('models/nn_regression_ta_all_vars.h5')

2023-06-23 08:57:17.550751: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2 AVX AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-06-23 08:57:19.912947: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2 AVX AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.


explainer_dic = shap.Explainer(model_dic.predict, X_dic)
shap_values_dic = explainer_dic(X_dic)

788/788 [==============================] - 3s 3ms/step
788/788 [==============================] - 3s 3ms/step
29/29 [==============================] - 0s 3ms/step

Exact explainer:   0%|                                                                       | 2/3970 [00:00<?, ?it/s]

788/788 [==============================] - 3s 3ms/step
29/29 [==============================] - 0s 5ms/step

Exact explainer:   0%|                                                             | 4/3970 [00:15<1:39:06,  1.50s/it]

788/788 [==============================] - 3s 3ms/step
29/29 [==============================] - 0s 4ms/step

Exact explainer:   0%|                                                             | 5/3970 [00:18<2:27:01,  2.22s/it]

788/788 [==============================] - 3s 3ms/step
29/29 [==============================] - 0s 4ms/step

Exact explainer:   0%|                                                             | 6/3970 [00:21<2:54:26,  2.64s/it]

788/788 [==============================] - 3s 3ms/step
29/29 [==============================] - 0s 3ms/step

Exact explainer:   0%|                                                             | 7/3970 [00:27<3:56:59,  3.59s/it]


shap.plots.bar(shap_values_dic)


shap.summary_plot(shap_values_dic)


shap.summary_plot(shap_values_dic, plot_type='violin')


explainer_ta = shap.Explainer(model_ta.predict, X_ta)
shap_values_ta = explainer_ta(X_ta)

788/788 [==============================] - 6s 7ms/step
788/788 [==============================] - 6s 8ms/step
29/29 [==============================] - 0s 7ms/step

Exact explainer:   0%|                                                                       | 2/3797 [00:00<?, ?it/s]

788/788 [==============================] - 6s 8ms/step
29/29 [==============================] - 0s 11ms/step

Exact explainer:   0%|                                                             | 4/3797 [00:25<4:13:30,  4.01s/it]

788/788 [==============================] - 7s 9ms/step
29/29 [==============================] - 0s 7ms/step

Exact explainer:   0%|                                                             | 5/3797 [00:34<6:10:04,  5.86s/it]

788/788 [==============================] - 6s 7ms/step
29/29 [==============================] - 0s 7ms/step

Exact explainer:   0%|                                                             | 6/3797 [00:41<6:35:54,  6.27s/it]

788/788 [==============================] - 7s 9ms/step
29/29 [==============================] - 0s 6ms/step

Exact explainer:   0%|                                                             | 7/3797 [00:52<8:14:11,  7.82s/it]


shap.plots.bar(shap_values_ta)


shap.summary_plot(shap_values_ta)


shap.summary_plot(shap_values_ta, plot_type='violin')

Evaluate feature importance using SHAP values¶

Load test data set¶

Load DIC bottle data¶

Load TA bottle data¶

Load trained neural network models¶

Fit explainer & compute SHAP values¶

DIC¶

TA¶