{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\ndata_dir = \"../input/qbi-preprocessing/\"\nresults_dir = \"../input/qbi-train-network/results/\"\nmodels_dir = \"../input/qbi-train-network/models/\"\n    \nprint(\"DATA:\", os.listdir(data_dir))\nprint(\"MODELS\", os.listdir(models_dir))\nprint(\"RESULTS\", os.listdir(results_dir))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle\ncrps_train = pickle.load(open(results_dir + \"crps_train.pickle\", \"rb\"))\ncrps_test = pickle.load(open(results_dir + \"crps_val.pickle\", \"rb\"))\nhistory_val_loss_systole, history_val_loss_diastole = pickle.load(open(models_dir + \"val_loss_history.txt\", \"rb\"))\nval_loss_systole, val_loss_diastole = pickle.load(open(models_dir + \"val_loss.txt\", \"rb\"))\n\nprint(\"Val Loss Systole\", val_loss_systole)\nprint(\"Val Loss Diastole\", val_loss_diastole)\nprint(\"CRPS (train)\", crps_train)\nprint(\"CRPS (val)\", crps_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Validation loss for training"},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=[18,5])\n\nax1.plot(history_val_loss_systole, linewidth=3)\nax1.set_title(\"Systole\")\nax1.set_xlabel(\"Iteration\")\nax1.set_ylabel(\"Validation loss (RMSE)\")\n\nax2.plot(history_val_loss_diastole, linewidth=3, color=\"green\")\nax2.set_title(\"Diastole\")\nax2.set_xlabel(\"Iteration\")\nax2.set_ylabel(\"Validation loss (RMSE)\")\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define model again (to load it)"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D\nfrom keras.layers.core import Activation, Dense, Flatten, Dropout\nfrom keras.optimizers import Adam\nfrom keras.regularizers import l2\nfrom keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def root_mean_squared_error(y_true, y_pred):\n    \"\"\"\n    RMSE loss function\n    \"\"\"\n    return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1))\n\ndef center_normalize(x):\n    \"\"\"\n    Custom activation for online sample-wise center and std. normalization\n    \"\"\"\n    return (x - K.mean(x)) / K.std(x)\n\ndef get_model(input_dims=(30, 64, 64)):\n    model = Sequential()\n    model.add(Activation(activation=center_normalize, input_shape=input_dims))\n\n    model.add(Convolution2D(64, 3, 3, border_mode='same', activation='relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\n    model.add(Dropout(0.25))\n    \n    model.add(Convolution2D(128, 3, 3, border_mode='same', activation='relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\n    model.add(Dropout(0.25))\n    \n    model.add(Flatten())\n    model.add(Dense(256, activation='relu'))\n    model.add(Dense(1))\n\n    model.compile(optimizer=\"adam\", loss=root_mean_squared_error)\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Util functions\n* Functions taken from https://github.com/jocicmarko/kaggle-dsb2-keras"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nfrom scipy.stats import norm\nfrom skimage.restoration import denoise_tv_chambolle\nfrom scipy import ndimage\nfrom keras.utils.generic_utils import Progbar","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crps(true, pred):\n    \"\"\"\n    Calculation of CRPS.\n\n    :param true: true values (labels)\n    :param pred: predicted values\n    \"\"\"\n    return np.sum(np.square(true - pred)) / true.size\n\n\ndef real_to_cdf(y, sigma=1e-10):\n    \"\"\"\n    Utility function for creating CDF from real number and sigma (uncertainty measure).\n\n    :param y: array of real values\n    :param sigma: uncertainty measure. The higher sigma, the more imprecise the prediction is, and vice versa.\n    Default value for sigma is 1e-10 to produce step function if needed.\n    \"\"\"\n    cdf = np.zeros((y.shape[0], 600))\n    for i in range(y.shape[0]):\n        cdf[i] = norm.cdf(np.linspace(0, 599, 600), y[i], sigma)\n    return cdf\n\ndef shuffle(x):\n    indices = np.arange(len(x))\n    np.random.seed(42)\n    np.random.shuffle(indices)\n\n    return x[indices]\n\n\ndef load_val_data(data_dir):\n    \"\"\"\n    Load validation data from .npy files.\n    \"\"\"\n    X = np.load(data_dir + 'X_validate.npz')['arr_0']\n    X = X.astype(np.float32)\n    y = np.load(data_dir + 'y_validate.npz')['arr_0']\n    return shuffle(X), shuffle(y)\n\ndef load_test_data(data_dir):\n    \"\"\"\n    Load validation data from .npy files.\n    \"\"\"\n    X = np.load(data_dir + 'X_test.npz')['arr_0']\n    X = X.astype(np.float32)\n    return shuffle(X)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluate on validation dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load data\nX_val, y_val = load_val_data(data_dir)\n\n# Load Model with saved parameters\nprint('Loading and compiling models...')\ninput_dims = (X_val.shape[1], X_val.shape[2], X_val.shape[3])\nmodel_systole = get_model(input_dims)\nmodel_diastole = get_model(input_dims)\n\nprint('Loading models weights...')\nmodel_systole.load_weights(models_dir + 'weights_systole_best.hdf5')\nmodel_diastole.load_weights(models_dir + 'weights_diastole_best.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_with_cdf(X, loss_systole, loss_diastole, batch_size = 32):\n    # Evaluate on data\n    print('Evaluating CRPS for dataset...')\n    pred_systole = model_systole.predict(X, batch_size=batch_size, verbose=1)\n    pred_diastole = model_diastole.predict(X, batch_size=batch_size, verbose=1)\n\n    # CDF for predicted data\n    cdf_systole = real_to_cdf(pred_systole, loss_systole)\n    cdf_diastole = real_to_cdf(pred_diastole, loss_diastole)\n    \n    return cdf_systole, cdf_diastole","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Val data - compare to labels in CSV"},{"metadata":{"trusted":true},"cell_type":"code","source":"# evaluate CRPS on validation data\ncdf_val_pred_systole, cdf_val_pred_diastole = predict_with_cdf(X_val, val_loss_systole, val_loss_diastole)\ncdf_val = real_to_cdf(np.concatenate((y_val[:, 0], y_val[:, 1])))\ncrps_val = crps(cdf_val, np.concatenate((cdf_val_pred_systole, cdf_val_pred_diastole)))\nprint('CRPS(val) = {0}'.format(crps_val))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Plot example predictions vs ground truth volumes"},{"metadata":{"trusted":true},"cell_type":"code","source":"example_cdf = [cdf_val_pred_systole[30], cdf_val_pred_diastole[30]]\nexample_y = y_val[30]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=[18,5])\n\nax1.plot(example_cdf[0], linewidth=3)\nax1.axvline(x=example_y[0], color=\"green\", linewidth=3)\nax1.legend([\"Predicted systole CDF\", \"True systole volume\"])\nax1.set_title(\"CDF Systole\")\nax1.set_xlabel(\"Volume (ml)\")\nax1.set_ylabel(\"Cumulative probability\")\n\nax2.plot(example_cdf[1], linewidth=3)\nax2.axvline(x=example_y[1], color=\"red\", linewidth=3)\nax2.legend([\"Predicted diastole CDF\", \"True diastole volume\"])\nax2.set_title(\"CDF Diastole\")\nax2.set_xlabel(\"Volume (ml)\")\nax2.set_ylabel(\"Cumulative probability\")\n               \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Predict on test dataset - TODO!"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"raw","source":""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}