{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow.keras.backend as K\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom skimage.transform import resize\nfrom skimage.measure import block_reduce\nimport tensorflow as tf\n#from tensorflow.keras.applications import *\nfrom tensorflow.keras.layers import *\nimport keras\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom numpy import where\nimport tensorflow_addons as tfa\nfrom tensorflow.keras import Model, Input\n#In this competition tensorflow is not updated, then we use keras to load model and weights\nfrom keras.models import model_from_json, load_model\nimport json\nfrom sklearn.model_selection import KFold","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pydicom\nimport os\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom sklearn.metrics import mean_absolute_error","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = \"../input/osic-pulmonary-fibrosis-progression\"\n#DESIRED_SIZE = 256 # Memory issue\nDESIRED_SIZE = 256","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = pd.read_csv(f\"{ROOT}/train.csv\")\ntr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\nchunk = pd.read_csv(f\"{ROOT}/test.csv\")\n\nprint(\"add infos\")\nsub = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\nsub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week']]\nsub = sub.merge(chunk.drop('Weeks', axis=1), on=\"Patient\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr['WHERE'] = 'train'\nchunk['WHERE'] = 'val'\nsub['WHERE'] = 'test'\ndata = tr.append([chunk, sub])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COLS = ['Sex','SmokingStatus']\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)\n#=================","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#\ndata['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\nFE += ['age','percent','week','BASE']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = data.loc[data.WHERE=='train']\nchunk = data.loc[data.WHERE=='val']\nsub = data.loc[data.WHERE=='test']\ndel data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr.shape, chunk.shape, sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Quick Image processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_images(df, how=\"train\"):\n    xo = []\n    p = []\n    w  = []\n    for i in tqdm(range(df.shape[0])):\n        patient = df.iloc[i,0]\n        week = df.iloc[i,1]\n        try:\n            img_path = f\"{ROOT}/{how}/{patient}/{week}.dcm\"\n            ds = pydicom.dcmread(img_path)\n            im = Image.fromarray(ds.pixel_array)\n            im = im.resize((DESIRED_SIZE,DESIRED_SIZE)) \n            im = np.array(im)\n            xo.append(im[np.newaxis,:,:])\n            p.append(patient)\n            w.append(week)\n        except:\n            pass\n    data = pd.DataFrame({\"Patient\":p,\"Weeks\":w})\n    return np.concatenate(xo, axis=0), data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x, df_tr = get_images(tr, how=\"train\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x.shape, df_tr.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = np.random.randint(x.shape[0])\nplt.imshow(x[idx], cmap=plt.cm.bone)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_tr = df_tr.merge(tr, how=\"left\", on=['Patient', 'Weeks'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = df_tr['FVC'].values\nz = df_tr[FE].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"z.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### BASELINE CNN "},{"metadata":{"trusted":true},"cell_type":"code","source":"# import tensorflow.keras.backend as K\n# import tensorflow.keras.layers as L\n# import tensorflow.keras.models as M","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n#=============================#\ndef kloss(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 1]\n    fvc_pred = y_pred[:, 0]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return K.mean(metric)\n#=============================#\ndef kmae(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    spread = tf.abs( (y_true[:, 0] -  y_pred[:, 0])  / y_true[:, 0] )\n    #spred = tf.square(y_true, y_pred[:, 0])\n    return K.mean(spread)\n#=============================#\n\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * kloss(y_true, y_pred) + (1 - _lambda)*kmae(y_true, y_pred)\n    return loss\n#=================\n\n# def make_model():\n#     inp = L.Input((DESIRED_SIZE,DESIRED_SIZE), name=\"input\")\n#     z = L.Input((9,), name=\"Patient\")\n#     x = L.Conv1D(50, 4, activation=\"relu\", name=\"conv1\")(inp)\n#     x = L.MaxPool1D(2, name='pool1')(x)\n    \n#     #x = L.Dropout(0.2)(x)\n#     x = L.Conv1D(50, 4, activation=\"relu\", name=\"conv2\")(x)\n#     x = L.MaxPool1D(2, name='pool2')(x)\n    \n#     #x = L.Dropout(0.2)(x)\n#     x = L.Conv1D(50, 4, activation=\"relu\", name=\"conv3\")(x)\n#     x = L.MaxPool1D(2, name='pool3')(x)\n    \n#     x = L.Flatten(name=\"features\")(x)\n#     x = L.Dense(50, activation=\"relu\", name=\"d1\")(x)\n#     l = L.Dense(10, activation=\"relu\", name=\"d2\")(z)\n#     x = L.Concatenate(name=\"combine\")([x, l])\n#     x = L.Dense(50, activation=\"relu\", name=\"d3\")(x)\n#     preds = L.Dense(2, activation=\"relu\", name=\"preds\")(x)\n    \n#     model = M.Model([inp, z], preds, name=\"CNN\")\n#     model.compile(loss=mloss(0.5), optimizer=\"adam\", metrics=[kloss])\n#     #model.compile(loss=kmae, optimizer=\"adam\", metrics=[kloss])\n#     #model.compile(loss=kloss, optimizer=\"adam\", metrics=[kmae])#\n#     return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Modified VGG19 use like baseline model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def VGG19(x):\n    \n    #Block 1\n    x = Conv2D(64, (3, 3),\n                      activation='relu',\n                      padding='same',\n                      name='block1_conv1', kernel_initializer='he_normal')(x)\n    x = Conv2D(64, (3, 3),\n                      activation='relu',\n                      padding='same',\n                      name='block1_conv2', kernel_initializer='he_normal')(x)\n    x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)\n\n    # Block 2\n    x = Conv2D(128, (3, 3),\n                      activation='relu',\n                      padding='same',\n                      name='block2_conv1', kernel_initializer='he_normal')(x)\n    x = Conv2D(128, (3, 3),\n                      activation='relu',\n                      padding='same',\n                      name='block2_conv2', kernel_initializer='he_normal')(x)\n#     x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)\n\n#     # Block 3\n#     x = Conv2D(256, (3, 3),\n#                       activation='relu',\n#                       padding='same',\n#                       name='block3_conv1', kernel_initializer='he_normal')(x)\n#     x = Conv2D(256, (3, 3),\n#                       activation='relu',\n#                       padding='same',\n#                       name='block3_conv2', kernel_initializer='he_normal')(x)\n#     x = Conv2D(256, (3, 3),\n#                       activation='relu',\n#                       padding='same',\n#                       name='block3_conv3', kernel_initializer='he_normal')(x)\n#     x = Conv2D(256, (3, 3),\n#                       activation='relu',\n#                       padding='same',\n#                       name='block3_conv4', kernel_initializer='he_normal')(x)\n#     x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)\n\n#     # Block 4\n#     x = Conv2D(512, (3, 3),\n#                       activation='relu',\n#                       padding='same',\n#                       name='block4_conv1', kernel_initializer='he_normal')(x)\n#     x = Conv2D(512, (3, 3),\n#                       activation='relu',\n#                       padding='same',\n#                       name='block4_conv2', kernel_initializer='he_normal')(x)\n#     x = Conv2D(512, (3, 3),\n#                       activation='relu',\n#                       padding='same',\n#                       name='block4_conv3', kernel_initializer='he_normal')(x)\n#     x = Conv2D(512, (3, 3),\n#                       activation='relu',\n#                       padding='same',\n#                       name='block4_conv4', kernel_initializer='he_normal')(x)\n#     x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)\n\n#     # Block 5\n#     x = Conv2D(512, (3, 3),\n#                       activation='relu',\n#                       padding='same',\n#                       name='block5_conv1', kernel_initializer='he_normal')(x)\n#     x = Conv2D(512, (3, 3),\n#                       activation='relu',\n#                       padding='same',\n#                       name='block5_conv2', kernel_initializer='he_normal')(x)\n#     x = Conv2D(512, (3, 3),\n#                       activation='relu',\n#                       padding='same',\n#                       name='block5_conv3', kernel_initializer='he_normal')(x)\n#     x = Conv2D(512, (3, 3),\n#                       activation='relu',\n#                       padding='same',\n#                       name='block5_conv4', kernel_initializer='he_normal')(x)\n#     x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)\n\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Global Constants\nLRN2D_NORM=True\nDATA_FORMAT='channels_last' # Theano:'channels_first' Tensorflow:'channels_last'\nUSE_BN=True\nDROPOUT=0.25\n#weight_decay=0.01 #None\nweight_decay=None\n\ndef conv2D_lrn2d(x,filters,kernel_size,strides=(1,1),padding='same',dilation_rate=(1,1),activation='relu',\n                 use_bias=True,kernel_initializer='he_normal',bias_initializer='zeros',\n                 kernel_regularizer=None,bias_regularizer=None,activity_regularizer=None,\n                 kernel_constraint=None,bias_constraint=None,lrn2d_norm=LRN2D_NORM,weight_decay=weight_decay):\n    #l2 normalization\n    if weight_decay:\n        kernel_regularizer=tf.keras.regularizers.l1(weight_decay)\n        bias_regularizer=tf.keras.regularizers.l1(weight_decay)\n    else:\n        kernel_regularizer=None\n        bias_regularizer=None\n    x=Conv2D(filters=filters,kernel_size=kernel_size,strides=strides,padding=padding,dilation_rate=dilation_rate,\n             activation=activation,use_bias=use_bias,kernel_initializer=kernel_initializer,\n             bias_initializer=bias_initializer,kernel_regularizer=kernel_regularizer,bias_regularizer=bias_regularizer,\n             activity_regularizer=activity_regularizer,kernel_constraint=kernel_constraint,bias_constraint=bias_constraint)(x)\n    if lrn2d_norm:\n        #batch normalization\n        x=BatchNormalization()(x)\n\n    return x\n\n#Defining InceptionV1 module\ndef inception_module(x,params,concat_axis,padding='same',dilation_rate=(1,1),activation='relu',\n                     use_bias=True,kernel_initializer='he_normal',bias_initializer='zeros',\n                     kernel_regularizer=None,bias_regularizer=None,activity_regularizer=None,kernel_constraint=None,\n                     bias_constraint=None,lrn2d_norm=LRN2D_NORM,weight_decay=weight_decay):\n    (branch1,branch2,branch3,branch4)=params\n    if weight_decay:\n        kernel_regularizer=tf.keras.regularizers.l1(weight_decay)\n        bias_regularizer=tf.keras.regularizers.l1(weight_decay)\n    else:\n        kernel_regularizer=None\n        bias_regularizer=None\n    #1x1\n    pathway1=Conv2D(filters=branch1[0],kernel_size=(1,1),strides=1,padding=padding,dilation_rate=dilation_rate,\n                    activation=activation,use_bias=use_bias,kernel_initializer=kernel_initializer,\n                    bias_initializer=bias_initializer,kernel_regularizer=kernel_regularizer,bias_regularizer=bias_regularizer,\n                    activity_regularizer=activity_regularizer,kernel_constraint=kernel_constraint,bias_constraint=bias_constraint)(x)\n    #1x1->3x3\n    pathway2=Conv2D(filters=branch2[0],kernel_size=(1,1),strides=1,padding=padding,dilation_rate=dilation_rate,\n                    activation=activation,use_bias=use_bias,kernel_initializer=kernel_initializer,\n                    bias_initializer=bias_initializer,kernel_regularizer=kernel_regularizer,bias_regularizer=bias_regularizer,\n                    activity_regularizer=activity_regularizer,kernel_constraint=kernel_constraint,bias_constraint=bias_constraint)(x)\n    pathway2=Conv2D(filters=branch2[1],kernel_size=(3,3),strides=1,padding=padding,dilation_rate=dilation_rate,\n                    activation=activation,use_bias=use_bias,kernel_initializer=kernel_initializer,\n                    bias_initializer=bias_initializer,kernel_regularizer=kernel_regularizer,bias_regularizer=bias_regularizer,\n                    activity_regularizer=activity_regularizer,kernel_constraint=kernel_constraint,bias_constraint=bias_constraint)(pathway2)\n    #1x1->5x5\n    pathway3=Conv2D(filters=branch3[0],kernel_size=(1,1),strides=1,padding=padding,dilation_rate=dilation_rate,\n                    activation=activation,use_bias=use_bias,kernel_initializer=kernel_initializer,\n                    bias_initializer=bias_initializer,kernel_regularizer=kernel_regularizer,bias_regularizer=bias_regularizer,\n                    activity_regularizer=activity_regularizer,kernel_constraint=kernel_constraint,bias_constraint=bias_constraint)(x)\n    pathway3=Conv2D(filters=branch3[1],kernel_size=(5,5),strides=1,padding=padding,dilation_rate=dilation_rate,\n                    activation=activation,use_bias=use_bias,kernel_initializer=kernel_initializer,\n                    bias_initializer=bias_initializer,kernel_regularizer=kernel_regularizer,bias_regularizer=bias_regularizer,\n                    activity_regularizer=activity_regularizer,kernel_constraint=kernel_constraint,bias_constraint=bias_constraint)(pathway3)\n    #3x3->1x1\n    pathway4=MaxPooling2D(pool_size=(3,3),strides=1,padding=padding,data_format=DATA_FORMAT)(x)\n    pathway4=Conv2D(filters=branch4[0],kernel_size=(1,1),strides=1,padding=padding,dilation_rate=dilation_rate,\n                    activation=activation,use_bias=use_bias,kernel_initializer=kernel_initializer,\n                    bias_initializer=bias_initializer,kernel_regularizer=kernel_regularizer,bias_regularizer=bias_regularizer,\n                    activity_regularizer=activity_regularizer,kernel_constraint=kernel_constraint,bias_constraint=bias_constraint)(pathway4)\n\n    return concatenate([pathway1,pathway2,pathway3,pathway4],axis=concat_axis)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Defining conv2d_bn to inceptionv3_module\ndef conv2d_bn(x, filters, num_row, num_col, padding='same', strides=(1, 1), name=None):\n    \"\"\"Utility function to apply conv + BN.\n    Arguments:\n    x: input tensor.\n    filters: filters in `Conv2D`.\n    num_row: height of the convolution kernel.\n    num_col: width of the convolution kernel.\n    padding: padding mode in `Conv2D`.\n    strides: strides in `Conv2D`.\n    name: name of the ops; will become `name + '_conv'`\n      for the convolution and `name + '_bn'` for the\n      batch norm layer.\n      Returns:\n      Output tensor after applying `Conv2D` and `BatchNormalization`.\n      \"\"\"\n    if name is not None:\n        bn_name = name + '_bn'\n        conv_name = name + '_conv'\n    else:\n        bn_name = None\n        conv_name = None\n    \n    x = Conv2D(filters, (num_row, num_col),strides=strides,padding=padding,use_bias=False,name=conv_name)(x)\n    x = BatchNormalization(scale=False, name=bn_name)(x)\n    x = Activation('relu', name=name)(x)\n    \n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"channel_axis=3\ndef inceptionV3_module(x):\n    x = conv2d_bn(x, 32, 3, 3, strides=(2, 2), padding='valid')\n    x = conv2d_bn(x, 32, 3, 3, padding='valid')\n    x = conv2d_bn(x, 64, 3, 3)\n    x = MaxPooling2D((3, 3), strides=(2, 2))(x)\n    \n    x = conv2d_bn(x, 80, 1, 1, padding='valid')\n    x = conv2d_bn(x, 192, 3, 3, padding='valid')\n    x = MaxPooling2D((3, 3), strides=(2, 2))(x)\n    \n    # mixed 0: 35 x 35 x 256\n    branch1x1 = conv2d_bn(x, 64, 1, 1)\n    \n    branch5x5 = conv2d_bn(x, 48, 1, 1)\n    branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)\n    \n    branch3x3dbl = conv2d_bn(x, 64, 1, 1)\n    branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)\n    branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)\n    \n    branch_pool = AveragePooling2D(\n        (3, 3), strides=(1, 1), padding='same')(x)\n    branch_pool = conv2d_bn(branch_pool, 32, 1, 1)\n    x = concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],\n                           axis=channel_axis,\n                           name='mixed0')\n    # mixed 1: 35 x 35 x 288\n    branch1x1 = conv2d_bn(x, 64, 1, 1)\n    \n    branch5x5 = conv2d_bn(x, 48, 1, 1)\n    branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)\n    \n    branch3x3dbl = conv2d_bn(x, 64, 1, 1)\n    branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)\n    branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)\n    \n    branch_pool = AveragePooling2D(\n        (3, 3), strides=(1, 1), padding='same')(x)\n    branch_pool = conv2d_bn(branch_pool, 64, 1, 1)\n    x = concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],\n                           axis=channel_axis,\n                           name='mixed1')\n    \n    # mixed 2: 35 x 35 x 288\n    branch1x1 = conv2d_bn(x, 64, 1, 1)\n    \n    branch5x5 = conv2d_bn(x, 48, 1, 1)\n    branch5x5 = conv2d_bn(branch5x5, 64, 5, 5)\n    \n    branch3x3dbl = conv2d_bn(x, 64, 1, 1)\n    branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)\n    branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)\n    \n    branch_pool = AveragePooling2D(\n        (3, 3), strides=(1, 1), padding='same')(x)\n    branch_pool = conv2d_bn(branch_pool, 64, 1, 1)\n    x = concatenate([branch1x1, branch5x5, branch3x3dbl, branch_pool],\n                           axis=channel_axis,\n                           name='mixed2')\n    \n    # mixed 3: 17 x 17 x 768\n    branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid')\n    \n    branch3x3dbl = conv2d_bn(x, 64, 1, 1)\n    branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3)\n    branch3x3dbl = conv2d_bn(\n        branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='valid')\n    \n    branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)\n    x = concatenate([branch3x3, branch3x3dbl, branch_pool],\n                           axis=channel_axis,\n                           name='mixed3')\n    \n    # mixed 4: 17 x 17 x 768\n    branch1x1 = conv2d_bn(x, 192, 1, 1)\n    \n    branch7x7 = conv2d_bn(x, 128, 1, 1)\n    branch7x7 = conv2d_bn(branch7x7, 128, 1, 7)\n    branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)\n    \n    branch7x7dbl = conv2d_bn(x, 128, 1, 1)\n    branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)\n    branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7)\n    branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1)\n    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)\n    \n    branch_pool = AveragePooling2D(\n        (3, 3), strides=(1, 1), padding='same')(x)\n    branch_pool = conv2d_bn(branch_pool, 192, 1, 1)\n    x = concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool],\n                           axis=channel_axis,\n                           name='mixed4')\n    \n    # mixed 5, 6: 17 x 17 x 768\n    for i in range(2):\n        branch1x1 = conv2d_bn(x, 192, 1, 1)\n        \n        branch7x7 = conv2d_bn(x, 160, 1, 1)\n        branch7x7 = conv2d_bn(branch7x7, 160, 1, 7)\n        branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)\n        \n        branch7x7dbl = conv2d_bn(x, 160, 1, 1)\n        branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)\n        branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7)\n        branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1)\n        branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)\n        \n        branch_pool = AveragePooling2D((3, 3),\n                                              strides=(1, 1),\n                                              padding='same')(x)\n        branch_pool = conv2d_bn(branch_pool, 192, 1, 1)\n        x = concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool],\n                               axis=channel_axis,\n                               name='mixed' + str(5 + i))\n        \n    # mixed 7: 17 x 17 x 768\n    branch1x1 = conv2d_bn(x, 192, 1, 1)\n    \n    branch7x7 = conv2d_bn(x, 192, 1, 1)\n    branch7x7 = conv2d_bn(branch7x7, 192, 1, 7)\n    branch7x7 = conv2d_bn(branch7x7, 192, 7, 1)\n    \n    branch7x7dbl = conv2d_bn(x, 192, 1, 1)\n    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)\n    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)\n    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1)\n    branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7)\n    \n    branch_pool = AveragePooling2D(\n        (3, 3), strides=(1, 1), padding='same')(x)\n    branch_pool = conv2d_bn(branch_pool, 192, 1, 1)\n    x = concatenate([branch1x1, branch7x7, branch7x7dbl, branch_pool],\n                           axis=channel_axis,\n                           name='mixed7')\n    \n    # mixed 8: 8 x 8 x 1280\n    branch3x3 = conv2d_bn(x, 192, 1, 1)\n    branch3x3 = conv2d_bn(branch3x3, 320, 3, 3, strides=(2, 2), padding='valid')\n    \n    branch7x7x3 = conv2d_bn(x, 192, 1, 1)\n    branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7)\n    branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1)\n    branch7x7x3 = conv2d_bn(\n        branch7x7x3, 192, 3, 3, strides=(2, 2), padding='valid')\n    \n    branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x)\n    x = concatenate([branch3x3, branch7x7x3, branch_pool],\n                           axis=channel_axis,\n                           name='mixed8')\n    \n    # mixed 9: 8 x 8 x 2048\n    for i in range(2):\n        branch1x1 = conv2d_bn(x, 320, 1, 1)\n        \n        branch3x3 = conv2d_bn(x, 384, 1, 1)\n        branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3)\n        branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1)\n        branch3x3 = concatenate([branch3x3_1, branch3x3_2],\n                                       axis=channel_axis,\n                                       name='mixed9_' + str(i))\n        \n        branch3x3dbl = conv2d_bn(x, 448, 1, 1)\n        branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3)\n        branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3)\n        branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1)\n        branch3x3dbl = concatenate([branch3x3dbl_1, branch3x3dbl_2],\n                                          axis=channel_axis)\n        \n        branch_pool = AveragePooling2D((3, 3),\n                                              strides=(1, 1),\n                                              padding='same')(x)\n        \n        branch_pool = conv2d_bn(branch_pool, 192, 1, 1)\n        x = concatenate([branch1x1, branch3x3, branch3x3dbl, branch_pool],\n                               axis=channel_axis,\n                               name='mixed' + str(9 + i))\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CONCAT_AXIS=3\n\n#Model InceptionV1\n\ndef InceptionV1():\n    inp = Input((DESIRED_SIZE,DESIRED_SIZE,1), name=\"input\")\n    x = inception_module(inp, params=[(128,), (128, 192), (32, 96), (64,)], concat_axis=CONCAT_AXIS)  # 3b\n    x = Flatten()(x)\n    x = Dropout(DROPOUT)(x)\n    preds = Dense(2, activation=\"relu\", name=\"preds\")(x)\n    \n    #model = tf.keras.Model([inp, z], preds, name=\"CNN\")\n    model = tf.keras.Model(inp, preds, name=\"CNN\")\n    #opt = tf.keras.optimizers.SGD(learning_rate=0.001)\n    #opt = tfa.optimizers.LAMB(learning_rate=0.001)\n    #opt = tfa.optimizers.SGDW(learning_rate=0.001,weight_decay=0.001, momentum=0.01) #weight_decay=0.001\n    #opt = tfa.optimizers.SWA(tf.keras.optimizers.Adam(lr=0.001), 100, 10)\n    #opt=tfa.optimizers.AveragedOptimizerWrapper(tf.keras.optimizers.Adam(lr=0.001))\n    #loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)\n    opt=tfa.optimizers.SWA(tf.keras.optimizers.Adam(lr=0.001))\n    #model.compile(loss=mloss(0.5), optimizer=\"adam\", metrics=[kloss])\n    model.compile(loss=mloss(0.5), optimizer=opt, metrics=[kloss])\n    #model.compile(optimizer=opt,loss=loss,metrics=['AUC'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CONCAT_AXIS=3\n\n#Model GoogleNet_InceptionV1\n\ndef GoogleNet():\n    inp = Input((DESIRED_SIZE,DESIRED_SIZE,1), name=\"input\")\n    z = Input((9,), name=\"Patient\")\n    #x = VGG19(inp)\n    x = conv2D_lrn2d(inp, 64, (7, 7), 2, padding='same', lrn2d_norm=False)\n    x = MaxPooling2D(pool_size=(2, 2), strides=2, padding='same')(x)\n    x = BatchNormalization()(x)\n\n    x = conv2D_lrn2d(x, 64, (1, 1), 1, padding='same', lrn2d_norm=False)\n\n    x = conv2D_lrn2d(x, 192, (3, 3), 1, padding='same', lrn2d_norm=True)\n    x = MaxPooling2D(pool_size=(2, 2), strides=2, padding='same')(x)\n\n    x = inception_module(x, params=[(64,), (96, 128), (16, 32), (32,)], concat_axis=CONCAT_AXIS)  # 3a\n    x = inception_module(x, params=[(128,), (128, 192), (32, 96), (64,)], concat_axis=CONCAT_AXIS)  # 3b\n    x = MaxPooling2D(pool_size=(2, 2), strides=2, padding='same')(x)\n    \n    x = inception_module(x, params=[(192,), (96, 208), (16, 48), (64,)], concat_axis=CONCAT_AXIS)  # 4a\n    x = inception_module(x, params=[(160,), (112, 224), (24, 64), (64,)], concat_axis=CONCAT_AXIS)  # 4b\n    x = inception_module(x, params=[(128,), (128, 256), (24, 64), (64,)], concat_axis=CONCAT_AXIS)  # 4c\n    x = inception_module(x, params=[(112,), (144, 288), (32, 64), (64,)], concat_axis=CONCAT_AXIS)  # 4d\n    x = inception_module(x, params=[(256,), (160, 320), (32, 128), (128,)], concat_axis=CONCAT_AXIS)  # 4e\n    x = MaxPooling2D(pool_size=(2, 2), strides=2, padding='same')(x)\n\n    x = inception_module(x, params=[(256,), (160, 320), (32, 128), (128,)], concat_axis=CONCAT_AXIS)  # 5a\n    x = inception_module(x, params=[(384,), (192, 384), (48, 128), (128,)], concat_axis=CONCAT_AXIS)  # 5b\n    x = AveragePooling2D(pool_size=(1, 1), strides=1, padding='valid')(x)\n\n    x = Flatten()(x)\n    x = Dropout(DROPOUT)(x)\n    preds = Dense(2, activation=\"relu\", name=\"preds\")(x)\n    \n    #model = tf.keras.Model([inp, z], preds, name=\"CNN\")\n    model = tf.keras.Model(inp, preds, name=\"CNN\")\n    #opt = tf.keras.optimizers.SGD(learning_rate=0.001)\n    #opt = tfa.optimizers.LAMB(learning_rate=0.001)\n    #opt = tfa.optimizers.SGDW(learning_rate=0.001,weight_decay=0.001, momentum=0.01) #weight_decay=0.001\n    #opt = tfa.optimizers.SWA(tf.keras.optimizers.Adam(lr=0.001), 100, 10)\n    #opt=tfa.optimizers.AveragedOptimizerWrapper(tf.keras.optimizers.Adam(lr=0.001))\n    #loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)\n    opt=tfa.optimizers.SWA(tf.keras.optimizers.Adam(lr=0.001))\n    #model.compile(loss=mloss(0.5), optimizer=\"adam\", metrics=[kloss])\n    model.compile(loss=mloss(0.5), optimizer=opt, metrics=[kloss])\n    #model.compile(optimizer=opt,loss=loss,metrics=['AUC'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CONCAT_AXIS=3\n\n#Model InceptionV3\n\ndef InceptionV3():\n    inp = Input((DESIRED_SIZE,DESIRED_SIZE,1), name=\"input\")\n    x = inceptionV3_module(inp)  # 3b\n    x = GlobalAveragePooling2D(name='avg_pool')(x)\n    preds = Dense(2, activation=\"relu\", name=\"preds\")(x)\n    \n    #model = tf.keras.Model([inp, z], preds, name=\"CNN\")\n    model = tf.keras.Model(inp, preds, name=\"CNN\")\n    #opt = tf.keras.optimizers.SGD(learning_rate=0.001)\n    #opt = tfa.optimizers.LAMB(learning_rate=0.001)\n    #opt = tfa.optimizers.SGDW(learning_rate=0.001,weight_decay=0.001, momentum=0.01) #weight_decay=0.001\n    #opt = tfa.optimizers.SWA(tf.keras.optimizers.Adam(lr=0.001), 100, 10)\n    #opt=tfa.optimizers.AveragedOptimizerWrapper(tf.keras.optimizers.Adam(lr=0.001))\n    #loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)\n    opt=tfa.optimizers.SWA(tf.keras.optimizers.Adam(lr=0.001))\n    #model.compile(loss=mloss(0.5), optimizer=\"adam\", metrics=[kloss])\n    model.compile(loss=mloss(0.5), optimizer=opt, metrics=[kloss])\n    #model.compile(optimizer=opt,loss=loss,metrics=['AUC'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CONCAT_AXIS=3\n\n#Model GoogleNet_InceptionV1\n\ndef InceptionV3_GoogleNet():\n    inp = Input((DESIRED_SIZE,DESIRED_SIZE,1), name=\"input\")\n    #x = VGG19(inp)\n    x = inceptionV3_module(inp)\n    x = conv2D_lrn2d(x, 64, (7, 7), 2, padding='same', lrn2d_norm=False)\n    x = MaxPooling2D(pool_size=(2, 2), strides=2, padding='same')(x)\n    x = BatchNormalization()(x)\n\n    x = conv2D_lrn2d(x, 64, (1, 1), 1, padding='same', lrn2d_norm=False)\n\n    x = conv2D_lrn2d(x, 192, (3, 3), 1, padding='same', lrn2d_norm=True)\n    x = MaxPooling2D(pool_size=(2, 2), strides=2, padding='same')(x)\n\n    x = inception_module(x, params=[(64,), (96, 128), (16, 32), (32,)], concat_axis=CONCAT_AXIS)  # 3a\n    x = inception_module(x, params=[(128,), (128, 192), (32, 96), (64,)], concat_axis=CONCAT_AXIS)  # 3b\n    x = MaxPooling2D(pool_size=(2, 2), strides=2, padding='same')(x)\n    \n    x = inception_module(x, params=[(192,), (96, 208), (16, 48), (64,)], concat_axis=CONCAT_AXIS)  # 4a\n    x = inception_module(x, params=[(160,), (112, 224), (24, 64), (64,)], concat_axis=CONCAT_AXIS)  # 4b\n    x = inception_module(x, params=[(128,), (128, 256), (24, 64), (64,)], concat_axis=CONCAT_AXIS)  # 4c\n    x = inception_module(x, params=[(112,), (144, 288), (32, 64), (64,)], concat_axis=CONCAT_AXIS)  # 4d\n    x = inception_module(x, params=[(256,), (160, 320), (32, 128), (128,)], concat_axis=CONCAT_AXIS)  # 4e\n    x = MaxPooling2D(pool_size=(2, 2), strides=2, padding='same')(x)\n\n    x = inception_module(x, params=[(256,), (160, 320), (32, 128), (128,)], concat_axis=CONCAT_AXIS)  # 5a\n    x = inception_module(x, params=[(384,), (192, 384), (48, 128), (128,)], concat_axis=CONCAT_AXIS)  # 5b\n    x = AveragePooling2D(pool_size=(1, 1), strides=1, padding='valid')(x)\n\n    x = Flatten()(x)\n    x = Dropout(DROPOUT)(x)\n    preds = Dense(2, activation=\"relu\", name=\"preds\")(x)\n    \n    #model = tf.keras.Model([inp, z], preds, name=\"CNN\")\n    model = tf.keras.Model(inp, preds, name=\"InceptionV3_GoogleNet\")\n    \n    global_step = tf.Variable(0, trainable=False)\n    initial_learning_rate = 0.01\n    lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(\n        initial_learning_rate,\n        decay_steps=1000,#100000\n        decay_rate=0.96,\n        staircase=True)\n    \n    #opt = tf.keras.optimizers.RMSprop(learning_rate=lr_schedule)\n    opt = tfa.optimizers.RectifiedAdam(learning_rate=lr_schedule)\n    #opt = tfa.optimizers.SGDW(learning_rate=0.001,weight_decay=0.001, momentum=0.01) #weight_decay=0.001\n    #opt = tfa.optimizers.SWA(tf.keras.optimizers.Adam(lr=0.001), 100, 10)\n    #loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)\n    #opt=tfa.optimizers.SWA(tf.keras.optimizers.Adam(lr=0.001))\n    #model.compile(loss=mloss(0.5), optimizer=\"adam\", metrics=[kloss])\n    model.compile(loss=mloss(0.65), optimizer=opt, metrics=[kloss])\n    #model.compile(optimizer=opt,loss=loss,metrics=['AUC'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Modified EfficientNet\n\n\"\"\"\n# Reference\n- [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks]\n   (https://arxiv.org/abs/1905.11946)\n\"\"\"\n\ndef get_top(x_input):\n    \"\"\"Block top operations\n    This functions apply Batch Normalization and Leaky ReLU activation to the input.\n    # Arguments:\n        x_input: Tensor, input to apply BN and activation  to.\n    # Returns:\n        Output tensor\n    \"\"\"\n    \n    x = BatchNormalization()(x_input)\n    x = LeakyReLU()(x)\n    return x\n\ndef get_block(x_input, input_channels, output_channels):\n    \"\"\"MBConv block\n    This function defines a mobile Inverted Residual Bottleneck block with BN and Leaky ReLU\n    # Arguments\n        x_input: Tensor, input tensor of conv layer.\n        input_channels: Integer, the dimentionality of the input space.\n        output_channels: Integer, the dimensionality of the output space.\n            \n    # Returns\n        Output tensor.\n    \"\"\"\n\n    x = Conv2D(input_channels, kernel_size=(1, 1), padding='same', use_bias=False)(x_input)\n    x = get_top(x)\n    x = DepthwiseConv2D(kernel_size=(1, 3), padding='same', use_bias=False)(x)\n    x = get_top(x)\n    x = MaxPooling2D(pool_size=(2, 1), strides=(2, 1))(x)\n    x = DepthwiseConv2D(kernel_size=(3, 1), padding='same', use_bias=False)(x)\n    x = get_top(x)\n    x = Conv2D(output_channels, kernel_size=(2, 1), strides=(1, 2), padding='same', use_bias=False)(x)\n    return x\n\n\ndef EffNet(num_classes=2):\n    \"\"\"EffNet\n    This function defines a EfficientNet architecture.\n    # Arguments\n        input_shape: An integer or tuple/list of 3 integers, shape\n            of input tensor.\n        num_classes: Integer, number of classes.\n        plot_model: Boolean, whether to plot model architecture or not\n    # Returns\n        EfficientNet model.\n    \"\"\"\n    \n    inp = Input((DESIRED_SIZE,DESIRED_SIZE,1), name=\"input\")\n    x = get_block(inp, 32, 64)\n    x = get_block(x, 64, 128)\n    x = get_block(x, 128, 256)\n    x = Flatten()(x)\n    preds = Dense(num_classes, activation='relu')(x)\n    \n    #model = tf.keras.Model([inp, z], preds, name=\"CNN\")\n    model = tf.keras.Model(inp, preds, name=\"EffNet\")\n    \n    global_step = tf.Variable(0, trainable=False)\n    initial_learning_rate = 0.01\n    lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(\n        initial_learning_rate,\n        decay_steps=1000,#100000\n        decay_rate=0.96,\n        staircase=True)\n\n    opt = tf.keras.optimizers.RMSprop(learning_rate=lr_schedule)\n    #opt = tfa.optimizers.RectifiedAdam(learning_rate=0.001)\n    #opt = tfa.optimizers.SGDW(learning_rate=0.001,weight_decay=0.001, momentum=0.01) #weight_decay=0.001\n    #opt = tfa.optimizers.SWA(tf.keras.optimizers.Adam(lr=0.001), 100, 10)\n    #loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)\n    #opt=tfa.optimizers.SWA(tf.keras.optimizers.Adam(lr=0.001))\n    #model.compile(loss=mloss(0.5), optimizer=\"adam\", metrics=[kloss])\n    model.compile(loss=mloss(0.5), optimizer=opt, metrics=[kloss])\n    #model.compile(optimizer=opt,loss=loss,metrics=['AUC'])\n\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = InceptionV3_GoogleNet()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# net = EffNet()\n# print(net.summary())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"x_min = np.min(x)\nx_max = np.max(x)\nxs = x - x_min / (x_max - x_min)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xs.shape, y.shape, x_min","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Redifining xs in order to agregate a one dimenssion\n#from array (1231, 128, 128) to (1231, 128, 128, 1)\nxs = np.expand_dims(xs, axis=3)\nxs.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Splitting data in train and developepment set\nfrom sklearn.model_selection import train_test_split\nX_train, X_oof, y_train, y_oof = train_test_split(xs, y, test_size=0.1, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Defining to save the best model\ndef checkpoint_callback(counter):\n    \n    model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n        filepath='./fold'+str(counter)+'.h5',\n        save_weights_only=True,\n        monitor='val_kloss',\n        mode='min',\n        save_best_only=True)\n    \n    return model_checkpoint_callback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_graph(history):\n    plt.plot(history.history['kloss'])\n    plt.plot(history.history['val_kloss'])\n    plt.legend(['train', 'valid.'])\n    plt.grid()\n    plt.title(\"Curves Comparision\")\n    #finding and plotting the min val_kloss value\n    min_val_kloss=np.min(history.history['val_kloss'])\n    x_min_val_kaloss=np.where(history.history['val_kloss']==np.min(history.history['val_kloss']))[0][0]\n    plt.scatter(x_min_val_kaloss,min_val_kloss, c='#ff7f0e', s=200)\n    bbox = dict(boxstyle=\"round\", fc=\"0.8\")\n    plt.annotate(\"min_val_kloss: \"+str(np.round(min_val_kloss, decimals=2)), (x_min_val_kaloss,min_val_kloss+0.35),bbox=bbox)\n    plt.annotate(\"Iteration: \"+str(x_min_val_kaloss+1), (x_min_val_kaloss,min_val_kloss+0.7),bbox=bbox)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED=42\nFOLDS=5\n\nskf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)\n\n#skf.get_n_splits(xs)\nskf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Trainning models using Kfolds"},{"metadata":{"trusted":true},"cell_type":"code","source":"def kloss1(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float64)\n    tf.dtypes.cast(y_pred, tf.float64)\n    sigma = y_pred[:, 1]\n    fvc_pred = y_pred[:, 0]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, tf.dtypes.cast(C1, dtype=\"float64\"))\n    delta = tf.abs(y_true - fvc_pred)\n    delta = tf.minimum(tf.dtypes.cast(delta, dtype=\"float64\"), tf.dtypes.cast(C2, dtype=\"float64\"))\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float64) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return K.mean(metric)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"counter=1\nEPOCHS=65\nfor train , test in skf.split(X_train):\n\t#print('train: %s, test: %s' % (train,test))\n    \n    net = InceptionV3_GoogleNet()\n\n    print()\n    print('Fold '+str(counter)+':')\n\n    history=net.fit(X_train[train], y_train[train],\n                    batch_size=32, epochs=EPOCHS,\n                    validation_data=(X_train[test], y_train[test]),\n                    callbacks=[checkpoint_callback(counter)])\n    print('kloss out of fold: '+str(kloss1(y_oof, net.predict(X_oof, batch_size=100, verbose=1)).numpy()))\n    display_graph(history)\n    \n    print()\n    print()\n\n    counter+=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#history=net.fit(X_train, y_train, batch_size=32, epochs=200, validation_data=(X_test, y_test), callbacks=[model_checkpoint_callback])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#net.fit([xs, z], y, batch_size=32, epochs=120) #, validation_split=0.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The model weights (that are considered the best) are loaded into the model.\n#net.load_weights('./weights.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#pred = net.predict([xs, z], batch_size=100, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net1 = InceptionV3_GoogleNet()\nnet2 = InceptionV3_GoogleNet()\nnet3 = InceptionV3_GoogleNet()\nnet4 = InceptionV3_GoogleNet()\nnet5 = InceptionV3_GoogleNet()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net1.load_weights('./fold1.h5')\nnet2.load_weights('./fold2.h5')\nnet3.load_weights('./fold3.h5')\nnet4.load_weights('./fold4.h5')\nnet5.load_weights('./fold5.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred1 = net1.predict(xs, batch_size=100, verbose=1)\npred2 = net2.predict(xs, batch_size=100, verbose=1)\npred3 = net3.predict(xs, batch_size=100, verbose=1)\npred4 = net4.predict(xs, batch_size=100, verbose=1)\npred5 = net5.predict(xs, batch_size=100, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred=np.mean(np.array([pred1,pred2,pred3,pred4,pred5]),axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sigma_opt = mean_absolute_error(y, pred[:, 0])\nsigma_mean = np.mean(pred[:, 1])\nprint(sigma_opt, sigma_mean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"plt.plot(y)\nplt.plot(pred[:, 0])\n#plt.plot(pred[:, 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred[:, 1].min(), pred[:, 1].max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(pred[:, 1])\nplt.title(\"uncertainty in prediction\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### PREDICTION"},{"metadata":{"trusted":true},"cell_type":"code","source":"xe, df_te = get_images(sub, how=\"test\")\ndf_te = df_te.merge(sub, how=\"left\", on=['Patient', 'Weeks'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_te = xe - x_min / (x_max - x_min)\n#Adding new dimenssion\nx_te = np.expand_dims(x_te, axis=3)\nze = df_te[FE].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pe1 = net1.predict(xs, batch_size=100, verbose=1)\npe2 = net2.predict(xs, batch_size=100, verbose=1)\npe3 = net3.predict(xs, batch_size=100, verbose=1)\npe4 = net4.predict(xs, batch_size=100, verbose=1)\npe5 = net5.predict(xs, batch_size=100, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pe=np.mean(np.array([pe1,pe2,pe3,pe4,pe5]),axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#pe = net.predict([x_te, ze], batch_size=100, verbose=1)\npe = net.predict(x_te, batch_size=100, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_te['FVC1'] = pe[:, 0]\ndf_te['Confidence1'] = pe[:, 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = sub.merge(df_te[['Patient','Weeks','FVC1','Confidence1']], how='left', \n                on=['Patient', 'Weeks'])\n#====================================================#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.loc[~subm.FVC1.isnull()].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.loc[~subm.FVC1.isnull(),'FVC'] = subm.loc[~subm.FVC1.isnull(),'FVC1']\nif sigma_mean<70:\n    subm['Confidence'] = sigma_opt\nelse:\n    subm.loc[~subm.FVC1.isnull(),'Confidence'] = subm.loc[~subm.FVC1.isnull(),'Confidence1']\n    subm.loc[subm.FVC1.isnull(),'Confidence'] = sigma_opt\n#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.describe().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.version.VERSION","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"keras.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import os\n# os.chdir(r'./')\n# from IPython.display import FileLink\n# FileLink(r'./weights.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}