{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n'''\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n'''\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n'''\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n'''\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-08T13:51:11.821919Z","iopub.execute_input":"2021-10-08T13:51:11.822229Z","iopub.status.idle":"2021-10-08T13:51:11.829298Z","shell.execute_reply.started":"2021-10-08T13:51:11.822188Z","shell.execute_reply":"2021-10-08T13:51:11.828194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nprint(os.listdir(\"/kaggle/input/notebookf30568c6d4\"))","metadata":{"execution":{"iopub.status.busy":"2021-10-08T13:51:11.831285Z","iopub.execute_input":"2021-10-08T13:51:11.831561Z","iopub.status.idle":"2021-10-08T13:51:11.842183Z","shell.execute_reply.started":"2021-10-08T13:51:11.831524Z","shell.execute_reply":"2021-10-08T13:51:11.841331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nprint(tf.__version__)\nprint(keras.__version__)","metadata":{"execution":{"iopub.status.busy":"2021-10-08T13:51:11.843775Z","iopub.execute_input":"2021-10-08T13:51:11.844423Z","iopub.status.idle":"2021-10-08T13:51:11.850820Z","shell.execute_reply.started":"2021-10-08T13:51:11.844291Z","shell.execute_reply":"2021-10-08T13:51:11.850010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport itertools\nimport os\nimport sys\nfrom prettytable import PrettyTable\nimport pickle\nfrom random import shuffle\nfrom sklearn.model_selection import train_test_split\nimport multiprocessing\nfrom multiprocessing.pool import ThreadPool\nfrom tqdm import tqdm_notebook\nprint(multiprocessing.cpu_count(),\" CPU cores\")\n\nimport seaborn as sns\n%matplotlib inline\nimport matplotlib.pyplot as plt\nplt.rcParams[\"axes.grid\"] = False\n\nfrom sklearn.metrics import confusion_matrix, cohen_kappa_score,accuracy_score\nfrom sklearn.preprocessing import MultiLabelBinarizer\n\nfrom PIL import Image\nimport cv2\nimport tensorflow as tf\n\nfrom tensorflow import keras\nfrom tensorflow.keras import applications\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import optimizers,Model,Sequential\nfrom tensorflow.keras.layers import Input,GlobalAveragePooling2D,Dropout,Dense,Activation,BatchNormalization,GlobalMaxPooling2D,concatenate,Flatten\nfrom tensorflow.keras.callbacks import EarlyStopping,ReduceLROnPlateau,Callback\nfrom tensorflow.keras.initializers import random_normal\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.losses import binary_crossentropy,categorical_crossentropy,mean_squared_error\nfrom tensorflow.keras import backend as K\n","metadata":{"execution":{"iopub.status.busy":"2021-10-08T13:51:11.853107Z","iopub.execute_input":"2021-10-08T13:51:11.853789Z","iopub.status.idle":"2021-10-08T13:51:11.871907Z","shell.execute_reply.started":"2021-10-08T13:51:11.853598Z","shell.execute_reply":"2021-10-08T13:51:11.871007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = open('/kaggle/input/notebookf30568c6d4/df_train_train', 'rb')\ndf_train_train = pickle.load(file)\nfile.close()\n\nfile = open('/kaggle/input/notebookf30568c6d4/df_train_test', 'rb')\ndf_train_test = pickle.load(file)\nfile.close()","metadata":{"execution":{"iopub.status.busy":"2021-10-08T13:51:11.873529Z","iopub.execute_input":"2021-10-08T13:51:11.873822Z","iopub.status.idle":"2021-10-08T13:51:11.887212Z","shell.execute_reply.started":"2021-10-08T13:51:11.873789Z","shell.execute_reply":"2021-10-08T13:51:11.886425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FACTOR = 4\nBATCH_SIZE = 8 * FACTOR\nIMG_SIZE = 512\nEPOCHS = 20\nWARMUP_EPOCHS = 5\nLEARNING_RATE = 1e-4 * FACTOR\nWARMUP_LEARNING_RATE = 1e-3 * FACTOR\nHEIGHT = 320\nWIDTH = 320\nCANAL = 3\nN_CLASSES = df_train_train['diagnosis'].nunique()\nES_PATIENCE = 5\nRLROP_PATIENCE = 3\nDECAY_DROP = 0.5\n\nLR_WARMUP_EPOCHS_1st = 2\nLR_WARMUP_EPOCHS_2nd = 5\nSTEP_SIZE = len(df_train_train) // BATCH_SIZE\nTOTAL_STEPS_1st = WARMUP_EPOCHS * STEP_SIZE\nTOTAL_STEPS_2nd = EPOCHS * STEP_SIZE\nWARMUP_STEPS_1st = LR_WARMUP_EPOCHS_1st * STEP_SIZE\nWARMUP_STEPS_2nd = LR_WARMUP_EPOCHS_2nd * STEP_SIZE","metadata":{"execution":{"iopub.status.busy":"2021-10-08T13:51:11.888738Z","iopub.execute_input":"2021-10-08T13:51:11.888933Z","iopub.status.idle":"2021-10-08T13:51:11.899051Z","shell.execute_reply.started":"2021-10-08T13:51:11.888909Z","shell.execute_reply":"2021-10-08T13:51:11.898220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def multiple_outputs(generator,dataframe, image_dir, batch_size, height,width, subset):\n    gen = generator.flow_from_dataframe(\n        dataframe = dataframe,\n        x_col = \"file_name\",\n        y_col = \"diagnosis\",\n        directory = image_dir,\n        target_size=(height, width),\n        batch_size=batch_size,\n        class_mode='categorical',\n        subset=subset)\n    \n    mlb = MultiLabelBinarizer(classes = range(N_CLASSES))\n    \n    while True:\n        gnext = gen.next()\n        yield gnext[0], [np.argmax(gnext[1],axis = -1),gnext[1],mlb.fit_transform([list(range(x+1)) for x in np.argmax(gnext[1],axis = -1)])]","metadata":{"execution":{"iopub.status.busy":"2021-10-08T13:51:11.900878Z","iopub.execute_input":"2021-10-08T13:51:11.902047Z","iopub.status.idle":"2021-10-08T13:51:11.911423Z","shell.execute_reply.started":"2021-10-08T13:51:11.901232Z","shell.execute_reply":"2021-10-08T13:51:11.910579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datagen=ImageDataGenerator(rescale=1./255, rotation_range=360,brightness_range=[0.5, 1.5],\n                                     zoom_range=[1, 1.2],zca_whitening=True,horizontal_flip=True,\n                                     vertical_flip=True,fill_mode='constant',cval=0.,validation_split = 0.2)\n\ntrain_generator = multiple_outputs(generator = train_datagen,dataframe = df_train_train,\n                                   image_dir=\"/kaggle/input/notebookf30568c6d4/train_images_resized_preprocessed/\",\n                                   batch_size=BATCH_SIZE,height = HEIGHT,width = WIDTH,\n                                   subset='training')\nvalid_generator=multiple_outputs(generator = train_datagen,dataframe = df_train_train,\n                                   image_dir=\"/kaggle/input/notebookf30568c6d4/train_images_resized_preprocessed/\",\n                                   batch_size=BATCH_SIZE,height = HEIGHT,width = WIDTH,\n                                   subset='validation')\n     \n# valid_generator = multiple_outputs(generator = train_datagen,dataframe = df_train_test,\n#                                    image_dir=\"/kaggle/input/notebookf30568c6d4/test_images_resized_preprocessed/\",\n#                                    batch_size=BATCH_SIZE,height = HEIGHT,width = WIDTH,\n#                                    subset='validation')","metadata":{"execution":{"iopub.status.busy":"2021-10-08T13:51:11.914476Z","iopub.execute_input":"2021-10-08T13:51:11.915062Z","iopub.status.idle":"2021-10-08T13:51:11.924743Z","shell.execute_reply.started":"2021-10-08T13:51:11.915035Z","shell.execute_reply":"2021-10-08T13:51:11.924001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_tensor = Input(shape=(HEIGHT, WIDTH, CANAL))\nbase_model = applications.ResNet50(weights=None, include_top=False,input_tensor=input_tensor)\nbase_model.load_weights('/kaggle/input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')\n\n\nx1 = GlobalAveragePooling2D()(base_model.output)\nx1 = BatchNormalization()(x1)\n\nx2 = GlobalMaxPooling2D()(base_model.output)\nx2 = BatchNormalization()(x2)\n\nx = concatenate([x1,x2])\n\n# Regression Head\nxr = Dense(2048, activation='relu')(x)\nxr = Dropout(0.5)(xr)\nxr = Dense(1,activation = 'linear',name = 'regression_output')(xr)\n\n# Classification Head\nxc = Dense(2048, activation='relu')(x)\nxc = Dropout(0.5)(xc)\nxc = Dense(N_CLASSES,activation = 'softmax',name = 'classification_output')(xc)\n\n# Ordinal Regression Head\nxo = Dense(2048, activation='relu')(x)\nxo = Dropout(0.5)(xo)\nxo = Dense(N_CLASSES,activation = 'softmax',name = 'ordinal_regression_output')(xo)\n\nmodel = Model(inputs = [input_tensor], outputs = [xr,xc,xo])","metadata":{"execution":{"iopub.status.busy":"2021-10-08T13:51:11.978604Z","iopub.execute_input":"2021-10-08T13:51:11.979165Z","iopub.status.idle":"2021-10-08T13:51:15.338436Z","shell.execute_reply.started":"2021-10-08T13:51:11.979125Z","shell.execute_reply":"2021-10-08T13:51:15.337711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for layer in model.layers:\n    layer.trainable = True\n# model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-10-08T13:51:15.341856Z","iopub.execute_input":"2021-10-08T13:51:15.342423Z","iopub.status.idle":"2021-10-08T13:51:15.352162Z","shell.execute_reply.started":"2021-10-08T13:51:15.342394Z","shell.execute_reply":"2021-10-08T13:51:15.351328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STEP_SIZE_TRAIN = len(df_train_train)//BATCH_SIZE\nSTEP_SIZE_VALID = len(df_train_test)//BATCH_SIZE\nprint(STEP_SIZE_TRAIN,STEP_SIZE_VALID)","metadata":{"execution":{"iopub.status.busy":"2021-10-08T13:51:15.353559Z","iopub.execute_input":"2021-10-08T13:51:15.354118Z","iopub.status.idle":"2021-10-08T13:51:15.361113Z","shell.execute_reply.started":"2021-10-08T13:51:15.354081Z","shell.execute_reply":"2021-10-08T13:51:15.360208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cosine_decay_with_warmup(global_step,\n                             learning_rate_base,\n                             total_steps,\n                             warmup_learning_rate=0.0,\n                             warmup_steps=0,\n                             hold_base_rate_steps=0):\n    \"\"\"\n    Cosine decay schedule with warm up period.\n    In this schedule, the learning rate grows linearly from warmup_learning_rate\n    to learning_rate_base for warmup_steps, then transitions to a cosine decay\n    schedule.\n    :param global_step {int}: global step.\n    :param learning_rate_base {float}: base learning rate.\n    :param total_steps {int}: total number of training steps.\n    :param warmup_learning_rate {float}: initial learning rate for warm up. (default: {0.0}).\n    :param warmup_steps {int}: number of warmup steps. (default: {0}).\n    :param hold_base_rate_steps {int}: Optional number of steps to hold base learning rate before decaying. (default: {0}).\n    :param global_step {int}: global step.\n    :Returns : a float representing learning rate.\n    :Raises ValueError: if warmup_learning_rate is larger than learning_rate_base, or if warmup_steps is larger than total_steps.\n    \"\"\"\n\n    if total_steps < warmup_steps:\n        raise ValueError('total_steps must be larger or equal to warmup_steps.')\n    learning_rate = 0.5 * learning_rate_base * (1 + np.cos(\n        np.pi *\n        (global_step - warmup_steps - hold_base_rate_steps\n         ) / float(total_steps - warmup_steps - hold_base_rate_steps)))\n    if hold_base_rate_steps > 0:\n        learning_rate = np.where(global_step > warmup_steps + hold_base_rate_steps,\n                                 learning_rate, learning_rate_base)\n    if warmup_steps > 0:\n        if learning_rate_base < warmup_learning_rate:\n            raise ValueError('learning_rate_base must be larger or equal to warmup_learning_rate.')\n        slope = (learning_rate_base - warmup_learning_rate) / warmup_steps\n        warmup_rate = slope * global_step + warmup_learning_rate\n        learning_rate = np.where(global_step < warmup_steps, warmup_rate,\n                                 learning_rate)\n    return np.where(global_step > total_steps, 0.0, learning_rate)\n\n\nclass WarmUpCosineDecayScheduler(Callback):\n    \"\"\"Cosine decay with warmup learning rate scheduler\"\"\"\n\n    def __init__(self,\n                 learning_rate_base,\n                 total_steps,\n                 global_step_init=0,\n                 warmup_learning_rate=0.0,\n                 warmup_steps=0,\n                 hold_base_rate_steps=0,\n                 verbose=0):\n        \"\"\"\n        Constructor for cosine decay with warmup learning rate scheduler.\n        :param learning_rate_base {float}: base learning rate.\n        :param total_steps {int}: total number of training steps.\n        :param global_step_init {int}: initial global step, e.g. from previous checkpoint.\n        :param warmup_learning_rate {float}: initial learning rate for warm up. (default: {0.0}).\n        :param warmup_steps {int}: number of warmup steps. (default: {0}).\n        :param hold_base_rate_steps {int}: Optional number of steps to hold base learning rate before decaying. (default: {0}).\n        :param verbose {int}: quiet, 1: update messages. (default: {0}).\n        \"\"\"\n\n        super(WarmUpCosineDecayScheduler, self).__init__()\n        self.learning_rate_base = learning_rate_base\n        self.total_steps = total_steps\n        self.global_step = global_step_init\n        self.warmup_learning_rate = warmup_learning_rate\n        self.warmup_steps = warmup_steps\n        self.hold_base_rate_steps = hold_base_rate_steps\n        self.verbose = verbose\n        self.learning_rates = []\n\n    def on_batch_end(self, batch, logs=None):\n        self.global_step = self.global_step + 1\n        lr = K.get_value(self.model.optimizer.lr)\n        self.learning_rates.append(lr)\n\n    def on_batch_begin(self, batch, logs=None):\n        lr = cosine_decay_with_warmup(global_step=self.global_step,\n                                      learning_rate_base=self.learning_rate_base,\n                                      total_steps=self.total_steps,\n                                      warmup_learning_rate=self.warmup_learning_rate,\n                                      warmup_steps=self.warmup_steps,\n                                      hold_base_rate_steps=self.hold_base_rate_steps)\n        K.set_value(self.model.optimizer.lr, lr)\n        if self.verbose > 0:\n            print('\\nBatch %02d: setting learning rate to %s.' % (self.global_step + 1, lr))","metadata":{"execution":{"iopub.status.busy":"2021-10-08T13:51:15.362867Z","iopub.execute_input":"2021-10-08T13:51:15.363253Z","iopub.status.idle":"2021-10-08T13:51:15.379972Z","shell.execute_reply.started":"2021-10-08T13:51:15.363208Z","shell.execute_reply":"2021-10-08T13:51:15.379362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cosine_lr = WarmUpCosineDecayScheduler(learning_rate_base = LEARNING_RATE,\n                                       total_steps=TOTAL_STEPS_1st,\n                                       warmup_learning_rate=0.0,\n                                       warmup_steps=TOTAL_STEPS_1st,\n                                       hold_base_rate_steps=(2 * STEP_SIZE))\n\ncallback_list = [cosine_lr]","metadata":{"execution":{"iopub.status.busy":"2021-10-08T13:51:15.383442Z","iopub.execute_input":"2021-10-08T13:51:15.383723Z","iopub.status.idle":"2021-10-08T13:51:15.392591Z","shell.execute_reply.started":"2021-10-08T13:51:15.383672Z","shell.execute_reply":"2021-10-08T13:51:15.391818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer = optimizers.SGD(lr=LEARNING_RATE),\n              loss={'regression_output': 'mean_absolute_error', \n                    'classification_output': 'categorical_crossentropy',\n                    'ordinal_regression_output' : 'binary_crossentropy'\n                    },\n              metrics = ['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-10-08T13:51:15.393864Z","iopub.execute_input":"2021-10-08T13:51:15.394181Z","iopub.status.idle":"2021-10-08T13:51:15.415929Z","shell.execute_reply.started":"2021-10-08T13:51:15.394147Z","shell.execute_reply":"2021-10-08T13:51:15.415244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit_generator(generator=train_generator,\n                              steps_per_epoch=STEP_SIZE_TRAIN,\n                              validation_data=valid_generator,\n                              validation_steps=STEP_SIZE_VALID,\n                              epochs=20,\n                              callbacks = callback_list,\n                              verbose=1)\n\nmodel.save(\"/kaggle/working/model_pre_training.h5\")\n# f = open(\"/kaggle/working/history_pre_training\",\"wb\")\n# pickle.dump(history,f)\n# f.close()","metadata":{"execution":{"iopub.status.busy":"2021-10-08T13:51:15.418470Z","iopub.execute_input":"2021-10-08T13:51:15.419069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = open(\"/kaggle/working/history_pre_training\",\"wb\")\npickle.dump(history.history,f)\nf.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,5))\nplt.plot(history.history['regression_output_loss'])\nplt.plot(history.history['val_regression_output_loss'])\nplt.title('Regression Model Loss - Pre Training')\nplt.ylabel('Loss (MAE)')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Validation'], loc='best')\nplt.xticks(range(1,21))\nplt.gca().ticklabel_format(axis='both', style='plain', useOffset=False)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,5))\nplt.plot(history.history['classification_output_loss'])\nplt.plot(history.history['val_classification_output_loss'])\nplt.title('Classification Model Loss - Pre Training')\nplt.ylabel('Loss (Categorical Cross Entropy)')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Validation'], loc='best')\nplt.xticks(range(1,21))\nplt.gca().ticklabel_format(axis='both', style='plain', useOffset=False)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,5))\nplt.plot(history.history['ordinal_regression_output_loss'])\nplt.plot(history.history['val_ordinal_regression_output_loss'])\nplt.title('Ordinal Regression Model Loss - Pre Training')\nplt.ylabel('Loss (Binary Cross Entropy)')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Validation'], loc='best')\nplt.xticks(range(1,21))\nplt.gca().ticklabel_format(axis='both', style='plain', useOffset=False)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for layer in model.layers:\n    layer.trainable = False\n\nfor i in range(-14,0):\n  model.layers[i].trainable = True\n\n# model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ref - https://github.com/umbertogriffo/focal-loss-keras/blob/master/losses.py\n\n'''Below Functions create custom loss functions - Categorical Focal Loss, Binary Focal Loss'''\n\ndef binary_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n\n        return -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\n    return binary_focal_loss_fixed","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def categorical_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Softmax version of focal loss.\n           m\n      FL = ∑  -alpha * (1 - p_o,c)^gamma * y_o,c * log(p_o,c)\n          c=1\n      where m = number of classes, c = class and o = observation\n    Parameters:\n      alpha -- the same as weighing factor in balanced cross entropy\n      gamma -- focusing parameter for modulating factor (1-p)\n    Default value:\n      gamma -- 2.0 as mentioned in the paper\n      alpha -- 0.25 as mentioned in the paper\n    References:\n        Official paper: https://arxiv.org/pdf/1708.02002.pdf\n        https://www.tensorflow.org/api_docs/python/tf/keras/backend/categorical_crossentropy\n    Usage:\n     model.compile(loss=[categorical_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def categorical_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred: A tensor resulting from a softmax\n        :return: Output tensor.\n        \"\"\"\n\n        # Scale predictions so that the class probas of each sample sum to 1\n        y_pred /= K.sum(y_pred, axis=-1, keepdims=True)\n\n        # Clip the prediction value to prevent NaN's and Inf's\n        epsilon = K.epsilon()\n        y_pred = K.clip(y_pred, epsilon, 1. - epsilon)\n\n        # Calculate Cross Entropy\n        cross_entropy = -y_true * K.log(y_pred)\n\n        # Calculate Focal Loss\n        loss = alpha * K.pow(1 - y_pred, gamma) * cross_entropy\n\n        # Compute mean loss in mini_batch\n        return K.mean(loss, axis=1)\n\n    return categorical_focal_loss_fixed","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer = optimizers.Adam(lr=WARMUP_LEARNING_RATE),\n              loss={'regression_output': mean_squared_error, \n                    'classification_output': categorical_focal_loss(alpha=.25, gamma=2) ,\n                    'ordinal_regression_output' : binary_focal_loss(alpha=.25, gamma=2)\n                    },\n              metrics = ['accuracy'])\n\nhistory = model.fit_generator(generator=train_generator,\n                              steps_per_epoch=STEP_SIZE_TRAIN,\n                              validation_data=valid_generator,\n                              validation_steps=STEP_SIZE_VALID,\n                              epochs=5,\n                              callbacks = callback_list,\n                              verbose=1).history\n\nmodel.save(\"/kaggle/working/model_main_training.h5\")\n# f = open(\"/kaggle/working/history_main_training\",\"wb\")\n# pickle.dump(history.history,f)\n# f.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for layer in model.layers:\n    layer.trainable = True\n# model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer = optimizers.Adam(lr=LEARNING_RATE),\n              loss={'regression_output': mean_squared_error, \n                    'classification_output': categorical_focal_loss(alpha=.25, gamma=2) ,\n                    'ordinal_regression_output' : binary_focal_loss(alpha=.25, gamma=2)\n                    },\n              metrics = ['accuracy'])\n\nhistory = model.fit_generator(generator=train_generator,\n                              steps_per_epoch=STEP_SIZE_TRAIN,\n                              validation_data=valid_generator,\n                              validation_steps=STEP_SIZE_VALID,\n                              epochs=45,\n                              callbacks = callback_list,\n                              verbose=1).history\n\nmodel.save(\"/kaggle/working/model_main_training.h5\")\n# f = open(\"/kaggle/working/history_main_training\",\"wb\")\n# pickle.dump(history.history,f)\n# f.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = open(\"/kaggle/working/history_main_training\",\"wb\")\npickle.dump(history,f)\nf.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.plot(history['regression_output_loss'])\nplt.plot(history['val_regression_output_loss'])\nplt.title('Regression Model Loss - Main Training')\nplt.ylabel('Loss (MSE)')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Validation'], loc='best')\nplt.xticks(range(1,46))\nplt.gca().ticklabel_format(axis='both', style='plain', useOffset=False)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.plot(history['classification_output_loss'])\nplt.plot(history['val_classification_output_loss'])\nplt.title('Classification Model Loss - Main Training')\nplt.ylabel('Loss (Categorical Focal Loss)')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Validation'], loc='best')\nplt.xticks(range(1,46))\nplt.gca().ticklabel_format(axis='both', style='plain', useOffset=False)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nplt.figure(figsize=(15,5))\nplt.plot(history['ordinal_regression_output_loss'])\nplt.plot(history['val_ordinal_regression_output_loss'])\nplt.title('Ordinal Regression Model Loss - Main Training')\nplt.ylabel('Loss (Binary Focal Loss)')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Validation'], loc='best')\nplt.xticks(range(1,46))\nplt.gca().ticklabel_format(axis='both', style='plain', useOffset=False)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('hello world')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}