{"cells":[{"metadata":{},"cell_type":"markdown","source":"**References**\n[https://www.kaggle.com/ragnar123/efficientnet-x-384](http://)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**Imports**","execution_count":null},{"metadata":{"id":"u-pXBTI5Tmsx","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom sklearn.preprocessing import LabelEncoder,LabelBinarizer\nfrom sklearn.model_selection import train_test_split,KFold\nfrom tensorflow.keras import *\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import *\nfrom kaggle_datasets import KaggleDatasets\nimport numpy as np\nimport pandas as pd\ngcs_path=KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')\nBATCH_SIZE=128","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Data Preprocessing**","execution_count":null},{"metadata":{"id":"2T-eddlmT_hm","trusted":true},"cell_type":"code","source":"train_csv=pd.read_csv(gcs_path+'/train.csv')\ntest_csv=pd.read_csv(gcs_path+'/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"id":"MPYdEnCMUSJJ","trusted":true},"cell_type":"code","source":"train_csv['age_approx']=train_csv['age_approx'].fillna(0)\ntrain_csv['sex']=train_csv['sex'].fillna('na')\ntrain_csv['anatom_site_general_challenge']=train_csv['anatom_site_general_challenge'].fillna('na')","execution_count":null,"outputs":[]},{"metadata":{"id":"5OLde6VUVJLn","outputId":"80b11075-819d-4187-b7ff-842f30120c24","trusted":true},"cell_type":"code","source":"train_csv.isna().any()","execution_count":null,"outputs":[]},{"metadata":{"id":"3m3hOQaVVMau","trusted":true},"cell_type":"code","source":"le=LabelEncoder()\nbi=LabelBinarizer()","execution_count":null,"outputs":[]},{"metadata":{"id":"8Qzuo0uTVRAX","trusted":true},"cell_type":"code","source":"train_csv['sex']=bi.fit_transform(train_csv['sex'])\ntrain_csv['anatom_site_general_challenge']=le.fit_transform(train_csv['anatom_site_general_challenge'])","execution_count":null,"outputs":[]},{"metadata":{"id":"5wP5b3H0VdX2","outputId":"ce88a3f9-7a85-481d-f1a8-4ce04c2caeab","trusted":true},"cell_type":"code","source":"train_csv.anatom_site_general_challenge.value_counts().plot(kind='barh')","execution_count":null,"outputs":[]},{"metadata":{"id":"Gf0msgJ_WvT_","trusted":true},"cell_type":"code","source":"test_csv['sex']=bi.fit_transform(test_csv['sex'])\ntest_csv['anatom_site_general_challenge']=test_csv['anatom_site_general_challenge'].fillna('na')\ntest_csv['anatom_site_general_challenge']=le.fit_transform(test_csv['anatom_site_general_challenge'])","execution_count":null,"outputs":[]},{"metadata":{"id":"yKusyaBGWr23","trusted":true},"cell_type":"code","source":"feat=['age_approx','sex','anatom_site_general_challenge']","execution_count":null,"outputs":[]},{"metadata":{"id":"-dasAlFtWlgI","trusted":true},"cell_type":"code","source":"X=train_csv[feat]\ny=train_csv['target']","execution_count":null,"outputs":[]},{"metadata":{"id":"tl6Q1l4BXZO1","trusted":true},"cell_type":"code","source":"#X_train,X_val,y_train,y_val=train_test_split(X,y,test_size=0.25,random_state=5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Making the Dataset**","execution_count":null},{"metadata":{"id":"0Uguuk2JX-aZ","trusted":true},"cell_type":"code","source":"def get_dataset(features,target,shuffle=False):\n   X=tf.data.Dataset.from_tensor_slices(tf.stack(features))\n   y=tf.data.Dataset.from_tensor_slices(target)\n   ds=tf.data.Dataset.zip((X,y))\n   ds=ds.repeat()\n   ds=ds.batch(BATCH_SIZE)\n   if shuffle:\n     ds=ds.shuffle(1234,reshuffle_each_iteration=True) #reshuffle_each_iteration=True\n   ds=ds.cache()\n   return ds","execution_count":null,"outputs":[]},{"metadata":{"id":"-HX4RJ5zyAIi","outputId":"c1fa4fa5-179c-4f42-988c-7232274e4fe1","trusted":true},"cell_type":"code","source":"\"\"\"train_X,train_y=X.iloc[train],y[train]\nvalid_X,valid_y=X.iloc[valid],y[valid]\ntrain_ds=get_dataset(X_train,y_train,shuffle=True)\nval_ds=get_dataset(X_val,y_val,shuffle=False)\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"id":"MlYZ65YJb47k","trusted":true},"cell_type":"code","source":"test_ds=tf.data.Dataset.from_tensor_slices(tf.stack(test_csv[feat]))\ntest_ds=test_ds.batch(BATCH_SIZE)\ntest_ds=test_ds.cache()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **Binary Focal Loss Function**","execution_count":null},{"metadata":{"id":"43SoAcuZ0ivR","trusted":true},"cell_type":"code","source":"def binary_focal_loss(gamma=2., alpha=.25):\n    \"\"\"\n    Binary form of focal loss.\n      FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)\n      where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.\n    References:\n        https://arxiv.org/pdf/1708.02002.pdf\n    Usage:\n     model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=[\"accuracy\"], optimizer=adam)\n    \"\"\"\n    def binary_focal_loss_fixed(y_true, y_pred):\n        \"\"\"\n        :param y_true: A tensor of the same shape as `y_pred`\n        :param y_pred:  A tensor resulting from a sigmoid\n        :return: Output tensor.\n        \"\"\"\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n\n        epsilon = K.epsilon()\n        # clip to prevent NaN's and Inf's\n        pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)\n        pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)\n\n        return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \\\n               -K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n\n    return binary_focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model Building**","execution_count":null},{"metadata":{"id":"vor1vg5fdZJ9","trusted":true},"cell_type":"code","source":"def create_model():\n  model=Sequential([\n                    Dense(256,activation='relu',input_shape=(3,),\n                          kernel_regularizer=regularizers.l2(0.001)),\n                    Dropout(0.2),\n                    BatchNormalization(),\n                    Dense(108,activation='relu',\n                          kernel_regularizer=regularizers.l2(0.001)),\n                    Dropout(0.2),\n                    Dense(182,activation='relu',\n                          kernel_regularizer=regularizers.l2(0.001)),\n                    Dropout(0.2),\n                    Dense(108,activation='relu',\n                         kernel_regularizer=regularizers.l2(0.001)),\n                    Dropout(0.2),\n                    Dense(108,activation='relu',\n                          kernel_regularizer=regularizers.l2(0.001)),\n                    Dense(1024,activation='relu',\n                          kernel_regularizer=regularizers.l2(0.001)),\n                    BatchNormalization(),\n                    Dropout(0.2),\n                    Dense(1,activation='sigmoid')\n  ])\n  model.compile(optimizer='sgd',\n                      loss=[binary_focal_loss(gamma = 2.2, alpha = 0.82)],\n                      metrics=[metrics.BinaryAccuracy(),metrics.AUC()]\n                )\n  return model","execution_count":null,"outputs":[]},{"metadata":{"id":"qucLIKmHfn6C","trusted":true},"cell_type":"code","source":"# Learning rate schedule for TPU, GPU and CPU.\n# Using an LR ramp up because fine-tuning a pre-trained model.\n# Starting with a high LR would break the pre-trained weights.\n\nLR_START = 0.004\nLR_MAX = 0.00005 * 16\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 4\nLR_SUSTAIN_EPOCHS = 4\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**KFOLD Cross Validation**","execution_count":null},{"metadata":{"id":"3gssNyrfvqBE","outputId":"e6d9b2db-b76c-4390-c12a-b41030a8b084","trusted":true},"cell_type":"code","source":"models=[]\noof_predictions=[]\noof_target=[]\nkf=KFold(n_splits=15,shuffle=True,random_state=1234)\n\nfor folds,(train,valid) in enumerate(kf.split(X,y)):\n  print('\\n')\n  print('-'*50)\n  print(f'Training fold {folds + 1}')\n  train_X,train_y=X.iloc[train],y[train]\n  valid_X,valid_y=X.iloc[valid],y[valid]\n  train_ds=get_dataset(train_X,train_y,True)\n  valid_ds=get_dataset(valid_X,valid_y,False)\n  K.clear_session()\n  model=create_model()\n  STEPS_PER_EPOCH=len(train_X)//BATCH_SIZE\n  VALIDATION_STEPS=len(valid_X)//BATCH_SIZE\n  es=tf.keras.callbacks.EarlyStopping(monitor = 'val_auc', mode = 'max', patience = 8, \n                                      verbose = 1, min_delta = 0.0001, restore_best_weights = True)\n  cb_schd=tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n  tb=tf.keras.callbacks.TensorBoard(log_dir=f'logs/{folds +1}')\n  history=model.fit(train_ds,\n          epochs=50,\n          steps_per_epoch=STEPS_PER_EPOCH,\n          validation_data=valid_ds,\n          validation_steps=VALIDATION_STEPS,\n          callbacks=[es,cb_schd,tb]\n          )\n  models.append(model)\n  probabilities = model.predict(valid_X)\n  oof_target.extend(list(valid_y))\n  oof_predictions.extend(list(np.concatenate(probabilities)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_curve,auc\nact,pred,threshold=roc_curve(oof_target,oof_predictions)\nprint(\"AUC SCORE : \",auc(act,pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Predictions**","execution_count":null},{"metadata":{"id":"9V1tQNirkwHn","trusted":true},"cell_type":"code","source":"sample_sub=pd.read_csv(gcs_path+'/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"id":"YsWt5AyTmI_A","trusted":true},"cell_type":"code","source":"sample_sub.head(5)","execution_count":null,"outputs":[]},{"metadata":{"id":"Tzq5Rh_Mm--3","trusted":true},"cell_type":"code","source":"df=sample_sub.copy()","execution_count":null,"outputs":[]},{"metadata":{"id":"miTytt1_mL9O","trusted":true},"cell_type":"code","source":"preds = np.average([np.concatenate(models[i].predict(test_ds)) for i in range(folds)], axis = 0)","execution_count":null,"outputs":[]},{"metadata":{"id":"1mn9u-XymWra","trusted":true},"cell_type":"code","source":"df.target=preds","execution_count":null,"outputs":[]},{"metadata":{"id":"yzy10kzxm0KW","trusted":true},"cell_type":"code","source":"df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"id":"MSX8b_87m6Ym","trusted":true},"cell_type":"code","source":"df.to_csv('sub.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}