{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Reading Data","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train=pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ntest=pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Checking for null values in data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(train.isnull().sum()/len(train.index))*100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dropping rows having null values\n\ntrain=train.dropna(axis=0).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(train.isnull().sum()/len(train.index))*100","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### As I will be using Keras to for implement Image Datagenerator and model building ,so the image so I will append .jpg in file ,so that it can be used with flow_from_dataframe method","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['image_name']=train['image_name'].apply(lambda x:x+'.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['image_name']=test['image_name'].apply(lambda x:x+'.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df=pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.target.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(train.target)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There is high class imbalance ,we will try to deal with this using weighted loss ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Importing important Libraries\n\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# import seaborn as sns\n# import matplotlib.pyplot as plt\n# import tensorflow as tf\n\n# from keras_preprocessing.image import ImageDataGenerator\n# # from keras.applications.densenet import DenseNet121\n# from keras.layers import Dense, GlobalAveragePooling2D\n# from keras.models import Model\n# from keras import backend as K\n\n# from keras.models import load_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# importing few important libraries for pre-processing,generator and model building\n\nimport tensorflow as tf\n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n# from keras.applications.densenet import DenseNet121\nfrom tensorflow.keras.layers import Dense, GlobalAveragePooling2D\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import backend as K\n\nfrom tensorflow.keras.models import load_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import Input,Add,Dense,Activation,ZeroPadding2D,BatchNormalization\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.layers import Dense,Dropout,Flatten\n# from keras.applications import resnet50\n# from keras.applications.resnet50 import preprocess_input\nK.set_image_data_format('channels_last')\nK.set_learning_phase","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.applications.resnet50  import ResNet50","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Checking Data leakage\n\nThere is a possible case that the image can be of same patient in training and test data ,we avoid it ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def checkLeakage(df1,df2,patient):\n    df1_unique = set(df1[patient].values)\n    df2_unique = set(df2[patient].values)\n    \n    patients_in_train_test = list(df1_unique.intersection(df2_unique))\n    if len(patients_in_train_test)>0:\n        leakage = True# boolean (true if there is at least 1 patient in both groups)\n        print(patients_in_train_test)\n    else:\n        leakage=False\n    \n    \n    return leakage","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"checkLeakage(train,test,'patient_id')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing Images","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"#### In the first part we will do build model without using any folds ,however in the second part we will improve it using stratified K-fold","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Train and Validation Generator","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_val_generator(df, image_dir, x_col, y_cols, shuffle=True, batch_size=8, seed=1, target_w = 240, target_h = 240):\n        print(\"getting train generator...\") \n    # normalize images\n        image_generator = ImageDataGenerator(\n            samplewise_center=True,\n            samplewise_std_normalization= True,\n            zoom_range=0.2,\n            shear_range=0.2,\n            validation_split=0.25)\n    \n    # flow from directory with specified batch size\n    # and target image size\n        train_generator = image_generator.flow_from_dataframe(\n            dataframe=train,\n            directory=image_dir,\n            validation_split=0.25,\n            x_col=x_col,\n            y_col=y_cols,\n            class_mode='raw',\n            batch_size=batch_size,\n            shuffle=shuffle,\n            seed=seed,\n            validate_filenames=True,\n            subset='training',\n            target_size=(target_w,target_h))\n        \n        valid_generator=image_generator.flow_from_dataframe(\n            dataframe=train,\n            directory=image_dir,\n            x_col=x_col,\n            y_col=y_cols,\n            validate_filenames=False,\n            batch_size=32,\n            shuffle=False,\n            class_mode='raw',\n            seed=seed,\n            subset='validation',\n            target_size=(target_w,target_h))\n        \n        return train_generator,valid_generator\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_DIR = \"../input/siim-isic-melanoma-classification/jpeg/train\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.makedirs('saved_models_noFold')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    \n#     image_input=Input(shape=(240, 240, 3))\n    base_model= ResNet50(weights='imagenet',include_top=False,input_shape=(240,240,3))\n    for layer in base_model.layers:\n        layer.trainable=False\n        \n#     Get base model output\n\n    base_model_output=base_model.output\n    \n#     Adding our own layers\n\n    x=GlobalAveragePooling2D()(base_model_output)\n#     adding fully connected layers\n\n    x=Dense(512,activation='relu')(x)\n    x=Dense(1,activation='sigmoid',name='fcnew')(x)\n    \n    model=Model(base_model.input,x)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating a function control gradient descent rate\n# Initially we will keep the loss high \n# will decrease it with the training\ndef lr_schedule(epoch):\n    lr = 1e-3\n    if epoch > 180:\n        lr *= 0.5e-3\n    elif epoch > 160:\n        lr *= 1e-3\n    elif epoch > 120:\n        lr *= 1e-2\n    elif epoch > 80:\n        lr *= 1e-1\n    print('Learning rate: ', lr)\n    return lr\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils import class_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data,valid_data=train_val_generator(train, IMAGE_DIR, \"image_name\", 'target')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weights = class_weight.compute_class_weight('balanced',np.unique(train_data.labels),train_data.labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weights_map={0:class_weights[0],1:class_weights[1]}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weight_map={}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get model\n\nmodel=get_model()\n\nbatch_size=32\n\n\n\n# Compile the model\n\nmodel.compile(loss='binary_crossentropy',optimizer ='sgd',metrics=[tf.keras.metrics.AUC()])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ncheckpoint = tensorflow.keras.callbacks.ModelCheckpoint('saved_models_noFold'+'.h5', verbose=1,save_best_only=True)\n\ncallbacks_list = [checkpoint]\n\n# model.compile(loss=get_weighted_loss(pos_weights,neg_weights),optimizer ='sgd',metrics=['accuracy'])\n\n\n\nhistory = model.fit_generator(train_data,\n                              validation_data=valid_data,\n                              steps_per_epoch=int(math.ceil(1. * len(train_data)// batch_size)),\n                              validation_steps=int(math.ceil(1. * len(valid_data)// batch_size)),\n                              callbacks=callbacks_list,\n                              class_weight=class_weights_map,\n#                                   workers=1,                        # maximum number of processes to spin up when using process-based threading\n#                                   use_  multiprocessing=False,\n                              epochs =10)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights(\"saved_models_noFold.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Writing test generator\ndef test_generator(df,image_dir, x_col, shuffle=False, seed=1, target_w = 240, target_h = 240):\n        print(\"getting test generator...\") \n        print(image_dir)\n    # normalize images\n        image_generator = ImageDataGenerator(rescale=1/255)\n        \n        \n        test_generator = image_generator.flow_from_dataframe(\n            dataframe=test,\n            directory=image_dir,\n            x_col=x_col,\n            y_col=None,\n#             classes=['test'],\n#             x_col=x_col,\n#             y_col=y_cols,\n            class_mode=None,\n#             batch_size=batch_size,\n            shuffle=False,\n            seed=seed,\n            target_size=(target_w,target_h))\n        \n        return test_generator\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_path='../input/siim-isic-melanoma-classification/jpeg/test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_gen=test_generator(test,test_path, \"image_name\",target_w = 240, target_h = 240)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# type(test_gen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predicted_vals = model.predict_generator(test_gen,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(predicted_vals)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"l=test['image_name'].apply(lambda x:x.split('.jpg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img=[im[0] for im in l]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img.index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_sub=test['image_name']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df=pd.DataFrame({'image_name':img})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['target']=predicted_vals","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -ltr\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_sub['target']=pd.DataFrame(data=predicted_vals)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predicted_class_indices = np.argmax(predicted_vals, axis=1)\n# labels = train_gen.class_indices\npredictions = [labels[k] for k in predicted_class_indices]\n\ntest_df['target'] = pd.DataFrame(data=predictions)\n# submission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compile the model\n    model.compile(loss='binary_crossentropy',optimizer ='sgd',metrics=[tf.keras.metrics.AUC()])\n    # CREATE CALLBACKS\n\t\n    checkpoint = keras.callbacks.ModelCheckpoint('model_'+str(fold_var)+'.h5', verbose=1,save_best_only=True)\n    \n    callbacks_list = [checkpoint]\n    \n    train_data_generator=train_val_generator(train, IMAGE_DIR, \"image_name\", 'target')\n    \n    history = model.fit_generator(train_data_generator,validation_data=valid_data_generator,\n                                  steps_per_epoch=int(math.ceil(1. * X_train.shape[0] // batch_size)),\n                                  validation_steps=int(math.ceil(1. * X_Val.shape[0] // batch_size)),\n                                  callbacks=callbacks_list,\n                                  class_weight=class_weights,\n#                                   workers=1,                        # maximum number of processes to spin up when using process-based threading\n                                  use_multiprocessing=False,\n                                  epochs =10)\n    \n    # LOAD BEST MODEL to evaluate the performance of the model\n   \n\t\n    model.load_weights(\"model_\"+str(fold_var)+\".h5\")\n\t","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Implementing stratified K-fold for best model\n\nfrom sklearn.model_selection import StratifiedKFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X =train.loc[:, ~train.columns.isin(['target'])].copy()\ny = train['target']\nskf = StratifiedKFold(n_splits=10,shuffle=False)\nskf.get_n_splits(X, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.index\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y.index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(skf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.makedirs('saved_models')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cd ../../outputs/saved_models\n# ../input/siim-isic-melanoma-classification","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -ltr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cd saved_models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -ltr\n!pwd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create the base pre-trained model\n# base_model = DenseNet121(weights='./nih/densenet.hdf5', include_top=False)\n\n# x = base_model.output\n\n# # add a global spatial average pooling layer\n# x = GlobalAveragePooling2D()(x)\n\n# # and a logistic layer\n# predictions = Dense(len(labels), activation=\"sigmoid\")(x)\n\n# model = Model(inputs=base_model.input, outputs=predictions)\n# model.compile(optimizer='adam', loss=get_weighted_loss(pos_weights, neg_weights))\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import Input,Add,Dense,Activation,ZeroPadding2D,BatchNormalization\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.layers import Dense,Dropout,Flatten\n# from keras.applications import resnet50\n# from keras.applications.resnet50 import preprocess_input\nK.set_image_data_format('channels_last')\nK.set_learning_phase","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.applications.resnet50  import ResNet50\n# from tensorflow.keras.preprocessing import image\n# from tensorflow.keras.applications.resnet50 import preprocess_input, decode_predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.applications.ResNet50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef get_model():\n    \n#     image_input=Input(shape=(240, 240, 3))\n    base_model= ResNet50(weights='imagenet',include_top=False,input_shape=(240,240,3))\n    for layer in base_model.layers:\n        layer.trainable=False\n        \n#     Get base model output\n\n    base_model_output=base_model.output\n    \n#     Adding our own layers\n\n    x=GlobalAveragePooling2D()(base_model_output)\n#     adding fully connected layers\n\n    x=Dense(512,activation='relu')(x)\n    x=Dense(1,activation='sigmoid',name='fcnew')(x)\n    \n    model=Model(base_model.input,x)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" model=get_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Writing function so that we speed up training \n# We will keep the learning rate high when the loss is high \n# As the loss decreases we will reduce the learning rate\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils import class_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VALIDATION_ACCURACY = []\nVALIDAITON_LOSS = []\n\nsave_dir = 'saved_models'\nfold_var = 1\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_generator = ImageDataGenerator(\n            samplewise_center=True,\n            samplewise_std_normalization= True,\n            zoom_range=0.2,\n            shear_range=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for train_index, val_index in skf.split(X, y):\n#     print(\"TRAIN:\", train_index, \"val:\", val_index)\n    batch_size=32\n    X_train, X_Val = X.iloc[train_index], X.iloc[val_index]\n    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n    train_df=pd.concat([X_train, y_train],axis=1)\n    val_df=pd.concat([X_Val, y_val],axis=1)\n#     train_generator,valid_generator =train_val_generator(train_df, IMAGE_DIR, \"image_name\", 'target',val_df)\n    train_data_generator = image_generator.flow_from_dataframe(train_df,\n                                                               directory = IMAGE_DIR,\n                                                               x_col = \"image_name\",\n                                                               y_col = \"target\",\n                                                               class_mode = \"raw\",\n                                                               batch_size=batch_size, target_size=(240,240),\n                                                               shuffle = True)\n    \n    valid_data_generator  = image_generator.flow_from_dataframe(val_df, \n                                                                directory = IMAGE_DIR,\n                                                                x_col = \"image_name\",\n                                                                y_col = \"target\",\n                                                                class_mode = \"raw\", \n                                                                batch_size=batch_size, \n                                                                target_size=(240,240),\n                                                                shuffle = True)\n    \n    class_weights = class_weight.compute_class_weight('balanced',np.unique(train_data_generator.labels),train_data_generator.labels)\n# create model \n    model=get_model()\n\n# Compile the model\n    model.compile(loss='binary_crossentropy',optimizer ='sgd',metrics=[tf.keras.metrics.AUC()])\n    # CREATE CALLBACKS\n\t\n    checkpoint = keras.callbacks.ModelCheckpoint('model_'+str(fold_var)+'.h5', verbose=1,save_best_only=True)\n    \n    callbacks_list = [checkpoint]\n    \n    history = model.fit_generator(train_data_generator,validation_data=valid_data_generator,\n                                  steps_per_epoch=int(math.ceil(1. * X_train.shape[0] // batch_size)),\n                                  validation_steps=int(math.ceil(1. * X_Val.shape[0] // batch_size)),\n                                  callbacks=callbacks_list,\n                                  class_weight=class_weights,\n#                                   workers=1,                        # maximum number of processes to spin up when using process-based threading\n                                  use_multiprocessing=False,\n                                  epochs =10)\n    \n    # LOAD BEST MODEL to evaluate the performance of the model\n   \n\t\n    model.load_weights(\"model_\"+str(fold_var)+\".h5\")\n\t\n    results = model.evaluate(valid_data_generator)\n    \n    results = dict(zip(model.metrics_names,results))\n\t\n    VALIDATION_ACCURACY.append(history.history['val_loss'])\n    VALIDATION_LOSS.append(history.history['val_acc'])\n\t\n    fold_var += 1\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator,valid_generator =train_val_generator(train, IMAGE_DIR, \"image_name\", 'target')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nx, y = train_generator.__getitem__(0)\nplt.imshow(x[0]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(glob.glob(\"../input/siim-isic-melanoma-classification/jpeg/train/*.jpg\"))\n\ndef compute_class_freqs(labels):\n    \"\"\"\n    Compute positive and negative frequences for each class.\n\n    Args:\n        labels (np.array): matrix of labels, size (num_examples, num_classes)\n    Returns:\n        positive_frequencies (np.array): array of positive frequences for each\n                                         class, size (num_classes)\n        negative_frequencies (np.array): array of negative frequences for each\n                                         class, size (num_classes)\n    \"\"\"\n    \n    # total number of patients (rows)\n    N = labels.shape\n    \n    \n    positive_frequencies = np.sum(labels,axis=0)/N[0]\n    negative_frequencies = (N[0]-np.sum(labels,axis=0))/N[0]\n\n    return positive_frequencies, negative_frequencies","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Checking class imbalance","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to get weight loss which we will be used for training model\n\ndef get_weighted_loss(pos_weights,neg_weight,epsilon=1e-7):\n    def weighted_loss(pred_pos,pred_neg):\n        loss=0#initialising the loss to 0\n        loss +=  K.mean(pos_weights * np.array(train['target'])*K.log(y_pred_1+epsilon) + neg_weights*(1- np.array(train['target'])*K.log(1-y_pred_2+epsilon)))\n        return loss       \n        \n    return weighted_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create the base pre-trained model\n# base_model = DenseNet121(weights='./nih/densenet.hdf5', include_top=False)\n\n# x = base_model.output\n\n# # add a global spatial average pooling layer\n# x = GlobalAveragePooling2D()(x)\n\n# # and a logistic layer\n# predictions = Dense(len(labels), activation=\"sigmoid\")(x)\n\n# model = Model(inputs=base_model.input, outputs=predictions)\n# model.compile(optimizer='adam', loss=get_weighted_loss(pos_weights, neg_weights))\n\nfrom keras import layers\nfrom keras.layers import Input,Add,Dense,Activation,ZeroPadding2D,BatchNormalization\nfrom keras import optimizers\nfrom keras.layers import Dense,Dropout,Flatten\n# from keras.applications import resnet50\n# from keras.applications.resnet50 import preprocess_input\nK.set_image_data_format('channels_last')\nK.set_learning_phase\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.applications import ResNet50\n# from tensorflow.keras.preprocessing import image\n# from tensorflow.keras.applications.resnet50 import preprocess_input, decode_predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ! wget https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef get_model():\n    \n    base_model= ResNet50(weights='imagenet',include_top=False)\n    for layer in base_model.layers:\n        layer.trainable=False\n        \n#     Get base model output\n\n    base_model_output=base_model.output\n    \n#     Adding our own layers\n\n    x=GlobalAveragePooling2D()(base_model_output)\n#     adding fully connected layers\n\n    x=Dense(512,activation='relu')(x)\n    x=Dense(1,activation='sigmoid',name='fcnew')(x)\n    \n    model=Model(input=base_model.input,output=x)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"set(train_generator.labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"freq_pos, freq_neg = compute_class_freqs(train_generator.labels)\nfreq_pos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"freq_neg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pos_weights = freq_neg\nneg_weights = freq_pos\npos_contribution = freq_pos * pos_weights \nneg_contribution = freq_neg * neg_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pos_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.utils import class_weight","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(train_generator.labels),\n                                                 train_generator.labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_weights={0:class_weights[0],1:class_weights[1]}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# y_pred_1  = K.constant(pos_weights*np.array(train['target']).reshape(33126,1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# y_pred_2  = K.constant(neg_weights*np.array(train['target']).reshape(33126,1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data = pd.DataFrame({\"Class\": labels, \"Label\": \"Positive\", \"Value\": freq_pos})\n# data = data.append([{\"Class\": labels[l], \"Label\": \"Negative\", \"Value\": v} for l,v in enumerate(freq_neg)], ignore_index=True)\n# plt.xticks(rotation=90)\n# f = sns.barplot(x=\"Class\", y=\"Value\", hue=\"Label\" ,data=data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get model\n\nmodel=get_model()\n\n# Compile the model\n\nmodel.compile(loss='binary_crossentropy',optimizer ='sgd',metrics=['accuracy'])\n\n# model.compile(loss=get_weighted_loss(pos_weights,neg_weights),optimizer ='sgd',metrics=['accuracy'])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(train_generator, \n                              steps_per_epoch=100,\n                              class_weight=class_weights,\n                              epochs =10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.ylabel(\"loss\")\nplt.xlabel(\"epoch\")\nplt.title(\"Training Loss Curve\")\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}