{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Introduction\n---\n---\nwhile creating an image classifier the very first approach we should go for is to look for a pre trained models instead of directly creating a model from scratch in this notebook we will look at how we can use a pre trained model and fine tune for our particular in this case melanoma classification. The model we are going to use is tensorflows InceptionV3","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom kaggle_datasets import KaggleDatasets\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow_addons as tfa\nfrom tensorflow.keras.layers import Dense , Dropout , BatchNormalization , concatenate\nfrom tensorflow.keras.layers import Activation , Input , GlobalAveragePooling2D  \nfrom tensorflow.keras.models import Model , Sequential\nfrom tensorflow.keras.applications import InceptionV3 , MobileNetV2\nfrom sklearn.model_selection import KFold\nimport re\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utitlity Functions\n---\nnow lets first create some our utility function for data preparation , image preprocessing and augumentation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_DIMS = 64 # inception net has a minimum input size of 75x75 thus 150 is good\nCHANNELS = 3\nBATCH_SIZE = 32\nSEED = 42\nSPLITS = 5\n\nAUTO  = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# getting training data gcs path\nGCS_PATH = KaggleDatasets().get_gcs_path('melanoma-256x256')\ntrain_datasets = tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')\n\nprint('number of TFRecords in train : ',len(train_datasets))\n\n# getting testing data\nGCS_PATH = KaggleDatasets().get_gcs_path('siim-isic-melanoma-classification')\ntest_datasets = tf.io.gfile.glob(GCS_PATH + '/tfrecords/test*.tfrec')\n\nprint('number of TFRecords in test : ' ,len(test_datasets))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# parse data from TF Records\ndef parse_TFR_data_labelled(sample):\n    features = {\n      'image': tf.io.FixedLenFeature([] , tf.string , default_value = ''),\n      'image_name': tf.io.FixedLenFeature([] , tf.string , default_value=''),\n      'patient_id': tf.io.FixedLenFeature([] , tf.int64 , default_value=0),\n      'sex': tf.io.FixedLenFeature([] , tf.int64 , default_value=0),\n      'age_approx': tf.io.FixedLenFeature([] , tf.int64 , default_value=0),\n      'anatom_site_general_challenge':tf.io.FixedLenFeature([] ,tf.int64 , default_value=0 ),\n      'diagnosis': tf.io.FixedLenFeature([] ,tf.int64 , default_value=0 ),\n      'target': tf.io.FixedLenFeature([] ,tf.int64 , default_value=0 ),\n      'width': tf.io.FixedLenFeature([] ,tf.int64 , default_value=0 ),\n      'height': tf.io.FixedLenFeature([] ,tf.int64 , default_value=0 )\n    }\n    \n    p = tf.io.parse_single_example(sample , features)\n    \n    img = p['image']\n    target = p['target']\n    \n    return img , target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# decode img\ndef decode_image(img , IMG_DIMS):\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img , [IMG_DIMS , IMG_DIMS])\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load data set for training and validation\ndef _get_ds(files , train=True , repeat=True , img_dims=64 , batch_size=32):\n    ds = tf.data.TFRecordDataset(files , num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    ds = ds.map(parse_TFR_data_labelled , num_parallel_calls=AUTO)     \n    ds = ds.map(lambda img ,label: (decode_image(img, img_dims),label) , num_parallel_calls=AUTO)\n    if train:\n        ds = ds.shuffle(buffer_size=1000)\n       \n    ds = ds.batch(batch_size*REPLICAS)\n    ds = ds.prefetch(AUTO)\n    return ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# preparing the testing data\ndef parsed_TFR_unlabelled_2(sample):\n    feature_description = {\n        'image': tf.io.FixedLenFeature([], tf.string, default_value=''),\n        'image_name': tf.io.FixedLenFeature([], tf.string, default_value=''),\n        'target': tf.io.FixedLenFeature([], tf.int64, default_value=0),\n    }\n    p = tf.io.parse_single_example(sample , feature_description)\n    img = p['image']\n    name = p['image_name']\n    return name , img\n\ntest_data= tf.data.TFRecordDataset(test_datasets)\ntest_data = test_data.map(parsed_TFR_unlabelled_2 , num_parallel_calls=AUTO)\ntest_data = test_data.map(lambda name , img: (name , decode_image(img , 64)))\n\nsub_df = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\n\nx_dict = {}\nfor p in test_data:\n    temp = {p[0].numpy().decode() : p[1].numpy()}\n    x_dict.update(temp)\n    \nprint(f'number of samples in testing data : {len(x_dict)}')\n\ntest = []\nfor i in sub_df['image_name']:\n    test.append(x_dict[i])\n    del(x_dict[i])\n    \ntest = np.array(test)\nprint(f'sahpe of testing set sorted according to the submission file : {test.shape}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now everything is ready lets\nthere are two ways we can use a pre trained model \n1. As a feature extractor - in this approach we don't train the pre trained model but use it as a feature extractor that is we will only use the trained model to process the image and produce the feature vector and then we will train a small inference model on these feature vectors. this approach is suitable when your dataset is very simple and very similar to what the model was trained on\n\n2. Fine Tuning - In this approach we retrrain the whole model on our dataset . now sometimes fine tuning also requires to adjust what layers you want to train and what you don't want. as most of the time the initial layers produce the same result for almost every data so you don't need to train those layers however the lower layers or the top layers are the place things changes so you need to train those layers\n\n# Using Pre Trained Model as a Feature extractor\n---\n---\n\nnow there are two ways of using a feature extractor\n1. use it as a part of the model - in this approach the features are regenerated in each epoch this computionally expensive if you want your model to work on  alow end device\n2. extract the features once - in this approach we extract the features of all the images and store them in a dataset and use that dataset to train the model. this approach is prefered when you are on a low end device but it takes more effort than the first one as you have to create a whole new dataset and makw sure that label of each features is accurately map to the correct feature\n\nnow here we are going for the first approach as we have resources to bear those computations thanks to kaggle for providing us with free TPU and GPUs \n\nfirst we need to create our base model and set all its layers to non trainable","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"now inception net requires each pixel in the image to be in the range of -1 and 1 so we inception_v3 preprocess layer to perform this transformation <br>\nand then we create a sequential model for image augumentaion which will augument the image and will feed it to base model and then base model will extract the features and will feed it to the inference model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# augumentaion model\naug = Sequential([\n    tf.keras.layers.experimental.preprocessing.RandomFlip('horizontal'),\n    tf.keras.layers.experimental.preprocessing.RandomRotation(0.2),\n    tf.keras.layers.experimental.preprocessing.RandomContrast(0.3)\n])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_Inf_model(IMG_DIMS , CHANNELS):\n    in_put = Input(shape=(IMG_DIMS , IMG_DIMS , 3))\n    # applying auggumentations\n    #pre = aug(in_put)\n    # pre process layer\n    pre_process_layer = tf.keras.applications.mobilenet_v2.preprocess_input\n    pre = pre_process_layer(in_put)\n    # base model non trainable\n    base_model = MobileNetV2(input_shape=(IMG_DIMS , IMG_DIMS , 3) , include_top=False , weights='imagenet')\n    x = base_model(pre , training = False)\n    # top trainable model layers\n    x = GlobalAveragePooling2D()(x)\n    x = Dense(128 , activation = 'relu')(x)\n    x = Dropout(0.3)(x)\n    x = Dense(1 , activation = 'sigmoid')(x)\n    model = Model(inputs=in_put , outputs=x)\n    # optimizer\n    opt = tf.keras.optimizers.Adam(0.0001)\n    model.compile(optimizer=opt , loss='binary_crossentropy' , metrics=['accuracy' , 'AUC'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now our model is reeady but since we don't have any validation set we will KFold cross validation technique to cross validate our model\n\nFirst lets allocate the TPUs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"TPU = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(TPU)\ntf.tpu.experimental.initialize_tpu_system(TPU)\nstrategy = tf.distribute.experimental.TPUStrategy(TPU)\n\nREPLICAS = strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nkf = KFold(n_splits=SPLITS)\noof_hist = []\noof_val = []\nfor f , (idxT , idxV) in enumerate(kf.split(train_datasets)):\n    train = []\n    val =[]\n    for idx in idxT:\n        train.append(train_datasets[idx])\n    for idx in idxV:\n        val.append(train_datasets[idx])\n\n    # instantiate model\n    with strategy.scope():\n         # cretae model check points\n        cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath='inf_model_fold_'+str(f)+'.hdf5' , \n                                                         monitor='val_auc',\n                                                         mode='max',\n                                                         save_best_only =True,\n                                                         verbose = 1 )\n        # early stopping\n        es_callback = tf.keras.callbacks.EarlyStopping(monitor='val_auc' , patience=5 , mode='max' )\n        model = create_Inf_model(IMG_DIMS , CHANNELS)\n        \n        history = model.fit(_get_ds(train) ,\n                        epochs = 20 ,\n                        steps_per_epoch = count_data_items(train)/BATCH_SIZE//REPLICAS,\n                        validation_data=_get_ds(val , train=False , repeat=False),\n                        callbacks = [cp_callback , es_callback],\n                        verbose = 0)\n    oof_hist.append(history)\n    oof_val.append(_get_ds(val, train=False))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(10,10))\ni = 1\nfor p in oof_hist:\n    fig.add_subplot(2,3,i)\n    plt.plot(p.history['loss'])\n    plt.plot(p.history['val_loss'])\n    plt.title('fold '+str(i)+ ' LOSS')\n    i +=1\n\nplt.legend(labels= ['loss' ,'val_loss'])\nfig.tight_layout(pad=3)\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(10,10))\ni = 1\nfor p in oof_hist:\n    fig.add_subplot(2,3,i)\n    plt.plot(p.history['auc'])\n    plt.plot(p.history['val_auc'])\n    plt.title('fold '+str(i) + ' AUC')\n    i +=1\n\nplt.legend(labels= ['auc' ,'val_auc'])\nfig.tight_layout(pad=3)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model = create_Inf_model(IMG_DIMS , CHANNELS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model.load_weights('inf_model_fold_0.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = best_model.predict(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df['target'] = preds\nsub_df.set_index('image_name' , inplace=True)\nsub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"so you can see with the results our dataset is not that simple . thus we would need to fine tune our model \n\n# fine tuning a pre trained model\nnow in this case we will be training our pre trained models except for its bottom layers so first lets look at how many layers are their in our model\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"base = InceptionV3(input_shape = (150, 150 , 3) , weights ='imagenet' , include_top=False)\nprint(f'# layers in base model = {len(base.layers)}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**IMPORTANT** now there is one important thing you need to know about while unfreezing your model is that if your pre trained model contains **BatchNormalization layer** then you must keep them frozen because BatchNormalization contains two non-trainable weights which get updated during training . so if you unfreeze these layers it will change these non trainable weights and will destroy the knowledge of pre trained model. thus must check if your model contains these layers and keep them frozen while unfreezing the model\n\nso now lets look how many BatchNormalization layers our model contains","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"count = 0\nfor layers in base.layers:\n    if type(layers) == tf.python.keras.layers.normalization_v2.BatchNormalization:\n        count +=1\n\nprint(f'no of BatchNormalization Layers in our base model : {count}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now lets create a simple function which will keep these layers frozen","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def freeze_bn_layers(model):\n    for layers in model.layers:\n        if type(layers) == tf.python.keras.layers.normalization_v2.BatchNormalization:\n            layers.trainable = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"so we won't train the bottom 100 layers. lets define a function to set these layers to non trainable","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_non_train_layers(model):\n    n = 100\n    for layers in model.layers[:100]:\n        layers.trainable = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now lets create a our fine tune model function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def trainable_model(IMG_DIMS , CHANNELS):\n    in_put = Input(shape=(IMG_DIMS , IMG_DIMS , 3))\n    # pre process layer\n    pre_process_layer = tf.keras.applications.mobilenet_v2.preprocess_input\n    pre = pre_process_layer(in_put)\n    # base model non trainable\n    base_model = MobileNetV2(input_shape=(IMG_DIMS , IMG_DIMS , 3) , include_top=False , weights='imagenet')\n    base_model.trainable = True\n    # set non trainable layers in the model\n    set_non_train_layers(base_model)\n    # freeze bn layers\n    freeze_bn_layers(base_model)\n    x = base_model(pre)\n    # top trainable model layers\n    x = GlobalAveragePooling2D()(x)\n    x = Dense(128 , activation = 'relu')(x)\n    x = Dropout(0.3)(x)\n    x = Dense(1 , activation = 'sigmoid')(x)\n    model = Model(inputs=in_put , outputs=x)\n    # optimizer\n    model.compile(optimizer='adam' , loss='binary_crossentropy' , metrics=['accuracy' , 'AUC'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now to train a deep network we should use a learning rate schedule for fine tuning the learning rate of our model and also speed up training so lets define that , this lr schedule I have copied from chris deotte's notebook stratifiedKfold with TFrecords\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def lr_schedule(batch_size= 16):\n    lr_start = 0.000005\n    lr_max = 0.00000125 * REPLICAS * batch_size\n    lr_min = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep = 0\n    lr_decay = 0.8\n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n\n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n\n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n\n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"now lets retrain this model ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"kf = KFold(n_splits=SPLITS)\noof_hist = []\noof_val = []\nfor f , (idxT , idxV) in enumerate(kf.split(train_datasets)):\n    train = []\n    val =[]\n    for idx in idxT:\n        train.append(train_datasets[idx])\n    for idx in idxV:\n        val.append(train_datasets[idx])\n\n    # instantiate model\n    with strategy.scope():\n        # lr callback\n        lr_callback = lr_schedule(BATCH_SIZE)\n        # cretae model check points\n        cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath='inf_model_fold_'+str(f)+'.hdf5' , \n                                                         monitor='val_auc',\n                                                         mode='max',\n                                                         save_best_only =True,\n                                                         verbose = 1 )\n        # early stopping\n        es_callback = tf.keras.callbacks.EarlyStopping(monitor='val_auc' , patience=5 , mode='max' )\n        model = trainable_model(IMG_DIMS , CHANNELS)\n        history = model.fit(_get_ds(train) ,\n                            epochs = 20 ,\n                            steps_per_epoch = count_data_items(train)/BATCH_SIZE//REPLICAS,\n                            validation_data=_get_ds(val , train=False , repeat=False),\n                            callbacks = [lr_callback , cp_callback , es_callback],\n                            verbose = 0)\n    oof_hist.append(history)\n    oof_val.append(_get_ds(val, train=False))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(10,10))\ni = 1\nfor p in oof_hist:\n    fig.add_subplot(2,3,i)\n    plt.plot(p.history['loss'])\n    plt.plot(p.history['val_loss'])\n    plt.title('fold '+str(i)+ ' LOSS')\n    i +=1\n\nplt.legend(labels= ['loss' ,'val_loss'])\nfig.tight_layout(pad=3)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(10,10))\ni = 1\nfor p in oof_hist:\n    fig.add_subplot(2,3,i)\n    plt.plot(p.history['auc'])\n    plt.plot(p.history['val_auc'])\n    plt.title('fold '+str(i) + ' AUC')\n    i +=1\n\nplt.legend(labels= ['auc' ,'val_auc'])\nfig.tight_layout(pad=3)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model2 = trainable_model(IMG_DIMS , CHANNELS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_model2.load_weights('inf_model_fold_0.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds2 = best_model2.predict(test)\nsub_df['target'] = preds2\nsub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.to_csv('submission2.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}