{"cells":[{"metadata":{},"cell_type":"markdown","source":"#   **1000 Each(Balanced)**"},{"metadata":{},"cell_type":"markdown","source":"****Vgg16 -Using Keras****"},{"metadata":{"trusted":true},"cell_type":"code","source":"#necessary Libraries\nimport numpy as np\nimport pandas as pd\nimport os\nfrom glob import glob \nimport matplotlib.pyplot as plt\nfrom sklearn.utils import shuffle\nimport shutil\n\nimport random\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_curve, auc\n\nfrom tqdm import tqdm\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nfrom keras.applications.vgg16 import VGG16, preprocess_input\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten\n\nfrom keras.optimizers import Adam, SGD\nfrom keras import optimizers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#for handling the data set\npath = \"../input/histopathologic-cancer-detection/\" \nlabels = pd.read_csv(path + 'train_labels.csv')\ntrain_path = path + 'train/'\ntest_path = path + 'test/'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Create a dataframe which contains every training examples path, id and label**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#fixing the dataframe\ndf = pd.DataFrame({'path': glob(os.path.join(train_path,'*.tif'))})\ndf['id'] = df.path.map(lambda x: ((x.split(\"in\")[2].split('.')[0])[1:]))\ndf = df.merge(labels, on = \"id\")\ndf.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SAMPLE_SIZE = 1000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#filter out class 0\ndf_0 = df[df['label'] == 0].sample(SAMPLE_SIZE, random_state = 101)\n# filter out class 1\ndf_1 = df[df['label'] == 1].sample(SAMPLE_SIZE, random_state = 101)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# concat the dataframes\ndf_data = pd.concat([df_0, df_1], axis=0).reset_index(drop=True)\n# shuffle\ndf_data = shuffle(df_data)\n\ndf_data['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"****Going to split 20% of the training set into a validation set****"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Use stratify= df['label'] to get balance ratio 1/1 in train and validation sets\ndf_train, df_val = train_test_split(df_data, test_size=0.2, stratify= df_data['label'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Moving images to directory"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Delete directory\nimport shutil\nshutil.rmtree('main', ignore_errors=True)\n\n# Create directory\nos.mkdir('main')\n\n# Create subfolder for train and val images\nos.mkdir(os.path.join('main', 'train'))\nos.mkdir(os.path.join('main', 'val'))\n\n# Create subfolders for true positive and true negative in train\nos.mkdir(os.path.join('main','train','true_positive'))\nos.mkdir(os.path.join('main','train','true_negative'))      \n         \n# Create subfolders for true positive and true negative in val\nos.mkdir(os.path.join('main','val','true_positive'))\nos.mkdir(os.path.join('main','val','true_negative'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Prepare image name classes for the directory structure\n# Save all train true positive names to list and add .tif\ntrain_df_1 = df_train[df_train[\"label\"] == 1]['id'].tolist()\ntrain_df_1 = [name + \".tif\" for name in train_df_1]\n\n# Save all train true negativeto names list and add .tif\ntrain_df_0 = df_train[df_train[\"label\"] == 0]['id'].tolist()\ntrain_df_0 = [name + \".tif\" for name in train_df_0]\n\n# Save all val true positive \"id\" to list and add .tif\nval_df_1 = df_val[df_val[\"label\"] == 1]['id'].tolist()\nval_df_1 = [name + \".tif\" for name in val_df_1]\n\n# Save all val true negative \"id\" to list\nval_df_0 = df_val[df_val[\"label\"] == 0]['id'].tolist()\nval_df_0 = [name + \".tif\" for name in val_df_0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Move images to directory structure\nimport shutil\nimport os\nfrom tqdm import tqdm\n\ndef transfer(source,destination,files):\n    for image in tqdm(files):\n        # source path to image\n        src = os.path.join(source,image)\n        dst = os.path.join(destination,image)\n        # copy the image from the source to the destination\n        shutil.copyfile(src,dst)\n        \n# transfer\ntransfer('../input/histopathologic-cancer-detection/train','main/train/true_positive',train_df_1)\ntransfer('../input/histopathologic-cancer-detection/train','main/train/true_negative',train_df_0)\ntransfer('../input/histopathologic-cancer-detection/train','main/val/true_positive',val_df_1)\ntransfer('../input/histopathologic-cancer-detection/train','main/val/true_negative',val_df_0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"****Increasing the size of the image results in a much higher performance****"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate batches of tensor image data with real-time data augmentation. \nimport numpy as np\nnum_train_samples = len(df_train)\nnum_val_samples = len(df_val)\ntrain_batch_size = 32\nval_batch_size = 32\n\ntrain_steps = np.ceil(num_train_samples / train_batch_size)\nval_steps = np.ceil(num_val_samples / val_batch_size)\n\nprint(train_steps)\nprint(val_steps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n# Augmentation \ntrain_datagen = ImageDataGenerator(\n                rescale=1./255,\n                vertical_flip=True,\n                horizontal_flip=True,\n                rotation_range=90,\n                shear_range=0.05)\ntest_datagen = ImageDataGenerator(rescale=1./255)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"****Resizing and dividing training data into train and validation****"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generator that will read pictures found in subfolers of 'main/train', and indefinitely generate batches of augmented image data\ntrain_generator = train_datagen.flow_from_directory('main/train',\n                                            target_size=(96,96),\n                                            batch_size=train_batch_size,\n                                            class_mode='categorical')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generator that will read pictures found in subfolers of 'main/train', and indefinitely generate batches of augmented image data\ntrain_generator = train_datagen.flow_from_directory('main/train',\n                                            target_size=(96,96),\n                                            batch_size=train_batch_size,\n                                            class_mode='categorical')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_generator = test_datagen.flow_from_directory('main/val',\n                                                  target_size=(96,96),\n                                            batch_size=val_batch_size,\n                                            class_mode='categorical')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !!! batch_size=1 & shuffle=False !!!!\ntest_generator = test_datagen.flow_from_directory('main/val',\n                                            target_size=(96,96),\n                                            batch_size=1,\n                                            class_mode='categorical',\n                                            shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Creating Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import VGG16 model, with weights pre-trained on ImageNet.\nfrom keras.applications.vgg16 import VGG16, preprocess_input\n\n# VGG model without the last classifier layers (include_top = False)\nvgg16_model = VGG16(include_top = False,\n                    input_shape = (96,96,3),\n                    #weights='../input/VGG16weights/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5')\n                    weights = 'imagenet')\n    \n# Freeze the layers \nfor layer in vgg16_model.layers[:-12]:\n    layer.trainable = False\n    \n# Check the trainable status of the individual layers\nfor layer in vgg16_model.layers:\n    print(layer, layer.trainable)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense,Flatten,Dropout\n\nmodel = Sequential()\nmodel.add(vgg16_model)\nmodel.add(Flatten())\nmodel.add(Dense(1024, activation=\"relu\"))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(512, activation=\"relu\"))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(2, activation=\"softmax\"))\n# Load previous weights to save the time\n#model.load_weights('../input/pretrained3/model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"****As we're using Vgg16 trained on ImageNet, we're going to train the last few layers instead of the just the last one. Cell images are quite different to what we see on ImageNet****"},{"metadata":{"trusted":true},"cell_type":"code","source":"#for last few layers\nfrom keras.optimizers import Adam, SGD\nfrom keras import optimizers\n\n\nmodel.compile(loss='binary_crossentropy',optimizer=optimizers.SGD(lr=0.00001, momentum=0.95),metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\n\n# starting time\nstart = time.time()\n\nhistory = model.fit_generator(\n                    train_generator, \n                    steps_per_epoch  = train_steps, \n                    validation_data  = val_generator,\n                    validation_steps = val_steps,\n                    epochs           = 30, \n                    verbose          = 1)\n# end time\nend = time.time()\n\n# total time taken\nprint(f\"Total Train Time is {end - start}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EVALUATION"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot validation and accuracies over epochs\nimport matplotlib.pyplot as plt\nepochs = [i for i in range(1, len(history.history['loss'])+1)]\n\nplt.plot(epochs, history.history['loss'], color='blue', label=\"training_loss\")\nplt.plot(epochs, history.history['val_loss'], color='red', label=\"validation_loss\")\nplt.legend(loc='best')\nplt.title('Loss')\nplt.xlabel('epoch')\nplt.show()\ntrain_acc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\n\nepochs = range(len(train_acc))\n\nplt.plot(epochs,train_acc,'b',label='Training accuracy')\nplt.plot(epochs,val_acc,'r',label='Validation accuracy')\nplt.title('Training and validation accuracy')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Validation Accuracy: \" + str(history.history['val_accuracy'][-1:]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Training Accuracy: \" + str(history.history['accuracy'][-1:]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Validation Loss: \" + str(history.history['val_loss'][-1:]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Training Loss: \" + str(history.history['loss'][-1:]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_predict = model.predict_generator(test_generator, steps=len(df_val), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ROC CURVE"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc\n\nfpr, tpr, thresholds = roc_curve(test_generator.classes, val_predict.argmax(axis=1))   \n# Compute ROC area\nprint(\"ROC area is: \" + str(auc(fpr, tpr)))\n\nplt.figure()\nplt.plot(fpr, tpr, color='darkred', label='ROC curve (area = %0.2f)' % auc(fpr, tpr))\nplt.plot([0, 1], [0, 1], color='darkblue', linestyle='--')\nplt.xlim([-0.01, 1.0])\nplt.ylim([0.0, 1.01])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('Receiver Operating Characteristic')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PREDICTIONS"},{"metadata":{},"cell_type":"markdown","source":"****Using Test Time Augmentation. For each test image we will augment it 3 ways and average the prediction.****"},{"metadata":{"trusted":true},"cell_type":"code","source":"testdf = pd.DataFrame({'path': glob(os.path.join(test_path, '*.tif'))})\ntestdf['id'] = testdf.path.map(lambda x: (x.split(\"st/\")[1].split('.')[0]))\ntestdf.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = testdf.sample(SAMPLE_SIZE, random_state = 101)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tta_datagen = ImageDataGenerator(rescale=1./255, #Normalise\n                                 vertical_flip = True,\n                                 horizontal_flip = True,\n                                 rotation_range=90,\n                                 zoom_range=0.2, \n                                 width_shift_range=0.1,\n                                 height_shift_range=0.1,\n                                 shear_range=0.05,\n                                 channel_shift_range=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nfrom IPython.display import clear_output\ntta_steps = 3\nfinal = pd.DataFrame()\nfor index in range(0, len(df)):\n    data_frame = pd.DataFrame({'path': df.iloc[index,0]}, index=[index])\n    data_frame['id'] = data_frame.path.map(lambda x: x.split('st/')[1].split('.')[0])\n    img_path = data_frame.iloc[0,0]\n    test_img = cv2.imread(img_path)\n    test_img = cv2.resize(test_img,(96,96))\n    test_img = np.expand_dims(test_img, axis = 0)  \n    predictionsTTA = []\n    for i in range(0, tta_steps):\n        preds = model.predict_generator(tta_datagen.flow_from_dataframe(dataframe = data_frame,\n                                                                           directory = None,\n                                                                           x_col = 'path',\n                                                                           target_size = (96, 96),\n                                                                           class_mode = None,\n                                                                           batch_size = 1,\n                                                                           shuffle = False), steps = 1)\n        predictionsTTA.append(preds)\n    clear_output()\n    prediction_entry = np.array(np.round(np.mean(predictionsTTA)))\n    data_frame['label'] = prediction_entry\n    final = pd.concat([final, data_frame[['id', 'label']]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final.set_index('id')\nfinal.head(3)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}