{"cells":[{"metadata":{"_uuid":"a6de38c0ff88b6738c5c8751c722c9b489602d88"},"cell_type":"markdown","source":"# Histopathologic Cancer Detection\n## Técnicas de Soft Computing para Aprendizaje y optimización. Redes Neuronales y Metaheurísticas, programación evolutiva y bioinspirada\n\nJaume Cloquell Capo"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport itertools\n\nimport os\nimport shutil\n\nfrom glob import glob \nfrom skimage.io import imread\nimport gc\n\nfrom keras.preprocessing.image import array_to_img, img_to_array, load_img\nimport numpy as np \nimport pandas as pd \nfrom fastai import *\nfrom fastai.vision import *\nfrom torchvision.models import *\nimport os \nimport path\n\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils import to_categorical\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d06b98f87cfa19e2548b0d3a558128d563942e1b"},"cell_type":"markdown","source":"# Load data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_path = '../input/train/'\ntest_path = '../input/test/'\n\n#base_tile_dir = '../input/train/'\n\ndf = pd.DataFrame({'path': glob(os.path.join(train_path,'*.tif'))})\n\ndf['id'] = df.path.map(lambda x: x.split('/')[3].split(\".\")[0])\n\nlabels = pd.read_csv(\"../input/train_labels.csv\")\ndf_data = df.merge(labels, on = \"id\")\n\n# removing this image because it caused a training error previously\ndf_data = df_data[df_data['id'] != 'dd6dfed324f9fcb6f93f46f32fc800f2ec196be2']\n\n# removing this image because it's black\ndf_data = df_data[df_data['id'] != '9369c7278ec8bcc6c880d99194de09fc2bd4efbe']\ndf_data.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_list = os.listdir(test_path)\ntrain_list = os.listdir(train_path)\nprint(\"There are \" + str(len(train_list)) + \" training examples.\")\nprint(\"There are \" + str(len(test_list)) + \" test examples.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Check distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# source: https://www.kaggle.com/gpreda/honey-bee-subspecies-classification\n\ndef draw_category_images(col_name,figure_cols, df, IMAGE_PATH):\n    categories = (df.groupby([col_name])[col_name].nunique()).index\n    f, ax = plt.subplots(nrows=len(categories),ncols=figure_cols, \n                         figsize=(4*figure_cols,4*len(categories))) # adjust size here\n    # draw a number of images for each location\n    for i, cat in enumerate(categories):\n        sample = df[df[col_name]==cat].sample(figure_cols) # figure_cols is also the sample size\n        for j in range(0,figure_cols):\n            file=IMAGE_PATH + sample.iloc[j]['id'] + '.tif'\n            im=cv2.imread(file)\n            ax[i, j].imshow(im, resample=True, cmap='gray')\n            ax[i, j].set_title(cat, fontsize=16)  \n    plt.tight_layout()\n    plt.show()\n    \ndraw_category_images('label',4, df_data, '../input/train/')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0bfaeb9d367e1b720fd81623e9248c3ff7bb0b5e"},"cell_type":"markdown","source":"# Split X and y in train/test and build folders"},{"metadata":{},"cell_type":"markdown","source":"Balance the target distribution\nWe will reduce the number of samples in class 0."},{"metadata":{"trusted":true,"_uuid":"d60413a93a292379227e2b8979d2abeed70ed718"},"cell_type":"code","source":"SAMPLE_SIZE = 80000# load 80k negative examples\n\n# take a random sample of class 0 with size equal to num samples in class 1\ndf_0 = df_data[df_data['label'] == 0].sample(SAMPLE_SIZE, random_state = 101)\n# filter out class 1\ndf_1 = df_data[df_data['label'] == 1].sample(SAMPLE_SIZE, random_state = 101)\n\n# concat the dataframes\ndf_data = shuffle(pd.concat([df_0, df_1], axis=0).reset_index(drop=True))\n\ndf_data['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_test_split # stratify=y creates a balanced validation set.\ny = df_data['label']\ndf_train, df_val = train_test_split(df_data, test_size=0.10, random_state=101, stratify=y)\n\n# Create directories\ntrain_path = 'base_dir/train'\nvalid_path = 'base_dir/valid'\n#test_path = '../input/test'\nfor fold in [train_path, valid_path]:\n    for subf in [\"0\", \"1\"]:\n        os.makedirs(os.path.join(fold, subf))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transfer the images into the folders"},{"metadata":{"trusted":true,"_uuid":"e85d943cfc261cca54b34550554549244f947400"},"cell_type":"code","source":"# Set the id as the index in df_data\ndf_data.set_index('id', inplace=True)\ndf_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6683c986522a604dc1a715687b1d1c621628e94"},"cell_type":"code","source":"for image in df_train['id'].values:\n    # the id in the csv file does not have the .tif extension therefore we add it here\n    fname = image + '.tif'\n    label = str(df_data.loc[image,'label']) # get the label for a certain image\n    src = os.path.join('../input/train', fname)\n    dst = os.path.join(train_path, label, fname)\n    shutil.copyfile(src, dst)\n\nfor image in df_val['id'].values:\n    fname = image + '.tif'\n    label = str(df_data.loc[image,'label']) # get the label for a certain image\n    src = os.path.join('../input/train', fname)\n    dst = os.path.join(valid_path, label, fname)\n    shutil.copyfile(src, dst)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Set Up the Generators"},{"metadata":{"trusted":true,"_uuid":"145ef9177524908eb5656fd1deb55a82aeb01973"},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\nIMAGE_SIZE = 96\nnum_train_samples = len(df_train)\nnum_val_samples = len(df_val)\ntrain_batch_size = 32\nval_batch_size = 32\n\ndatagen  = ImageDataGenerator(rescale=1.0/255.0,\n                                  vertical_flip = True,\n                                  horizontal_flip = True)\n# don't perform augmentation on validation data\ntest_datagen = ImageDataGenerator(rescale=1.0/255.0)\n\n\ntrain_gen = datagen.flow_from_directory(train_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=train_batch_size,\n                                        class_mode='binary')\n\nval_gen = test_datagen.flow_from_directory(valid_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=val_batch_size,\n                                        class_mode='binary')\n\n# Note: shuffle=False causes the test dataset to not be shuffled\ntest_gen = test_datagen.flow_from_directory(valid_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=1,\n                                        class_mode='binary',\n                                        shuffle=False)\n\ntrain_steps=train_gen.n//train_gen.batch_size\nval_steps=val_gen.n//val_gen.batch_size\n#STEP_SIZE_TEST=test_gen.n//test_gen.batch_size\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Example images generate to ImageDataGenerator"},{"metadata":{"trusted":true},"cell_type":"code","source":"count = 8\n\nimgs = os.listdir('../input/train')[:1]\nimage = img_to_array(load_img('../input/train/'+imgs[0]))\nimage = image.reshape((1,) + image.shape)\n\nfig = plt.figure(figsize=(14, 12))\nfig.subplots_adjust()\n\n# let's create infinite flow of images\nimages_flow = datagen.flow(image, batch_size=1)\n\nfor i, new_images in enumerate(images_flow):\n    # we access only first image because of batch_size=1\n    new_image = array_to_img(new_images[0], scale=True)\n    \n    ax = fig.add_subplot(3,3, i + 1)\n    ax.imshow(new_image)\n    ax.set_title(i)\n\n    if i >= count:\n        break","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"037766b4d6d71232ff280ee04aee45671db93b7a"},"cell_type":"markdown","source":"# Create the Model Architecture"},{"metadata":{"trusted":true,"_uuid":"1d252eb588aaf888171ff82b332efd4a0880cab3"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, BatchNormalization, Activation\nfrom keras.layers import Conv2D, MaxPool2D, AveragePooling2D\nfrom keras.optimizers import RMSprop, Adam\n\nkernel_size = (3,3)\npool_size= (2,2)\n\n#base_filters = 16\nfirst_filters = 32\nsecond_filters = 64\nthird_filters = 128\n\ndropout_conv = 0.3\ndropout_dense = 0.5\n\nmodel = Sequential()\n# (CONV => RELU => POOL) * 2\nmodel.add(Conv2D(first_filters, kernel_size, activation = 'relu', input_shape = (IMAGE_SIZE, IMAGE_SIZE, 3)))\nmodel.add(Conv2D(first_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPool2D(pool_size = pool_size)) \nmodel.add(Dropout(dropout_conv))\n\n# (CONV => RELU => POOL) * 2\nmodel.add(Conv2D(second_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(Conv2D(second_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPool2D(pool_size = pool_size))\nmodel.add(Dropout(dropout_conv))\n\n# (CONV => RELU => POOL) * 2\nmodel.add(Conv2D(third_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(Conv2D(third_filters, kernel_size, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(MaxPool2D(pool_size = pool_size))\nmodel.add(Dropout(dropout_conv))\n\n# first (and only) set of FC => RELU layers\nmodel.add(Flatten())\nmodel.add(Dense(256, use_bias=False))\nmodel.add(BatchNormalization())\nmodel.add(Activation(\"relu\"))\nmodel.add(Dropout(dropout_dense))\n\n# sigmoid classifier\nmodel.add(Dense(1, activation = \"sigmoid\"))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compile the model\nmodel.compile(Adam(0.01), loss = \"binary_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"56a7cce49809eae68d588a542620e16368e4fd1a"},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true,"_uuid":"a6dc621f8fc4f4558cb22289986a4886fe9e64c8"},"cell_type":"code","source":"from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfilepath = \"model.h5\"\n#checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, \n#                             save_best_only=True, mode='max')\nearlystopper = EarlyStopping(monitor='val_acc', patience=2, verbose=1, restore_best_weights=True)\nreducel = ReduceLROnPlateau(monitor='val_loss', patience=1, verbose=1, factor=0.1)\n\nhistory = model.fit_generator(train_gen, steps_per_epoch=train_steps, \n                    validation_data=val_gen,\n                    validation_steps=val_steps,\n                    epochs=13,\n                    use_multiprocessing=True,\n                   workers=16,\n                   max_queue_size=32,\n                   callbacks=[reducel, earlystopper])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model.load_weights('model.h5')\n\nval_loss, val_acc = \\\nmodel.evaluate_generator(test_gen, \n                        steps=len(df_val))\n\nprint('val_loss:', val_loss)\nprint('val_acc:', val_acc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_history(history):\n    loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' not in s]\n    val_loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' in s]\n    acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' not in s]\n    val_acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' in s]\n    \n    if len(loss_list) == 0:\n        print('Loss is missing in history')\n        return \n    \n    ## As loss always exists\n    epochs = range(1,len(history.history[loss_list[0]]) + 1)\n    \n    ## Loss\n    plt.figure(1)\n    for l in loss_list:\n        plt.plot(epochs, history.history[l], 'b', label='Training loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))\n    for l in val_loss_list:\n        plt.plot(epochs, history.history[l], 'g', label='Validation loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))\n    \n    plt.title('Loss')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n    \n    ## Accuracy\n    plt.figure(2)\n    for l in acc_list:\n        plt.plot(epochs, history.history[l], 'b', label='Training accuracy (' + str(format(history.history[l][-1],'.5f'))+')')\n    for l in val_acc_list:    \n        plt.plot(epochs, history.history[l], 'g', label='Validation accuracy (' + str(format(history.history[l][-1],'.5f'))+')')\n\n    plt.title('Accuracy')\n    plt.xlabel('Epochs')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    plt.show()\nplot_history(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c8099fc4994ae9792c74477711794c83db3aeee"},"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc, roc_auc_score\nimport matplotlib.pyplot as plt\n\n# make a prediction\ny_pred_keras = model.predict_generator(test_gen, steps=len(df_val), verbose=1)\nfpr_keras, tpr_keras, thresholds_keras = roc_curve(test_gen.classes, y_pred_keras)\nauc_keras = auc(fpr_keras, tpr_keras)\nauc_keras","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f755d935752d007d21a59ac111cd0d604880c5fa"},"cell_type":"markdown","source":"# Plot ROC Curve"},{"metadata":{"trusted":true,"_uuid":"2397d1c4956ea8615140b9cc40bb857e8a5deeae"},"cell_type":"code","source":"plt.figure(1)\nplt.plot([0, 1], [0, 1], 'k--')\nplt.plot(fpr_keras, tpr_keras, label='area = {:.3f}'.format(auc_keras))\nplt.xlabel('False positive rate')\nplt.ylabel('True positive rate')\nplt.title('ROC curve')\nplt.legend(loc='best')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d52bccbbd7d4a8114b5bc9f53ce26db2601fa238"},"cell_type":"markdown","source":"# Load test data and predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport shutil\n\nshutil.rmtree(train_path)\nshutil.rmtree(valid_path)\n# create test_dir\ntest_dir = 'test_dir'\nos.mkdir(test_dir)\n    \n# create test_images inside test_dir\ntest_images = os.path.join(test_dir, 'test_images')\nos.mkdir(test_images)\ntest_list = os.listdir('../input/test')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfor image in test_list:\n    \n    fname = image\n    \n    # source path to image\n    src = os.path.join('../input/test', fname)\n    # destination path to image\n    dst = os.path.join(test_images, fname)\n    # copy the image from the source to the destination\n    shutil.copyfile(src, dst)\n    \nlen(os.listdir('test_dir/test_images'))\ntest_path ='test_dir'\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Here we change the path to point to the test_images folder.\ntest_gen = test_datagen.flow_from_directory(test_path,\n                                        target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                        batch_size=1,\n                                        class_mode='binary',\n                                        shuffle=False)\nnum_test_images = 57458\n\npredictions = model.predict_generator(test_gen, steps=num_test_images, verbose=1)\ndf_preds = pd.DataFrame(predictions)\n\ndf_preds.head()\n# This outputs the file names in the sequence in which \n# the generator processed the test images.\ntest_filenames = test_gen.filenames\n\n# add the filenames to the dataframe\ndf_preds['file_names'] = test_filenames\n\ndf_preds.head()\ndef extract_id(x):\n    \n    # split into a list\n    a = x.split('/')\n    # split into a list\n    b = a[1].split('.')\n    extracted_id = b[0]\n    \n    return extracted_id\n\ndf_preds['id'] = df_preds['file_names'].apply(extract_id)\n\ndf_preds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'id':df_preds['id'], \n                           'label':df_preds[0], \n                          }).set_index('id')\n\nsubmission.to_csv('patch_preds.csv', columns=['label']) \nshutil.rmtree('test_dir')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}