{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport os,shutil,math\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport glob\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix,roc_curve,auc\nfrom sklearn.utils import shuffle\nfrom keras.optimizers import Adam,SGD\nimport os\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.applications.vgg19 import VGG19\nfrom tensorflow.keras.models import Model, Sequential\nfrom keras.layers import Dense, Activation, Flatten , Dropout , GlobalAveragePooling2D","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data = pd.read_csv('../input/histopathologic-cancer-detection/train_labels.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#import seaborn as sns\n#sns.countplot(df_data.label)\ndf_data.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SAMPLE_SIZE= 80000\n# take a random sample of class 0 with size equal to num samples in class 1\ndf_0 = df_data[df_data['label'] == 0].sample(SAMPLE_SIZE, random_state = 101)\n# filter out class 1\ndf_1 = df_data[df_data['label'] == 1].sample(SAMPLE_SIZE, random_state = 101)\n\n# concat the dataframes\ndf_data = pd.concat([df_0, df_1], axis=0).reset_index(drop=True)\n# shuffle\ndf_data = shuffle(df_data)\n\ndf_data['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_test_split\n\n# stratify=y creates a balanced validation set.\ny = df_data['label']\n\ndf_train, df_val = train_test_split(df_data, test_size=0.10, random_state=101, stratify=y)\n\nprint(df_train.shape)\nprint(df_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create a new directory\nbase_dir = 'base_dir'\nos.mkdir(base_dir)\n\n\n#[CREATE FOLDERS INSIDE THE BASE DIRECTORY]\n\n# now we create 2 folders inside 'base_dir':\n\n# train_dir\n    # a_no_tumor_tissue\n    # b_has_tumor_tissue\n\n# val_dir\n    # a_no_tumor_tissue\n    # b_has_tumor_tissue\n\n\n\n# create a path to 'base_dir' to which we will join the names of the new folders\n# train_dir\ntrain_dir = os.path.join(base_dir, 'train_dir')\nos.mkdir(train_dir)\n\n# val_dir\nval_dir = os.path.join(base_dir, 'val_dir')\nos.mkdir(val_dir)\n\n\n\n# [CREATE FOLDERS INSIDE THE TRAIN AND VALIDATION FOLDERS]\n# Inside each folder we create seperate folders for each class\n\n# create new folders inside train_dir\nno_tumor_tissue = os.path.join(train_dir, 'a_no_tumor_tissue')\nos.mkdir(no_tumor_tissue)\nhas_tumor_tissue = os.path.join(train_dir, 'b_has_tumor_tissue')\nos.mkdir(has_tumor_tissue)\n\n\n# create new folders inside val_dir\nno_tumor_tissue = os.path.join(val_dir, 'a_no_tumor_tissue')\nos.mkdir(no_tumor_tissue)\nhas_tumor_tissue = os.path.join(val_dir, 'b_has_tumor_tissue')\nos.mkdir(has_tumor_tissue)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('base_dir/train_dir')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data.set_index('id', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Get a list of train and val images\ntrain_list = list(df_train['id'])\nval_list = list(df_val['id'])\n\n\n\n# Transfer the train images\n\nfor image in train_list:\n    \n    # the id in the csv file does not have the .tif extension therefore we add it here\n    fname = image + '.tif'\n    # get the label for a certain image\n    target = df_data.loc[image,'label']\n    \n    # these must match the folder names\n    if target == 0:\n        label = 'a_no_tumor_tissue'\n    if target == 1:\n        label = 'b_has_tumor_tissue'\n    \n    # source path to image\n    src = os.path.join('../input/histopathologic-cancer-detection/train', fname)\n    # destination path to image\n    dst = os.path.join(train_dir, label, fname)\n    # copy the image from the source to the destination\n    shutil.copyfile(src, dst)\n\n\n# Transfer the val images\n\nfor image in val_list:\n    \n    # the id in the csv file does not have the .tif extension therefore we add it here\n    fname = image + '.tif'\n    # get the label for a certain image\n    target = df_data.loc[image,'label']\n    \n    # these must match the folder names\n    if target == 0:\n        label = 'a_no_tumor_tissue'\n    if target == 1:\n        label = 'b_has_tumor_tissue'\n    \n\n    # source path to image\n    src = os.path.join('../input/histopathologic-cancer-detection/train', fname)\n    # destination path to image\n    dst = os.path.join(val_dir, label, fname)\n    # copy the image from the source to the destination\n    shutil.copyfile(src, dst)\n    \n\n\n   \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#------Generators------------#\ntrain_augs = ImageDataGenerator(\n    rescale=1./255,\n    horizontal_flip=True,\n    vertical_flip=True,\n    rotation_range=90,\n    zoom_range=0.2, \n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    shear_range=0.05,\n    channel_shift_range=0.1\n)\n\n\nval_augs = ImageDataGenerator(rescale=1./255)\n\ntrain_gen = train_augs.flow_from_directory(\n    \"./base_dir/train_dir\",\n    target_size=(96,96),\n    batch_size=10,\n    class_mode='binary')\n\nval_gen = val_augs.flow_from_directory(\n    \"./base_dir/val_dir\",\n    target_size=(96,96),\n    batch_size=10,\n    class_mode='binary')\n#print(train_gen.class_indices)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor='val_loss',\n    factor=0.3,\n    patience=2,\n    verbose=1, \n    mode='auto',\n    cooldown=1 \n)\nearlystop = tf.keras.callbacks.EarlyStopping(\n    monitor='val_loss',\n    min_delta=0.001,\n    patience=10,\n    verbose=1,\n    mode='auto'\n)\ntensorboard = tf.keras.callbacks.TensorBoard(\n    log_dir = './logs',\n    histogram_freq=0,\n    batch_size=16,\n    write_graph=True,\n    write_grads=True,\n    write_images=False,\n)\ncallbacks=[c,earlystop,tensorboard]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_model = VGG19(include_top=False,\n                  input_shape = (96,96,3),\n                  weights = 'imagenet')\n\nfor layer in base_model.layers[:-15]:\n    layer.trainable = False\n    \nx = base_model.output\nx = Flatten()(x)\nx = Dense(1024, activation='relu')(x)\nx = Dropout(0.5)(x)\npredictions = Dense(1, activation='sigmoid')(x)\nmodel = Model(inputs=base_model.input,outputs=predictions)\nmodel.summary()\nopt = SGD(lr=1e-4,momentum=0.99)\nmodel.compile(\n    loss='binary_crossentropy',\n    optimizer=opt,\n    metrics=['accuracy']\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(\n    train_gen,  \n    steps_per_epoch  = 2000, \n    validation_data  = val_gen,\n    epochs = 30, \n    verbose = 1,\n    callbacks=callbacks\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_final_history(history):\n    fig, ax = plt.subplots(1, 2, figsize=(15,5))\n    ax[0].set_title('loss')\n    ax[0].plot(history.epoch, history.history[\"loss\"], label=\"Train loss\")\n    ax[0].plot(history.epoch, history.history[\"val_loss\"], label=\"Validation loss\")\n    ax[1].set_title('acc')\n    ax[1].plot(history.epoch, history.history[\"accuracy\"], label=\"Train acc\")\n    ax[1].plot(history.epoch, history.history[\"val_accuracy\"], label=\"Validation acc\")\n    ax[0].legend()\n    ax[1].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_final_history(history)\nprint(\"Validation Accuracy: \" + str(history.history['val_accuracy'][-1:]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shutil.rmtree('base_dir')\ntest_dir = 'test_dir'\nos.mkdir(test_dir)\n    \n# create test_images inside test_dir\ntest_images = os.path.join(test_dir, 'test_images')\nos.mkdir(test_images)\nos.listdir('test_dir')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_list = os.listdir('../input/histopathologic-cancer-detection/test')\n\nfor image in test_list:\n    \n    fname = image\n    \n    # source path to image\n    src = os.path.join('../input/histopathologic-cancer-detection/test', fname)\n    # destination path to image\n    dst = os.path.join(test_images, fname)\n    # copy the image from the source to the destination\n    shutil.copyfile(src, dst)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(os.listdir('test_dir/test_images'))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_path ='test_dir'\ndatagen = ImageDataGenerator(rescale=1.0/255)\n\n\n# Here we change the path to point to the test_images folder.\n\ntest_gen = datagen.flow_from_directory(test_path,\n                                        target_size=(96,96),\n                                        batch_size=1,\n                                        class_mode='binary',\n                                        shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_test_images = 57458\n\npredictions = model.predict_generator(test_gen, steps=num_test_images, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Put the predictions into a dataframe\n\ndf_preds = pd.DataFrame(predictions, columns=[\"tumor\"])\n\ndf_preds.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This outputs the file names in the sequence in which \n# the generator processed the test images.\ntest_filenames = test_gen.filenames\n\n# add the filenames to the dataframe\ndf_preds['file_names'] = test_filenames\n\ndf_preds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_id(x):\n    \n    # split into a list\n    a = x.split('/')\n    # split into a list\n    b = a[1].split('.')\n    extracted_id = b[0]\n    \n    return extracted_id\n\ndf_preds['id'] = df_preds['file_names'].apply(extract_id)\n\ndf_preds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'id':df_preds.id, \n                           'label':df_preds.tumor, \n                          }).set_index('id')\n\nsubmission.to_csv('patch_preds.csv', columns=['label']) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"shutil.rmtree('test_dir')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}