{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Histopathologic Cancer Detection\n## Identify metastatic tissue in histopathologic scans of lymph node sections¶","metadata":{}},{"cell_type":"markdown","source":"# About the images","metadata":{}},{"cell_type":"markdown","source":"#### There are 220,025 training images and 57,456 test images.\n#### The images are 96x96 pixels and are full color.","metadata":{}},{"cell_type":"markdown","source":"# Import packages","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom sklearn.model_selection import train_test_split\nimport pickle\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import *\n\nimport zipfile ","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:23:23.752198Z","iopub.execute_input":"2021-11-13T16:23:23.752531Z","iopub.status.idle":"2021-11-13T16:23:29.177559Z","shell.execute_reply.started":"2021-11-13T16:23:23.752439Z","shell.execute_reply":"2021-11-13T16:23:29.176768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Number of images in the train and test folder","metadata":{}},{"cell_type":"code","source":"print('Number of images in train set',len(os.listdir('../input/histopathologic-cancer-detection/train')))\nprint('Number of images in test set',len(os.listdir('../input/histopathologic-cancer-detection/test')))","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:23:29.179206Z","iopub.execute_input":"2021-11-13T16:23:29.179488Z","iopub.status.idle":"2021-11-13T16:23:33.912872Z","shell.execute_reply.started":"2021-11-13T16:23:29.179452Z","shell.execute_reply":"2021-11-13T16:23:33.911999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load training dataframe","metadata":{}},{"cell_type":"code","source":"# Load the training data into a DataFrame named 'train'.\ntrain = pd.read_csv(f'../input/histopathologic-cancer-detection/train_labels.csv',dtype = 'str')\n\n# Print the shape of the resulting DataFrame.\nprint('Training set size', train.shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:23:33.914187Z","iopub.execute_input":"2021-11-13T16:23:33.914448Z","iopub.status.idle":"2021-11-13T16:23:34.403252Z","shell.execute_reply.started":"2021-11-13T16:23:33.914414Z","shell.execute_reply":"2021-11-13T16:23:34.402500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display the first few rows of the dataframe.\ntrain.head(10) ","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:23:34.405558Z","iopub.execute_input":"2021-11-13T16:23:34.405814Z","iopub.status.idle":"2021-11-13T16:23:34.424923Z","shell.execute_reply.started":"2021-11-13T16:23:34.405779Z","shell.execute_reply":"2021-11-13T16:23:34.424224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#The id in the csv file does not have .tif extension, let's add it.\ntrain['id'] = train['id'].apply(lambda x:f'{x}.tif')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:23:34.425935Z","iopub.execute_input":"2021-11-13T16:23:34.426197Z","iopub.status.idle":"2021-11-13T16:23:34.514120Z","shell.execute_reply.started":"2021-11-13T16:23:34.426161Z","shell.execute_reply":"2021-11-13T16:23:34.513377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Label distribution","metadata":{}},{"cell_type":"code","source":"#Let's check the class distribution\n#train['label'].value_counts()\ntrain.label.value_counts() ","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:23:34.515674Z","iopub.execute_input":"2021-11-13T16:23:34.515909Z","iopub.status.idle":"2021-11-13T16:23:34.548205Z","shell.execute_reply.started":"2021-11-13T16:23:34.515876Z","shell.execute_reply":"2021-11-13T16:23:34.547299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Let's check the class distribution in proportion\n#y_train = train.label\nround((train.label.value_counts() / len(train)).to_frame()*100,2)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:23:34.549886Z","iopub.execute_input":"2021-11-13T16:23:34.550173Z","iopub.status.idle":"2021-11-13T16:23:34.584968Z","shell.execute_reply.started":"2021-11-13T16:23:34.550135Z","shell.execute_reply":"2021-11-13T16:23:34.584183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# View sample of images","metadata":{}},{"cell_type":"code","source":"#display 16 images\n\nsample = train.sample(n=16).reset_index()\nplt.figure(figsize=(6,6)) # specifying the overall grid size\n\nfor i, row in sample.iterrows():  \n    img = mpimg.imread(f'../input/histopathologic-cancer-detection/train/{row.id}')\n    label = row.label\n    \n    plt.subplot(4,4,i+1)    # the number of images in the grid is 6*6 (16)\n    plt.imshow(img)\n    plt.text(0, -5, f'Class {label}', color='k')\n    plt.axis('off')\n    \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:23:34.586352Z","iopub.execute_input":"2021-11-13T16:23:34.586812Z","iopub.status.idle":"2021-11-13T16:23:35.650598Z","shell.execute_reply.started":"2021-11-13T16:23:34.586777Z","shell.execute_reply":"2021-11-13T16:23:35.649840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's balance the label distribution","metadata":{}},{"cell_type":"code","source":"from sklearn.utils import shuffle\n\nsample_size = 80000\ntrain_0 = train[train['label'] == '0'].sample(sample_size, random_state=1)\ntrain_1 = train[train['label'] == '1'].sample(sample_size, random_state=1)\n\n#combine the two dataframe\ntrain_set = pd.concat([train_0, train_1], axis=0).reset_index(drop=True)\n\n#Shuffle\ntrain = shuffle(train_set)\n\ntrain['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:23:35.651559Z","iopub.execute_input":"2021-11-13T16:23:35.651782Z","iopub.status.idle":"2021-11-13T16:23:35.789159Z","shell.execute_reply.started":"2021-11-13T16:23:35.651753Z","shell.execute_reply":"2021-11-13T16:23:35.788324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data generator","metadata":{}},{"cell_type":"code","source":"# Split the dataframe train into two DataFrames named train_df and valid_df. \n\ntrain_df, valid_df = train_test_split(train, test_size=0.20, random_state=1, stratify=train.label)\n\nprint(train_df.shape)\nprint(valid_df.shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:23:35.792472Z","iopub.execute_input":"2021-11-13T16:23:35.792732Z","iopub.status.idle":"2021-11-13T16:23:36.049745Z","shell.execute_reply.started":"2021-11-13T16:23:35.792705Z","shell.execute_reply":"2021-11-13T16:23:36.048986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create image data generators for both the training set and the validation set. \n# Here we use the data generators to scale the pixel values by a factor of 1/255. \n\ntrain_datagen = ImageDataGenerator(rescale=1/255)\nvalid_datagen = ImageDataGenerator(rescale=1/255)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:23:36.051060Z","iopub.execute_input":"2021-11-13T16:23:36.051480Z","iopub.status.idle":"2021-11-13T16:23:36.056453Z","shell.execute_reply.started":"2021-11-13T16:23:36.051441Z","shell.execute_reply":"2021-11-13T16:23:36.055657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nBATCH_SIZE = 64\n\ntrain_loader = train_datagen.flow_from_dataframe(\n    dataframe = train_df,\n    directory = '../input/histopathologic-cancer-detection/train/',\n    x_col = 'id',\n    y_col = 'label',\n    batch_size = BATCH_SIZE,\n    seed = 1,\n    shuffle = True,\n    class_mode = 'categorical',\n    target_size = (96,96)\n)\n\nvalid_loader = train_datagen.flow_from_dataframe(\n    dataframe = valid_df,\n    directory = '../input/histopathologic-cancer-detection/train/',\n    x_col = 'id',\n    y_col = 'label',\n    batch_size = BATCH_SIZE,\n    seed = 1,\n    shuffle = True,\n    class_mode = 'categorical',\n    target_size = (96,96)\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:23:36.057843Z","iopub.execute_input":"2021-11-13T16:23:36.058121Z","iopub.status.idle":"2021-11-13T16:27:57.332870Z","shell.execute_reply.started":"2021-11-13T16:23:36.058082Z","shell.execute_reply":"2021-11-13T16:27:57.332117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's determine the number of training and validation batches. \n\nTR_STEPS = len(train_loader)\nVA_STEPS = len(valid_loader)\n\nprint('Number of batches in the training set:',TR_STEPS)\nprint('Number of batches in the validation set:',VA_STEPS)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:27:57.334148Z","iopub.execute_input":"2021-11-13T16:27:57.334571Z","iopub.status.idle":"2021-11-13T16:27:57.340485Z","shell.execute_reply.started":"2021-11-13T16:27:57.334533Z","shell.execute_reply":"2021-11-13T16:27:57.339804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build network","metadata":{}},{"cell_type":"code","source":"np.random.seed(1)\ntf.random.set_seed(1)\n\ncnn_model = Sequential([\n    Cropping2D(cropping=((32, 32), (32, 32)), input_shape=(96,96,3)),\n    \n    Conv2D(filters=32, kernel_size=(3,3), padding='same', activation='relu'),\n    Conv2D(filters=32, kernel_size=(3,3), padding='same', activation='relu'),\n    MaxPooling2D(2,2),\n    Dropout(0.25),\n    BatchNormalization(),\n\n    Conv2D(filters=64, kernel_size=(3,3), padding='same', activation='relu'),\n    Conv2D(filters=64, kernel_size=(3,3), padding='same', activation='relu'),\n    MaxPooling2D(2,2),\n    Dropout(0.25),\n    BatchNormalization(),\n\n    Flatten(),\n    \n    Dense(128, activation='relu'),\n    Dropout(0.25),\n    Dense(64, activation='relu'),\n    Dropout(0.25),\n    Dense(32, activation='relu'),\n    Dropout(0.25),\n    Dense(16, activation='relu'),\n    Dropout(0.25),\n    BatchNormalization(),\n    Dense(2, activation='softmax')\n])\n \n\ncnn_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:27:57.341751Z","iopub.execute_input":"2021-11-13T16:27:57.341994Z","iopub.status.idle":"2021-11-13T16:27:59.846355Z","shell.execute_reply.started":"2021-11-13T16:27:57.341961Z","shell.execute_reply":"2021-11-13T16:27:59.845668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train network","metadata":{}},{"cell_type":"code","source":"# Define an optimizer and select a learning rate. \n# And then compile the model. \nimport tensorflow as tf\n\nopt = tf.keras.optimizers.Adam(0.001)\ncnn_model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy',tf.keras.metrics.AUC()])","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:27:59.847758Z","iopub.execute_input":"2021-11-13T16:27:59.848004Z","iopub.status.idle":"2021-11-13T16:27:59.866921Z","shell.execute_reply.started":"2021-11-13T16:27:59.847969Z","shell.execute_reply":"2021-11-13T16:27:59.866269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\nh1 = cnn_model.fit(\n    x = train_loader, \n    steps_per_epoch = TR_STEPS, \n    epochs = 25,\n    validation_data = valid_loader, \n    validation_steps = VA_STEPS, \n    verbose = 1\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T16:27:59.867946Z","iopub.execute_input":"2021-11-13T16:27:59.868206Z","iopub.status.idle":"2021-11-13T18:03:38.616947Z","shell.execute_reply.started":"2021-11-13T16:27:59.868171Z","shell.execute_reply":"2021-11-13T18:03:38.616234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = h1.history\nprint(history.keys())","metadata":{"execution":{"iopub.status.busy":"2021-11-13T18:03:38.618492Z","iopub.execute_input":"2021-11-13T18:03:38.619014Z","iopub.status.idle":"2021-11-13T18:03:38.624425Z","shell.execute_reply.started":"2021-11-13T18:03:38.618975Z","shell.execute_reply":"2021-11-13T18:03:38.623745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epoch_range = range(1, len(history['loss'])+1)\n\nplt.figure(figsize=[14,4])\nplt.subplot(1,3,1)\nplt.plot(epoch_range, history['loss'], label='Training')\nplt.plot(epoch_range, history['val_loss'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Loss'); plt.title('Loss')\nplt.legend()\nplt.subplot(1,3,2)\nplt.plot(epoch_range, history['accuracy'], label='Training')\nplt.plot(epoch_range, history['val_accuracy'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Accuracy'); plt.title('Accuracy')\nplt.legend()\nplt.subplot(1,3,3)\nplt.plot(epoch_range, history['auc'], label='Training')\nplt.plot(epoch_range, history['val_auc'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('AUC'); plt.title('AUC')\nplt.legend()\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-13T18:03:38.625584Z","iopub.execute_input":"2021-11-13T18:03:38.626225Z","iopub.status.idle":"2021-11-13T18:03:39.141842Z","shell.execute_reply.started":"2021-11-13T18:03:38.626189Z","shell.execute_reply":"2021-11-13T18:03:39.141160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we decrease the learning rate\ntf.keras.backend.set_value(cnn_model.optimizer.learning_rate, 0.0001)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T18:03:39.143104Z","iopub.execute_input":"2021-11-13T18:03:39.143365Z","iopub.status.idle":"2021-11-13T18:03:39.147556Z","shell.execute_reply.started":"2021-11-13T18:03:39.143330Z","shell.execute_reply":"2021-11-13T18:03:39.146785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\nh2 = cnn_model.fit(\n    x = train_loader, \n    steps_per_epoch = TR_STEPS, \n    epochs = 30,\n    validation_data = valid_loader, \n    validation_steps = VA_STEPS, \n    verbose = 1\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-13T18:03:39.148947Z","iopub.execute_input":"2021-11-13T18:03:39.149385Z","iopub.status.idle":"2021-11-13T19:48:29.961094Z","shell.execute_reply.started":"2021-11-13T18:03:39.149349Z","shell.execute_reply":"2021-11-13T19:48:29.960341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in history.keys():\n    history[k] += h2.history[k]","metadata":{"execution":{"iopub.status.busy":"2021-11-13T19:48:29.964027Z","iopub.execute_input":"2021-11-13T19:48:29.965470Z","iopub.status.idle":"2021-11-13T19:48:29.970150Z","shell.execute_reply.started":"2021-11-13T19:48:29.965426Z","shell.execute_reply":"2021-11-13T19:48:29.969362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epoch_range = range(1, len(history['loss'])+1)\n\nplt.figure(figsize=[14,4])\nplt.subplot(1,3,1)\nplt.plot(epoch_range, history['loss'], label='Training')\nplt.plot(epoch_range, history['val_loss'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Loss'); plt.title('Loss')\nplt.legend()\nplt.subplot(1,3,2)\nplt.plot(epoch_range, history['accuracy'], label='Training')\nplt.plot(epoch_range, history['val_accuracy'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('Accuracy'); plt.title('Accuracy')\nplt.legend()\nplt.subplot(1,3,3)\nplt.plot(epoch_range, history['auc'], label='Training')\nplt.plot(epoch_range, history['val_auc'], label='Validation')\nplt.xlabel('Epoch'); plt.ylabel('AUC'); plt.title('AUC')\nplt.legend()\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-13T19:48:29.972030Z","iopub.execute_input":"2021-11-13T19:48:29.972963Z","iopub.status.idle":"2021-11-13T19:48:30.708768Z","shell.execute_reply.started":"2021-11-13T19:48:29.972926Z","shell.execute_reply":"2021-11-13T19:48:30.708127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save model and history","metadata":{}},{"cell_type":"code","source":"# save the model and the combined history dictionary to files.\ncnn_model.save('cancer_model_v07.h5')\npickle.dump(history, open(f'cancer_history_v07.pkl', 'wb'))","metadata":{"execution":{"iopub.status.busy":"2021-11-13T19:57:32.966549Z","iopub.execute_input":"2021-11-13T19:57:32.967299Z","iopub.status.idle":"2021-11-13T19:57:33.052640Z","shell.execute_reply.started":"2021-11-13T19:57:32.967262Z","shell.execute_reply":"2021-11-13T19:57:33.051920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}}]}