{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":11848,"databundleVersionId":862157,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Description\n\nIn the Histopathologic Cancer Detection competition, we must create an model that can accurately identify metastatic cancer in small image patches taken from larger digital pathology scans. Accurate detection is crucial for early diagnosis and treatment planning in cancer patients.\nhttps://www.kaggle.com/competitions/histopathologic-cancer-detection/overview\n\n## Data Characteristics\nA positive label indicates that the center 32x32px region of a patch contains at least one pixel of tumor tissue. Tumor tissue in the outer region of the patch does not influence the label.<br>\n**Image Size:** 96x96px<br>\n**Color Channels:** 3 (RGB)<br>\n**Format:** .tif files<br>","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nimport tensorflow as tf\nimport keras\nfrom keras.models import Sequential\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, BatchNormalization\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2024-11-03T01:45:28.134872Z","iopub.execute_input":"2024-11-03T01:45:28.135604Z","iopub.status.idle":"2024-11-03T01:45:44.402885Z","shell.execute_reply.started":"2024-11-03T01:45:28.135548Z","shell.execute_reply":"2024-11-03T01:45:44.401893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Paths to the data directories\ntrain_dir = '/kaggle/input/histopathologic-cancer-detection/train/'\ntest_dir = '/kaggle/input/histopathologic-cancer-detection/test/'\n\n# Load the labels\nlabels = pd.read_csv('/kaggle/input/histopathologic-cancer-detection/train_labels.csv')","metadata":{"execution":{"iopub.status.busy":"2024-11-03T01:46:04.097809Z","iopub.execute_input":"2024-11-03T01:46:04.098704Z","iopub.status.idle":"2024-11-03T01:46:04.440352Z","shell.execute_reply.started":"2024-11-03T01:46:04.098661Z","shell.execute_reply":"2024-11-03T01:46:04.439544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pathlib\n\ntrain_path = '/kaggle/input/histopathologic-cancer-detection/train'\ntest_path = '/kaggle/input/histopathologic-cancer-detection/test'\ntrain_dir = pathlib.Path(train_path).with_suffix('')\ntest_dir = pathlib.Path(test_path).with_suffix('')\n\ntrain_imgs = list(train_dir.glob('*.tif'))\ntest_imgs = list(test_dir.glob('*.tif'))\n\nprint(len(train_imgs))\nprint(len(test_imgs))","metadata":{"execution":{"iopub.status.busy":"2024-11-03T01:46:08.345219Z","iopub.execute_input":"2024-11-03T01:46:08.345647Z","iopub.status.idle":"2024-11-03T01:46:12.635324Z","shell.execute_reply.started":"2024-11-03T01:46:08.345578Z","shell.execute_reply":"2024-11-03T01:46:12.634385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n# Get training and testing files\ntrain_files = os.listdir(train_dir)\ntest_files = os.listdir(test_dir)\n\nprint('Training image files: ')\nprint(train_files[:10])\nprint('Testing image files: ')\nprint(test_files[:10])","metadata":{"execution":{"iopub.status.busy":"2024-11-03T01:48:39.940140Z","iopub.execute_input":"2024-11-03T01:48:39.940845Z","iopub.status.idle":"2024-11-03T01:48:40.129411Z","shell.execute_reply.started":"2024-11-03T01:48:39.940807Z","shell.execute_reply":"2024-11-03T01:48:40.128476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preview","metadata":{}},{"cell_type":"code","source":"def show_samples(label, num_samples=5):\n    samples = labels[labels['label'] == label].sample(num_samples)\n    plt.figure(figsize=(15, 3))\n    for idx, img_name in enumerate(samples['id']):\n        img_path = os.path.join(train_dir, img_name + '.tif')\n        img = Image.open(img_path)\n        plt.subplot(1, num_samples, idx+1)\n        plt.imshow(img)\n        plt.axis('off')\n    plt.suptitle(f'Sample Images - Label {label}')\n    plt.show()\n\nshow_samples(label=0)\nshow_samples(label=1)","metadata":{"execution":{"iopub.status.busy":"2024-11-02T17:30:45.915885Z","iopub.execute_input":"2024-11-02T17:30:45.916561Z","iopub.status.idle":"2024-11-02T17:30:46.931247Z","shell.execute_reply.started":"2024-11-02T17:30:45.916520Z","shell.execute_reply":"2024-11-02T17:30:46.930284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Missing values\nlabels.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-11-02T17:30:50.189837Z","iopub.execute_input":"2024-11-02T17:30:50.190771Z","iopub.status.idle":"2024-11-02T17:30:50.219350Z","shell.execute_reply.started":"2024-11-02T17:30:50.190728Z","shell.execute_reply":"2024-11-02T17:30:50.218391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distribution\nplt.figure(figsize=(4, 4))\nsns.countplot(x='label', data=labels)\nplt.title('Class Distribution')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-02T17:30:52.263019Z","iopub.execute_input":"2024-11-02T17:30:52.263800Z","iopub.status.idle":"2024-11-02T17:30:52.505736Z","shell.execute_reply.started":"2024-11-02T17:30:52.263759Z","shell.execute_reply":"2024-11-02T17:30:52.504886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build the Convolutional Neural Network models\n\n### Model 1 : Baseline","metadata":{}},{"cell_type":"code","source":"# Convolution and Pooling layers\nconfig = [Conv2D(32, (3,3), activation='relu', input_shape=(96, 96, 3)),\n          MaxPooling2D(2,2),\n          \n          Conv2D(64, (3,3), activation='relu'),\n          MaxPooling2D(2,2),\n          \n          Flatten(),\n          Dense(128, activation='relu'),\n          Dense(1, activation='sigmoid')]\n\nmodel = Sequential(config)","metadata":{"execution":{"iopub.status.busy":"2024-11-02T17:31:35.736725Z","iopub.execute_input":"2024-11-02T17:31:35.737375Z","iopub.status.idle":"2024-11-02T17:31:36.675979Z","shell.execute_reply.started":"2024-11-02T17:31:35.737336Z","shell.execute_reply":"2024-11-02T17:31:36.675081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile\nlearning_rate = 0.001\nopt = keras.optimizers.Adam(learning_rate = learning_rate)\nmodel.compile(optimizer = opt,\n              loss = 'binary_crossentropy',\n              metrics = ['accuracy'])\n\n# Summary\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2024-11-02T17:32:43.978459Z","iopub.execute_input":"2024-11-02T17:32:43.978860Z","iopub.status.idle":"2024-11-02T17:32:44.012909Z","shell.execute_reply.started":"2024-11-02T17:32:43.978820Z","shell.execute_reply":"2024-11-02T17:32:44.012058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_labels, val_labels = train_test_split(labels,\n                                            test_size = 0.2,\n                                            random_state = 0)","metadata":{"execution":{"iopub.status.busy":"2024-11-03T01:49:03.097308Z","iopub.execute_input":"2024-11-03T01:49:03.098057Z","iopub.status.idle":"2024-11-03T01:49:03.299175Z","shell.execute_reply.started":"2024-11-03T01:49:03.098018Z","shell.execute_reply":"2024-11-03T01:49:03.298326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.head()","metadata":{"execution":{"iopub.status.busy":"2024-11-03T01:49:04.510208Z","iopub.execute_input":"2024-11-03T01:49:04.510622Z","iopub.status.idle":"2024-11-03T01:49:04.525129Z","shell.execute_reply.started":"2024-11-03T01:49:04.510570Z","shell.execute_reply":"2024-11-03T01:49:04.523952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert labels to strings\ntrain_labels['label'] = train_labels['label'].astype(str)\nval_labels['label'] = val_labels['label'].astype(str)\n\n# Create a new 'filename' column by adding '.tif' extension\ntrain_labels['filename'] = train_labels['id'] + '.tif'\nval_labels['filename'] = val_labels['id'] + '.tif'","metadata":{"execution":{"iopub.status.busy":"2024-11-03T01:49:07.655563Z","iopub.execute_input":"2024-11-03T01:49:07.656274Z","iopub.status.idle":"2024-11-03T01:49:07.805470Z","shell.execute_reply.started":"2024-11-03T01:49:07.656233Z","shell.execute_reply":"2024-11-03T01:49:07.804491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.head()","metadata":{"execution":{"iopub.status.busy":"2024-11-03T01:49:08.423544Z","iopub.execute_input":"2024-11-03T01:49:08.424083Z","iopub.status.idle":"2024-11-03T01:49:08.434364Z","shell.execute_reply.started":"2024-11-03T01:49:08.424039Z","shell.execute_reply":"2024-11-03T01:49:08.433448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data Generators\ntrain_datagen = ImageDataGenerator(\n    rescale=1./255,\n    horizontal_flip=True,\n    vertical_flip=True)\n\nval_datagen = ImageDataGenerator(rescale=1./255)\n\n# Train data\ntrain_generator = train_datagen.flow_from_dataframe(\n    dataframe=train_labels,\n    directory=train_dir,\n    x_col='filename',\n    y_col='label',\n    target_size=(96, 96),\n    batch_size=32,\n    class_mode='binary')\n\n# Validation data\nval_generator = val_datagen.flow_from_dataframe(\n    dataframe=val_labels,\n    directory=train_dir,\n    x_col='filename',\n    y_col='label',\n    target_size=(96, 96),\n    batch_size=32,\n    class_mode='binary')","metadata":{"execution":{"iopub.status.busy":"2024-11-03T01:49:14.462838Z","iopub.execute_input":"2024-11-03T01:49:14.463216Z","iopub.status.idle":"2024-11-03T01:56:39.221171Z","shell.execute_reply.started":"2024-11-03T01:49:14.463180Z","shell.execute_reply":"2024-11-03T01:56:39.220392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model fitting\nhistory = model.fit(train_generator,\n                    epochs = 10,\n                    validation_data = val_generator)\nhistory","metadata":{"execution":{"iopub.status.busy":"2024-11-02T17:41:29.375692Z","iopub.execute_input":"2024-11-02T17:41:29.375999Z","iopub.status.idle":"2024-11-02T19:19:43.183676Z","shell.execute_reply.started":"2024-11-02T17:41:29.375966Z","shell.execute_reply":"2024-11-02T19:19:43.182602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def learning_curve(history):\n    acc = history.history['accuracy']\n    val_acc = history.history['val_accuracy']\n\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n\n    plt.figure(figsize=(8, 10))\n    plt.subplot(2, 1, 1)\n    plt.plot(acc, label='Training Accuracy')\n    plt.plot(val_acc, label='Validation Accuracy')\n    plt.legend(loc='lower right')\n    plt.ylabel('Accuracy')\n    plt.ylim([min(plt.ylim()),1])\n    plt.title('Training and Validation Accuracy')\n\n    plt.subplot(2, 1, 2)\n    plt.plot(loss, label='Training Loss')\n    plt.plot(val_loss, label='Validation Loss')\n    plt.legend(loc='upper right')\n    plt.ylabel('Cross Entropy')\n    plt.ylim([0,1.0])\n    plt.title('Training and Validation Loss')\n    plt.xlabel('epoch')","metadata":{"execution":{"iopub.status.busy":"2024-11-03T04:05:04.919669Z","iopub.execute_input":"2024-11-03T04:05:04.920065Z","iopub.status.idle":"2024-11-03T04:05:04.928402Z","shell.execute_reply.started":"2024-11-03T04:05:04.920029Z","shell.execute_reply":"2024-11-03T04:05:04.927513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learning_curve(history)","metadata":{"execution":{"iopub.status.busy":"2024-11-02T19:19:43.194959Z","iopub.execute_input":"2024-11-02T19:19:43.195263Z","iopub.status.idle":"2024-11-02T19:19:43.784699Z","shell.execute_reply.started":"2024-11-02T19:19:43.195230Z","shell.execute_reply":"2024-11-02T19:19:43.783711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model 2 : Deeper layers with Batch Normalization and Dropout\n\nThe baseline model has 2 convolution layers. Now, let's increase the number of convolution layers to 4.<br>\nIn addition, in order to enhance the stability of training and potentially speed up convergence, I apply He-initialization to the hidden layer and Xavier-initialization to the output layer.","metadata":{}},{"cell_type":"code","source":"# Layers\nconfig_2 = [Conv2D(32, (3,3), activation='relu', input_shape=(96, 96, 3), kernel_initializer='he_normal'),\n            BatchNormalization(),\n            MaxPooling2D(2,2),\n            \n            Conv2D(64, (3,3), activation='relu', kernel_initializer='he_normal'),\n            BatchNormalization(),\n            MaxPooling2D(2,2),\n            \n            Conv2D(128, (3,3), activation='relu', kernel_initializer='he_normal'),\n            BatchNormalization(),\n            MaxPooling2D(2,2),\n            \n            Conv2D(256, (3,3), activation='relu', kernel_initializer='he_normal'),\n            BatchNormalization(),\n            MaxPooling2D(2,2),\n            \n            Flatten(),\n            # Fully connected layer with Dropout\n            Dense(512, activation='relu'),\n            Dropout(0.5),\n            Dense(1, activation='sigmoid', kernel_initializer='glorot_uniform')]\n\nmodel_2 = Sequential(config_2)\n\n# Compile\nlearning_rate = 0.001\nopt = keras.optimizers.Adam(learning_rate = learning_rate)\nmodel_2.compile(optimizer = opt,\n              loss = 'binary_crossentropy',\n              metrics = ['accuracy'])\n\n# Summary\nmodel_2.summary()","metadata":{"execution":{"iopub.status.busy":"2024-11-02T19:19:43.787042Z","iopub.execute_input":"2024-11-02T19:19:43.787429Z","iopub.status.idle":"2024-11-02T19:19:43.929042Z","shell.execute_reply.started":"2024-11-02T19:19:43.787389Z","shell.execute_reply":"2024-11-02T19:19:43.928042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model fitting\nhistory_2 = model_2.fit(train_generator,\n                        epochs = 6,\n                        validation_data = val_generator)\nhistory_2","metadata":{"execution":{"iopub.status.busy":"2024-11-02T19:19:43.930303Z","iopub.execute_input":"2024-11-02T19:19:43.930729Z","iopub.status.idle":"2024-11-02T20:03:08.090281Z","shell.execute_reply.started":"2024-11-02T19:19:43.930678Z","shell.execute_reply":"2024-11-02T20:03:08.089338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learning_curve(history_2)","metadata":{"execution":{"iopub.status.busy":"2024-11-02T20:03:08.091585Z","iopub.execute_input":"2024-11-02T20:03:08.091977Z","iopub.status.idle":"2024-11-02T20:03:08.688806Z","shell.execute_reply.started":"2024-11-02T20:03:08.091932Z","shell.execute_reply":"2024-11-02T20:03:08.687787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model 3 : Reducing the learning rate and Increase the number of epochs\n\nA learning rate that is too high can cause the model to overshoot the optimal solution, leading to unstable loss. Therefore, it seems worthwhile to reduce the learning rate to 0.0001.<br>\nSince each update becomes smaller, the model may require more epochs to effectively learn the patterns in the data.","metadata":{}},{"cell_type":"code","source":"# Layers\nconfig_3 = [Conv2D(32, (3,3), activation='relu', input_shape=(96, 96, 3), kernel_initializer='he_normal'),\n            BatchNormalization(),\n            MaxPooling2D(2,2),\n            \n            Conv2D(64, (3,3), activation='relu', kernel_initializer='he_normal'),\n            BatchNormalization(),\n            MaxPooling2D(2,2),\n            \n            Conv2D(128, (3,3), activation='relu', kernel_initializer='he_normal'),\n            BatchNormalization(),\n            MaxPooling2D(2,2),\n            \n            Conv2D(256, (3,3), activation='relu', kernel_initializer='he_normal'),\n            BatchNormalization(),\n            MaxPooling2D(2,2),\n            \n            Flatten(),\n            # Fully connected layer with Dropout\n            Dense(512, activation='relu'),\n            Dropout(0.5),\n            Dense(1, activation='sigmoid', kernel_initializer='glorot_uniform')]\n\nmodel_3 = Sequential(config_3)\n\n# Compile\nlearning_rate = 0.0001 # Reduce the learning rate\nopt = keras.optimizers.Adam(learning_rate = learning_rate)\nmodel_3.compile(optimizer = opt,\n                loss = 'binary_crossentropy',\n                metrics = ['accuracy'])\n\n# Summary\nmodel_3.summary()","metadata":{"execution":{"iopub.status.busy":"2024-11-03T01:57:00.708772Z","iopub.execute_input":"2024-11-03T01:57:00.709123Z","iopub.status.idle":"2024-11-03T01:57:01.772948Z","shell.execute_reply.started":"2024-11-03T01:57:00.709091Z","shell.execute_reply":"2024-11-03T01:57:01.772063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping\n\n# Early stopping\nearly_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n\n# Model fitting\nhistory_3 = model_3.fit(train_generator,\n                        epochs = 25,\n                        validation_data = val_generator,\n                        callbacks = [early_stopping])\n\nhistory_3","metadata":{"execution":{"iopub.status.busy":"2024-11-03T01:57:06.655895Z","iopub.execute_input":"2024-11-03T01:57:06.656352Z","iopub.status.idle":"2024-11-03T03:59:23.716820Z","shell.execute_reply.started":"2024-11-03T01:57:06.656316Z","shell.execute_reply":"2024-11-03T03:59:23.715937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learning_curve(history_3)","metadata":{"execution":{"iopub.status.busy":"2024-11-03T04:05:09.928385Z","iopub.execute_input":"2024-11-03T04:05:09.929035Z","iopub.status.idle":"2024-11-03T04:05:10.569372Z","shell.execute_reply.started":"2024-11-03T04:05:09.928997Z","shell.execute_reply":"2024-11-03T04:05:10.568442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conclusion and Submission\n\nI compared three models: one consisting of two convolution layers, the second incorporating four convolution layers plus Batch Normalisation and Dropout, and the third varying the learning rate and the number of epochs of the second model.\n\nIn the validation using training data, Model 3 recorded the highest score, and the cross-entropy error also more steadily decreased than Model 2. By reducing the learning rate of unstable Model 2 to 0.0001 and increasing the number of iterations (epochs), I was able to obtain better weights. But it took a very long time to train Model 3 although I used GPU.\n\nIt seems that Model 3 overfitted the train data, resulting in the gap between validation accuracy and test accuracy.　It may have been necessary to generate more types of image data with `ImageDataGenerator`.","metadata":{}},{"cell_type":"code","source":"# Prepare test data generator\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\ntest_generator = test_datagen.flow_from_dataframe(\n    dataframe=pd.DataFrame({'id': os.listdir(test_dir)}),\n    directory=test_dir,\n    x_col='id',\n    y_col=None,\n    target_size=(96, 96),\n    batch_size=32,\n    class_mode=None,\n    shuffle=False\n)\n\n# Predict on test data with model_3\ntest_generator.reset()\npredictions = model_3.predict(test_generator, steps=test_generator.samples)\n\n# Prepare the submission DataFrame\nfilenames = test_generator.filenames\nids = [filename.split('.')[0] for filename in filenames]\npredicted_labels = (predictions > 0.5).astype(int).reshape(-1)\n\nsubmission_df = pd.DataFrame({\n    'id': ids,\n    'label': predicted_labels\n})\n\n# Save submission.csv\nsubmission_df.to_csv('submission_model3_final.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-11-03T04:05:20.920080Z","iopub.execute_input":"2024-11-03T04:05:20.920439Z","iopub.status.idle":"2024-11-03T04:11:52.367246Z","shell.execute_reply.started":"2024-11-03T04:05:20.920403Z","shell.execute_reply":"2024-11-03T04:11:52.366291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Result\n|Model|val-accuracy|Private Score|\n|:-----:|:-----:|:-----:|\n|1|0.8876|0.8424|\n|2|0.8225|0.6899|\n|3|0.9347|0.8601|","metadata":{}}]}