{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":11848,"databundleVersionId":862157,"sourceType":"competition"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## The aim of the model is to identify metastatic cancer in small image patches taken from larger digital pathology scans","metadata":{}},{"cell_type":"code","source":"import os\nimport shutil\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport cv2\nimport matplotlib.pyplot as plt\n\nimport pandas as pd\nimport numpy as np\nfrom numpy.random import seed\nseed(123)\n\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Flatten, Activation\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.optimizers import Adam\ntf.random.set_seed(123)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-22T07:56:34.333746Z","iopub.execute_input":"2024-07-22T07:56:34.334384Z","iopub.status.idle":"2024-07-22T07:56:46.477896Z","shell.execute_reply.started":"2024-07-22T07:56:34.334343Z","shell.execute_reply":"2024-07-22T07:56:46.477060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('../input/histopathologic-cancer-detection')","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:56:46.479542Z","iopub.execute_input":"2024-07-22T07:56:46.480097Z","iopub.status.idle":"2024-07-22T07:56:46.489413Z","shell.execute_reply.started":"2024-07-22T07:56:46.480068Z","shell.execute_reply":"2024-07-22T07:56:46.488240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"print(len(os.listdir('../input/histopathologic-cancer-detection/train')))\nprint(len(os.listdir('../input/histopathologic-cancer-detection/test')))","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:56:46.490700Z","iopub.execute_input":"2024-07-22T07:56:46.491053Z","iopub.status.idle":"2024-07-22T07:56:56.105181Z","shell.execute_reply.started":"2024-07-22T07:56:46.491022Z","shell.execute_reply":"2024-07-22T07:56:56.104266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/histopathologic-cancer-detection/train_labels.csv')\ndf_sample_submission = pd.read_csv('../input/histopathologic-cancer-detection/sample_submission.csv')\nprint(df_train.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:56:56.107574Z","iopub.execute_input":"2024-07-22T07:56:56.107871Z","iopub.status.idle":"2024-07-22T07:56:56.634312Z","shell.execute_reply.started":"2024-07-22T07:56:56.107847Z","shell.execute_reply":"2024-07-22T07:56:56.633400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:56:56.635554Z","iopub.execute_input":"2024-07-22T07:56:56.635845Z","iopub.status.idle":"2024-07-22T07:56:56.652333Z","shell.execute_reply.started":"2024-07-22T07:56:56.635821Z","shell.execute_reply":"2024-07-22T07:56:56.651218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## It is difficult for untrained eye to determine malignent cells, let's use CNN to extract features and see it's performance","metadata":{}},{"cell_type":"code","source":"plt.imshow(cv2.imread('../input/histopathologic-cancer-detection/train/f38a6374c348f90b587e046aac6079959adf3835.tif'))\nplt.show()\nplt.imshow(cv2.imread('../input/histopathologic-cancer-detection/train/c18f2d887b7ae4f6742ee445113fa1aef383ed77.tif'))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:56:56.653869Z","iopub.execute_input":"2024-07-22T07:56:56.654163Z","iopub.status.idle":"2024-07-22T07:56:57.132578Z","shell.execute_reply.started":"2024-07-22T07:56:56.654138Z","shell.execute_reply":"2024-07-22T07:56:57.131663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:56:57.134071Z","iopub.execute_input":"2024-07-22T07:56:57.134401Z","iopub.status.idle":"2024-07-22T07:56:57.148292Z","shell.execute_reply.started":"2024-07-22T07:56:57.134375Z","shell.execute_reply":"2024-07-22T07:56:57.147497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## It will take too long is we use the entire population and the label will also be unbalanced. Let's take 50,000 with label 0 and another 50,000 with label 1","metadata":{}},{"cell_type":"code","source":"df0 = df_train[df_train['label']==0].sample(500)\ndf1 = df_train[df_train['label']==1].sample(500)\ndf_data = pd.concat([df0, df1], axis=0).reset_index(drop=True)\n\ndf_data = shuffle(df_data)\n\n\ndf_data['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:56:57.149439Z","iopub.execute_input":"2024-07-22T07:56:57.149771Z","iopub.status.idle":"2024-07-22T07:56:57.183286Z","shell.execute_reply.started":"2024-07-22T07:56:57.149736Z","shell.execute_reply":"2024-07-22T07:56:57.182409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = df_data['label']\n\ndf_train, df_val = train_test_split(df_data, test_size=0.20, stratify=y)\n\nprint(df_train.shape)\nprint(df_val.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:56:57.184461Z","iopub.execute_input":"2024-07-22T07:56:57.184742Z","iopub.status.idle":"2024-07-22T07:56:57.192459Z","shell.execute_reply.started":"2024-07-22T07:56:57.184717Z","shell.execute_reply":"2024-07-22T07:56:57.191502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('base')\nos.mkdir('base/train')\nos.mkdir('base/val')\nos.mkdir('base/train/0')\nos.mkdir('base/train/1')\nos.mkdir('base/val/0')\nos.mkdir('base/val/1')","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:56:57.196904Z","iopub.execute_input":"2024-07-22T07:56:57.197215Z","iopub.status.idle":"2024-07-22T07:56:57.202659Z","shell.execute_reply.started":"2024-07-22T07:56:57.197190Z","shell.execute_reply":"2024-07-22T07:56:57.201790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for image in list(df_train[df_train['label']==0]['id']):\n    shutil.copyfile('../input/histopathologic-cancer-detection/train/'+image+'.tif', 'base/train/0/'+image+'.tif')\n\nfor image in list(df_train[df_train['label']==1]['id']):\n    shutil.copyfile('../input/histopathologic-cancer-detection/train/'+image+'.tif', 'base/train/1/'+image+'.tif')\n    \nfor image in list(df_val[df_val['label']==0]['id']):\n    shutil.copyfile('../input/histopathologic-cancer-detection/train/'+image+'.tif', 'base/val/0/'+image+'.tif')\n    \nfor image in list(df_val[df_val['label']==1]['id']):\n    shutil.copyfile('../input/histopathologic-cancer-detection/train/'+image+'.tif', 'base/val/1/'+image+'.tif')","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:56:57.203696Z","iopub.execute_input":"2024-07-22T07:56:57.203954Z","iopub.status.idle":"2024-07-22T07:57:06.043858Z","shell.execute_reply.started":"2024-07-22T07:56:57.203932Z","shell.execute_reply":"2024-07-22T07:57:06.042910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(os.listdir('base/train/0')))\nprint(len(os.listdir('base/train/1')))\nprint(len(os.listdir('base/val/0')))\nprint(len(os.listdir('base/val/1')))","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:57:06.046624Z","iopub.execute_input":"2024-07-22T07:57:06.046916Z","iopub.status.idle":"2024-07-22T07:57:06.052994Z","shell.execute_reply.started":"2024-07-22T07:57:06.046892Z","shell.execute_reply":"2024-07-22T07:57:06.052175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set up the generators\ntrain_path = 'base/train'\nvalid_path = 'base/val'\ntest_path = '../input/histopathologic-cancer-detection/test'\n\nnum_train_samples = len(df_train)\nnum_val_samples = len(df_val)\ntrain_batch_size = 10\nval_batch_size = 10\n\n\ntrain_steps = int(np.ceil(num_train_samples // train_batch_size))\nval_steps = int(np.ceil(num_val_samples // val_batch_size))","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:57:06.054292Z","iopub.execute_input":"2024-07-22T07:57:06.054592Z","iopub.status.idle":"2024-07-22T07:57:06.073901Z","shell.execute_reply.started":"2024-07-22T07:57:06.054568Z","shell.execute_reply":"2024-07-22T07:57:06.073026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen = ImageDataGenerator(rescale=1.0/255)\n\ntrain_gen = datagen.flow_from_directory(train_path,\n                                        target_size=(96,96),\n                                        batch_size=train_batch_size,\n                                        class_mode='categorical')\n\nval_gen = datagen.flow_from_directory(valid_path,\n                                        target_size=(96,96),\n                                        batch_size=val_batch_size,\n                                        class_mode='categorical')\n\n# Note: shuffle=False causes the test dataset to not be shuffled\ntest_gen = datagen.flow_from_directory('../input/histopathologic-cancer-detection',\n                                        target_size=(96,96),\n                                        batch_size=1,\n                                        classes=['test'],\n                                        shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:57:06.075028Z","iopub.execute_input":"2024-07-22T07:57:06.075333Z","iopub.status.idle":"2024-07-22T07:59:19.851622Z","shell.execute_reply.started":"2024-07-22T07:57:06.075311Z","shell.execute_reply":"2024-07-22T07:59:19.850661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kernel_size = (3,3)\npool_size= (2,2)\nfirst_filters = 64\nsecond_filters = 128\nthird_filters = 256\nfourth_filters = 512\n\ndropout_conv = 0.5\ndropout_dense = 0.5\n\nmodel = Sequential()\nmodel.add(Conv2D(first_filters, kernel_size, activation = 'relu',padding='same', input_shape = (96, 96, 3)))\nmodel.add(Conv2D(first_filters, kernel_size, activation = 'relu',padding='same'))\nmodel.add(MaxPooling2D(pool_size = pool_size)) \n\nmodel.add(Conv2D(second_filters, kernel_size, activation ='relu',padding='same'))\nmodel.add(Conv2D(second_filters, kernel_size, activation ='relu',padding='same'))\nmodel.add(MaxPooling2D(pool_size = pool_size))\n\nmodel.add(Conv2D(third_filters, kernel_size, activation ='relu',padding='same'))\nmodel.add(Conv2D(third_filters, kernel_size, activation ='relu',padding='same'))\nmodel.add(Conv2D(third_filters, kernel_size, activation ='relu',padding='same'))\nmodel.add(MaxPooling2D(pool_size = pool_size))\n\nmodel.add(Conv2D(fourth_filters, kernel_size, activation ='relu',padding='same'))\nmodel.add(Conv2D(fourth_filters, kernel_size, activation ='relu',padding='same'))\nmodel.add(Conv2D(fourth_filters, kernel_size, activation ='relu',padding='same'))\n\nmodel.add(Flatten())\nmodel.add(Dense(4096, activation = \"relu\"))\nmodel.add(Dropout(dropout_dense))\nmodel.add(Dense(4096, activation = \"relu\"))\nmodel.add(Dropout(dropout_dense))\nmodel.add(Dense(2, activation = \"softmax\"))\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:59:19.852907Z","iopub.execute_input":"2024-07-22T07:59:19.853268Z","iopub.status.idle":"2024-07-22T07:59:20.801444Z","shell.execute_reply.started":"2024-07-22T07:59:19.853237Z","shell.execute_reply":"2024-07-22T07:59:20.800573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(Adam(learning_rate=0.0001), loss='binary_crossentropy', \n              metrics=['AUC'])","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:59:20.802560Z","iopub.execute_input":"2024-07-22T07:59:20.802839Z","iopub.status.idle":"2024-07-22T07:59:20.817137Z","shell.execute_reply.started":"2024-07-22T07:59:20.802815Z","shell.execute_reply":"2024-07-22T07:59:20.816171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(val_gen.class_indices)","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:59:20.820050Z","iopub.execute_input":"2024-07-22T07:59:20.820363Z","iopub.status.idle":"2024-07-22T07:59:20.825060Z","shell.execute_reply.started":"2024-07-22T07:59:20.820333Z","shell.execute_reply":"2024-07-22T07:59:20.824130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# filepath = \"model.keras\"\n# checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, \n#                              save_best_only=True, mode='max')\n\n# reduce_lr = ReduceLROnPlateau(monitor='val_accuracy', factor=0.5, patience=2, \n#                                    verbose=1, mode='max', min_lr=0.00001)\n                              \n                              \n# callbacks_list = [checkpoint, reduce_lr]\n\nhistory = model.fit(train_gen, \n                    validation_data=val_gen,\n                    epochs=10, verbose=1)\n# ,\n#                    callbacks=callbacks_list)","metadata":{"execution":{"iopub.status.busy":"2024-07-22T07:59:20.826230Z","iopub.execute_input":"2024-07-22T07:59:20.826552Z","iopub.status.idle":"2024-07-22T08:00:29.159108Z","shell.execute_reply.started":"2024-07-22T07:59:20.826519Z","shell.execute_reply":"2024-07-22T08:00:29.158270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr_acc = history.history['AUC']\nval_acc = history.history['val_AUC']\n\nepoc = range(1, len(tr_acc) + 1)\n\nplt.plot(epoc, tr_acc, label='Training acc')\nplt.plot(epoc, val_acc, label='Validation acc')\nplt.title('Accuracy')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-22T08:00:29.160471Z","iopub.execute_input":"2024-07-22T08:00:29.161087Z","iopub.status.idle":"2024-07-22T08:00:29.448579Z","shell.execute_reply.started":"2024-07-22T08:00:29.161051Z","shell.execute_reply":"2024-07-22T08:00:29.447648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(test_gen, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2024-07-22T08:00:29.449606Z","iopub.execute_input":"2024-07-22T08:00:29.449880Z","iopub.status.idle":"2024-07-22T08:10:26.330749Z","shell.execute_reply.started":"2024-07-22T08:00:29.449856Z","shell.execute_reply":"2024-07-22T08:10:26.329721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions","metadata":{"execution":{"iopub.status.busy":"2024-07-22T08:10:26.332055Z","iopub.execute_input":"2024-07-22T08:10:26.332346Z","iopub.status.idle":"2024-07-22T08:10:26.339181Z","shell.execute_reply.started":"2024-07-22T08:10:26.332322Z","shell.execute_reply":"2024-07-22T08:10:26.338272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_preds = pd.DataFrame(predictions, columns=['0', '1'])\n\ndf_preds.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-22T08:10:26.340224Z","iopub.execute_input":"2024-07-22T08:10:26.340467Z","iopub.status.idle":"2024-07-22T08:10:26.354749Z","shell.execute_reply.started":"2024-07-22T08:10:26.340446Z","shell.execute_reply":"2024-07-22T08:10:26.353888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_preds[df_preds['1']>0.5]","metadata":{"execution":{"iopub.status.busy":"2024-07-22T08:10:26.355928Z","iopub.execute_input":"2024-07-22T08:10:26.357913Z","iopub.status.idle":"2024-07-22T08:10:26.371812Z","shell.execute_reply.started":"2024-07-22T08:10:26.357886Z","shell.execute_reply":"2024-07-22T08:10:26.370955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_preds['file_names'] = test_gen.filenames","metadata":{"execution":{"iopub.status.busy":"2024-07-22T08:10:26.373097Z","iopub.execute_input":"2024-07-22T08:10:26.373717Z","iopub.status.idle":"2024-07-22T08:10:26.382290Z","shell.execute_reply.started":"2024-07-22T08:10:26.373682Z","shell.execute_reply":"2024-07-22T08:10:26.381358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_preds['id'] = df_preds['file_names'].str[5:-4]\ndf_preds[['id','1']].rename(columns={'1':'label'}).to_csv('submission.csv', columns=['id','label'],index=False) ","metadata":{"execution":{"iopub.status.busy":"2024-07-22T08:10:26.383668Z","iopub.execute_input":"2024-07-22T08:10:26.384052Z","iopub.status.idle":"2024-07-22T08:10:26.621281Z","shell.execute_reply.started":"2024-07-22T08:10:26.384002Z","shell.execute_reply":"2024-07-22T08:10:26.620477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-07-22T08:10:26.622487Z","iopub.execute_input":"2024-07-22T08:10:26.622827Z","iopub.status.idle":"2024-07-22T08:10:26.687716Z","shell.execute_reply.started":"2024-07-22T08:10:26.622795Z","shell.execute_reply":"2024-07-22T08:10:26.686798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shutil.rmtree('base')","metadata":{"execution":{"iopub.status.busy":"2024-07-22T08:10:26.688968Z","iopub.execute_input":"2024-07-22T08:10:26.689338Z","iopub.status.idle":"2024-07-22T08:10:26.729383Z","shell.execute_reply.started":"2024-07-22T08:10:26.689305Z","shell.execute_reply":"2024-07-22T08:10:26.728339Z"},"trusted":true},"execution_count":null,"outputs":[]}]}