{"cells":[{"metadata":{"_uuid":"6b0118dc818119b911aeeeb41d8e54999a4618f2"},"cell_type":"markdown","source":"Check out corresponding Medium article:\n\n[Histopathologic Cancer Detector - Machine Learning in Medicine](https://towardsdatascience.com/histopathologic-cancer-detector-finding-cancer-cells-with-machine-learning-b77ce1ee9b0a)"},{"metadata":{"trusted":true,"_uuid":"812221f6046eb6002d9835e040d8d50675c4718e"},"cell_type":"code","source":"%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --upgrade pip","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install livelossplot\nfrom livelossplot import PlotLossesKeras","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"# Imports\nimport numpy as np \nimport pandas as pd \nfrom glob import glob \nfrom skimage.io import imread \nimport os\nimport shutil\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import roc_curve, auc, roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.applications.nasnet import NASNetMobile\nfrom keras.applications.xception import Xception\nfrom keras.layers import Dropout, Flatten, Dense, GlobalAveragePooling2D, Input, Concatenate, GlobalMaxPooling2D\nfrom keras.models import Model\nfrom keras.callbacks import CSVLogger, ReduceLROnPlateau, ModelCheckpoint\nfrom keras.optimizers import Adam\n#!pip install livelossplot\n#from livelossplot import PlotLossesKeras","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cada1e5e63e66eaccab61c40422b7d7f8fc481c6"},"cell_type":"code","source":"# Output files\nTRAINING_LOGS_FILE = \"training_logs.csv\"\nMODEL_SUMMARY_FILE = \"model_summary.txt\"\nMODEL_FILE = \"histopathologic_cancer_detector.h5\"\nTRAINING_PLOT_FILE = \"training.png\"\nVALIDATION_PLOT_FILE = \"validation.png\"\nROC_PLOT_FILE = \"roc.png\"\nKAGGLE_SUBMISSION_FILE = \"kaggle_submission.csv\"\nINPUT_DIR = '../input/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b8d7a9899440da705f78f32d5c7a94c9b5434fb"},"cell_type":"code","source":"# Hyperparams\nSAMPLE_COUNT = 8500\nTRAINING_RATIO = 0.9\nIMAGE_SIZE = 96\nEPOCHS = 12\nBATCH_SIZE = 18\nVERBOSITY = 1\nTESTING_BATCH_SIZE = 5000","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Data setup\ntraining_dir = INPUT_DIR + 'train/'\ndata_frame = pd.DataFrame({'path': glob(os.path.join(training_dir,'*.tif'))})\ndata_frame['id'] = data_frame.path.map(lambda x: x.split('/')[3].split('.')[0]) \nlabels = pd.read_csv(INPUT_DIR + 'train_labels.csv')\ndata_frame = data_frame.merge(labels, on = 'id')\nnegatives = data_frame[data_frame.label == 0].sample(SAMPLE_COUNT)\npositives = data_frame[data_frame.label == 1].sample(SAMPLE_COUNT)\ndata_frame = pd.concat([negatives, positives]).reset_index()\ndata_frame = data_frame[['path', 'id', 'label']]\ndata_frame['image'] = data_frame['path'].map(imread)\n\ntraining_path = '../training'\nvalidation_path = '../validation'\n\nfor folder in [training_path, validation_path]:\n    for subfolder in ['0', '1']:\n        path = os.path.join(folder, subfolder)\n        os.makedirs(path, exist_ok=True)\n\ntraining, validation = train_test_split(data_frame, train_size=TRAINING_RATIO, stratify=data_frame['label'])\n\ndata_frame.set_index('id', inplace=True)\n\nfor images_and_path in [(training, training_path), (validation, validation_path)]:\n    images = images_and_path[0]\n    path = images_and_path[1]\n    for image in images['id'].values:\n        file_name = image + '.tif'\n        label = str(data_frame.loc[image,'label'])\n        destination = os.path.join(path, label, file_name)\n        if not os.path.exists(destination):\n            source = os.path.join(INPUT_DIR + 'train', file_name)\n            shutil.copyfile(source, destination)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cec14fe7a59727d5606263f17aeff81699d912e0"},"cell_type":"code","source":"# Data augmentation\ntraining_data_generator = ImageDataGenerator(rescale=1./255,\n                                             horizontal_flip=True,\n                                             vertical_flip=True,\n                                             rotation_range=90,\n                                             zoom_range=0.2, \n                                             width_shift_range=0.1,\n                                             height_shift_range=0.1,\n                                             shear_range=0.05,\n                                             channel_shift_range=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccad1f37cc2bfae0ba735b0b8e1ddb20a5930dae"},"cell_type":"code","source":"# Data generation\ntraining_generator = training_data_generator.flow_from_directory(training_path,\n                                                                 target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                                                 batch_size=BATCH_SIZE,\n                                                                 class_mode='binary')\nvalidation_generator = ImageDataGenerator(rescale=1./255).flow_from_directory(validation_path,\n                                                                              target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                                                              batch_size=BATCH_SIZE,\n                                                                              class_mode='binary')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1dd6a67856557351a02e4ebcfc87e1c94c7706dc","scrolled":false},"cell_type":"code","source":"# Model (LB 0.9558)\ninput_shape = (IMAGE_SIZE, IMAGE_SIZE, 3)\ninputs = Input(input_shape)\n\nxception = Xception(include_top=False, input_shape=input_shape)  \nnas_net = NASNetMobile(include_top=False, input_shape=input_shape)\n\noutputs = Concatenate(axis=-1)([GlobalAveragePooling2D()(xception(inputs)),\n                                GlobalAveragePooling2D()(nas_net(inputs))])\noutputs = Dropout(0.5)(outputs)\noutputs = Dense(1, activation='sigmoid')(outputs)\n\nmodel = Model(inputs, outputs)\nmodel.compile(optimizer=Adam(lr=0.0001, decay=0.00001),\n              loss='binary_crossentropy',\n              metrics=['accuracy'])\nmodel.summary()\n\nfrom IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot\nSVG(model_to_dot(model).create(prog='dot', format='svg'))\n\nfrom keras.utils.vis_utils import plot_model\nplot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True, expand_nested=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"8a854c31ea90b6dada058e8048cc20a0a5907357"},"cell_type":"code","source":"#  Training\nhistory = model.fit_generator(training_generator,\n                              steps_per_epoch=len(training_generator), \n                              validation_data=validation_generator,\n                              validation_steps=len(validation_generator),\n                              epochs=EPOCHS,\n                              verbose=VERBOSITY,\n                              callbacks=[PlotLossesKeras(),\n                                         ModelCheckpoint(MODEL_FILE,\n                                                         monitor='val_acc',\n                                                         verbose=VERBOSITY,\n                                                         save_best_only=True,\n                                                         mode='max'),\n                                         CSVLogger(TRAINING_LOGS_FILE,\n                                                   append=False,\n                                                   separator=';')])\nmodel.load_weights(MODEL_FILE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd1582290e369f0f6bed6c6856a1c850b6ec3ef4"},"cell_type":"code","source":"# Training plots\nepochs = [i for i in range(1, len(history.history['loss'])+1)]\n\nplt.plot(epochs, history.history['loss'], color='blue', label=\"training_loss\")\nplt.plot(epochs, history.history['val_loss'], color='red', label=\"validation_loss\")\nplt.legend(loc='best')\nplt.title('training')\nplt.xlabel('epoch')\nplt.savefig(TRAINING_PLOT_FILE, bbox_inches='tight')\nplt.show()\n\nplt.plot(epochs, history.history['acc'], color='blue', label=\"training_accuracy\")\nplt.plot(epochs, history.history['val_acc'], color='red',label=\"validation_accuracy\")\nplt.legend(loc='best')\nplt.title('validation')\nplt.xlabel('epoch')\nplt.savefig(VALIDATION_PLOT_FILE, bbox_inches='tight')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d366ba6530d8d2ae88f9a2947abb38c15e040a3"},"cell_type":"code","source":"# ROC validation plot\nroc_validation_generator = ImageDataGenerator(rescale=1./255).flow_from_directory(validation_path,\n                                                                                  target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                                                                  batch_size=BATCH_SIZE,\n                                                                                  class_mode='binary',\n                                                                                  shuffle=False)\npredictions = model.predict_generator(roc_validation_generator, steps=len(roc_validation_generator), verbose=VERBOSITY)\nfalse_positive_rate, true_positive_rate, threshold = roc_curve(roc_validation_generator.classes, predictions)\narea_under_curve = auc(false_positive_rate, true_positive_rate)\n\nplt.plot([0, 1], [0, 1], 'k--')\nplt.plot(false_positive_rate, true_positive_rate, label='AUC = {:.3f}'.format(area_under_curve))\nplt.xlabel('False positive rate')\nplt.ylabel('True positive rate')\nplt.title('ROC curve')\nplt.legend(loc='best')\nplt.savefig(ROC_PLOT_FILE, bbox_inches='tight')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dae7e6683cde39444ff098777222b55dd130ee29"},"cell_type":"code","source":"# Kaggle testing\ntesting_files = glob(os.path.join(INPUT_DIR+'test/','*.tif'))\nsubmission = pd.DataFrame()\nfor index in range(0, len(testing_files), TESTING_BATCH_SIZE):\n    data_frame = pd.DataFrame({'path': testing_files[index:index+TESTING_BATCH_SIZE]})\n    data_frame['id'] = data_frame.path.map(lambda x: x.split('/')[3].split(\".\")[0])\n    data_frame['image'] = data_frame['path'].map(imread)\n    images = np.stack(data_frame.image, axis=0)\n    predicted_labels = [model.predict(np.expand_dims(image/255.0, axis=0))[0][0] for image in images]\n    predictions = np.array(predicted_labels)\n    data_frame['label'] = predictions\n    submission = pd.concat([submission, data_frame[[\"id\", \"label\"]]])\nsubmission.to_csv(KAGGLE_SUBMISSION_FILE, index=False, header=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}