{"cells":[{"metadata":{"_uuid":"6b0118dc818119b911aeeeb41d8e54999a4618f2"},"cell_type":"markdown","source":"Check out corresponding Medium article:\n\n[Histopathologic Cancer Detector - Machine Learning in Medicine](https://towardsdatascience.com/histopathologic-cancer-detector-finding-cancer-cells-with-machine-learning-b77ce1ee9b0a)"},{"metadata":{"trusted":true,"_uuid":"812221f6046eb6002d9835e040d8d50675c4718e"},"cell_type":"code","source":"%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"# Imports\nimport numpy as np \nimport pandas as pd \nfrom glob import glob \nfrom skimage.io import imread \nimport os\nimport shutil\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import roc_curve, auc, roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.applications.nasnet import NASNetMobile\nfrom keras.applications.xception import Xception\nfrom keras.layers import Dropout, Flatten, Dense, GlobalAveragePooling2D, Input, Concatenate, GlobalMaxPooling2D\nfrom keras.models import Model\nfrom keras.callbacks import CSVLogger, ReduceLROnPlateau, ModelCheckpoint\nfrom keras.optimizers import Adam\n!pip install --upgrade pip\n!pip install livelossplot\nfrom livelossplot import PlotLossesKeras","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cada1e5e63e66eaccab61c40422b7d7f8fc481c6"},"cell_type":"code","source":"# Output files\nTRAINING_LOGS_FILE = \"training_logs.csv\"\nMODEL_SUMMARY_FILE = \"model_summary.txt\"\nMODEL_FILE = \"histopathologic_cancer_detector.h5\"\nTRAINING_PLOT_FILE = \"training.png\"\nVALIDATION_PLOT_FILE = \"validation.png\"\nROC_PLOT_FILE = \"roc.png\"\nKAGGLE_SUBMISSION_FILE = \"kaggle_submission.csv\"\nINPUT_DIR = '../input/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b8d7a9899440da705f78f32d5c7a94c9b5434fb"},"cell_type":"code","source":"# Hyperparams\nSAMPLE_COUNT = 85000\nTRAINING_RATIO = 0.9\nIMAGE_SIZE = 96\nEPOCHS = 12\nBATCH_SIZE = 216\nVERBOSITY = 1\nTESTING_BATCH_SIZE = 5000","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Data setup\ntraining_dir = INPUT_DIR + 'train/'\ndata_frame = pd.DataFrame({'path': glob(os.path.join(training_dir,'*.tif'))})\ndata_frame['id'] = data_frame.path.map(lambda x: x.split('/')[3].split('.')[0]) \nlabels = pd.read_csv(INPUT_DIR + 'train_labels.csv')\ndata_frame = data_frame.merge(labels, on = 'id')\nnegatives = data_frame[data_frame.label == 0].sample(SAMPLE_COUNT)\npositives = data_frame[data_frame.label == 1].sample(SAMPLE_COUNT)\ndata_frame = pd.concat([negatives, positives]).reset_index()\ndata_frame = data_frame[['path', 'id', 'label']]\ndata_frame['image'] = data_frame['path'].map(imread)\n\ntraining_path = '../training'\nvalidation_path = '../validation'\n\nfor folder in [training_path, validation_path]:\n    for subfolder in ['0', '1']:\n        path = os.path.join(folder, subfolder)\n        os.makedirs(path, exist_ok=True)\n\ntraining, validation = train_test_split(data_frame, train_size=TRAINING_RATIO, stratify=data_frame['label'])\n\ndata_frame.set_index('id', inplace=True)\n\nfor images_and_path in [(training, training_path), (validation, validation_path)]:\n    images = images_and_path[0]\n    path = images_and_path[1]\n    for image in images['id'].values:\n        file_name = image + '.tif'\n        label = str(data_frame.loc[image,'label'])\n        destination = os.path.join(path, label, file_name)\n        if not os.path.exists(destination):\n            source = os.path.join(INPUT_DIR + 'train', file_name)\n            shutil.copyfile(source, destination)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data augmentation\nfrom imgaug import augmenters as iaa \nimport imgaug as ia\nfrom random import shuffle\nsometimes = lambda aug: iaa.Sometimes(0.5, aug)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_seq():\n    sometimes = lambda aug: iaa.Sometimes(0.5, aug)\n    seq = iaa.Sequential(\n        [\n            # apply the following augmenters to most images\n            iaa.Fliplr(0.5), # horizontally flip 50% of all images\n            iaa.Flipud(0.2), # vertically flip 20% of all images\n            sometimes(iaa.Affine(\n                scale={\"x\": (0.9, 1.1), \"y\": (0.9, 1.1)}, # scale images to 80-120% of their size, individually per axis\n                translate_percent={\"x\": (-0.1, 0.1), \"y\": (-0.1, 0.1)}, # translate by -20 to +20 percent (per axis)\n                rotate=(-10, 10), # rotate by -45 to +45 degrees\n                shear=(-5, 5), # shear by -16 to +16 degrees\n                order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)\n                cval=(0, 255), # if mode is constant, use a cval between 0 and 255\n                mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)\n            )),\n            # execute 0 to 5 of the following (less important) augmenters per image\n            # don't execute all of them, as that would often be way too strong\n            iaa.SomeOf((0, 5),\n                [\n                    sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation\n                    iaa.OneOf([\n                        iaa.GaussianBlur((0, 1.0)), # blur images with a sigma between 0 and 3.0\n                        iaa.AverageBlur(k=(3, 5)), # blur image using local means with kernel sizes between 2 and 7\n                        iaa.MedianBlur(k=(3, 5)), # blur image using local medians with kernel sizes between 2 and 7\n                    ]),\n                    iaa.Sharpen(alpha=(0, 1.0), lightness=(0.9, 1.1)), # sharpen images\n                    iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images\n                    # search either for all edges or for directed edges,\n                    # blend the result with the original image using a blobby mask\n                    iaa.SimplexNoiseAlpha(iaa.OneOf([\n                        iaa.EdgeDetect(alpha=(0.5, 1.0)),\n                        iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)),\n                    ])),\n                    iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.01*255), per_channel=0.5), # add gaussian noise to images\n                    iaa.OneOf([\n                        iaa.Dropout((0.01, 0.05), per_channel=0.5), # randomly remove up to 10% of the pixels\n                        iaa.CoarseDropout((0.01, 0.03), size_percent=(0.01, 0.02), per_channel=0.2),\n                    ]),\n                    iaa.Invert(0.01, per_channel=True), # invert color channels\n                    iaa.Add((-2, 2), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)\n                    iaa.AddToHueAndSaturation((-1, 1)), # change hue and saturation\n                    # either change the brightness of the whole image (sometimes\n                    # per channel) or change the brightness of subareas\n                    iaa.OneOf([\n                        iaa.Multiply((0.9, 1.1), per_channel=0.5),\n                        iaa.FrequencyNoiseAlpha(\n                            exponent=(-1, 0),\n                            first=iaa.Multiply((0.9, 1.1), per_channel=True),\n                            second=iaa.ContrastNormalization((0.9, 1.1))\n                        )\n                    ]),\n                    sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)\n                    sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))), # sometimes move parts of the image around\n                    sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))\n                ],\n                random_order=True\n            )\n        ],\n        random_order=True\n    )\n    return seq","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def chunker(seq, size):\n    return (seq[pos:pos + size] for pos in range(0, len(seq), size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_data_generator = ImageDataGenerator(rescale=1./255,\n                                             horizontal_flip=True,\n                                             vertical_flip=True,\n                                             rotation_range=90,\n                                             zoom_range=0.2, \n                                             width_shift_range=0.1,\n                                             height_shift_range=0.1,\n                                             shear_range=0.05,\n                                             channel_shift_range=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccad1f37cc2bfae0ba735b0b8e1ddb20a5930dae"},"cell_type":"code","source":"# Data generation\ntraining_generator = training_data_generator.flow_from_directory(training_path,\n                                                                 target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                                                 batch_size=BATCH_SIZE,\n                                                                 class_mode='binary')\nvalidation_generator = ImageDataGenerator(rescale=1./255).flow_from_directory(validation_path,\n                                                                              target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                                                              batch_size=BATCH_SIZE,\n                                                                              class_mode='binary')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1dd6a67856557351a02e4ebcfc87e1c94c7706dc","scrolled":false},"cell_type":"code","source":"# Model (LB 0.9558)\ninput_shape = (IMAGE_SIZE, IMAGE_SIZE, 3)\ninputs = Input(input_shape)\n\ninput_tensor = Input(shape=(96,96,3))\nxception = Xception(include_top=False, input_shape=input_shape)  \nnas_net = NASNetMobile(input_tensor=input_tensor , include_top=False, weights='imagenet')\n\noutputs = Concatenate(axis=-1)([GlobalAveragePooling2D()(xception(inputs)),\n                                GlobalAveragePooling2D()(nas_net(inputs))])\noutputs = Dropout(0.5)(outputs)\noutputs = Dense(1, activation='sigmoid')(outputs)\n\nmodel = Model(inputs, outputs)\nmodel.compile(optimizer=Adam(lr=0.0001, decay=0.00001),\n              loss='binary_crossentropy',\n              metrics=['accuracy'])\nmodel.summary()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"8a854c31ea90b6dada058e8048cc20a0a5907357"},"cell_type":"code","source":"#  Training\nhistory = model.fit_generator(training_generator,\n                              steps_per_epoch=len(training_generator), \n                              validation_data=validation_generator,\n                              validation_steps=len(validation_generator),\n                              epochs=EPOCHS,\n                              verbose=VERBOSITY,\n                              callbacks=[PlotLossesKeras(),\n                                         ModelCheckpoint(MODEL_FILE,\n                                                         monitor='val_acc',\n                                                         verbose=VERBOSITY,\n                                                         save_best_only=True,\n                                                         mode='max'),\n                                         CSVLogger(TRAINING_LOGS_FILE,\n                                                   append=False,\n                                                   separator=';')])\nmodel.load_weights(MODEL_FILE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd1582290e369f0f6bed6c6856a1c850b6ec3ef4"},"cell_type":"code","source":"# Training plots\nepochs = [i for i in range(1, len(history.history['loss'])+1)]\n\nplt.plot(epochs, history.history['loss'], color='blue', label=\"training_loss\")\nplt.plot(epochs, history.history['val_loss'], color='red', label=\"validation_loss\")\nplt.legend(loc='best')\nplt.title('training')\nplt.xlabel('epoch')\nplt.savefig(TRAINING_PLOT_FILE, bbox_inches='tight')\nplt.show()\n\nplt.plot(epochs, history.history['acc'], color='blue', label=\"training_accuracy\")\nplt.plot(epochs, history.history['val_acc'], color='red',label=\"validation_accuracy\")\nplt.legend(loc='best')\nplt.title('validation')\nplt.xlabel('epoch')\nplt.savefig(VALIDATION_PLOT_FILE, bbox_inches='tight')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d366ba6530d8d2ae88f9a2947abb38c15e040a3"},"cell_type":"code","source":"# ROC validation plot\nroc_validation_generator = ImageDataGenerator(rescale=1./255).flow_from_directory(validation_path,\n                                                                                  target_size=(IMAGE_SIZE,IMAGE_SIZE),\n                                                                                  batch_size=BATCH_SIZE,\n                                                                                  class_mode='binary',\n                                                                                  shuffle=False)\npredictions = model.predict_generator(roc_validation_generator, steps=len(roc_validation_generator), verbose=VERBOSITY)\nfalse_positive_rate, true_positive_rate, threshold = roc_curve(roc_validation_generator.classes, predictions)\narea_under_curve = auc(false_positive_rate, true_positive_rate)\n\nplt.plot([0, 1], [0, 1], 'k--')\nplt.plot(false_positive_rate, true_positive_rate, label='AUC = {:.3f}'.format(area_under_curve))\nplt.xlabel('False positive rate')\nplt.ylabel('True positive rate')\nplt.title('ROC curve')\nplt.legend(loc='best')\nplt.savefig(ROC_PLOT_FILE, bbox_inches='tight')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dae7e6683cde39444ff098777222b55dd130ee29"},"cell_type":"code","source":"# Kaggle testing\ntesting_files = glob(os.path.join(INPUT_DIR+'test/','*.tif'))\nsubmission = pd.DataFrame()\nfor index in range(0, len(testing_files), TESTING_BATCH_SIZE):\n    data_frame = pd.DataFrame({'path': testing_files[index:index+TESTING_BATCH_SIZE]})\n    data_frame['id'] = data_frame.path.map(lambda x: x.split('/')[3].split(\".\")[0])\n    data_frame['image'] = data_frame['path'].map(imread)\n    images = np.stack(data_frame.image, axis=0)\n    predicted_labels = [model.predict(np.expand_dims(image/255.0, axis=0))[0][0] for image in images]\n    predictions = np.array(predicted_labels)\n    data_frame['label'] = predictions\n    submission = pd.concat([submission, data_frame[[\"id\", \"label\"]]])\nsubmission.to_csv(KAGGLE_SUBMISSION_FILE, index=False, header=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}