{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport random\nfrom sklearn.utils import shuffle\nfrom tqdm import tqdm_notebook\n\npath = \"../input/histopathologic-cancer-detection/\"\n\ndata = pd.read_csv(path +\"train_labels.csv\")\ntrain_path = path +'train/'\ntest_path = path + 'test/'\n# quick look at the label stats\ndata['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def readImage(path):\n    # OpenCV reads the image in bgr format by default\n    bgr_img = cv2.imread(path)\n    # We flip it to rgb for visualization purposes\n    b,g,r = cv2.split(bgr_img)\n    rgb_img = cv2.merge([r,g,b])\n    return rgb_img","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"visualisation from : https://www.kaggle.com/qitvision/a-complete-ml-pipeline-fast-ai"},{"metadata":{"trusted":true},"cell_type":"code","source":"# random sampling\nshuffled_data = shuffle(data)\n\nfig, ax = plt.subplots(2,5, figsize=(20,8))\nfig.suptitle('Histopathologic scans of lymph node sections',fontsize=20)\n# Negatives\nfor i, idx in enumerate(shuffled_data[shuffled_data['label'] == 0]['id'][:5]):\n    path = os.path.join(train_path, idx)\n    ax[0,i].imshow(readImage(path + '.tif'))\n    # Create a Rectangle patch\n    box = patches.Rectangle((32,32),32,32,linewidth=4,edgecolor='b',facecolor='none', linestyle=':', capstyle='round')\n    ax[0,i].add_patch(box)\nax[0,0].set_ylabel('Negative samples', size='large')\n# Positives\nfor i, idx in enumerate(shuffled_data[shuffled_data['label'] == 1]['id'][:5]):\n    path = os.path.join(train_path, idx)\n    ax[1,i].imshow(readImage(path + '.tif'))\n    # Create a Rectangle patch\n    box = patches.Rectangle((32,32),32,32,linewidth=4,edgecolor='r',facecolor='none', linestyle=':', capstyle='round')\n    ax[1,i].add_patch(box)\nax[1,0].set_ylabel('Tumor tissue samples', size='large')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -Uqq fastbook\nimport fastbook\nfrom fastai.vision.all import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = \"../input/histopathologic-cancer-detection/\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_x(r): return path+'train/'+r['id']+'.tif'\ndef get_y(r): return r['label']\n\n\n# start with creatinga datablock\n\ndblock =  DataBlock(blocks=(ImageBlock, CategoryBlock),\n                    splitter=RandomSplitter(valid_pct=0.2,seed=42), \n                    get_x=get_x, \n                    get_y=get_y, \n                    item_tfms=RandomResizedCrop(128, min_scale=0.35))\ndls = dblock.dataloaders(data)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dls.show_batch(nrows=1, ncols=3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So we are able to create a dataloader however the images are too big right nowm we need a way to reduce the size of images. lets try for a crop to 48x48 at center with flip. and brightness. https://docs.fast.ai/vision.augment#PadMode"},{"metadata":{"trusted":true},"cell_type":"code","source":"dblock =  DataBlock(blocks=(ImageBlock, CategoryBlock),\n                    splitter=RandomSplitter(valid_pct=0.2,seed=42), \n                    get_x=get_x, \n                    get_y=get_y, \n                    item_tfms= (CropPad(48, pad_mode='zeros'),DihedralItem(p=1.0, nm=None, before_call=None) ))\ndls = dblock.dataloaders(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dls.show_batch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = cnn_learner(dls, resnet34, metrics=error_rate)\nlearn.fine_tune(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"interp = ClassificationInterpretation.from_learner(learn)\ninterp.plot_confusion_matrix()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"interp.plot_top_losses(16, nrows=8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.predict('../input/histopathologic-cancer-detection/test/00006537328c33e284c973d7b39d340809f7271b.tif')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Problems with testing data\n\nfor submission again from : https://www.kaggle.com/qitvision/a-complete-ml-pipeline-fast-ai. Maybe U sould consider differnet way to evaluate any ideas ?"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = \"../input/histopathologic-cancer-detection/\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"from : https://www.kaggle.com/mamamot/fastai-v2-example and https://www.kaggle.com/mentalwanderer/image-classification-workflow-with-fast-ai\n\nfor more info on tta\nhttps://docs.fast.ai/learner#Learner.tta"},{"metadata":{"trusted":true},"cell_type":"code","source":"preds, y = learn.tta()\nacc = accuracy(preds, y)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\ndef auc_score(y_pred,y_true,tens=True):\n    score = roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])\n    if tens:\n        score = tensor(score)\n    return score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('The validation accuracy is {} %.'.format(acc * 100))\npred_score = auc_score(preds,y).item()\nprint('The validation AUC is {}.'.format(pred_score))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Since definition of tta has changed I am unable currently to make it work, however creating a dataloader for test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # doesnt work\n# tf_fns = get_image_files(path + 'test')\n# test_data = DataBlock(get_items=get_image_files,\n#                  item_tfms=(CropPad(48, pad_mode='zeros'),DihedralItem(p=1.0, nm=None, before_call=None)))\n# dl_test = test_data.dataloaders(path+'test')\n# dl_test.show_batch()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntest_images = get_image_files(path + 'test')\npreds,y = learn.get_preds(dl=dls.test_dl(test_images, shuffle=False, drop_last=False))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_list = list(preds[:,1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(pred_list), len(test_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submissions = pd.read_csv(path + 'sample_submission.csv')\nid_list = list(submissions.id)\nid_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_images_dict = {}\nfor i in range(len(test_images)):\n    test_images_dict[str(str(test_images[i]).split('/')[-1].split('.')[0])] = float(pred_list[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_images_dict['88a12685148c0d876fed1fba8228afc6e7ee937f']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_list  = []\n\nfor i in id_list:\n    prediction_list.append(test_images_dict[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_list[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nsubmissions = pd.DataFrame({'id':id_list,'label':prediction_list})\nsubmissions.to_csv(\"submission.csv\".format(pred_score),index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"baseline model has an accuracy of \\n\nScore!\nsubmission.csv\njust now\n1 seconds\n1 seconds\n0.9269\n\n[Screenshot%20%28119%29.png](attachment:Screenshot%20%28119%29.png)","attachments":{}},{"metadata":{},"cell_type":"markdown","source":"Rest not needed (down below) \nbut you can upload your own kaggle score image if you want !"},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.vision.widgets import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"btn_upload = widgets.FileUpload()\nbtn_upload","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img =   PILImage.create(btn_upload.data[-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"out_pl = widgets.Output()\nout_pl.clear_output()\nwith out_pl: display(img.to_thumb(600,600))\nout_pl","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}