{"cells":[{"metadata":{"_uuid":"a3d5d1c13787b85b2dd7d7e7219e653d0f169a6c"},"cell_type":"markdown","source":"This is a Fork of my [Densenet169](https://www.kaggle.com/guntherthepenguin/fastai-v1-densenet169/) kernel. The only thing new is I included the [WSI ids](https://www.kaggle.com/tywangty/histopathologiccancerwsi) from this [discussion](https://www.kaggle.com/c/histopathologic-cancer-detection/discussion/83760) to reduce overfitting and correlations between validation and training set.\nThanks to [SM](https://www.kaggle.com/sermakarevich) for the WSI set and Idea and [Taylor](https://www.kaggle.com/tywangty) for uploading it on kaggle"},{"metadata":{"trusted":true,"_uuid":"a503853adef770ddb1513e85ad2d377f8965c1db","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\n\nimport numpy as np\nimport os\nfrom sklearn.metrics import f1_score\n\nfrom fastai import *\nfrom fastai.vision import *\n\nimport torch\nimport torch.nn as nn\nimport torchvision\nimport cv2\n\nfrom tqdm import tqdm\nfrom skmultilearn.model_selection import iterative_train_test_split\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import MultiLabelBinarizer\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom sklearn.metrics import roc_auc_score\n%load_ext autoreload\n%autoreload","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95ac8eb66b25e87a0f4828640a7d6ef405e313f1"},"cell_type":"code","source":"from torchvision.models import *","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6b220f04907e1c156e1e1235dc0b53f56b63661e"},"cell_type":"markdown","source":"Defining a metric so after epoch I get the validation ROC-AUC score"},{"metadata":{"trusted":true,"_uuid":"5caf6cf236cf9d098331763b434806cd73d13823"},"cell_type":"code","source":"model_path='.'\npath='../input/histopathologic-cancer-detection/'\ntrain_folder=f'{path}train'\ntest_folder=f'{path}test'\ntrain_lbl=f'{path}train_labels.csv'\nORG_SIZE=96\n\nbs=64\nnum_workers=None # Apprently 2 cpus per kaggle node, so 4 threads I think\nsz=96","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b1c089e47c2325cf783092cc235b9b5e9831022a"},"cell_type":"markdown","source":"In Case I want to run quick tests use a subsample:"},{"metadata":{"trusted":true,"_uuid":"818e942f47e2dbf4b7eb9a75c372b35d5ecd7511"},"cell_type":"code","source":"from pathlib import Path\ntest_fnames=[str(file) for file in Path(test_folder).iterdir()]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ecb840384eb1e0aa90fe881f8c212beb135ad6c4"},"cell_type":"code","source":"df_trn=pd.read_csv(train_lbl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55ffc97adace3390ca59f715a996d0a17cc10eb4"},"cell_type":"code","source":"df_WSI=pd.read_csv('../input/histopathologiccancerwsi/patch_id_wsi.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7982793320648eb1ecf956b46da3f2da0d81aaef"},"cell_type":"code","source":"tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=0.0, max_zoom=.15,\n                      max_lighting=0.1, max_warp=0.15)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3eb2fee617a5d8012161678f53bdb30bc09c8a0e"},"cell_type":"markdown","source":"# Count per WSI id"},{"metadata":{"_uuid":"fe67edcdaba2195437d7064b0385cbba395866d1"},"cell_type":"markdown","source":"## Take only Ids of which we know the WSI"},{"metadata":{"_uuid":"800ade386c9370cc1e0c75a4d135549dfa4bd975"},"cell_type":"markdown","source":"## extract all images we dont know the WSI id of (will go into validation set)"},{"metadata":{"trusted":true,"_uuid":"33f3f69d99e88457d512f7828575c90d0abe72ba"},"cell_type":"code","source":"df_notinWSI=df_trn.set_index('id').drop(df_WSI.id)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"645241d88fa8180aa18805e71e97be3aa25bdc58"},"cell_type":"markdown","source":"## Get 20% of the WSIs as validation set (maybe later make sure its stratified)"},{"metadata":{"trusted":true,"_uuid":"e23f7153a2596b8b0b614452e4815d5450ef69b5"},"cell_type":"code","source":"valWSI=df_WSI.groupby(by='wsi')['id'].count().sample(frac=0.23).index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e88aa04d45980367fa021aa56358db22abdd53b"},"cell_type":"code","source":"trnWSI=[i[0] for i in df_WSI.groupby(by='wsi')['id'] if i[0] not in valWSI]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eac0a47126779243628accd92a8a63501171c6be"},"cell_type":"code","source":"len(trnWSI),len(valWSI)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"510aeaad58766e7b7b48da0fdbd41c11542f6fd7"},"cell_type":"markdown","source":"\n## get the integer index for all the images with WSIs of the validation set"},{"metadata":{"trusted":true,"_uuid":"5f7d1af5b67d5b54cb08e64d0fb6f2bc3ed27d95"},"cell_type":"code","source":"val_idx=np.hstack([df_WSI.groupby(by='wsi')['id'].indices[WSI] for WSI in valWSI])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6173c856876b240a7420ba94bc7cf019d6ce6c84"},"cell_type":"code","source":"val_idx=np.append(df_notinWSI.index.values,df_WSI.id[val_idx])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee28a818e1909b5a9e4b46d4a62a8bd452720201"},"cell_type":"code","source":"trn_idx=np.hstack([df_WSI.groupby(by='wsi')['id'].indices[WSI] for WSI in trnWSI])\ntrn_idx=df_WSI.id[trn_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50bac254c2e2f2518dd48cd5cf4fa5218374d92a"},"cell_type":"code","source":"val_idx=df_trn.reset_index().set_index('id').loc[val_idx,'index'].values\ntrn_idx=df_trn.reset_index().set_index('id').loc[trn_idx,'index'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f6316009e58ae90a7f5a25a01dedea6575d3def"},"cell_type":"code","source":"np.random.shuffle(val_idx)\nnp.random.shuffle(trn_idx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed36c59281f8377952a7ef185619affc79143e46"},"cell_type":"code","source":"src = (ImageList.from_df(df_trn, path=path, suffix='.tif',folder='train')                \n                .split_by_idxs(trn_idx,val_idx)\n                .label_from_df(label_delim=' '))\nsrc.add_test(test_fnames);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a57139c31c9dbdd6d6c6c7fcad30a23170a26de"},"cell_type":"code","source":"data=ImageDataBunch.create_from_ll(src, ds_tfms=tfms, size=sz,bs=bs)\nstats=data.batch_stats()        \ndata.normalize(stats);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"881873d04d04ba407205e66c9f535dce0938adb2"},"cell_type":"code","source":"def auc_score(y_pred,y_true,tens=True):\n    score=roc_auc_score(y_true[:,1],torch.sigmoid(y_pred)[:,1])\n    if tens:\n        score=tensor(score)\n    else:\n        score=score\n    return score\n\n\n\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, logits=False, reduce=True):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.logits = logits\n        self.reduce = reduce\n    def forward(self, inputs, targets):\n        if self.logits:\n            BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduce=False)\n        else:\n            BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)\n        pt = torch.exp(-BCE_loss)\n        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n\n        if self.reduce:\n            return torch.mean(F_loss)\n        else:\n            return F_loss\n        \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e87abf1fbf9b061cb26fb05afc6932e253d4135c"},"cell_type":"code","source":"learn = create_cnn(\n    data,\n    densenet169,\n    path='.',    \n    metrics=[auc_score], \n    #loss_func=FocalLoss(logits=True,gamma=1),\n    ps=0.5\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0af0326b82e4419c0572ac17d53f381ccb804d6e"},"cell_type":"code","source":"x,y=learn.get_preds()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a15744e5298e8fb67138e6ceaa2cf6e2e75a74f"},"cell_type":"code","source":"auc_score(x,y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8f689bfd03e79e5f8f130032468a85788b7acce","scrolled":true},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fcfa743233605eafacfcbc383da90c7bc5fbd473"},"cell_type":"code","source":"learn.fit_one_cycle(1,1e-2)\nlearn.recorder.plot()\nlearn.recorder.plot_losses()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c87540b8006dffcda2f33bab39add4a704b1740d"},"cell_type":"code","source":"learn.unfreeze()\nlearn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c87540b8006dffcda2f33bab39add4a704b1740d"},"cell_type":"code","source":"learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"237be2254fe3cb29c8e04430963e6883995b277c"},"cell_type":"markdown","source":"### Warm up with frozen weight is done on a subset so we dont have to waste an entire epoch"},{"metadata":{"trusted":true,"_uuid":"b63495c9c8dd4e12d91cda8a2702f6c1a5bdc168"},"cell_type":"code","source":"learn.fit_one_cycle(10,slice(1e-4,1e-3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b63495c9c8dd4e12d91cda8a2702f6c1a5bdc168"},"cell_type":"code","source":"learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b63495c9c8dd4e12d91cda8a2702f6c1a5bdc168"},"cell_type":"code","source":"learn.recorder.plot_losses()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"947809ccf88a5065841714c36c7339567059c55f"},"cell_type":"code","source":"preds,y,losses = learn.get_preds(with_loss=True)\ninterp = ClassificationInterpretation(learn, preds, y.long(), losses)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5c4e009e09fb77b08997ac8e9ee69660c28676be"},"cell_type":"markdown","source":"### Predit the validation data using TTA\nHere for every image we want to predict on, n_augs images are augmented form the original image.\nWe can then compare the predictions on for example the image and the image flipped / roated / slightly different crop/ lighting/stretched etc. \nFor now only the diherdral and rotations are used. THis gives a nice extra percent or two when compared to the auc above after training where not TTA is used. \nI also test if mean or max is better to use on the image and its augments but it can't conclude anything yet."},{"metadata":{"trusted":true,"_uuid":"b6917d2f83141d472e04073b59a9b5e1e6b26c57"},"cell_type":"code","source":"preds,y=learn.get_preds()\npred_score=auc_score(preds,y)\npred_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6917d2f83141d472e04073b59a9b5e1e6b26c57"},"cell_type":"code","source":"preds,y=learn.TTA()\npred_score_tta=auc_score(preds,y)\npred_score_tta","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bdfd3b80d33057d155e7da32391817a7e88ff80f"},"cell_type":"markdown","source":"### Now predict on test set"},{"metadata":{"trusted":true,"_uuid":"6c1890c9509ca13a01b6bba3bba296b5a9b34fa7"},"cell_type":"code","source":"preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a85d720a61b996e3ee308c4a012345ef69fc461","scrolled":true},"cell_type":"code","source":"preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"88a566de210643baa4a44519e67b6e377c9d26d7"},"cell_type":"markdown","source":"### prepare submission\nI now load in the sample submission and put my predictions in the label column and save to a new file."},{"metadata":{"_uuid":"c7f2e327a7943ed6e8d7f8bfc7bd3f9682186b7c"},"cell_type":"markdown","source":"Sometimes its important in which order the ids in the submissions are so to make sure I don't mess up I put them in the same order. My first submission had a 50% score so I somewhere messed up the order oder the matching of id to label.\nsince fname_clean is the id we can just use that as index when adding the correct label in our dataframe. "},{"metadata":{"trusted":true,"_uuid":"7c64426066b5d158024e4bbf05a6a05c10777ce7"},"cell_type":"code","source":"sub=pd.read_csv(f'{path}/sample_submission.csv').set_index('id')\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20c89b9c62d4d98fe4cae6195e80338abfb3a553"},"cell_type":"code","source":"clean_fname=np.vectorize(lambda fname: str(fname).split('/')[-1].split('.')[0])\nfname_cleaned=clean_fname(data.test_ds.items)\nfname_cleaned=fname_cleaned.astype(str)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"25f186dca18d7855cf8b3910a45acd3de9f1f34e"},"cell_type":"markdown","source":"## I add the score to the name of the file so I can later plot the leaderboard score versus my validation score\nIn the fastai course Jeremy mentions that if you have a monotonic relation between validation and LB score the way you set up your validation set matches what the test set consists of."},{"metadata":{"trusted":true,"_uuid":"1e9f92573996df4ff7a5f047e7ffae6ef6e53b73"},"cell_type":"code","source":"sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])\nsub.to_csv(f'submission_{pred_score}.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a934db6adbf52935aab26a9c6400bf0530d3bb9"},"cell_type":"code","source":"sub.loc[fname_cleaned,'label']=to_np(preds_test_tta[:,1])\nsub.to_csv(f'submission_{pred_score_tta}.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15af9ff7c16ef0e5bda33d4595b482c91d21f305"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3584d685b32adf5ca42d8d2df7451e26750bc82"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}