{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## install packages","metadata":{}},{"cell_type":"markdown","source":"Kaggle notebook: https://www.kaggle.com/benayang/fastai-classifier-v2-563bc4","metadata":{}},{"cell_type":"code","source":"#!pip install iterative_stratification -q\n#!pip install \"../input/hpacellsegmentatormaster/HPA-Cell-Segmentation-master\"\n!pip install \"../input/hpapytorchzoozip/pytorch_zoo-master\"","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('../input/iterative-stratification/iterative-stratification-master')\nsys.path.append('../input/hpacellsegmentatormaster/HPA-Cell-Segmentation-master')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.vision import *\nfrom fastai.vision.all import *\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport hpacellseg.cellsegmentator as cellsegmentator\nfrom hpacellseg.utils import label_cell, label_nuclei\nimport cv2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from https://www.kaggle.com/c/hpa-single-cell-image-classification/data\n# not actually needed\n\nspecified_class_names = \"\"\"0. Nucleoplasm\n1. Nuclear membrane\n2. Nucleoli\n3. Nucleoli fibrillar center\n4. Nuclear speckles\n5. Nuclear bodies\n6. Endoplasmic reticulum\n7. Golgi apparatus\n8. Intermediate filaments\n9. Actin filaments \n10. Microtubules\n11. Mitotic spindle\n12. Centrosome\n13. Plasma membrane\n14. Mitochondria\n15. Aggresome\n16. Cytosol\n17. Vesicles and punctate cytosolic patterns\n18. Negative\"\"\"\n\nclass_names = [class_name.split('. ')[1] for class_name in specified_class_names.split('\\n')]\nclass_names","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## training setup","metadata":{}},{"cell_type":"code","source":"path = Path('../input/hpa512x512dataset')\ndf = pd.read_csv(path/'train.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_size = 1\nseed = 42\nstats = ([0.07237246, 0.04476176, 0.07661699], [0.17179589, 0.10284516, 0.14199627])\nitem_tfms = RandomResizedCrop(448, min_scale=0.75, ratio=(1.,1.))\nbatch_tfms = [*aug_transforms(flip_vert=True, max_warp=0), Normalize.from_stats(*stats)]\nbs = 32\nlr = 3e-2\nepochs = 5\ncbs = None\n\n\npath = Path('../input/hpa512x512dataset')\ndf = pd.read_csv(path/'train.csv')\n\nlabels = [str(i) for i in range(19)]\n# one-hot encoding of labels\nfor x in labels: df[x] = df['Label'].apply(lambda r: int(x in r.split('|')))\ndfs = df.sample(frac=sample_size, random_state=seed).reset_index(drop=True)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dfs = df.sample(frac=sample_size, random_state=seed).reset_index(drop=True)\ny = dfs[labels].values\nX = dfs['ID'].values\ndfs['fold'] = np.nan\n\nmskf = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=seed)\nfor i, (_, test_index) in enumerate(mskf.split(X, y)):\n    dfs.iloc[test_index, -1] = i","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs.fold.value_counts().plot.bar();","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs['fold'] = dfs['fold'].astype('int')\ndfs['is_valid'] = False\ndfs['is_valid'][dfs.fold == 0] = True\ndfs.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cheat_id = list(dfs[(dfs[\"0\"]==1) & (dfs[\"3\"]==1)].ID)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_x(r): return path/\"rgb_train\"/f'{r[\"ID\"]}.png'\ndef get_y(r): return list(set(r['Label'].split('|')))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dblock = DataBlock(blocks=(ImageBlock, MultiCategoryBlock(vocab=labels)),\n                    splitter=ColSplitter(col='is_valid'),\n                    #splitter=TrainTestSplitter(test_size=0.2, random_state=seed, stratify=None, shuffle=True), \n                    get_x=get_x,\n                    get_y=get_y,\n                    item_tfms=item_tfms,\n                    batch_tfms=batch_tfms\n                    )\ndls = dblock.dataloaders(dfs, bs=bs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.train.show_batch()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.valid.show_batch()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = dfs[dfs.is_valid==False]\nvalid_df = dfs[dfs.is_valid==True]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def oversample(frame: pd.DataFrame):\n    max_size = frame['Label'].value_counts().max()\n    lst = [frame]\n    for class_index, group in frame.groupby('Label'):\n        lst.append(group.sample(max_size-len(group), replace=True))\n    return pd.concat(lst)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oversampled_train_df = oversample(train_df) # <------- TRAINING DATAFRAME ONLY\n# oversampled_train_df['Label'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oversampled_df = pd.concat((oversampled_train_df, valid_df))\n# oversampled_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = dfs[dfs.is_valid==False]\nvalid_df = dfs[dfs.is_valid==True]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vals,counts=np.unique(oversampled_train_df['Label'], return_counts=True)\nfig = plt.figure(figsize=(50,5))\nplt.bar(vals,counts,width=0.75)\nplt.xticks(rotation = 90)\nplt.title(\"Oversampled\")\nplt.savefig('Oversampled_class_dist.png')\n\nvals,counts=np.unique(train_df['Label'], return_counts=True)\nfig = plt.figure(figsize=(50,5))\nplt.bar(vals,counts,width=0.75)\nplt.xticks(rotation = 90)\nplt.title(\"Original\")\nplt.savefig('Original_class_dist.png')\n","metadata":{"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oversampled_dls = dblock.dataloaders(oversampled_df) # Updated - Pass the dataframe, not the folder","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oversampled_dls.train.show_batch()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# oversampled_dls.valid.show_batch()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## training part - done","metadata":{}},{"cell_type":"code","source":"#train_labels = list(dls.train_ds.items.Label)\n#unique_train_labels, counts = np.unique(train_labels,return_counts=True)\n#class_weights = 1./counts\n#class_weight_dict = dict(zip(unique_train_labels, class_weights))\n#weights = [class_weight_dict[x] for x in train_labels]\n#total_len_oversample = int(dls.train_ds.c*np.max(counts))","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# learn = cnn_learner(oversampled_dls, resnet18, metrics=[accuracy_multi, APScoreMulti()]).to_fp16()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# resnet34 (lr_min=0.025118863582611083, lr_steep=0.03981071710586548)\n#learn.lr_find()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cbs=[SaveModelCallback()]\n# #learn.fit_one_cycle(2, cbs=cbs)\n# learn.fine_tune(epochs, base_lr=2.5e-2, cbs = cbs)\n# learn.export('./resnet18_2.5e-2_oversample (4_26_21).pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### load trained learner for analysis","metadata":{}},{"cell_type":"code","source":"#learn.export(fname='./resnet50_2.5e-2.pkl')\n#learn.save('./resnet18_2.5e-2')\n#learn.export('./resnet18_2.5e-2_oversample (4_24_21).pkl')\nlearn = load_learner('../input/resnet18-oversample/resnet18_2.5e-2_oversample(4_27_21).pkl')\n#learn = load_learner('./resnet18_2.5e-2_rotate.pkl')\n#learn.load('resnet18_2.5e-2_oversample(4_27_21)')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### get predictions - takes some time, so saved","metadata":{}},{"cell_type":"code","source":"preds1 = learn.get_preds(dl=oversampled_dls.valid, with_input=False, with_loss=False, with_decoded=True, act=None)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### load predictions","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import multilabel_confusion_matrix, precision_score,f1_score\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#np.save(arr=pd.DataFrame(preds), file='./resnet18_2.5e-2_oversample_predictions.npy')\npreds = np.load('../input/resnet18oversamplepredictions/resnet18_2.5e-2_oversample_predictions.npy', allow_pickle=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_raw = preds[0][0]\ny_true = preds[1][0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fast heatmap of labels\n# k = 3\n# fig,ax = plt.subplots(figsize=(k*2,k*3))\n# sns.heatmap(pred_raw.numpy())\n# plt.savefig(\"raw_pred_scores.png\",dpi=300)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### plotting various metrics","metadata":{}},{"cell_type":"code","source":"# multi-label accuracy\naccuracy_multi(pred_raw, y_true, thresh=0.9,sigmoid=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pscore = []\nf1 = []\nfor thresh in np.arange(0.05,1,0.05):\n    y_pred = (pred_raw.numpy() > thresh).astype(int)\n    f1.append(f1_score(y_true,y_pred,average=\"weighted\"))\n    pscore.append(precision_score(y_true,y_pred,average=\"weighted\"))\n\n\nplt.plot(np.arange(0.05,1,0.05),pscore,label=\"precision\")\nplt.plot(np.arange(0.05,1,0.05),f1,label=\"f1\")\nplt.xlabel(\"threshold\")\nplt.legend()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prediction heatmaps for various thresholds\n# for i in range(5,10):\n#     thresh = 0.1*i\n    \n#     y_pred = (pred_raw.numpy() > thresh).astype(int)\n#     fig,ax = plt.subplots(figsize=(k*2,k*3))\n#     sns.heatmap(y_pred,cbar=False)\n#     plt.savefig(f\"pres_thresh_{i}.png\",dpi=300)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\nxs = np.linspace(0.5,1,100)\ncl_reports = []\nfor i in xs:\n    cl_rep = classification_report(y_true=preds[0][1].numpy(), y_pred=(preds[0][0]>i).numpy().astype(int), \n                                  output_dict=True, target_names=class_names)\n    cl_reports.append(cl_rep)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accs = [accuracy_multi(preds[0][0], preds[0][1], thresh=i, sigmoid=False) for i in xs]\ndropout = [sum(np.sum((preds[0][0]>i).numpy().astype(int), axis=1)==0)/len(preds[0][0]) for i in xs]\n\nfig = plt.figure(figsize=(5,4), facecolor='white')\nplt.plot(xs, [x['weighted avg']['f1-score'] for x in cl_reports], label='F1-Score')\nplt.plot(xs, [x['weighted avg']['recall'] for x in cl_reports], label='Recall')\nplt.plot(xs, [x['weighted avg']['precision'] for x in cl_reports], label='Precision')\nplt.plot(xs, accs, label='Multi-label Accuracy')\nplt.plot(xs, dropout, label='Dropout Rate')\nplt.xlabel('Classification Threshold')\n#plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))\nplt.legend(loc='center', prop={\"size\":10}, bbox_to_anchor=(0.5,1.15), ncol=3)\nplt.tight_layout()\nplt.show()\n#fig.savefig('classification_thresh_metrics.png', dpi=300)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### confusion matrix","metadata":{}},{"cell_type":"code","source":"# confusion matrix setup\ny_pred = (pred_raw.numpy() > 0.8).astype(int)\nconfusion_matrix = multilabel_confusion_matrix(y_true, y_pred)","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(rc={'figure.facecolor':'white'})\n\ndef print_confusion_matrix(confusion_matrix, axes, class_label, class_names, fontsize=14):\n\n    df_cm = pd.DataFrame(\n        confusion_matrix, index=class_names, columns=class_names,\n    )\n\n    try:\n        heatmap = sns.heatmap(df_cm, annot=True, vmin=0,vmax=1,fmt=\".2f\", cbar=False, ax=axes,cmap=\"Blues_r\")\n    except ValueError:\n        raise ValueError(\"Confusion matrix values must be integers.\")\n    heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=fontsize)\n    heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=45, ha='right', fontsize=fontsize)\n    axes.set_ylabel('True label')\n    axes.set_xlabel('Predicted label')\n    axes.set_title(\"Confusion Matrix for class \" + class_label)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(4, 5, figsize=(20, 10))\n    \nfor axes, cfs_matrix, label in zip(ax.flatten(), confusion_matrix, labels):\n    # changed this bit here to plot fractions - otherwise the high number of true negatives would\n    # obscure some of the patterns\n    norm_cfs_matrix = cfs_matrix/cfs_matrix.sum(axis=1, keepdims=True)\n    print_confusion_matrix(norm_cfs_matrix, axes, label, [\"N\", \"Y\"])\n\nfig.tight_layout()\nplt.savefig('./confusion_matrix_probs_oversample.png',dpi=300)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Look at top losses","metadata":{}},{"cell_type":"code","source":"losses = preds[0][3].numpy()\nsorted_loss_idx = losses.argsort()[::-1]\ntop_losses = valid_df.Label.iloc[sorted_loss_idx[:1000]] # get true classes with top losses\nlosses_df = pd.DataFrame({'cl':np.array(top_losses), 'loss':losses[sorted_loss_idx[:1000]]})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"means = {}\nfor cl in np.unique(losses_df['cl']):\n    tmp = losses_df.loc[losses_df.cl==cl]\n    means[cl] = tmp['loss'].mean()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams.update({'font.size': 15})\n\nvals,counts=np.unique(top_losses, return_counts=True)\nidx = counts.argsort()[::-1]\nvals_sorted = vals[idx]\ncounts_sorted = counts[idx]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.rcParams.update({'font.size': 20})\n\nfig,ax1 = plt.subplots(figsize=(7,7))\n\nax1.barh(vals_sorted[:15],counts_sorted[:15])\nax1.invert_yaxis()\nax1.set_ylabel(\"Classes\")\nax1.set_xlabel(\"Number of Images\")\n\n#ax2 = plt.twinx()\n#ax2.plot([means[x] for x in vals_sorted[:15]], vals_sorted[:15], color='k', label='Sine')\n#ax2.invert_yaxis()\n#ax2.set_ylabel('Line plot')\n#ax2.plot([means[x] for x in vals_sorted[:15]],vals_sorted[:15])\n#ax2.invert_yaxis()\n\n#plt.xticks(rotation = 90)\n#plt.xticks(x_pos,vals_sorted,rotation = 90)\nplt.title(\"Top 15 Mislabeled Classes\")\nfig.set_facecolor('white')\nplt.tight_layout()\n#plt.show()\nplt.savefig(\"Top_15_Losses.png\", dpi=300)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_thresh = 0.6\n\n# Most incorrect or top losses\nnum_cl = 15\ntop_cl = vals_sorted[:num_cl]\ncl_thresh = 0.6\n\nplt.subplots(figsize=(10,7), facecolor='white')\nfor i in range(num_cl): \n    curr_idx = valid_df.loc[valid_df.Label==top_cl[i]].index[1] # get index of first match with class\n    tag = valid_df.ID.loc[curr_idx]\n    img = plt.imread(path/\"rgb_train\"/f'{tag}.png')\n    \n    img,true_cl,_,img_preds = learn.predict(path/\"rgb_train\"/f'{tag}.png', with_input=True)\n    pred_cl = (img_preds>cl_thresh).numpy().astype(int)\n    pred_cl = np.array2string(np.asarray(np.where(pred_cl>0)))\n    true_cl = valid_df.loc[curr_idx].Label\n    #true_cl = top_cl[i]\n    \n    plt.subplot(3,5,i+1)\n    plt.imshow(img.permute(2,1,0))\n    #plt.imshow(img)\n    plt.xticks([])\n    plt.yticks([])\n    plt.title(f'{true_cl} / {pred_cl}')\n    \nplt.suptitle(f'True Classes / Predicted Classes (Thresh={cl_thresh})', fontsize=20)\nplt.tight_layout()\n#plt.show()\nplt.savefig('./True_and_predicted_imgs_top1000.png', dpi=300)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## segmentator","metadata":{}},{"cell_type":"code","source":"class Hook():\n    def __init__(self, m):\n        self.hook = m.register_forward_hook(self.hook_func)   \n    def hook_func(self, m, i, o): self.stored = o.detach().clone()\n    def __enter__(self, *args): return self\n    def __exit__(self, *args): self.hook.remove()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HookBwd():\n    def __init__(self, m):\n        self.hook = m.register_backward_hook(self.hook_func)   \n    def hook_func(self, m, gi, go): self.stored = go[0].detach().clone()\n    def __enter__(self, *args): return self\n    def __exit__(self, *args): self.hook.remove()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs.drop(dfs.columns[0],axis=1,inplace=True)\ndfs.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_img(tag,show_img=False):\n    \"\"\"imagefile getter for a given tag. Can be an integer (df row index)\n        or string (the actual image ID)\"\"\"\n    if type(tag) == int:\n        sample = dfs.iloc[tag]\n        label = sample.Label\n        ID = sample.ID\n        header = f'ID: {ID}, Label: {label}'\n        img = PILImage.create(get_x(sample))\n        \n    elif type(tag) == str:\n        sample = dfs[dfs.ID == tag].reset_index(drop=True)\n        label = sample.Label.loc[0]\n        ID = sample.ID.loc[0]\n        header = f'ID: {ID}, Label: {label}'\n        img = PILImage.create(get_x(sample.loc[0]))\n        \n    if show_img: \n        print(header)\n        img.show(figsize=(5,5))\n    label = np.array([int(i) for i in label.split(\"|\")])\n    return img,ID,label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_grad_cam_cpu(x, cls,return_cam=True):\n    \"\"\"plot and return the grad-cam heatmap\"\"\"\n    with HookBwd(learn.model[0]) as hookg:\n        with Hook(learn.model[0]) as hook:\n            output = learn.model.eval()(x.cpu())\n            act = hook.stored\n        output[0,cls].backward()\n        grad = hookg.stored\n    \n    w = grad[0].mean(dim=[1,2], keepdim=True)\n    cam_map = (w * act[0]).sum(0)\n\n\n    _,ax = plt.subplots()\n    x_dec = x[0,:,:,:].permute(1, 2, 0).cpu()\n    x_dec.show(ctx=ax,alpha=0.5)\n    ax.imshow(cam_map.detach().cpu(), alpha=0.6, extent=(0,448,448,0),\n                  interpolation='bilinear', cmap='magma');\n    \n    if return_cam:\n        return cam_map.detach().cpu()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# old code. not used.\ndef visualize_grad_cam(x, cls,show_img=False):\n    with HookBwd(learn.model[0]) as hookg:\n        with Hook(learn.model[0]) as hook:\n            output = learn.model.cuda().eval()(x.cuda())\n            act = hook.stored\n        output[0,cls].backward()\n        grad = hookg.stored\n    \n    w = grad[0].mean(dim=[1,2], keepdim=True)\n    cam_map = (w * act[0]).sum(0)\n\n    plt_size = np.ceil(len(cls)/2)\n    x_img = TensorImage(dls.train.decode((x,))[0][0])\n\n    if plt_size > 1:\n        fig = plt.figure()\n        for c in np.arange(len(cls)):\n            ax = fig.add_subplot(1,1,c)\n            ax.imshow(x_img.numpy().transpose(1,2,0))\n            ax.imshow(\n                cam_map.detach().cpu(), alpha=0.6, extent=(0, 448, 448,0),\n                interpolation='bicubic', cmap='magma'\n            )\n            ax.set_title(class_names[c])\n    else:\n        _,ax = plt.subplots()\n        ax.imshow(x_img.numpy().transpose(1,2,0))\n        ax.imshow(\n            cam_map.detach().cpu(), alpha=0.6, extent=(0, 448, 448,0),\n            interpolation='bicubic', cmap='magma'\n        )\n        ax.set_title(str(cls[0])+': '+class_names[cls[0]])\n\n    plt.tight_layout()\n    if show_img:\n        plt.show()\n        \n    #_,ax = plt.subplots(len(cls))\n    #x_img = TensorImage(dls.train.decode((x,))[0][0])\n    #x_img.show(ax=ax)\n    #ax.imshow(x_img.numpy().transpose(1,2,0))\n    #ax.imshow(\n    #    cam_map.detach().cpu(), alpha=0.6, extent=(0, 448, 448,0),\n    #    interpolation='bicubic', cmap='magma'\n    #)\n    #ax.set_axis_off()\n    \n    #ax.imshow(cam_map.detach().cpu(), alpha=0.6, extent=(0,448,448,0),\n    #              interpolation='bilinear', cmap='magma');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Single cell segmentation","metadata":{}},{"cell_type":"code","source":"\nNUC_MODEL = '../input/hpacellsegmentatormodelweights/dpn_unet_nuclei_v1.pth'\nCELL_MODEL = '../input/hpacellsegmentatormodelweights/dpn_unet_cell_3ch_v1.pth'\n\nsegmentator = cellsegmentator.CellSegmentator(\n    NUC_MODEL,\n    CELL_MODEL,\n    scale_factor=0.25,\n    device='cuda',\n    padding=True,\n    multi_channel_model=True\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# helper function to get files *** from the large version - 512x512 ones don't segment well ***\ndef build_image_names(image_id: str,dataset: str) -> list:\n    root_dir = \"/kaggle/input/hpa-single-cell-image-classification\"\n    # mt is the microtubules\n    mt = f'{root_dir}/{dataset}/{image_id}_red.png'\n    # er is the endoplasmic reticulum\n    er = f'{root_dir}/{dataset}/{image_id}_yellow.png'\n    # nu is the nuclei\n    nu = f'{root_dir}/{dataset}/{image_id}_blue.png'\n    \n    return [[mt], [er], [nu]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_mask(img_ID):\n    \"\"\"get 512x512 masks for a given image ID\n        this uses the non-resized images, then downsizes the mask to 512x512.\"\"\"\n    images = build_image_names(image_id=img_ID,dataset=\"train\")    \n    # For nuclei\n    nuc_segmentations = segmentator.pred_nuclei(images[2])\n\n    # For full cells\n    cell_segmentations = segmentator.pred_cells(images)\n    nuclei_mask, cell_mask = label_cell(nuc_segmentations[0], cell_segmentations[0])\n    mask_small = cv2.resize(cell_mask,dsize=(512,512))\n    \n    return mask_small","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## merging everything","metadata":{}},{"cell_type":"code","source":"import numpy.ma as ma","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# takes the ID/df row number, and prediction threshold.\nvalid_ids = list(dfs[dfs.is_valid].ID)\n\n\ntag = \"4f47bb78-bbb7-11e8-b2ba-ac1f6b6435d0\"\n#tag = '08705684-bbc5-11e8-b2bc-ac1f6b6435d0'\npred_thresh = 0.6","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get the image \nimg,ID,true_label = get_img(tag,True)\n\n#sample_pred, _ = learn.get_preds(dl=dls.test_dl([img]))\ntrue_cl,_,sample_pred = learn.predict(path/\"rgb_train\"/f'{tag}.png', with_input=False)\npred_label = np.where(sample_pred> pred_thresh)[0]\nprint(pred_label)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get learner-friendly image format and mask\nx, = first(oversampled_dls.test_dl([img]))\nmask_512 = get_mask(ID)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loop over all the predicted classes here.\n\nint_dict = {\"cell\":range(1,np.max(mask_512)+1)}\n# for each class:\nfor p_cls in pred_label:\n    # get acivation map and resize\n    class_cam = visualize_grad_cam_cpu(x, p_cls)\n    class_cam_512 = cv2.resize(class_cam.numpy(),dsize=(512,512))\n    class_cam_intensities = []\n    # loop through cells\n    for i in range(1,np.max(mask_512)+1):\n        # get the region of overlap between cell location and CAM, and take the mean intensity\n        intensity = np.mean(ma.masked_where(mask_512 != i, class_cam_512).compressed())\n        class_cam_intensities.append(abs(intensity))\n    int_dict[str(p_cls)] = class_cam_intensities","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#make dataframe. Each row is a cell, and avg intensities for each class.\nresults = pd.DataFrame(data=int_dict)\nlabels = np.array(list(results)[1:])\nresults.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate cell-level labels - if a cell has intensity > cam_thresh for a class, it'\ncam_thresh = 0.015\nall_cell_labels = []\nfor r in results.itertuples():\n    use_inds = np.where(np.array(r[2:])>cam_thresh)[0]\n    cell_label = [int(labels[i]) for i in use_inds]\n    all_cell_labels.append(cell_label)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display final results\nresults[\"pred_label\"] = all_cell_labels\nresults","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### for plotting the report figures","metadata":{}},{"cell_type":"code","source":"# for p_cls in pred_label:\n\np_cls = pred_label[0]\nclass_cam = visualize_grad_cam_cpu(x, p_cls)\nclass_cam_512_0 = cv2.resize(class_cam.numpy(),dsize=(512,512))\n\np_cls = pred_label[1]\nclass_cam = visualize_grad_cam_cpu(x, p_cls)\nclass_cam_512_1 = cv2.resize(class_cam.numpy(),dsize=(512,512))\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clipped_mask_0 = ma.masked_where(mask_512 == 0, class_cam_512_0)\nclipped_mask_1 = ma.masked_where(mask_512 == 0, class_cam_512_1)\nk = 10\nfsize=16\nfig,ax = plt.subplots(2,2,figsize=(k,k))\nax[0,0].imshow(mask_512,cmap=\"viridis\")\n\nax[0,0].set_title(\"Cell Segmentation\",fontsize=fsize)\nax[0,1].imshow(class_cam_512_0,cmap=\"inferno\")\nax[0,1].set_title(f\"Grad-CAM, class {pred_label[0]}\",fontsize=fsize)\n\nax[1,0].imshow(clipped_mask_0,cmap=\"inferno\")\nax[1,0].set_title(f\"Segmented Grad-CAM, class {pred_label[0]}\",fontsize=fsize)\n\nax[1,1].imshow(clipped_mask_1,cmap=\"inferno\")\nax[1,1].set_title(f\"Segmented Grad-CAM, class {pred_label[1]}\",fontsize=fsize)\n\nfor iax in ax:\n    for jax in iax:\n        jax.grid(False)\n        jax.set_xticks([])\n        jax.set_yticks([])\n        \n#plt.savefig(\"grad_cam_seg_example.png\",dpi=300)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## a plot for training stats for un-augmented resnet model","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport pandas as pd\n\ntrain_df = pd.read_csv(\"../input/resnetnoaugtrainstats/resnet_no_aug_training_data.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,axs = plt.subplots(1,2,figsize=(15,5))\naxs[0].plot(train_df.epoch,train_df.train_loss,label=\"training loss\")\naxs[0].plot(train_df.epoch,train_df.valid_loss,label=\"validation loss\")\naxs[0].legend()\n\naxs[1].plot(train_df.epoch,train_df.accuracy_multi,label=\"accuracy\")\naxs[1].plot(train_df.epoch,train_df.average_precision_score,label=\"avg precision\")\naxs[1].legend()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## everything below is old and not used","metadata":{}},{"cell_type":"code","source":"def Components(Image, shade = 230, cutoff = 5000, distance = 20):\n    V = np.array((Image[:,:,2]> shade), dtype = np.int)\n    V[:,0] = 0\n    V[:, -1] = 0\n    W = V[:,1:]- V[:,:-1]\n    v = np.where(W==1)\n    w = np.where(W == -1)\n    \n    n = len(v[0])\n    In = []\n    Out = []\n    Comp = [i for i in range(n)]\n    row = 0\n    for i in range(n):\n        new_row = 0 + v[0][i]\n       \n        if new_row == row:\n            In.append(i)\n        elif new_row == row+1:\n            Out.append(i)\n        \n        else:\n            for p in In:\n                for q in Out:\n                    a = v[1][p]\n                    b = w[1][p]\n                    c = v[1][q]\n                    d = w[1][q]\n             \n                    if ((a <= c and c <=b) or (c <= a and a <= d)) and (b-a > distance and d-c > distance):\n                    \n                        if Comp[p] !=  Comp[q]:\n                            # compute root of q:\n                            root1 = p+0\n                            root2 = Comp[p]\n                            while root2 < root1:\n                                root2, root1 = Comp[root2], root2\n                            root3 = q+0\n                            root4 = Comp[q]\n                            while root4<root3:\n                                root4, root3 = Comp[root4], root4\n                            if root1 < root3:\n                                Comp[root3] = root1\n                                Comp[q] = root1\n                                Comp[p] = root1\n                            else:\n                                Comp[root1] = root3\n                                Comp[p] = root3\n                                Comp[q] = root3\n                                \n                    \n            \n            if new_row == row+2:\n                In, Out = Out, [i]\n                row = row + 1\n            else:\n                In, Out = [i], []\n                row = 0 + new_row\n    \n    \n    \n    for i in range(n):\n        a = 0 + i\n        b = Comp[i]\n        while b < a:\n            b, a  = Comp[b], b\n        Comp[i] = b\n    \n    L1 = list(set(Comp)) \n    \n    D1 = {i:0 for i in L1}\n    Total_Weight = 0\n    for i in range(n):\n        a = v[1][i]\n        b = w[1][i]\n        Total_Weight+=(b-a+1)\n        D1[Comp[i]]+=(b-a+1)\n    \n    L2 = [(-b,a) for a,b in D1.items() if b > cutoff]\n    \n    L2.sort()\n    L2 = L2[:100]\n    L2 = [a for b,a in L2]\n    \n    D2 = {}\n    \n    for i in range(len(L2)):\n        D2[L2[i]] = i\n    \n    t = len(L2)\n    x_max = [0 for _ in range(t)]\n    x_min = [V.shape[1] for _ in range(t)]\n    y_max = [0 for _ in range(t)]\n    y_min = [V.shape[0] for _ in range(t)]\n    \n    Segmented_Image = np.zeros((Image.shape[0],Image.shape[1]), dtype = np.uint8)\n    for i in range(n):\n        if Comp[i] in L2:\n            value = t - D2[Comp[i]] \n            row = v[0][i]\n            a = v[1][i]\n            b = w[1][i]\n            Segmented_Image[row,a:b] = value\n         \n\n    return Segmented_Image      ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_names","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_image_names(image_id: str) -> list:\n    path = '../input/hpa512x512dataset/train/'\n    # mt is the mitchondria\n    mt = f'{path}{image_id}_red.png'\n    \n    # er is the endoplasmic reticulum\n    er = f'{path}{image_id}_yellow.png'\n    \n    # nu is the nuclei\n    nu = f'{path}{image_id}_blue.png'\n    \n    return [[mt], [er], [nu]]\ndef Factory_Segmentation(file_id):\n    images = build_image_names(file_id)\n    cell = segmentator.pred_cells(images)[0]\n    nuclei = segmentator.pred_nuclei(images[2])[0]\n    fine_grained_segmentation = label_cell(nuclei,cell)[1]\n    return fine_grained_segmentation\ndef Coarse_Segmentation(file_id):\n    images = build_image_names(file_id)\n    cell = segmentator.pred_cells(images)[0]\n    coarse_segmentation = Components(cell)\n    return coarse_segmentation\ndef color_image(file_id):\n    a, b, c = build_image_names(file_id)\n    image0 = cv2.imread(a[0])\n    image1 = cv2.imread(b[0])\n    image2 = cv2.imread(c[0])\n    image = np.zeros_like(image0)\n    image[:,:,0] = image0[:,:,0]\n    image[:,:,1] = image1[:,:,0]\n    image[:,:,2] = image2[:,:,0]\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_0 = df['ID'][0]\ncoarse_segmentation_0 = Coarse_Segmentation(file_0)\nprecise_segmentation_0 = Factory_Segmentation(file_0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"build_image_names(file_0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_,(ax1,ax2) = plt.subplots(1,2)\nax1.imshow(precise_segmentation_0)\nax2.imshow(color_image(file_0))","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}