{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!mkdir -p /root/.cache/torch/hub/checkpoints\n!cp -r ../input/landmark-additional-packages/rwightman_gen-efficientnet-pytorch_master/rwightman_gen-efficientnet-pytorch_master /root/.cache/torch/hub\n!cp ../input/landmark-additional-packages/tf_efficientnet_b3_aa-84b4657e.pth /root/.cache/torch/hub/checkpoints/\n!cp ../input/landmark-additional-packages/tf_efficientnet_b5_ra-9a3e5369.pth /root/.cache/torch/hub/checkpoints/\n!cp ../input/landmark-additional-packages/se_resnext50_32x4d-a260b3a4.pth /root/.cache/torch/hub/checkpoints/\n!cp ../input/landmark-additional-packages/resnet50d_ra2-464e36ba.pth /root/.cache/torch/hub/checkpoints/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q ../input/landmark-additional-packages/timm-0.3.4-py3-none-any.whl\n!pip install -q ../input/landmark-additional-packages/geffnet-1.0.0-py3-none-any.whl\n!pip install -q ../input/landmark-additional-packages/EfficientNet-PyTorch/EfficientNet-PyTorch-master\n!pip install -q ../input/landmark-additional-packages/pycocotools-2.0.2/dist/pycocotools-2.0.2.tar\n!pip install -q ../input/landmark-additional-packages/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install \"/kaggle/input/hpamisc/pytorch_zoo-master\"\n!pip install \"/kaggle/input/hpamisc/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl\"\n!pip install \"/kaggle/input/hpamisc/faiss_gpu-1.7.0-cp37-cp37m-manylinux2014_x86_64.whl\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp ../input/segment-cam-draft-of-sub-of-jakiro-model-zhizi/submission.csv .","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('../input/hpa-code-for-cam-cell-level/hpa_singlecell-double_level_valid_all/')\n\nfrom torch import nn\nimport torch\nimport torch.nn.functional as F\nimport torchvision\nimport timm\nfrom torch.nn.parameter import Parameter\nimport albumentations as A\n\nfrom utils import parse_args, prepare_for_result\nfrom torch.utils.data import DataLoader, Dataset\nfrom losses import get_loss, get_class_balanced_weighted\nfrom dataloaders import get_dataloader\nfrom utils import load_matched_state\nfrom configs import Config\nfrom models import get_model\nfrom dataloaders.transform_loader import get_tfms\nfrom skimage.io import imsave\n\ntensor_tfms = torchvision.transforms.Compose([\n            torchvision.transforms.ToTensor(),\n            torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406, 0.406], std=[0.229, 0.224, 0.225, 0.225]),\n        ])\n\ntta_tfms = A.Compose([\n    A.Resize(always_apply=False, p=1, height=256, width=256, interpolation=1),\n    A.HorizontalFlip(always_apply=False, p=0.5),\n    A.ShiftScaleRotate(always_apply=False, p=0.7, shift_limit_x=(-0.06, 0.06), shift_limit_y=(-0.06, 0.06), scale_limit=(-0.3, 0.3), rotate_limit=(-22.5, 22.5), interpolation=1, border_mode=2, value=None, mask_value=None),\n    A.RandomBrightnessContrast(always_apply=False, p=0.5, brightness_limit=(-0.2, 0.2), contrast_limit=(-0.2, 0.2), brightness_by_max=True),\n])\n\n\nimport base64\nimport zlib\nfrom pycocotools import _mask as coco_mask\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport cv2\nimport tqdm\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def binary_mask_to_ascii(mask, mask_val=1):\n    \"\"\"Converts a binary mask into OID challenge encoding ascii text.\"\"\"\n    mask = np.where(mask==mask_val, 1, 0).astype(np.bool)\n    \n    # check input mask --\n    if mask.dtype != np.bool:\n        raise ValueError(f\"encode_binary_mask expects a binary mask, received dtype == {mask.dtype}\")\n\n    mask = np.squeeze(mask)\n    if len(mask.shape) != 2:\n        raise ValueError(f\"encode_binary_mask expects a 2d mask, received shape == {mask.shape}\")\n\n    # convert input mask to expected COCO API input --\n    mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n    mask_to_encode = mask_to_encode.astype(np.uint8)\n    mask_to_encode = np.asfortranarray(mask_to_encode)\n\n    # RLE encode mask --\n    encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n    # compress and base64 encoding --\n    binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n    base64_str = base64.b64encode(binary_str)\n    return base64_str.decode()\n\ndef process(x):\n    iid, msk, img, sz = x\n    img = cv2.resize(img, (2048, 2048))\n    enc_msk = cv2.resize(msk, (sz, sz))\n    cell_mask = msk\n    subs = {}\n    results = []\n    for i in range(1, cell_mask.max() + 1):\n        enc = binary_mask_to_ascii(enc_msk, i)\n        sub = cv2.resize((cell_mask == i).astype(np.float), (2048, 2048), cv2.INTER_LINEAR)\n        xr, yr = np.where(sub == 1)\n        xmin, xmax, ymin, ymax = xr.min(), xr.max(), yr.min(), yr.max()\n        subs[i] = (img * np.repeat((sub == 1).astype(np.int)[:, :, np.newaxis], 4, 2))[xmin:xmax, ymin: ymax]\n#         imsave(f'./seg_png_fix_test/{iid}_{i}.png', (255 * subs[i]).astype(np.uint8))\n        results.append(((255 * subs[i]).astype(np.uint8), enc, sz, sz))\n    return results\n\ndef squarify(M,val):\n    (a,b,c)=M.shape\n    if a>b:\n        padding=((0,0),((a-b)//2,a-b-(a-b)//2),(0, 0))\n    else:\n        padding=(((b-a)//2,b-a-(b-a)//2),(0,0),(0, 0))\n    return np.pad(M,padding,mode='constant',constant_values=val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading models\n* b3\n* b5\n* r50d\n* r200d\n* se50","metadata":{}},{"cell_type":"code","source":"ckpt = {\n    0: 13, 1: 12, 2: 12, 3: 11, 4: 14\n}\n\nmodels = []\nfor i in range(5):\n    cfg = Config.load_json('../input/hpa-single-cell-b3-philandrare-5f/5f_double_sin_exp5_rare.yaml/config.json')\n    model = get_model(cfg).cuda()\n    load_matched_state(model, torch.load(\n        f'../input/hpa-single-cell-b3-philandrare-5f/5f_double_sin_exp5_rare.yaml/f{i}_epoch-{ckpt[i]}.pth'))\n    _ = model.eval()\n    models.append(model)\n    break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ckpt = {\n#     0: 18, 1: 14, 2: 14, 3: 15, 4: 15\n# }\n\n# # models = []\n# for i in range(5):\n#     cfg = Config.load_json('../input/hpa-b5-final-model/b5_final_hpa_0504/config.json')\n#     model = get_model(cfg).cuda()\n#     load_matched_state(model, torch.load(\n#         f'../input/hpa-b5-final-model/b5_final_hpa_0504/checkpoints/f{i}_epoch-{ckpt[i]}.pth'))\n#     _ = model.eval()\n#     models.append(model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(models)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## If we read from a csv","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('submission.csv')\n\nimgs = []\nfor i, x in df.iterrows():\n    label = x.PredictionString.split(' ')[0::3]\n    prob = x.PredictionString.split(' ')[1::3]\n    encodes = x.PredictionString.split(' ')[2::3]\n    for idx, enc in enumerate(list(set(encodes))):\n        imgs.append({\n            'image_id': x.ID,\n            'cell_id': idx+1,\n            'enc': enc,\n            'fname': f'{x.ID}_{idx+1}',\n        })\n\ntm = pd.DataFrame(imgs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probs = []\nfor i, x in df.iterrows():\n    label = x.PredictionString.split(' ')[0::3]\n    prob = x.PredictionString.split(' ')[1::3]\n    encodes = x.PredictionString.split(' ')[2::3]\n    for idx, enc in enumerate(encodes):\n        probs.append({\n            'enc': enc,\n            'predict': int(label[idx]),\n            'prob': float(prob[idx])\n        })\n\nprob = pd.DataFrame(probs)\ntm_pred = prob.groupby(['enc', 'predict']).mean().unstack()['prob']\ntm_pred.columns.name = ''\nteam = tm[['enc', 'fname']].merge(tm_pred.reset_index(), on='enc', how='inner').drop('enc', 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/hpa-single-cell-image-classification/sample_submission.csv', index_col=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"team_pred = team.set_index('fname')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SliceInferenceDataset(torch.utils.data.Dataset):\n    def __init__(self, df, tta=16, cfg=None, tfms=None):\n        self.df = df\n        self.iids = self.df.image_id.unique()\n        self.tta = tta\n        \n    def __len__(self):\n        return len(self.iids)\n\n    def __getitem__(self, idx):\n        iid = self.iids[idx]\n        mt = f'../input/hpa-cams/{iid}_red.png'\n        er = f'../input/hpa-cams/{iid}_yellow.png'\n        nu = f'../input/hpa-cams/{iid}_blue.png'\n        pr = f'../input/hpa-cams/{iid}_green.png'\n        r = cv2.imread(mt, 0).astype(np.float) / 255.0\n        g = cv2.imread(pr, 0).astype(np.float) / 255.0\n        b = cv2.imread(nu, 0).astype(np.float) / 255.0\n        a = cv2.imread(er, 0).astype(np.float) / 255.0\n        sz = r.shape[0]\n        img = np.stack([r, g, b, a], -1)\n        img2 = np.stack([cv2.resize(r, (512, 512)), cv2.resize(g, (512, 512)), cv2.resize(b, (512, 512)), cv2.resize(a, (512, 512))], -1)\n        sli = []\n        for i, x in self.df[self.df.image_id == iid].iterrows():\n            bd = base64.b64decode(x.enc)\n            zd = zlib.decompress(bd)\n            encoded = [{'counts': zd, 'size': (sz, sz)}]\n            ded = coco_mask.decode(encoded)[:, :, 0]\n\n            xr, yr = np.where(ded == 1)\n            sub = img[xr.min(): xr.max(), yr.min(): yr.max()]\n            crop_sub_mask = ded[xr.min(): xr.max(), yr.min(): yr.max()]\n            crop_sub_mask = np.repeat(crop_sub_mask[:, :, np.newaxis], 4, axis=2)\n            r = sub * crop_sub_mask\n            sli.append((cv2.resize(squarify(r, 0), (256, 256)).astype(np.float32), x.fname))\n        BS, tta=len(sli) + 1, self.tta\n        ipts = []\n        raw_ipt = [e[0] for e in sli]\n        for tt in range(tta):\n            ipts.append(torch.stack([tensor_tfms(tta_tfms(image=x)['image']) for x in raw_ipt]).float())\n        return ipts, BS, len(sli), tta, iid, [x[1] for x in sli], [x[0] for x in sli], img2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sid = SliceInferenceDataset(tm, tta=8)\ndl = torch.utils.data.DataLoader(sid, batch_size=1, num_workers=2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pdfs = []\nwhole_dfs = []\nfor ipts, BS, lsli, tta, iid, fnames_raw, image_raw, full_img in tqdm.tqdm(dl):\n    BS, tta, iid, fnames, lsli = BS.item(), tta.item(), iid[0], [e[0] for e in fnames_raw], lsli.item()\n    predicted_ps = []\n    exp_ps = []\n    for i in range(0, lsli, BS):\n#         ipt = torch.stack([tensor_tfms(cv2.resize(squarify(s[0], 0), (256, 256))) for s in ress[i: BS+i]]).cuda()\n        ipt = ipts[0][0].cuda()\n    break\n#         with torch.no_grad():\n#             res = []\n#             exp = []\n#             for tt in range(tta):\n#                 ipt = ipts[tt][0].cuda()\n#                 for model in models:\n#                     with torch.cuda.amp.autocast():\n#                         ifr = model(ipt, len(ipt))\n#                     res.append(ifr[0].float())\n#                     exp.append(ifr[1].float())\n#         predict_p = [torch.sigmoid(r.cpu()) for r in res]\n#         exp_p = [torch.sigmoid(r.cpu()) for r in exp]\n#         predict_p = np.stack(predict_p).mean(0)\n#         exp_p = np.stack(exp_p).mean(0)\n#         predicted_ps.append(predict_p)\n#         exp_ps.append(exp_p)\n#     p = np.concatenate(predicted_ps)\n#     image_df = pd.DataFrame(p, index=fnames)\n#     whole_df = pd.DataFrame(np.concatenate(exp_ps).mean(0).reshape(1, 19), index=[iid])\n#     whole_dfs.append(whole_df)\n#     pdfs.append(image_df) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ipt = tensor_tfms(full_img[0].numpy())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ipt = ipt.reshape(1, ipt.shape[0], ipt.shape[1], ipt.shape[2])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ipt.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install grad-cam\n!pip install ttach","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pytorch_grad_cam import GradCAM, ScoreCAM, GradCAMPlusPlus, AblationCAM, XGradCAM, EigenCAM\nfrom pytorch_grad_cam.utils.image import show_cam_on_image\nfrom torchvision.models import resnet50","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_null = models[0].model\n\nmodel_null.global_pool = models[0].pool\nmodel_null.classifier = models[0].last_linear2\n\nmodel_null.train()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cam = GradCAM(model=model_null, target_layer=model_null.conv_head, use_cuda=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cls_cams = {}\nfor cat in range(19):\n    grayscale_cam = cam(input_tensor=ipt[:, :, :, :].float().cuda(), target_category=cat)\n    cls_cams[cat] = grayscale_cam[0, :]\n# In this example grayscale_cam has only one image in the batch:\n# grayscale_cam = grayscale_cam[0, :]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(4, 5, figsize=(20, 20))\nfor i in range(19):\n    ax[i//5][i%5].imshow(full_img[0].numpy()[:, :, :3])\n    ax[i//5][i%5].imshow(cls_cams[i], alpha=0.3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir cams","metadata":{"execution":{"iopub.status.busy":"2021-10-17T11:00:21.898548Z","iopub.execute_input":"2021-10-17T11:00:21.898886Z","iopub.status.idle":"2021-10-17T11:00:22.619622Z","shell.execute_reply.started":"2021-10-17T11:00:21.898859Z","shell.execute_reply":"2021-10-17T11:00:22.618677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pdfs = []\nwhole_dfs = []\nfor ipts, BS, lsli, tta, iid, fnames_raw, image_raw, full_img in tqdm.tqdm(dl):\n    BS, tta, iid, fnames, lsli = BS.item(), tta.item(), iid[0], [e[0] for e in fnames_raw], lsli.item()\n#     predicted_ps = []\n#     exp_ps = []\n#     for i in range(0, lsli, BS):\n# #         ipt = torch.stack([tensor_tfms(cv2.resize(squarify(s[0], 0), (256, 256))) for s in ress[i: BS+i]]).cuda()\n#         ipt = ipts[0][0].cuda()\n    ipt = tensor_tfms(full_img[0].numpy())\n    ipt = ipt.reshape(1, ipt.shape[0], ipt.shape[1], ipt.shape[2])\n    cls_cams = {}\n    for cat in range(19):\n        grayscale_cam = cam(input_tensor=ipt[:, :, :, :].float().cuda(), target_category=cat)\n        cls_cams[cat] = grayscale_cam[0, :]\n    for idx, v in cls_cams.items():\n        imsave('./cams/{}_{}.png'.format('_'.join(fnames_raw[0][0].split('_')[:-1]), idx), (v * 255.0).astype(np.uint8))","metadata":{"execution":{"iopub.status.busy":"2021-10-17T11:00:23.078093Z","iopub.execute_input":"2021-10-17T11:00:23.078454Z","iopub.status.idle":"2021-10-17T11:02:18.57711Z","shell.execute_reply.started":"2021-10-17T11:00:23.078414Z","shell.execute_reply":"2021-10-17T11:02:18.576009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r cams.zip cams","metadata":{"execution":{"iopub.status.busy":"2021-10-18T11:39:01.204965Z","iopub.execute_input":"2021-10-18T11:39:01.205271Z","iopub.status.idle":"2021-10-18T11:39:01.892061Z","shell.execute_reply.started":"2021-10-18T11:39:01.205198Z","shell.execute_reply":"2021-10-18T11:39:01.891101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r cams","metadata":{},"execution_count":null,"outputs":[]}]}