{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### strategy 2:\n\n1. use shajiayu's models to generate predictions\n\n2. then ensemble with 3d models","metadata":{}},{"cell_type":"code","source":"import sys\nimport re\nimport gc\nimport time\n\nTHRES_HIGH = 12\nTHRES_LOW = 12\n\n\ninclude_2d = True\nsys.path.append('../input/monai-v081/')\nif include_2d:\n    sys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\n    sys.path.append(\"../input/segmentation-models-pytorch/segmentation_models.pytorch-0.2.1\")\n    sys.path.append(\"../input/pretrainedmodels/pretrainedmodels-0.7.4\")\n    sys.path.append(\"../input/efficientnet-pytorch/EfficientNet-PyTorch-master\")\n\n    !pip install ../input/mmdetection/addict-2.4.0-py3-none-any.whl > /dev/null\n    !pip install ../input/mmdetection/yapf-0.31.0-py2.py3-none-any.whl > /dev/null\n    !pip install ../input/mmdetection/terminaltables-3.1.0-py3-none-any.whl > /dev/null\n    !pip install ../input/mmdetection/einops* > /dev/null\n    !pip install ../input/mmdetection/mmcv_full-1.3.17-cp37-cp37m-linux_x86_64.whl > /dev/null\n    !pip install ../input/tract-submission/mmseg* > /dev/null\n\nfrom glob import glob\nfrom tqdm.notebook import tqdm\nimport torch\nimport numpy as np\nimport matplotlib.pyplot as plt  \nfrom skimage import measure\nfrom monai.inferers import sliding_window_inference\nfrom monai.networks.nets import UNet, SegResNet\nfrom monai.data import CacheDataset, DataLoader\nfrom monai.transforms import *","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:56:36.237812Z","iopub.execute_input":"2022-07-12T14:56:36.238185Z","iopub.status.idle":"2022-07-12T14:59:49.913918Z","shell.execute_reply.started":"2022-07-12T14:56:36.238106Z","shell.execute_reply":"2022-07-12T14:59:49.913086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create 3d models","metadata":{}},{"cell_type":"code","source":"default_3d_tta = [[2],[3]]\nclass cfg_3d:\n    img_size = (224, 224, 80)\n    in_channels = 1\n    out_channels = 3\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    weights = [\n        dict(\n            weight = \"../input/tract-submission/segres_all_s2.pth\",\n            model_type = SegResNet, spatial_dims = 3, in_channels = 1, out_channels = 3, \n            act = 'PRELU', norm ='BATCH',\n            init_filters = 12,\n            tta = default_3d_tta + [[2,3]],\n        ),\n        dict(\n            weight = \"../input/tract-submission/segres20_all_s2.pth\",\n            model_type = SegResNet, spatial_dims = 3, in_channels = 1, out_channels = 3, \n            act = 'PRELU', norm ='BATCH',\n            init_filters = 20,\n            tta = default_3d_tta,\n        ),\n        dict(\n            weight = \"../input/tract-submission/segresnet_alldata.pth\",\n            model_type = SegResNet, spatial_dims = 3, in_channels = 1, out_channels = 3, \n            act = 'PRELU', norm ='BATCH',\n            init_filters = 32,\n            tta = default_3d_tta,\n        ),\n    ]\n    batch_size = 1\n    sw_batch_size = 2\n    \ntest_transforms_3d = Compose(\n    [\n        AddChanneld(keys=\"image\"), # c, d, h, w\n        Transposed(keys=\"image\", indices=[0, 3, 2, 1]), # c, w, h, d\n        Lambdad(keys=\"image\", func=lambda x: x / x.max()),\n        EnsureTyped(keys=\"image\", dtype=torch.float32),\n    ]\n)\n\ndef get_model(cfg, weight):\n    weight_path = weight.pop(\"weight\")\n    model = weight.pop(\"model_type\")(**weight)\n    stt = torch.load(weight_path)\n    if \"model\" in stt: stt = stt[\"model\"]\n    if all([k.startswith(\"module.\") for k in stt]): stt = {k[7:]: v for k, v in stt.items()}\n    model.load_state_dict(stt)\n    model.eval()\n    return model\n    \nmodels_3d = []\ndims_3d_tta = []\n\nfor weight in tqdm(cfg_3d.weights):\n    dims_3d_tta.append(weight.pop(\"tta\"))\n    model = get_model(cfg_3d, weight)\n    models_3d.append(model)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:10:08.493489Z","iopub.execute_input":"2022-07-12T15:10:08.493744Z","iopub.status.idle":"2022-07-12T15:10:09.070751Z","shell.execute_reply.started":"2022-07-12T15:10:08.493713Z","shell.execute_reply":"2022-07-12T15:10:09.069914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create 2.5d models (shajiayu)","metadata":{}},{"cell_type":"code","source":"if include_2d:\n    from mmseg.apis import init_segmentor, inference_segmentor\n    from mmcv.utils import config\n\n    cfgs = [\n        dict(\n            cfg = \"../input/kfold8-model-dice/fold_all_mask_b6_bs16_dice/config_uwm.py\",\n            ckpt = \"../input/kfold8-model-dice/fold_all_mask_b6_bs16_dice/iter_26000.pth\",  #0.884\n            weight = 0.2,\n            tta = True,\n        ),\n        dict(\n            cfg = \"../input/kfold8-model-dice/fold_all_mask_b4_bs25_dice/config_uwm.py\",\n            ckpt = \"../input/kfold8-model-dice/fold_all_mask_b4_bs25_dice/iter_16170.pth\",  #0.883\n            weight = 0.2,\n            tta = True,\n        ),\n        dict(\n            cfg = \"../input/kfold8-model-dice/fold_all_bs14_b7_mask_60epoch2/config_uwm.py\",\n            ckpt = \"../input/kfold8-model-dice/fold_all_bs14_b7_mask_60epoch2/iter_34810.pth\",# 0.884\n            weight = 0.3,\n            tta = True,\n        ),\n        dict(\n            cfg = \"../input/kfold8-model-dice/fold_all_bs20_b5_mask_60epoch/config_uwm.py\",\n            ckpt = \"../input/kfold8-model-dice/fold_all_bs20_b5_mask_60epoch/iter_23240.pth\", #0.883\n            weight = 0.2,\n            tta = True,\n        ),\n        dict(\n            cfg = \"../input/model-mask/fold_all_mask_bs20_b5_dice/config_uwm.py\",\n            ckpt = \"../input/model-mask/fold_all_mask_bs20_b5_dice/iter_24900.pth\",\n            weight = 0.1,\n            tta = True,\n        ),\n        dict(\n            cfg = \"../input/tract-submission/upt_cvb_448_grd_20k_opt_2.5d_all.py\",\n            ckpt = \"../input/tract-submission/upt_cvb_448_grd_20k_opt_2.5d_all.pth\",\n            weight = 0.2 / 3,\n            tta = False,\n        ),\n        dict(\n            cfg = \"../input/tract-submission/upt_cvs_448_grd_20k_opt_2.5d_all.py\",\n            ckpt = \"../input/tract-submission/upt_cvs_448_grd_20k_opt_2.5d_all.pth\",\n            weight = 0.2 / 3,\n            tta = False,\n        ),\n        dict(\n            cfg = \"../input/all-data-model/fold_case_b5_60epoch_dice_bce/config_uwm.py\",\n            ckpt = \"../input/all-data-model/fold_case_b5_60epoch_dice_bce/iter_49400.pth\",\n            weight = 0.2 / 3,\n            tta = True,\n        ),\n    ]\n\n    weights = []\n    models = []\n    for cfg_dic in cfgs:\n        cfg = cfg_dic[\"cfg\"]\n        ckpt = cfg_dic[\"ckpt\"]\n        weights.append(cfg_dic[\"weight\"])\n        \n        cfg = config.Config.fromfile(cfg)\n        cfg.model.backbone.pretrained = None\n        cfg.model.test_cfg.logits = True\n        # TTA >>>>>\n        cfg.data.test.pipeline[1].flip = cfg_dic[\"tta\"]\n        # <<<<<<<<<\n        cfg.data.test.pipeline[1].transforms.insert(2, dict(type=\"Normalize\", mean=[0,0,0], std=[1,1,1], to_rgb=False))\n\n        model = init_segmentor(cfg, ckpt)\n        models.append(model)\n    weights = np.array(weights) / sum(weights)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:59:58.177343Z","iopub.execute_input":"2022-07-12T14:59:58.177808Z","iopub.status.idle":"2022-07-12T15:00:50.051071Z","shell.execute_reply.started":"2022-07-12T14:59:58.177770Z","shell.execute_reply":"2022-07-12T15:00:50.050318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create 2.5d classification models","metadata":{}},{"cell_type":"code","source":"if include_2d:\n    from mmseg.apis import init_segmentor, inference_segmentor\n    from mmcv.utils import config\n    cfgs = [\n        \"../input/all-data-model/fold_case_b5_60epoch_dice_bce/config_uwm.py\",\n        \"../input/all-data-model/fold_all_b7_bs14_20epoch/config_uwm.py\",\n        \"../input/all-data-model/fold_case_b4_bs20/config_uwm.py\",\n        \"../input/all-data-model/fold_case_b6_bs18_19epoch/config_uwm.py\",  \n\n    ]\n\n    ckpts = [\n        \"../input/all-data-model/fold_case_b5_60epoch_dice_bce/iter_49400.pth\",\n        \"../input/all-data-model/fold_all_b7_bs14_20epoch/iter_24750.pth\",\n        \"../input/all-data-model/fold_case_b4_bs20/iter_38200.pth\",\n        \"../input/all-data-model/fold_case_b6_bs18_19epoch/iter_16960.pth\",\n    ]\n\n    cls_models = []\n    for cfg, ckpt in zip(cfgs, ckpts):\n        cfg = config.Config.fromfile(cfg)\n        cfg.model.backbone.pretrained = None\n        cfg.model.test_cfg.logits = True\n        \n        cfg.data.test.pipeline[1].transforms.insert(2, dict(type=\"Normalize\", mean=[0,0,0], std=[1,1,1], to_rgb=False))\n\n        model = init_segmentor(cfg, ckpt)\n        cls_models.append(model)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:00:50.053327Z","iopub.execute_input":"2022-07-12T15:00:50.053756Z","iopub.status.idle":"2022-07-12T15:01:00.888894Z","shell.execute_reply.started":"2022-07-12T15:00:50.053715Z","shell.execute_reply":"2022-07-12T15:01:00.888147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare test dataframe","metadata":{}},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport glob\nfrom tqdm.auto import tqdm\nfrom scipy.ndimage import binary_closing, binary_opening, measurements\n\ndef rle_encode(img):\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\nclasses = ['large_bowel', 'small_bowel', 'stomach']\ndata_dir = \"../input/uw-madison-gi-tract-image-segmentation/\"\ntest_dir = os.path.join(data_dir, \"test\")\nsub = pd.read_csv(os.path.join(data_dir, \"sample_submission.csv\"))\ntest_images = glob.glob(os.path.join(test_dir, \"**\", \"*.png\"), recursive = True)\n\nif len(test_images) == 0:\n    test_dir = os.path.join(data_dir, \"train\")\n    sub = pd.read_csv(os.path.join(data_dir, \"train.csv\"))[[\"id\", \"class\"]].iloc[:144 * 3]\n    sub[\"predicted\"] = \"\"\n    test_images = glob.glob(os.path.join(test_dir, \"**\", \"*.png\"), recursive = True)\n    \nid2img = {_.rsplit(\"/\", 4)[2] + \"_\" + \"_\".join(_.rsplit(\"/\", 4)[4].split(\"_\")[:2]): _ for _ in test_images}\nsub[\"file_name\"] = sub.id.map(id2img)\nsub[\"days\"] = sub.id.apply(lambda x: \"_\".join(x.split(\"_\")[:2]))\nfname2index = {f + c: i for f, c, i in zip(sub.file_name, sub[\"class\"], sub.index)}\nsub","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:01:00.890705Z","iopub.execute_input":"2022-07-12T15:01:00.890978Z","iopub.status.idle":"2022-07-12T15:01:05.874643Z","shell.execute_reply.started":"2022-07-12T15:01:00.890941Z","shell.execute_reply":"2022-07-12T15:01:05.873956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make prediction","metadata":{}},{"cell_type":"code","source":"def make_3d_prediction(imgs, cnts_cls):\n    im_vol = np.transpose(imgs, [2, 0, 1])\n    im_vol = test_transforms_3d({'image': im_vol})['image'].unsqueeze(0).to(cfg_3d.device)\n\n    sliding_args = {\"overlap\": 0.7, \"mode\": \"gaussian\"} if include_2d else {}\n    pred_all_3d = 0\n    tta_cnt = 0\n    for model_idx, model in enumerate(models_3d):\n        model.to(cfg_3d.device)\n        with torch.no_grad():\n            pred_all_3d += sliding_window_inference(\n                im_vol, \n                cfg_3d.img_size, \n                cfg_3d.sw_batch_size, \n                model,\n                **sliding_args,\n            ).cpu().sigmoid()\n            tta_cnt += 1\n        for dims in dims_3d_tta[model_idx]:\n            with torch.no_grad():\n                pred_all_3d += torch.flip(\n                    sliding_window_inference(\n                        torch.flip(im_vol, dims=dims), \n                        cfg_3d.img_size, \n                        cfg_3d.sw_batch_size, \n                        model,\n                        **sliding_args,\n                    ).cpu().sigmoid(),\n                    dims=dims) \n                tta_cnt += 1\n        model.to('cpu')\n    pred_all_3d = torch.permute(pred_all_3d[0], [3, 0, 2, 1]) / tta_cnt # from (c,w,h,d) -> to (d,c,h,w)\n    pred_all_3d = pred_all_3d.numpy()\n    del im_vol\n    gc.collect()\n    return pred_all_3d\n\ndef make_2d_classification_predictions(imgs, old_size):\n    preds = []; cls_cnts = []\n    for i in range(imgs.shape[-1]):\n        if include_2d:\n            img = imgs[...,[max(0, i - 2), i, min(imgs.shape[-1] - 1, i + 2)]]\n            res = []\n            new_img = img.astype(np.float32) / img.max()\n            res = [inference_segmentor(model, new_img)[0][0] for model in cls_models]\n            res = sum(res) / len(res)\n            \n            cls_cnt = (res > 0.6).astype(np.uint8)\n            cls_cnt = cv2.resize(cls_cnt, old_size[::-1], interpolation = cv2.INTER_NEAREST).sum((0,1))\n            \n            res = (res > 0.5).astype(np.uint8)\n            res = cv2.resize(res, old_size[::-1], interpolation = cv2.INTER_NEAREST)\n            preds.append(res)\n            cls_cnts.append(cls_cnt)\n    \n    if include_2d:\n        preds = np.stack(preds, 0)\n        cls_cnts = np.stack(cls_cnts, 0)\n    else:\n        preds = None; cls_cnts = None\n    return preds, cls_cnts\n\ndef make_2d_predictions_carnoandnam(imgs, old_size, preds_3d, preds_cls, cnts_cls):\n    preds = []\n    for i in range(imgs.shape[-1]):\n        \n        res_3d = preds_3d[i]\n        res_3d = np.transpose(res_3d, (1,2,0)) # make channel last (h,w,c)\n        \n        if include_2d:\n            cnt = cnts_cls[i]\n            pred_cls = preds_cls[i]\n            if (cnt > THRES_HIGH).any():\n                img = imgs[...,[max(0, i - 2), i, min(imgs.shape[-1] - 1, i + 2)]]\n                new_img = img.astype(np.float32) / img.max()\n\n                res = []\n                for model, weight in zip(models, weights):\n                    res.append(inference_segmentor(model, new_img)[0][0] * weight)\n                res = sum(res) / sum(weights)\n                \n                res = cv2.resize(res, old_size[::-1], interpolation = cv2.INTER_NEAREST)\n                res_ens = 0.6 * res + 0.4 * res_3d\n                #### 0.5 thres ####\n                # res_ens = (res_ens > 0.5).astype(np.uint8)\n                #### 0.4/0.5 thres ####\n                res1 = (res_ens > [0.4, 0.4, 0.4]).astype(np.uint8)\n                res1[...,res_ens.max((0,1)) < [0.5, 0.5, 0.5]] = 0\n                res_ens = res1\n                for j in range(3):\n                    if cnt[j] < THRES_LOW:\n                        res_ens[...,j] = 0\n                    elif cnt[j] <= THRES_HIGH:\n                        res_ens[...,j] = pred_cls[...,j]\n            else:\n                res_ens = pred_cls\n                for j in range(3):\n                    if cnt[j] < THRES_LOW:\n                        res_ens[...,j] = 0\n        else:\n            res_ens = (res_3d > 0.5).astype(int)\n        preds.append(res_ens)\n    \n    preds = np.stack(preds, 0)\n    \n#     for org in range(3):\n#         connect = measure.label(preds[...,org])\n#         connect_label, connect_cnt = np.unique(connect, return_counts = True)\n#         small_cnt_label = connect_label[connect_cnt < 20]\n#         preds[np.isin(connect, small_cnt_label),org] = 0\n    \n    return preds","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:01:05.876022Z","iopub.execute_input":"2022-07-12T15:01:05.876369Z","iopub.status.idle":"2022-07-12T15:01:05.904522Z","shell.execute_reply.started":"2022-07-12T15:01:05.876324Z","shell.execute_reply":"2022-07-12T15:01:05.903750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_time = False\nsubs = []\nfor day, group in tqdm(sub.groupby(\"days\")):\n    imgs = []\n    file_names = []\n    old_sizes = []\n    for file_name in sorted(group.file_name.unique()):\n        img = cv2.imread(file_name, cv2.IMREAD_ANYDEPTH)\n        old_size = img.shape[:2]\n        imgs.append(img)\n        file_names.append(file_name)\n        old_sizes.append(old_size)\n        \n    imgs = np.stack(imgs, -1)\n    \n    ###########################\n    #                         #\n    #    2.5D classification  #\n    #                         #\n    ###########################\n    if show_time: stamp = time.time()\n    preds_cls, cnts_cls = make_2d_classification_predictions(imgs, old_size)\n    if show_time: print(\"2d classification: \" , time.time() - stamp)\n    \n    ###########################\n    #                         #\n    #      3D prediction      #\n    #                         #\n    ###########################\n    if show_time: stamp = time.time()\n    pred_3d = make_3d_prediction(imgs, cnts_cls)\n    if show_time: print(\"3d prediction: \", time.time() - stamp)\n    \n    ###########################\n    #                         #\n    #      2.5D prediction    #\n    #                         #\n    ###########################\n    if show_time: stamp = time.time()\n    preds = make_2d_predictions_carnoandnam(imgs, old_size, pred_3d, preds_cls, cnts_cls)\n    if show_time: print(\"2d prediction: \", time.time() - stamp)\n    \n    \n    ###########################\n    #                         #\n    #         output          #\n    #                         #\n    ###########################\n    for i, res in enumerate(preds):\n        file_name = file_names[i]\n        old_size = old_sizes[i]\n        for j in range(3):\n            rle = rle_encode(res[...,j])\n            index = fname2index[file_name + classes[j]]\n            sub.loc[index, \"predicted\"] = rle\n            \n    del preds_cls, cnts_cls, imgs, pred_3d, preds\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:10:15.214070Z","iopub.execute_input":"2022-07-12T15:10:15.214405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = sub[[\"id\", \"class\", \"predicted\"]]\nsub.to_csv(\"submission.csv\", index = False)\nsub","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:01:32.652461Z","iopub.status.idle":"2022-07-12T15:01:32.653305Z","shell.execute_reply.started":"2022-07-12T15:01:32.653022Z","shell.execute_reply":"2022-07-12T15:01:32.653051Z"},"trusted":true},"execution_count":null,"outputs":[]}]}