{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"## import packages\n# import packages\nimport os, sys\nimport gc\nimport numpy as np      \nimport pandas as pd\nfrom glob import glob\nfrom tqdm.notebook import tqdm\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport cv2\n#import rasterio\n#from rasterio.windows import Window\nimport tifffile as tiff\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, SequentialSampler","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-15T16:04:26.591348Z","iopub.execute_input":"2022-08-15T16:04:26.592004Z","iopub.status.idle":"2022-08-15T16:04:28.654892Z","shell.execute_reply.started":"2022-08-15T16:04:26.591913Z","shell.execute_reply":"2022-08-15T16:04:28.653614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append(\"../input/upernetswinmodels/upernet-swin-models\")\nsys.path.append(\"../input/upernetswinmodels/timm_0.6.7\")\n\nimport timm\nfrom model import Net","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:04:28.657318Z","iopub.execute_input":"2022-08-15T16:04:28.657741Z","iopub.status.idle":"2022-08-15T16:04:30.579958Z","shell.execute_reply.started":"2022-08-15T16:04:28.657714Z","shell.execute_reply":"2022-08-15T16:04:30.578805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_arch = \"swin_small_patch4_window7_224_22k\"\n# # model_arch = \"swin_tiny_patch4_window7_224\" \n\n# device = torch.device(\"cpu\")\n# model = Net(arch=model_arch).to(device)     # ^ Swin-tiny/small + UperDecode\n# print(\"model initiated\")","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:04:30.583716Z","iopub.execute_input":"2022-08-15T16:04:30.584040Z","iopub.status.idle":"2022-08-15T16:04:30.589681Z","shell.execute_reply.started":"2022-08-15T16:04:30.584011Z","shell.execute_reply":"2022-08-15T16:04:30.588672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class config:\n    \n    device = torch.device(f\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    debug = False\n    \n    # data related\n    img_size = 768\n    img_norm_cfg = {\n            \"mean\" : np.array([0.485, 0.456, 0.406]),  # RGB_MEAN\n            \"std\" : np.array([0.229, 0.224, 0.225]),   # RGB_STD\n    }\n    \n    test_bs = 1\n    num_workers = 4\n    \n    # postprocessing of inference\n#     thres = 0.40\n    # -------- Exp2\n#     thres = {\n#         \"prostate\" : 0.45, \n#         \"spleen\" : 0.5, \n#         \"lung\" : 0.35, \n#         \"kidney\" : 0.45, \n#         \"largeintestine\" : 0.5,\n#     }\n    # ---------- Exp 6\n#     thres = {\n#         \"prostate\" : 0.40, \n#         \"spleen\" : 0.45, \n#         \"lung\" : 0.25, \n#         \"kidney\" : 0.45, \n#         \"largeintestine\" : 0.5,\n#     }\n    # ---------- Exp 7\n    thres = {\n        \"prostate\" : 0.40, \n        \"spleen\" : 0.40, \n        \"lung\" : 0.15, \n        \"kidney\" : 0.45, \n        \"largeintestine\" : 0.45,\n    }\n    \n    TTA = False           # ! FUCK it!\n    slice_tile = False\n    use_ema = False\n    \n    # ^ seprate HPA & HuBMAP images\n    data_source = [\"HPA\", \"Hubmap\"]\n#     data_source = [\"Hubmap\"]\n#     data_source = [\"HPA\"]\n    organ = [\"prostate\", \"spleen\", \"lung\", \"kidney\", \"largeintestine\"]   # organ you want to probe\n    \n    # define model \n    #model_arch = \"swin_tiny_patch4_window7_224\" \n    model_arch = \"swin_small_patch4_window7_224_22k\"\n    \n    pass","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:04:30.593287Z","iopub.execute_input":"2022-08-15T16:04:30.593695Z","iopub.status.idle":"2022-08-15T16:04:30.664514Z","shell.execute_reply.started":"2022-08-15T16:04:30.593660Z","shell.execute_reply":"2022-08-15T16:04:30.663400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config.img_norm_cfg, config.img_size, config.TTA, config.use_ema, config.thres, config.model_arch, config.data_source","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:04:30.665961Z","iopub.execute_input":"2022-08-15T16:04:30.667342Z","iopub.status.idle":"2022-08-15T16:04:30.681786Z","shell.execute_reply.started":"2022-08-15T16:04:30.667305Z","shell.execute_reply":"2022-08-15T16:04:30.680813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FOLDER = \"../input/upernetswinsubmissions/Exp1_upernet-swin-v1-tiny-aux5-768\"\n# FOLDER = \"../input/upernetswinsubmissions/Exp2_upernet-swin-v1-small-aux5-768\"\n# FOLDER = \"../input/upernetswinsubmissions/Exp3_upernet-swin-v1-tiny_auxALL-ModPPL-768-B12\"\n# FOLDER = \"../input/upernetswinsubmissions/Exp4_upernet-swin-v1-small-auxALL-ModPPL-768-B8\"\n# FOLDER = \"../input/upernetswinsubmissions/Exp5_upernet-swin-v1-tiny_auxALL-ModPPL_StratifiedK4_768-B4\"\n# FOLDER = \"../input/upernetswinsubmissions/Exp6_upernet-swin-v1-small_auxALL-ModPPL_StratifiedK4_768-B8\"\nFOLDER = \"../input/upernetswinsubmissions/Exp7_upernet-swin-v1-small_auxALL-wgt005-ModPPL-768-B4\"\n\nmodel_weights = sorted(glob(f\"{FOLDER}/*.pth\"))\nmodel_weights","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:04:30.683394Z","iopub.execute_input":"2022-08-15T16:04:30.684968Z","iopub.status.idle":"2022-08-15T16:04:30.704314Z","shell.execute_reply.started":"2022-08-15T16:04:30.684933Z","shell.execute_reply":"2022-08-15T16:04:30.703315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Selecte 3 out of K4 folds to submit\n# selected = [0, 1, 2]\n# selected = [0, 1, 3]\n# selected = [0, 2, 3]\nselected = [1, 2, 3]\n\nmodel_weights = [model_weights[k] for k in selected]\nmodel_weights","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:04:30.706467Z","iopub.execute_input":"2022-08-15T16:04:30.707214Z","iopub.status.idle":"2022-08-15T16:04:30.714281Z","shell.execute_reply.started":"2022-08-15T16:04:30.707178Z","shell.execute_reply":"2022-08-15T16:04:30.713164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define model & load model","metadata":{}},{"cell_type":"code","source":"device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n# !nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:04:30.715968Z","iopub.execute_input":"2022-08-15T16:04:30.716606Z","iopub.status.idle":"2022-08-15T16:04:30.722743Z","shell.execute_reply.started":"2022-08-15T16:04:30.716573Z","shell.execute_reply":"2022-08-15T16:04:30.721818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Load in models\n# MODELS = []\n# for ckpt_path in model_weights:\n    \n#     # build model --> modify model.output_type = ['inference']!!\n#     model = Net(arch=config.model_arch) # ^ Swin-tiny/small + UperDecoder\n#     model.output_type = ['inference']\n    \n#     # load weights\n#     ckpt = torch.load(ckpt_path, map_location=\"cpu\")\n#     if config.use_ema and \"ema_state_dict\" in ckpt:\n#         model.load_state_dict(ckpt[\"ema_state_dict\"])\n#         print(f\"Load in 'ema_state_dict' from {ckpt_path}...\")\n#     else:\n#         model.load_state_dict(ckpt[\"state_dict\"])\n#         print(f\"Load in 'state_dict' from {ckpt_path}...\")\n    \n#     # make model ready\n#     model.float()\n#     model.eval()  \n#     model.to(config.device)\n#     MODELS.append(model)\n#     pass\n\n# del ckpt","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:04:30.724505Z","iopub.execute_input":"2022-08-15T16:04:30.724858Z","iopub.status.idle":"2022-08-15T16:04:30.732503Z","shell.execute_reply.started":"2022-08-15T16:04:30.724826Z","shell.execute_reply":"2022-08-15T16:04:30.731526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:04:30.736512Z","iopub.execute_input":"2022-08-15T16:04:30.737056Z","iopub.status.idle":"2022-08-15T16:04:30.742899Z","shell.execute_reply.started":"2022-08-15T16:04:30.737027Z","shell.execute_reply":"2022-08-15T16:04:30.741698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare data","metadata":{}},{"cell_type":"code","source":"def rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.T.flatten()    # ! encode row by row --> but submission requires col by col!\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\n\n# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\ndef rle_decode(mask_rle, shape):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width, height) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)     # ! W x H\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T  # ! rle encoded col by col --> transpose to row by row; ","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:04:30.744208Z","iopub.execute_input":"2022-08-15T16:04:30.745158Z","iopub.status.idle":"2022-08-15T16:04:30.755528Z","shell.execute_reply.started":"2022-08-15T16:04:30.745124Z","shell.execute_reply":"2022-08-15T16:04:30.754564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load in csvs\ndf_sample = pd.read_csv(\"../input/hubmap-organ-segmentation/test.csv\")\nDATA = '../input/hubmap-organ-segmentation/test_images'\n\n# df_sample = pd.read_csv(\"../input/hubmap-organ-segmentation/train.csv\")\n# df_sample = df_sample[:5]\n# DATA = '../input/hubmap-organ-segmentation/train_images'\n\nprint(f\"--> submission_df = {df_sample.head()}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:04:30.757173Z","iopub.execute_input":"2022-08-15T16:04:30.757771Z","iopub.status.idle":"2022-08-15T16:04:30.787288Z","shell.execute_reply.started":"2022-08-15T16:04:30.757720Z","shell.execute_reply":"2022-08-15T16:04:30.786419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict images","metadata":{}},{"cell_type":"code","source":"def image_to_tensor(image, mode='bgr'): #image mode\n    if mode=='bgr':\n        image = image[:,:,::-1]\n    x = image\n    x = x.transpose(2,0,1)      # [C, H, W]\n    x = np.ascontiguousarray(x)\n    x = torch.tensor(x, dtype=torch.float)\n    return x\n\ndef tensor_to_image(x, mode='bgr'):\n    image = x.data.cpu().numpy()\n    image = image.transpose(1,2,0)\n    if mode=='bgr':\n        image = image[:,:,::-1]\n    image = np.ascontiguousarray(image)\n    image = image.astype(np.float32)\n    return image\n\ndef mask_to_tensor(mask):\n    x = mask\n    x = torch.tensor(x, dtype=torch.float)\n    return x\n\ndef tensor_to_mask(x):\n    mask = x.data.cpu().numpy()\n    mask = mask.astype(np.float32)\n    return mask\n","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:04:30.788543Z","iopub.execute_input":"2022-08-15T16:04:30.788866Z","iopub.status.idle":"2022-08-15T16:04:30.797504Z","shell.execute_reply.started":"2022-08-15T16:04:30.788841Z","shell.execute_reply":"2022-08-15T16:04:30.796399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### v3 TTA only with H/V-flips","metadata":{}},{"cell_type":"code","source":"names, preds = [], []\n\nfor idx, row in tqdm(df_sample.iterrows(), total=len(df_sample)):\n    idx = str(row['id'])\n    dsource = row[\"data_source\"]\n    dorgan = row['organ']\n    dthres = config.thres[dorgan] if isinstance(config.thres, dict) else config.thres\n    dheight, dwidth = row[\"img_height\"], row[\"img_width\"]  # original tiff_image size\n    dpxsize = row[\"pixel_size\"]\n    px_scaler = dpxsize / 0.4       # need to resize tiff image\n    print(f\"--> idx={idx}, source={dsource}, organ={dorgan}, thres={dthres}\" \\\n          f\"pixelsize={dpxsize}, pixelsize scaler={px_scaler}\")\n    \n    if (dsource in config.data_source) and (dorgan in config.organ):\n        \n        # ^ load in original tiff_image\n        img_path = f\"{DATA}/{idx}.tiff\"\n        tiff_image = tiff.imread(img_path) # [h, w, c]\n        \n        # ^ normalize pixel_size to HPA(0.4)\n        norm_tiff_image = cv2.resize(tiff_image, None, fx=px_scaler, fy=px_scaler)\n        print(f\"After PixelSize normalize, norm_tiff_image = {norm_tiff_image.shape}\")\n        \n        # ^ resize to img_size = 768x768\n        image = cv2.resize(norm_tiff_image, dsize=(config.img_size, config.img_size))\n        image = image.astype(np.float32) / 255.0\n        \n        # convert to tensor\n        image = image_to_tensor(image)   # [C, H, W]\n        image = image.to(device)\n        # ^ if TTA, stack [image, V-flipped, H-flipped] together\n        if config.TTA:\n            image = torch.stack([image, torch.flip(image, [1]), torch.flip(image, [2])])\n        else:\n            image = image.unsqueeze(0)   # [1, C, H, W]\n        print(f\"Load in image = {image.size()}, mean = {image.mean()}, max = {image.max()}\")\n        \n        # & put batch image in a dict\n        batch = {\"image\" : image}\n        py = None\n        times = 0\n        \n        # & loop over each model & infer every time, otherwise OOM!\n        for ckpt_path in model_weights:\n            # build model --> modify model.output_type = ['inference']!!\n            model = Net(arch=config.model_arch) # ^ Swin-tiny/small + UperDecoder\n            model.output_type = ['inference']\n\n            # load weights\n            ckpt = torch.load(ckpt_path, map_location=\"cpu\")\n            if config.use_ema and \"ema_state_dict\" in ckpt:\n                model.load_state_dict(ckpt[\"ema_state_dict\"])\n                print(f\"Load in 'ema_state_dict' from {ckpt_path}...\")\n            else:\n                model.load_state_dict(ckpt[\"state_dict\"])\n                print(f\"Load in 'state_dict' from {ckpt_path}...\")\n\n            # make model ready\n            model.float()\n            model.eval()  \n            model.to(config.device)\n\n            # & infer once\n            with torch.no_grad():\n                output = model(batch) # {\"probability\" : tensor}\n                p = output[\"probability\"]  # [B, C, H ,W]\n                pass\n            \n            # flip flipped images back\n            if config.TTA:\n                p[1] = torch.flip(p[1], [1])  # VFlip\n                p[2] = torch.flip(p[2], [2])   # HFlip\n            \n            # Way 1 = directly resize probs back to original tiff_image size & Avg by TTA times\n            p = F.upsample(p, size=(dheight, dwidth), mode=\"bilinear\")\n            p = p.permute(0, 2, 3, 1).float().mean(0).detach().cpu().numpy()  # [H, W, C]\n\n            if py is None: \n                py = p\n            else: \n                py += p\n            \n            # release GPU memory\n            # torch.cuda.empty_cache()\n            pass\n\n        # ^ Average by # of model_weights\n        py /= len(model_weights)\n    \n        # ^ cast probs to mask\n        mask = (py > dthres).astype(np.uint8)\n        rle = rle_encode(mask)\n        print(f\"--> final mask={mask.shape} --> encoded rle = {len(rle)}\")\n        \n    else:\n        rle = \"\"        # organ you don't want to probe\n    \n    names.append(idx)\n    preds.append(rle)\n              \n    del mask\n    gc.collect()\n    pass","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:04:30.799155Z","iopub.execute_input":"2022-08-15T16:04:30.799823Z","iopub.status.idle":"2022-08-15T16:05:01.393151Z","shell.execute_reply.started":"2022-08-15T16:04:30.799788Z","shell.execute_reply":"2022-08-15T16:05:01.392183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.DataFrame({'id': names, 'rle' : preds})\nsub_df.to_csv('submission.csv', index=False)\nsub_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:05:01.394802Z","iopub.execute_input":"2022-08-15T16:05:01.395492Z","iopub.status.idle":"2022-08-15T16:05:01.412834Z","shell.execute_reply.started":"2022-08-15T16:05:01.395456Z","shell.execute_reply":"2022-08-15T16:05:01.411800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Visualize on images\ndef read_tiff(path, scale=None, verbose=0): \n    image = tiff.imread(path)\n    if len(image.shape) == 5:\n        image = image.squeeze().transpose(1, 2, 0)\n    \n    if verbose:\n        print(f\"[{path}] Image shape: {image.shape}\")\n    \n    if scale:\n        new_size = (image.shape[1] // scale, image.shape[0] // scale)\n        image = cv2.resize(image, new_size)\n        if verbose:\n            print(f\"[{path}] Resized Image shape: {image.shape}\")\n        \n    # For visualize only!\n    mx = np.max(image)\n    image = image.astype(np.float32)\n    if mx:\n        image /= mx # scale image to [0, 1]\n    return image\n\ndef visualize_image(image, mask):\n    plt.figure(figsize=(16,16))\n    # visualize orig image\n    plt.subplot(1, 2, 1)\n    plt.imshow(image, vmin=0, vmax=255, alpha=1.0)\n    plt.subplot(1, 2, 2)\n    plt.imshow(image, vmin=0, vmax=255, alpha=0.3)\n    plt.imshow(mask * 255, alpha=0.5)\n#     plt.axis(\"off\")\n    pass","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:05:01.414105Z","iopub.execute_input":"2022-08-15T16:05:01.414522Z","iopub.status.idle":"2022-08-15T16:05:01.424062Z","shell.execute_reply.started":"2022-08-15T16:05:01.414489Z","shell.execute_reply":"2022-08-15T16:05:01.422989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ID = min(3, sub_df.shape[0] - 1)\n\ntest_id = sub_df.loc[ID, \"id\"]\ntest_rle = sub_df.loc[ID, \"rle\"]\n\n# load in images\nimg_path = os.path.join(DATA, test_id +'.tiff')\ntest_image = read_tiff(img_path)\nprint(f\"--> {test_id} image = {test_image.shape}\")\n\n# load in masks\ntest_mask = rle_decode(test_rle, test_image.shape[:-1])\nprint(f\"--> pred mask = {test_mask.shape}, min = {test_mask.min()}, max={test_mask.max()}\")\n\n# Show\nvisualize_image(test_image, test_mask)\n\nif \"rle\" in df_sample.columns:\n    gt_rle = df_sample.loc[ID, \"rle\"]\n    gt_mask = rle_decode(gt_rle, test_image.shape[:-1])\n    visualize_image(test_image, gt_mask)\n    pass","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:05:01.425463Z","iopub.execute_input":"2022-08-15T16:05:01.426315Z","iopub.status.idle":"2022-08-15T16:05:03.765977Z","shell.execute_reply.started":"2022-08-15T16:05:01.426277Z","shell.execute_reply":"2022-08-15T16:05:03.765111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Done!\")","metadata":{"execution":{"iopub.status.busy":"2022-08-15T16:05:03.766961Z","iopub.execute_input":"2022-08-15T16:05:03.768005Z","iopub.status.idle":"2022-08-15T16:05:03.773770Z","shell.execute_reply.started":"2022-08-15T16:05:03.767969Z","shell.execute_reply":"2022-08-15T16:05:03.772782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}