{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl\n!pip install ../input/hpapytorchzoozip/pytorch_zoo-master\n!pip install ../input/hpacellsegmentatormaster/HPA-Cell-Segmentation-master","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import base64\nimport gc\nimport os\nimport pickle\nimport random\nimport sys\nimport typing as t\nimport zlib\nfrom itertools import groupby\nfrom multiprocessing import Pool\nfrom operator import itemgetter\nfrom pathlib import Path\n\nfrom IPython.display import display\nimport albumentations as A\nimport cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom albumentations.pytorch import ToTensorV2\nfrom pycocotools import _mask as coco_mask\nfrom pycocotools import mask as mutils\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm import tqdm\n\nrandom.seed(0)\n\n# =============================================================================\n# setting\n# =============================================================================\n\"\"\"\nPrecomputed Public + Dummy Private: COMPUTE_PUBLIC=False, COMPUTE_PRIVATE=False\nPrecomputed Public + Private      : COMPUTE_PUBLIC=False, COMPUTE_PRIVATE=True\nPublic + Private                  : COMPUTE_PUBLIC=True,  COMPUTE_PRIVATE=True\n\"\"\"\n\nLOCAL = False\nCOMPUTE_PUBLIC = False\nCOMPUTE_PRIVATE = True\n\n# ==========================\n# 1 st stage\n# ==========================\nEXP_NAME_1ST = [\"exp049\", \"exp050\"]\nMODEL_NAMES_1ST = [\n    \"model_best_0.pth\", # \"model_tmp_0.pth\", \n    \"model_best_1.pth\", # \"model_tmp_1.pth\", \n    \"model_best_2.pth\", # \"model_tmp_2.pth\",\n    \"model_best_3.pth\", # \"model_tmp_3.pth\"\n              ]\nTEST_LOCAL_COMPUTED_1ST = [\n    \"pred_0.csv\", \n    \"pred_1.csv\", \n    \"pred_2.csv\", \n    \"pred_3.csv\"\n]\n\n\n# ==========================\n# 2nd stage\n# ==========================\nBACKBONE_NAME = \"seresnet152d\"\n\nEXP_NAME = [\"exp068\", \"exp071\", \"exp072\", \"exp073\"]\nMODEL_NAMES = [\n    # index 0, exp068\n    [\n        \"model_best_0.pth\", \"model_tmp_0.pth\", \n        \"model_best_1.pth\", \"model_tmp_1.pth\", \n        \"model_best_2.pth\", \"model_tmp_2.pth\",\n        \"model_best_3.pth\", \"model_tmp_3.pth\"\n\n    ],\n    # index 1, exp071\n    [\n        \"model_0_25.pth\", \"model_0_21.pth\", \n        \"model_1_25.pth\", \"model_1_21.pth\", \n    ],\n    # index 2, exp072\n    [\n        \"model_0_25.pth\", \"model_0_21.pth\", \n#         \"model_1_25.pth\", \"model_1_21.pth\", \n    ],\n    # index 3, exp073\n    [\n        \"model_0_25.pth\", \"model_0_21.pth\", \n#         \"model_1_25.pth\", \"model_1_21.pth\", \n    ],\n]\nTEST_LOCAL_COMPUTED = [\n    # index 0, exp068\n    [\n        \"pred_0.csv\", \n        \"pred_1.csv\", \n        \"pred_2.csv\", \n        \"pred_3.csv\"\n    ],\n    # index 1, exp071\n    [\n        \"pred_0.csv\", \n        \"pred_1.csv\", \n#         \"pred_2.csv\", \n#         \"pred_3.csv\"\n    ],\n    # index 2, exp072\n    [\n        \"pred_0.csv\", \n#         \"pred_1.csv\", \n#         \"pred_2.csv\", \n#         \"pred_3.csv\"\n    ],\n    # index 3, exp073\n    [\n        \"pred_0.csv\", \n#         \"pred_1.csv\", \n#         \"pred_2.csv\", \n#         \"pred_3.csv\"\n    ],\n]\n\n\n# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n# image level\n# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\nMODEL_NAMES_IMAGE = [\n    \"model_best_0.pth\", \"model_tmp_0.pth\", \n    \"model_best_1.pth\", \"model_tmp_1.pth\", \n    \"model_best_2.pth\", \"model_tmp_2.pth\",\n    \"model_best_3.pth\", \"model_tmp_3.pth\",\n    \"model_best_4.pth\", \"model_tmp_4.pth\",\n    \"model_best_5.pth\", \"model_tmp_5.pth\",\n    \"model_best_6.pth\", \"model_tmp_6.pth\",\n    \"model_best_7.pth\", \"model_tmp_7.pth\",\n              ]\n\nMODEL_PATHS_IMAGE = [f\"../input/hpa-image-level-weight/exp102/{p}\" for p in MODEL_NAMES_IMAGE]\n\nTEST_LOCAL_COMPUTED_IMAGE = [\n    \"pred_0.csv\", \n    \"pred_1.csv\", \n    \"pred_2.csv\", \n    \"pred_3.csv\",\n    \"pred_4.csv\",\n    \"pred_5.csv\",\n    \"pred_6.csv\",\n    \"pred_7.csv\",\n]\n\nTEST_LOCAL_COMPUTED_PATHS_IMAGE = [f\"../input/hpa-image-level-weight/exp102/{p}\" for p in TEST_LOCAL_COMPUTED_IMAGE]\n\n\n\nCOLS_TARGET = [f\"label_{i}\" for i in range(19)]\n\nBATCH_SIZE = 32\nIMAGE_SIZE = 512\n\nMARGIN = 100\nW_MASK = True\nIN_CHANS = 4\n\nKEEP_CELL_AREA_MIN = 0.005\nKEEP_NUC_AREA_MIN = 0.001\nKEEP_EDGE_CELL_AREA_MIN = 0.01\nNUC_AREA_MIN_0to5 = 0.12\n\nWEIGHT_CELL_LEVEL_VS_IMAGE_MEAN_VS_IMAGE_PRED = [0.6, 0., 0.4]\n\nRATE_OF_WEIGHT_1ST_2ND = [0.2, 0.8]\n\nGPUS = torch.cuda.device_count()\nGPU = 0\n\nROOT = Path.cwd().parent\nif LOCAL:\n    INPUT = ROOT / \"input\"\n    MODEL_PATHS = [ROOT / \"output\" / EXP_NAME / p for p in MODEL_NAMES]\n    TEST_LOCAL_COMPUTED_PATH = ROOT / \"output\" / EXP_NAME / TEST_LOCAL_COMPUTED\n    TEST_IMG_DIR = ROOT / \"data\" / \"test_rgby_images\"\n    MASK_DIR = ROOT / \"data\" / \"mask\"\n\n    NUC_MODEL = MASK_DIR / \"dpn_unet_nuclei_v1.pth\"\n    CELL_MODEL = MASK_DIR / \"dpn_unet_cell_3ch_v1.pth\"\n\n    MAX_THRE = 40\n\n    import timm\nelse:\n    INPUT = ROOT / \"input\" / \"hpa-single-cell-image-classification\"\n    LIB_DIR = ROOT / \"input\" / \"hpa2021-libs\"\n    \n    # ============================\n    # 1 st stage\n    # ============================\n    MODEL_PATHS = []\n    TEST_LOCAL_COMPUTED_PATHS_1ST = []\n    for e_name in EXP_NAME_1ST:\n        MODEL_PATHS += [LIB_DIR / e_name / p for p in MODEL_NAMES_1ST]\n        TEST_LOCAL_COMPUTED_PATHS_1ST += [LIB_DIR / e_name / p for p in TEST_LOCAL_COMPUTED_1ST]\n        \n    LEN_1ST = len(MODEL_PATHS)\n        \n    # ============================\n    # 2nd stage\n    # ============================\n    TEST_LOCAL_COMPUTED_PATHS = []\n    for i, e_name in enumerate(EXP_NAME):\n        MODEL_PATHS += [LIB_DIR / e_name / p for p in MODEL_NAMES[i]]\n        TEST_LOCAL_COMPUTED_PATHS += [LIB_DIR / e_name / p for p in TEST_LOCAL_COMPUTED[i]]\n        \n    # ===========================\n    # Weight 1st vs 2nd\n    # ===========================\n    WEIGHT_1ST_2ND = np.ones(len(MODEL_PATHS)).reshape(-1 , 1, 1)\n    WEIGHT_1ST_2ND[:LEN_1ST] = len(MODEL_PATHS) * (1 / LEN_1ST) * RATE_OF_WEIGHT_1ST_2ND[0]\n    WEIGHT_1ST_2ND[LEN_1ST:] = len(MODEL_PATHS) * (1 / (len(MODEL_PATHS) - LEN_1ST)) * RATE_OF_WEIGHT_1ST_2ND[1]\n\n\n    OUTPUT = ROOT / \"temp\"\n    MASK_DIR = OUTPUT / \"mask\"\n    MASK_DIR.mkdir(exist_ok=True, parents=True)\n    NUCEIL_DIR = MASK_DIR / \"test\" / \"nuclei\"\n    NUCEIL_DIR.mkdir(exist_ok=True, parents=True)\n    CELL_DIR = MASK_DIR / \"test\" / \"cell\"\n    CELL_DIR.mkdir(exist_ok=True, parents=True)\n\n    NUC_MODEL = (\n        ROOT / \"input\" / \"hpacellsegmentatormodelweights\" / \"dpn_unet_nuclei_v1.pth\"\n    )\n    CELL_MODEL = (\n        ROOT / \"input\" / \"hpacellsegmentatormodelweights\" / \"dpn_unet_cell_3ch_v1.pth\"\n    )\n\n    MAX_THRE = 2\n\n    sys.path.append(str(ROOT / \"input\" / \"hpa2021-libs\"))\n    import timm\n    from tqdm.notebook import tqdm\n\n\nsample_submission = pd.read_csv(INPUT / \"sample_submission.csv\")\n\n\nprint(\"NUC_MODEL:\", NUC_MODEL.exists())\nprint(\"CELL_MODEL:\", CELL_MODEL.exists())\nprint(\"MODEL_PATHS:\", [p.exists() for p in MODEL_PATHS])\nprint(\"TEST_LOCAL_COMPUTED_PATHS:\", [p.exists() for p in TEST_LOCAL_COMPUTED_PATHS])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =============================================================================\n# def\n# =============================================================================\ndef decode_binary_mask(decoded_base64_str, width, height):\n    \"\"\"Converts a OID challenge encoding ascii text into binary mask.\"\"\"\n\n    binary_str = base64.b64decode(decoded_base64_str)\n    rle_encoded_mask = zlib.decompress(binary_str)\n    # print(rle_encoded_mask)\n    decoding_dict = {\n        \"size\": [height, width],  # [im_height, im_width],\n        \"counts\": rle_encoded_mask,\n    }\n    mask_tensor = mutils.decode(decoding_dict).astype(bool)\n    return mask_tensor\n\n\ndef coco_rle_encode(mask):\n    rle = {\"counts\": [], \"size\": list(mask.shape)}\n    counts = rle.get(\"counts\")\n    for i, (value, elements) in enumerate(groupby(mask.ravel(order=\"F\"))):\n        if i == 0 and value == 1:\n            counts.append(0)\n        counts.append(len(list(elements)))\n    return rle\n\n\ndef read_img(image_id, color, train_or_test=\"train\"):\n    filename = f\"{INPUT}/{train_or_test}/{image_id}_{color}.png\"\n    assert os.path.exists(filename), f\"not found {filename}\"\n    img = cv2.imread(filename, cv2.IMREAD_UNCHANGED)\n    if img.dtype == \"uint16\":\n        img = (img / 256).astype(\"uint8\")\n    return img\n\n\ndef load_RGB_image(image_id, train_or_test=\"test\"):\n    red = read_img(image_id, \"red\", train_or_test)\n    green = read_img(image_id, \"green\", train_or_test)\n    blue = read_img(image_id, \"blue\", train_or_test)\n    # using rgb only here\n    # yellow = read_img(image_id, \"yellow\", train_or_test, image_size)\n    stacked_images = np.transpose(np.array([red, green, blue]), (1, 2, 0))\n    return stacked_images\n\n\ndef load_RGBY_image(image_id, train_or_test=\"test\"):\n    red = read_img(image_id, \"red\", train_or_test)\n    green = read_img(image_id, \"green\", train_or_test)\n    blue = read_img(image_id, \"blue\", train_or_test)\n    # using rgb only here\n    yellow = read_img(image_id, \"yellow\", train_or_test)\n    stacked_images = np.transpose(np.array([red, green, blue, yellow]), (1, 2, 0))\n    return stacked_images\n\n\ndef print_masked_img(image_id, mask):\n    img = load_RGB_image(image_id, \"test\")\n\n    plt.figure(figsize=(15, 15))\n    plt.subplot(1, 3, 1)\n    plt.imshow(img)\n    plt.title(\"Image\")\n    plt.axis(\"off\")\n\n    plt.subplot(1, 3, 2)\n    plt.imshow(mask)\n    plt.title(\"Mask\")\n    plt.axis(\"off\")\n\n    plt.subplot(1, 3, 3)\n    plt.imshow(img)\n    plt.imshow(mask, alpha=0.6)\n    plt.title(\"Image + Mask\")\n    plt.axis(\"off\")\n    plt.show()\n\n\ndef split_list(l, n):\n    for idx in range(0, len(l), n):\n        yield l[idx : idx + n]\n\n\n# =============================================================================\n# Transforms\n# =============================================================================\ndef get_transforms():\n    return A.Compose(\n        [\n            A.Resize(IMAGE_SIZE, IMAGE_SIZE),\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406, 0.456], std=[0.229, 0.224, 0.225, 0.225],\n            ),\n            ToTensorV2(),\n        ]\n    )\n\n\n# =============================================================================\n# Dataset\n# =============================================================================\ndef load_bmask(cell_mask_dir, image_id, cell_id):\n    mask = np.load(f\"{cell_mask_dir}/{image_id}.npz\")[\"arr_0\"]\n    bmask = mask == cell_id\n    return bmask * 1\n\n\nclass MyDataset(Dataset):\n    def __init__(self, df, mode, w_mask=False):\n        self.df = df.reset_index(drop=True)\n        self.mode = mode\n        self.w_mask = w_mask\n        self.transform = get_transforms()\n\n        if self.mode in [\"train\", \"valid\"]:\n            self.targets = self.df[COLS_TARGET].values\n            self.cell_mask_dir = MASK_DIR / \"train\" / \"cell\"\n        else:\n            self.cell_mask_dir = MASK_DIR / \"test\" / \"cell\"\n\n    def crop(self, image, idx):\n        y0, x0, y1, x1 = self.df.loc[idx, [\"y0\", \"x0\", \"y1\", \"x1\"]].values.astype(int)\n\n        y0 = max(0, y0 - MARGIN)\n        x0 = max(0, x0 - MARGIN)\n        y1 = min(image.shape[0], y1 + MARGIN)\n        x1 = min(image.shape[1], x1 + MARGIN)\n\n        image = image[y0:y1, x0:x1]\n        return image\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        image_id = self.df.loc[idx, \"image_id\"]\n\n        image = load_RGBY_image(image_id, \"test\")\n        image = self.crop(image, idx)\n\n        if self.w_mask:\n            cell_id = self.df.loc[idx, \"cell_id\"]\n            bmask = load_bmask(self.cell_mask_dir, image_id, cell_id)\n            bmask = self.crop(bmask, idx)\n\n            image = image * np.stack([bmask] * IN_CHANS, 2)\n\n        else:\n            pass\n\n        augmented = self.transform(image=image.astype(\"uint8\"))\n        image = augmented[\"image\"]\n\n        if self.mode in [\"train\", \"valid\"]:\n            targets = self.df.loc[idx, COLS_TARGET].values\n            return image, torch.FloatTensor(targets.astype(\"float32\"))\n        else:\n            return image\n\n\n# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n# image level\n# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n\ndef get_transforms_image():\n    return A.Compose(\n        [\n            A.Resize(768, 768),\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406, 0.456], std=[0.229, 0.224, 0.225, 0.225],\n            ),\n            ToTensorV2(),\n        ]\n    )\n\n\nclass ImageDataset(Dataset):\n    def __init__(self, df):\n        self.df = df.reset_index(drop=True)\n        self.transform = get_transforms_image()\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        image_id = self.df.loc[idx, \"image_id\"]\n\n        image = load_RGBY_image(image_id, \"test\")\n\n        augmented = self.transform(image=image.astype(\"uint8\"))\n        image = augmented[\"image\"]\n\n        return image\n\n# =============================================================================\n# Network\n# =============================================================================\ndef get_model(model_path):\n    model = timm.create_model(\n        BACKBONE_NAME, pretrained=False, in_chans=IN_CHANS, num_classes=19\n    )\n    load_model(model_path, model, GPU)\n    return model.cuda(GPU)\n\n\ndef load_model(model_path, model, rank):\n    # print(f\"loading... {model_path}\")\n    model.load_state_dict(torch.load(model_path, map_location=torch.device(rank)))\n    return\n\n\ndef predict(models, test_loader, rank):\n    [model.eval() for model in models]\n    pred_list1 = []\n    with torch.no_grad():\n        for images in tqdm(test_loader):\n            images = images.cuda(rank)\n            pred_list2 = []\n            for model in models:\n                preds = model(images).cpu().sigmoid()\n\n                pred_list2.append(preds.numpy())\n\n            pred_list1.append(np.stack(pred_list2, 0).mean(0))\n\n        preds = np.concatenate(pred_list1)\n    # torch.cuda.empty_cache()\n    return preds\n\n\ndef predict_weighted(models, test_loader, rank, weights:np.ndarray=None):\n    [model.eval() for model in models]\n    if weights is None:\n        weights = np.ones(len(models)).reshape(-1 , 1, 1)\n    pred_list1 = []\n    with torch.no_grad():\n        for images in tqdm(test_loader):\n            images = images.cuda(rank)\n            pred_list2 = []\n            for i, model in enumerate(models):\n                preds = model(images).cpu().sigmoid().numpy()\n                if i < LEN_1ST:\n                    preds[:, 11] = 0\n                pred_list2.append(preds)\n            stack = np.stack(pred_list2, 0)\n            preds = np.multiply(stack, weights).mean(0)\n            pred_list1.append(preds)\n        preds = np.concatenate(pred_list1)\n    # torch.cuda.empty_cache()\n    return preds\n\n\ndef get_test_shortage(test):\n    test_shortage = sample_submission[~sample_submission[\"ID\"].isin(test[\"image_id\"])]\n    return test_shortage\n\n\ndef write_submission(test, fill_shortage=False):\n    if test.isnull().sum().sum() > 0:\n        sample_submission.to_csv(\"submission.csv\", index=False)\n        return\n\n    gr = test[[\"image_id\", \"w\", \"h\", \"encoded_mask\"] + COLS_TARGET].groupby(\"image_id\")\n\n    if fill_shortage:\n        test_shortage = get_test_shortage(test)\n        len_test = test[\"image_id\"].nunique()\n        len_test_shortage = test_shortage[\"ID\"].nunique()\n        print(f\"fill shortage: {len_test} => {len_test + len_test_shortage}\")\n        test_shortage = get_test_shortage(test)\n    else:\n        test_shortage = []\n        if len(sample_submission) == 559:\n            pass\n        elif sample_submission[\"ID\"].isin(test[\"image_id\"]).mean() != 1.0:\n            sample_submission.to_csv(\"submission.csv\", index=False)\n            return\n\n    classes = list(map(str, range(19)))\n\n    with open(\"submission.csv\", \"w\") as outf:\n        print(\"ID,ImageWidth,ImageHeight,PredictionString\", file=outf)\n        for image_id, df in gr:\n            if len(df) == 0:\n                continue\n            w = df.iloc[0, 1]\n            h = df.iloc[0, 2]\n\n            pred_strs = []\n            for i, row in df.iterrows():\n                cnfs = [row[c] for c in COLS_TARGET]\n                emasks = [row[\"encoded_mask\"]] * len(cnfs)\n                pred_strs += list(zip(classes, cnfs, emasks))\n\n            pred_strs = sorted(pred_strs, key=itemgetter(1), reverse=True)\n            pred_strs = \" \".join(map(lambda x: f\"{x[0]} {x[1]} {x[2]}\", pred_strs))\n\n            print(f\"{image_id},{w},{h},{pred_strs}\", file=outf)\n\n        if len(test_shortage) > 0:\n            for i, row in test_shortage.iterrows():\n                print(\n                    f\"{row['ID']},{row['ImageWidth']},{row['ImageHeight']},{row['PredictionString']}\",\n                    file=outf,\n                )\n\n    return\n\n\ndef post_process1(test):\n    len1 = len(test)\n\n    keep_condition = (\n        (test[\"cell_area_ratio\"] > KEEP_CELL_AREA_MIN) &\n        (test[\"nuc_area_ratio\"] > KEEP_NUC_AREA_MIN)\n    )\n    test = test[keep_condition].reset_index(drop=True)\n\n    # edge\n    edge = (\n        (test['y0'].between(0,1)) |\n        (test['h'] - test['y1']).between(0,1) |\n        (test['x0'].between(0,1)) |\n        (test['w'] - test['x1']).between(0,1)\n    )\n    drop_condition = (edge &\n      (test[\"cell_area_ratio\"] < KEEP_EDGE_CELL_AREA_MIN)\n      )\n    test = test[~drop_condition].reset_index(drop=True)\n\n    len2 = len(test)\n\n    print(f\"remove cell with \")\n    print(f\"cell_area_ratio({KEEP_CELL_AREA_MIN:.6f}) and \")\n    print(f\"nuc_area_ratio ({KEEP_NUC_AREA_MIN:.6f}), \")\n    print(f\"small cell in edge({KEEP_EDGE_CELL_AREA_MIN:.6f}), \")\n    print(f\"{len1} => {len2}\")\n\n    return test\n\n\ndef post_process2(test):\n    condition = (test[\"nuc_area_ratio\"] / test[\"cell_area_ratio\"]) < NUC_AREA_MIN_0to5\n\n    test.loc[condition, COLS_TARGET[:6]] = test.loc[condition, COLS_TARGET[:6]] * 0.5\n\n    print(\n        f\"decrease conf with NUC_AREA_MIN_0to5({NUC_AREA_MIN_0to5:.6f}) / \"\n        f\"subject to update: {condition.sum()}\"\n    )\n\n    return test\n\n\n# def post_process3(test):\n#     # groupby image id -> mean\n#     image_mean = test.groupby(\"image_id\")[COLS_TARGET].transform('mean')\n#     image_mean[\"label_11\"] = test[\"label_11\"]\n#     w_cell = WEIGHT_CELL_LEVEL_VS_IMAGE_MEAN[0]\n#     w_image = WEIGHT_CELL_LEVEL_VS_IMAGE_MEAN[1]\n\n#     test[COLS_TARGET] = w_cell * test[COLS_TARGET] + w_image * image_mean\n\n#     print(\n#         \"Add image level prediction (aggregating-average of image_id) to cell level prediction\"\n#     )\n#     print(f\"cell-level-pred : image-level-pred = {w_cell} : {w_image}\")\n\n#     return test\n\n\ndef post_process3(test, test_image):\n    # groupby image id -> mean\n    image_mean = test.groupby(\"image_id\")[COLS_TARGET].transform('mean')\n    image_mean[\"label_11\"] = test[\"label_11\"]\n\n    # match shape of image_level_prediction to cell_level_prediction\n    image_pred = test.loc[:, [\"image_id\"]].merge(test_image, on=\"image_id\", how=\"left\")[COLS_TARGET]\n    image_pred[\"label_11\"] = test[\"label_11\"]\n\n    w_cell = WEIGHT_CELL_LEVEL_VS_IMAGE_MEAN_VS_IMAGE_PRED[0]\n    w_image_mean = WEIGHT_CELL_LEVEL_VS_IMAGE_MEAN_VS_IMAGE_PRED[1]\n    w_image_pred = WEIGHT_CELL_LEVEL_VS_IMAGE_MEAN_VS_IMAGE_PRED[2]\n\n    test[COLS_TARGET] = w_cell * test[COLS_TARGET] + w_image_mean * image_mean + w_image_pred * image_pred\n\n    print(\n        \"Add image level prediction (aggregating-average of image_id) to cell level prediction\"\n    )\n    print(f\"cell-level-pred : image-level-mean : image-level-pred = {w_cell} : {w_image_mean} : {w_image_pred}\")\n\n    return test","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1st Stage","metadata":{}},{"cell_type":"code","source":"cmd = \"python ../input/hpa2021-libs/generate_test_mask_kernel_faster7.py \"\n\nif COMPUTE_PUBLIC:\n    cmd += \"--compute_public \"\nif COMPUTE_PRIVATE:\n    cmd += \"--compute_private \"\n\ncmd","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! {cmd}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(MASK_DIR / \"test_bbox.csv\")\ntest","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = post_process1(test)\ntest","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image level prediction","metadata":{}},{"cell_type":"code","source":"# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n# image level prediction\n# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n\ntest_image = test.groupby(\"image_id\").first().reset_index()\n\nif LOCAL:\n    pass\n\nelse:\n\n    test_loader = DataLoader(\n        ImageDataset(test_image),\n        batch_size=BATCH_SIZE,\n        num_workers=MAX_THRE,\n        pin_memory=True,\n    )\n\n    models = []\n    for p in MODEL_PATHS_IMAGE:\n        model = get_model(p)\n        model.eval()\n        models.append(model)\n\n    preds = predict(models, test_loader, GPU)\n\n    display(pd.crosstab(test_image[\"image_id\"], preds.argmax(1)))\n\n    test_image[COLS_TARGET] = preds\n\n    del models, model, test_loader\n    torch.cuda.empty_cache()\n    gc.collect()\n\n    display(test_image)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2nd Stage","metadata":{}},{"cell_type":"code","source":"if LOCAL:\n    pass\n\nelse:\n\n    test_loader = DataLoader(\n        MyDataset(test, mode=\"test\", w_mask=W_MASK),\n        batch_size=BATCH_SIZE,\n        num_workers=MAX_THRE,\n        pin_memory=True,\n    )\n\n    models = []\n    for p in MODEL_PATHS:\n        model = get_model(p)\n        model.eval()\n        models.append(model)\n\n    preds = predict_weighted(models, test_loader, GPU, WEIGHT_1ST_2ND)\n\n    display(pd.crosstab(test[\"image_id\"], preds.argmax(1)))\n\n    test[COLS_TARGET] = preds\n    test = post_process2(test)\n    test = post_process3(test, test_image)\n\n    del models, model\n    torch.cuda.empty_cache()\n    gc.collect()\n\n    display(test)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm -rf ../temp","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check Submission","metadata":{}},{"cell_type":"code","source":"# =============================================================================\n# submission\n# =============================================================================\ncols = sample_submission.columns\n\n\nif COMPUTE_PUBLIC is False and COMPUTE_PRIVATE is False:\n\n    print(\"dryrun, replace with local computed file\")\n    # ===========================\n    # cell level 2nd\n    # ===========================\n    test_2nd = pd.read_csv(TEST_LOCAL_COMPUTED_PATHS[0])\n    for p in TEST_LOCAL_COMPUTED_PATHS[1:]:\n        test_2nd.loc[:, COLS_TARGET] += pd.read_csv(p).loc[:, COLS_TARGET]\n    test_2nd.loc[:, COLS_TARGET] /= len(TEST_LOCAL_COMPUTED_PATHS)\n    \n    # ===========================\n    # cell level 1st\n    # ===========================\n    test_1st = pd.read_csv(TEST_LOCAL_COMPUTED_PATHS_1ST[0])\n    for p in TEST_LOCAL_COMPUTED_PATHS_1ST[1:]:\n        test_1st.loc[:, COLS_TARGET] += pd.read_csv(p).loc[:, COLS_TARGET]\n    test_1st.loc[:, COLS_TARGET] /= len(TEST_LOCAL_COMPUTED_PATHS_1ST)\n    \n    # ensemble 1st + 2nd\n    test = test_2nd.copy()\n    test_1st.iloc[:, 11] = 0\n    test.loc[:, COLS_TARGET] = test_1st.loc[:, COLS_TARGET] * RATE_OF_WEIGHT_1ST_2ND[0] + test_2nd.loc[:, COLS_TARGET] * RATE_OF_WEIGHT_1ST_2ND[1]\n    \n    # image level\n    test_image = pd.read_csv(TEST_LOCAL_COMPUTED_PATHS_IMAGE[0])\n    for p in TEST_LOCAL_COMPUTED_PATHS_IMAGE[1:]:\n        test_image.loc[:, COLS_TARGET] += pd.read_csv(p).loc[:, COLS_TARGET]\n    test_image.loc[:, COLS_TARGET] /= len(TEST_LOCAL_COMPUTED_PATHS_IMAGE)\n        \n    test = post_process1(test)\n    test = post_process2(test)\n    test = post_process3(test, test_image)\n\n    write_submission(test, fill_shortage=True)\n\n\nelif COMPUTE_PUBLIC is False and COMPUTE_PRIVATE is True:\n\n    print(\"only private\")\n    \n    # ===========================\n    # cell level 2nd\n    # ===========================\n    test_2nd = pd.read_csv(TEST_LOCAL_COMPUTED_PATHS[0])\n    for p in TEST_LOCAL_COMPUTED_PATHS[1:]:\n        test_2nd.loc[:, COLS_TARGET] += pd.read_csv(p).loc[:, COLS_TARGET]\n    test_2nd.loc[:, COLS_TARGET] /= len(TEST_LOCAL_COMPUTED_PATHS)\n    \n    # ===========================\n    # cell level 1st\n    # ===========================\n    test_1st = pd.read_csv(TEST_LOCAL_COMPUTED_PATHS_1ST[0])\n    for p in TEST_LOCAL_COMPUTED_PATHS_1ST[1:]:\n        test_1st.loc[:, COLS_TARGET] += pd.read_csv(p).loc[:, COLS_TARGET]\n    test_1st.loc[:, COLS_TARGET] /= len(TEST_LOCAL_COMPUTED_PATHS_1ST)\n    \n    # ensemble 1st + 2nd\n    test_local = test_2nd.copy()\n    test_1st.iloc[:, 11] = 0\n    test_local.loc[:, COLS_TARGET] = test_1st.loc[:, COLS_TARGET] * RATE_OF_WEIGHT_1ST_2ND[0] + test_2nd.loc[:, COLS_TARGET] * RATE_OF_WEIGHT_1ST_2ND[1]\n    \n    # image level\n    test_image_local = pd.read_csv(TEST_LOCAL_COMPUTED_PATHS_IMAGE[0])\n    for p in TEST_LOCAL_COMPUTED_PATHS_IMAGE[1:]:\n        test_image_local.loc[:, COLS_TARGET] += pd.read_csv(p).loc[:, COLS_TARGET]\n    test_image_local.loc[:, COLS_TARGET] /= len(TEST_LOCAL_COMPUTED_PATHS_IMAGE)\n    \n    test_local = post_process1(test_local)\n    test_local = post_process2(test_local)\n    test_local = post_process3(test_local, test_image_local)\n\n    if len(sample_submission) == 559:\n        write_submission(test_local, fill_shortage=False)\n\n    else:\n        test2 = pd.concat([test_local, test[test_local.columns]], ignore_index=True)\n        write_submission(test2, fill_shortage=False)\n\n\nelif COMPUTE_PUBLIC is True and COMPUTE_PRIVATE is True:\n    print(\"full compute\")\n    write_submission(test, fill_shortage=False)\n\n\nelif COMPUTE_PUBLIC is True and COMPUTE_PRIVATE is False:\n    print(\"only public\")\n    write_submission(test, fill_shortage=True)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(sample_submission) == 559:\n    sub = pd.read_csv(\"submission.csv\")\n    display(sub)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(sample_submission) == 559:\n    sub = pd.read_csv(\"submission.csv\")\n    for index, row in sub.head(3).iterrows():\n        image_id = row[\"ID\"]\n        w = row[\"ImageWidth\"]\n        h = row[\"ImageHeight\"]\n        pred_strs = row[\"PredictionString\"].split()\n        pred_strs = list(split_list(pred_strs, 3))\n        for i, pred in enumerate(pred_strs):\n            class_id, cnf, encoded_mask = pred\n            class_id = int(class_id)\n            cnf = float(cnf)\n\n            print(f\"class_id:{class_id}, image_id:{image_id}, confidence:{cnf}\")\n            mask = decode_binary_mask(encoded_mask, w, h)\n            print_masked_img(image_id, mask)\n            if i == 9:\n                break\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -lh","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}