{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"},{"sourceId":7392733,"sourceType":"datasetVersion","datasetId":4297749}],"dockerImageVersionId":30636,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About\n\nIn this notebook, I'll share an image clacification approach for given spectrograms.  \n\nI tried several experiments, but didn't obtain good results :(\n\n\n* **version 1**: naive approach\n* **version 2**: For comparing with [Chris's EfficientNetB2 Starter](https://www.kaggle.com/code/cdeotte/efficientnetb2-starter-lb-0-57), I added **log transform** and **LR scheduling**.\n\n## Experimental Settings\n\n### model\n* backbone: resnet34d (use the pretrained model provided by [timm](https://github.com/huggingface/pytorch-image-models))\n* head classifier: one linear layer\n* num of input channels: 1\n\n### data augmentation¶\n* implemented by [albumentations](https://albumentations.ai/)\n* Train\n    * Resize\n* Val, Test\n    * Resize\n    \n### learning settings\n* CV Strategy: Stratified Group KFold (K=5)\n    * y: `expert_consensus`\n    * group: `patient_id`\n* max epochs: 9\n* data:\n    * input image size: 1x512x512\n    * batch size: 32\n* loss: [KLDivLoss](https://pytorch.org/docs/stable/generated/torch.nn.KLDivLoss.html)\n* optimizer: AdamW\n    * learning rate: 1.0e-03\n    * weight decay: 1.0e-02\n    \n* lr scheduler: OneCycleLR\n    * max lr: 1.0e-03\n    * min lr: 1.0e-04\n    \n### NOTE: I normalized spectrograms per image\n```python\nimg = np.load(path)  # shape: (Hz, Time) = (400, 300)\neps = 1e-6\nimg_mean = img.mean(axis=(0, 1))\nimg = img - img_mean\nimg_std = img.std(axis=(0, 1))\nimg = img / (img_std + eps)\n```","metadata":{}},{"cell_type":"markdown","source":"# Prepare","metadata":{}},{"cell_type":"markdown","source":"## import","metadata":{}},{"cell_type":"code","source":"import sys\nimport os\nimport gc\nimport copy\nimport yaml\nimport random\nimport shutil\nfrom time import time\nimport typing as tp\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import StratifiedGroupKFold\n\nimport torch\nfrom torch import nn\nfrom torch import optim\nfrom torch.optim import lr_scheduler\nfrom torch.cuda import amp\n\nimport timm\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-01-24T03:48:28.205937Z","iopub.execute_input":"2024-01-24T03:48:28.206210Z","iopub.status.idle":"2024-01-24T03:48:35.654462Z","shell.execute_reply.started":"2024-01-24T03:48:28.206186Z","shell.execute_reply":"2024-01-24T03:48:35.653669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"","metadata":{"execution":{"iopub.status.busy":"2024-01-24T03:48:35.655861Z","iopub.execute_input":"2024-01-24T03:48:35.656297Z","iopub.status.idle":"2024-01-24T03:48:35.660479Z","shell.execute_reply.started":"2024-01-24T03:48:35.656253Z","shell.execute_reply":"2024-01-24T03:48:35.659571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT = Path.cwd().parent\nINPUT = ROOT / \"input\"\nOUTPUT = ROOT / \"output\"\nSRC = ROOT / \"src\"\n\nDATA = INPUT / \"hms-harmful-brain-activity-classification\"\nTRAIN_SPEC = DATA / \"train_spectrograms\"\nTEST_SPEC = DATA / \"test_spectrograms\"\n\nTMP = ROOT / \"tmp\"\nTRAIN_SPEC_SPLIT = TMP / \"train_spectrograms_split\"\nTEST_SPEC_SPLIT = TMP / \"test_spectrograms_split\"\nTMP.mkdir(exist_ok=True)\nTRAIN_SPEC_SPLIT.mkdir(exist_ok=True)\nTEST_SPEC_SPLIT.mkdir(exist_ok=True)\n\n\nRANDAM_SEED = 1086\nCLASSES = [\"seizure_vote\", \"lpd_vote\", \"gpd_vote\", \"lrda_vote\", \"grda_vote\", \"other_vote\"]\nN_CLASSES = len(CLASSES)\nFOLDS = [0, 1, 2, 3, 4]\nN_FOLDS = len(FOLDS)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-01-24T03:48:35.662247Z","iopub.execute_input":"2024-01-24T03:48:35.662566Z","iopub.status.idle":"2024-01-24T03:48:35.675216Z","shell.execute_reply.started":"2024-01-24T03:48:35.662536Z","shell.execute_reply":"2024-01-24T03:48:35.674338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read Data, Split Folds, Split Spectrograms","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(DATA / \"train.csv\")\n\n# convert vote to probability\ntrain[CLASSES] /= train[CLASSES].sum(axis=1).values[:, None]\n\nprint(train.shape)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-01-24T03:48:35.679175Z","iopub.execute_input":"2024-01-24T03:48:35.679644Z","iopub.status.idle":"2024-01-24T03:48:35.948464Z","shell.execute_reply.started":"2024-01-24T03:48:35.679612Z","shell.execute_reply":"2024-01-24T03:48:35.947525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### NOTE: I used the **first** `spectrogram_sub_id` for each `spectrogram_id` in order to train model faster.","metadata":{"execution":{"iopub.execute_input":"2024-01-14T00:22:27.085297Z","iopub.status.busy":"2024-01-14T00:22:27.084935Z","iopub.status.idle":"2024-01-14T00:22:27.093145Z","shell.execute_reply":"2024-01-14T00:22:27.091394Z","shell.execute_reply.started":"2024-01-14T00:22:27.085268Z"}}},{"cell_type":"code","source":"train = train.groupby(\"spectrogram_id\").head(1).reset_index(drop=True)\nprint(train.shape)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-01-24T03:48:35.949688Z","iopub.execute_input":"2024-01-24T03:48:35.950435Z","iopub.status.idle":"2024-01-24T03:48:35.967934Z","shell.execute_reply.started":"2024-01-24T03:48:35.950390Z","shell.execute_reply":"2024-01-24T03:48:35.967104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### split folds","metadata":{}},{"cell_type":"code","source":"sgkf = StratifiedGroupKFold(n_splits=N_FOLDS, shuffle=True, random_state=RANDAM_SEED)\n\ntrain[\"fold\"] = -1\n\nfor fold_id, (_, val_idx) in enumerate(\n    sgkf.split(train, y=train[\"expert_consensus\"], groups=train[\"patient_id\"])\n):\n    train.loc[val_idx, \"fold\"] = fold_id","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-01-24T03:48:35.968976Z","iopub.execute_input":"2024-01-24T03:48:35.969310Z","iopub.status.idle":"2024-01-24T03:48:36.939620Z","shell.execute_reply.started":"2024-01-24T03:48:35.969264Z","shell.execute_reply":"2024-01-24T03:48:36.938829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby(\"fold\")[CLASSES].sum()","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-01-24T03:48:36.940820Z","iopub.execute_input":"2024-01-24T03:48:36.941163Z","iopub.status.idle":"2024-01-24T03:48:36.963219Z","shell.execute_reply.started":"2024-01-24T03:48:36.941132Z","shell.execute_reply":"2024-01-24T03:48:36.962408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### split sepectogram files","metadata":{}},{"cell_type":"code","source":"for spec_id, df in tqdm(train.groupby(\"spectrogram_id\")):\n    spec = pd.read_parquet(TRAIN_SPEC / f\"{spec_id}.parquet\")\n    \n    spec_arr = spec.fillna(0).values[:, 1:].T.astype(\"float32\")  # (Hz, Time) = (400, 300)\n    \n    for spec_offset, label_id in df[\n        [\"spectrogram_label_offset_seconds\", \"label_id\"]\n    ].astype(int).values:\n        spec_offset = spec_offset // 2\n        split_spec_arr = spec_arr[:, spec_offset: spec_offset + 300]\n        np.save(TRAIN_SPEC_SPLIT / f\"{label_id}.npy\" , split_spec_arr)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-01-24T03:48:36.964263Z","iopub.execute_input":"2024-01-24T03:48:36.964591Z","iopub.status.idle":"2024-01-24T03:57:07.763352Z","shell.execute_reply.started":"2024-01-24T03:48:36.964565Z","shell.execute_reply":"2024-01-24T03:57:07.762261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Difinition, Model, Dataset, Metric","metadata":{}},{"cell_type":"markdown","source":"### model","metadata":{}},{"cell_type":"code","source":"class HMSHBACSpecModel(nn.Module):\n\n    def __init__(\n            self,\n            model_name: str,\n            pretrained: bool,\n            in_channels: int,\n            num_classes: int,\n        ):\n        super().__init__()\n        self.model = timm.create_model(\n            model_name=model_name, pretrained=pretrained,\n            num_classes=num_classes, in_chans=in_channels)\n\n    def forward(self, x):\n        h = self.model(x)      \n\n        return h","metadata":{"_kg_hide-input":false,"tags":[],"execution":{"iopub.status.busy":"2024-01-24T03:57:07.764649Z","iopub.execute_input":"2024-01-24T03:57:07.764982Z","iopub.status.idle":"2024-01-24T03:57:07.771602Z","shell.execute_reply.started":"2024-01-24T03:57:07.764953Z","shell.execute_reply":"2024-01-24T03:57:07.770592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### dataset","metadata":{}},{"cell_type":"code","source":"FilePath = tp.Union[str, Path]\nLabel = tp.Union[int, float, np.ndarray]\n\nclass HMSHBACSpecDataset(torch.utils.data.Dataset):\n\n    def __init__(\n        self,\n        image_paths: tp.Sequence[FilePath],\n        labels: tp.Sequence[Label],\n        transform: A.Compose,\n    ):\n        self.image_paths = image_paths\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_paths)\n\n    def __getitem__(self, index: int):\n        img_path = self.image_paths[index]\n        label = self.labels[index]\n\n        img = np.load(img_path)  # shape: (Hz, Time) = (400, 300)\n        # log transform\n        img = np.clip(img,np.exp(-4), np.exp(8))\n        img = np.log(img)\n        \n        # normalize per image\n        eps = 1e-6\n        img_mean = img.mean(axis=(0, 1))\n        img = img - img_mean\n        img_std = img.std(axis=(0, 1))\n        img = img / (img_std + eps)\n\n        img = img[..., None] # shape: (Hz, Time) -> (Hz, Time, Channel)\n        img = self._apply_transform(img)\n\n        return {\"data\": img, \"target\": label}\n\n    def _apply_transform(self, img: np.ndarray):\n        \"\"\"apply transform to image and mask\"\"\"\n        transformed = self.transform(image=img)\n        img = transformed[\"image\"]\n        return img","metadata":{"_kg_hide-input":false,"tags":[],"execution":{"iopub.status.busy":"2024-01-24T03:57:07.776016Z","iopub.execute_input":"2024-01-24T03:57:07.776343Z","iopub.status.idle":"2024-01-24T03:57:07.921502Z","shell.execute_reply.started":"2024-01-24T03:57:07.776308Z","shell.execute_reply":"2024-01-24T03:57:07.920546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### loss","metadata":{}},{"cell_type":"code","source":"class KLDivLossWithLogits(nn.KLDivLoss):\n\n    def __init__(self):\n        super().__init__(reduction=\"batchmean\")\n\n    def forward(self, y, t):\n        y = nn.functional.log_softmax(y,  dim=1)\n        loss = super().forward(y, t)\n\n        return loss\n\n\nclass KLDivLossWithLogitsForVal(nn.KLDivLoss):\n    \n    def __init__(self):\n        \"\"\"\"\"\"\n        super().__init__(reduction=\"batchmean\")\n        self.log_prob_list  = []\n        self.label_list = []\n\n    def forward(self, y, t):\n        y = nn.functional.log_softmax(y, dim=1)\n        self.log_prob_list.append(y.numpy())\n        self.label_list.append(t.numpy())\n        \n    def compute(self):\n        log_prob = np.concatenate(self.log_prob_list, axis=0)\n        label = np.concatenate(self.label_list, axis=0)\n        final_metric = super().forward(\n            torch.from_numpy(log_prob),\n            torch.from_numpy(label)\n        ).item()\n        self.log_prob_list = []\n        self.label_list = []\n        \n        return final_metric","metadata":{"_kg_hide-input":false,"tags":[],"execution":{"iopub.status.busy":"2024-01-24T03:57:07.922510Z","iopub.execute_input":"2024-01-24T03:57:07.922770Z","iopub.status.idle":"2024-01-24T03:57:07.937105Z","shell.execute_reply.started":"2024-01-24T03:57:07.922747Z","shell.execute_reply":"2024-01-24T03:57:07.936200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"class CFG:\n    model_name = \"resnet34d\"\n    img_size = 512\n    max_epoch = 1\n    batch_size = 32\n    lr = 1.0e-03\n    weight_decay = 1.0e-02\n    es_patience =  5\n    seed = 1086\n    deterministic = True\n    enable_amp = True\n    device = \"cuda\"","metadata":{"execution":{"iopub.status.busy":"2024-01-24T03:57:07.938359Z","iopub.execute_input":"2024-01-24T03:57:07.938755Z","iopub.status.idle":"2024-01-24T03:57:07.951257Z","shell.execute_reply.started":"2024-01-24T03:57:07.938722Z","shell.execute_reply":"2024-01-24T03:57:07.950336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Functions for training","metadata":{}},{"cell_type":"code","source":"def set_random_seed(seed: int = 42, deterministic: bool = False):\n    \"\"\"Set seeds\"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)  # type: ignore\n    torch.backends.cudnn.deterministic = deterministic  # type: ignore\n    \ndef to_device(\n    tensors: tp.Union[tp.Tuple[torch.Tensor], tp.Dict[str, torch.Tensor]],\n    device: torch.device, *args, **kwargs\n):\n    if isinstance(tensors, tuple):\n        return (t.to(device, *args, **kwargs) for t in tensors)\n    elif isinstance(tensors, dict):\n        return {\n            k: t.to(device, *args, **kwargs) for k, t in tensors.items()}\n    else:\n        return tensors.to(device, *args, **kwargs)","metadata":{"_kg_hide-input":false,"tags":[],"execution":{"iopub.status.busy":"2024-01-24T03:57:07.952424Z","iopub.execute_input":"2024-01-24T03:57:07.954123Z","iopub.status.idle":"2024-01-24T03:57:07.963577Z","shell.execute_reply.started":"2024-01-24T03:57:07.954096Z","shell.execute_reply":"2024-01-24T03:57:07.962818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_path_label(val_fold, train_all: pd.DataFrame):\n    \"\"\"Get file path and target info.\"\"\"\n    \n    train_idx = train_all[train_all[\"fold\"] != val_fold].index.values\n    val_idx   = train_all[train_all[\"fold\"] == val_fold].index.values\n    img_paths = []\n    labels = train_all[CLASSES].values\n    for label_id in train_all[\"label_id\"].values:\n        img_path = TRAIN_SPEC_SPLIT / f\"{label_id}.npy\"\n        img_paths.append(img_path)\n\n    train_data = {\n        \"image_paths\": [img_paths[idx] for idx in train_idx],\n        \"labels\": [labels[idx].astype(\"float32\") for idx in train_idx]}\n\n    val_data = {\n        \"image_paths\": [img_paths[idx] for idx in val_idx],\n        \"labels\": [labels[idx].astype(\"float32\") for idx in val_idx]}\n    \n    return train_data, val_data, train_idx, val_idx\n\n\ndef get_transforms(CFG):\n    train_transform = A.Compose([\n        A.Resize(p=1.0, height=CFG.img_size, width=CFG.img_size),\n        ToTensorV2(p=1.0)\n    ])\n    val_transform = A.Compose([\n        A.Resize(p=1.0, height=CFG.img_size, width=CFG.img_size),\n        ToTensorV2(p=1.0)\n    ])\n    return train_transform, val_transform","metadata":{"_kg_hide-input":false,"tags":[],"execution":{"iopub.status.busy":"2024-01-24T03:57:07.964699Z","iopub.execute_input":"2024-01-24T03:57:07.965031Z","iopub.status.idle":"2024-01-24T03:57:07.974579Z","shell.execute_reply.started":"2024-01-24T03:57:07.965007Z","shell.execute_reply":"2024-01-24T03:57:07.973725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_one_fold(CFG, val_fold, train_all, output_path,fold_id):\n    \"\"\"Main\"\"\"\n    torch.backends.cudnn.benchmark = True\n    set_random_seed(CFG.seed, deterministic=CFG.deterministic)\n    device = torch.device(CFG.device)\n    \n    train_path_label, val_path_label, _, _ = get_path_label(val_fold, train_all)\n    train_transform, val_transform = get_transforms(CFG)\n    \n    train_dataset = HMSHBACSpecDataset(**train_path_label, transform=train_transform)\n    val_dataset = HMSHBACSpecDataset(**val_path_label, transform=val_transform)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_dataset, batch_size=CFG.batch_size, num_workers=4, shuffle=True, drop_last=True)\n    val_loader = torch.utils.data.DataLoader(\n        val_dataset, batch_size=CFG.batch_size, num_workers=4, shuffle=False, drop_last=False)\n    \n    model = HMSHBACSpecModel(\n        model_name=CFG.model_name, pretrained=True, num_classes=6, in_channels=1)\n    model.to(device)\n    \n    optimizer = optim.AdamW(params=model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay)\n    scheduler = lr_scheduler.OneCycleLR(\n        optimizer=optimizer, epochs=CFG.max_epoch,\n        pct_start=0.0, steps_per_epoch=len(train_loader),\n        max_lr=CFG.lr, div_factor=25, final_div_factor=4.0e-01\n    )\n    \n    loss_func = KLDivLossWithLogits()\n    loss_func.to(device)\n    loss_func_val = KLDivLossWithLogitsForVal()\n    \n    use_amp = CFG.enable_amp\n    scaler = amp.GradScaler(enabled=use_amp)\n    \n    best_val_loss = 1.0e+09\n    best_epoch = 0\n    train_loss = 0\n    \n    for epoch in range(1, CFG.max_epoch + 1):\n        epoch_start = time()\n        model.train()\n        bar = tqdm(enumerate(train_loader), total=len(train_loader))\n        bs = 0 \n        for step, batch in bar:\n            bs+=1\n#         for batch in train_loader:\n            batch = to_device(batch, device)\n            x, t = batch[\"data\"], batch[\"target\"]\n                \n            optimizer.zero_grad()\n            with amp.autocast(use_amp):\n                y = model(x)\n                loss = loss_func(y, t)\n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            scheduler.step()\n            train_loss += loss.item()\n            loss_show = float(train_loss/bs)\n#             bar.set_postfix(Epoch=epoch,LR=optimizer.param_groups[0]['lr'])\n            bar.set_postfix(Epoch=epoch, Train_Loss=loss_show, LR=optimizer.param_groups[0]['lr'])\n\n        train_loss /= len(train_loader)\n            \n        model.eval()\n        bar = tqdm(enumerate(val_loader), total=len(val_loader))\n        bs = 0 \n        for step, batch in bar:\n            bs+=1\n#         for batch in val_loader:\n            x, t = batch[\"data\"], batch[\"target\"]\n            x = to_device(x, device)\n            with torch.no_grad(), amp.autocast(use_amp):\n                y = model(x)\n            y = y.detach().cpu().to(torch.float32)\n            loss_func_val(y, t)\n#             val_loss1 = loss_func_val.compute() \n            bar.set_postfix(Epoch=epoch, LR=optimizer.param_groups[0]['lr'])\n        val_loss = loss_func_val.compute()        \n        if val_loss < best_val_loss:\n            best_epoch = epoch\n            best_val_loss = val_loss\n            # print(\"save model\")\n            torch.save(model.state_dict(), str(output_path / f'snapshot_epoch_{epoch}.pth'))\n        \n        elapsed_time = time() - epoch_start\n        print(\n            f\"[fold {fold_id}  epoch {epoch}] train loss: {train_loss: .6f}, val loss: {val_loss: .6f},best loss: {best_val_loss: .6f},best_epoch: {best_epoch}, elapsed_time: {elapsed_time: .3f}\")\n        \n        if epoch - best_epoch > CFG.es_patience:\n            print(\"Early Stopping!\")\n            break\n            \n        train_loss = 0\n            \n    return val_fold, best_epoch, best_val_loss","metadata":{"tags":[],"execution":{"iopub.status.busy":"2024-01-24T03:57:07.975799Z","iopub.execute_input":"2024-01-24T03:57:07.976333Z","iopub.status.idle":"2024-01-24T03:57:07.995729Z","shell.execute_reply.started":"2024-01-24T03:57:07.976273Z","shell.execute_reply":"2024-01-24T03:57:07.994850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Run Training","metadata":{}},{"cell_type":"code","source":"score_list = []\nfor fold_id in FOLDS:\n    output_path = Path(f\"fold{fold_id}\")\n    output_path.mkdir(exist_ok=True)\n    print(f\"[fold{fold_id}]\")\n    score_list.append(train_one_fold(CFG, fold_id, train, output_path,fold_id))","metadata":{"execution":{"iopub.status.busy":"2024-01-24T03:57:07.996736Z","iopub.execute_input":"2024-01-24T03:57:07.997075Z","iopub.status.idle":"2024-01-24T03:57:16.850713Z","shell.execute_reply.started":"2024-01-24T03:57:07.997049Z","shell.execute_reply":"2024-01-24T03:57:16.849121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference Out Of Fold","metadata":{}},{"cell_type":"markdown","source":"## Copy best models","metadata":{}},{"cell_type":"code","source":"print(score_list)","metadata":{"execution":{"iopub.status.busy":"2024-01-24T03:57:16.852093Z","iopub.status.idle":"2024-01-24T03:57:16.852468Z","shell.execute_reply.started":"2024-01-24T03:57:16.852271Z","shell.execute_reply":"2024-01-24T03:57:16.852304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_log_list = []\nfor (fold_id, best_epoch, _) in score_list:\n    \n    exp_dir_path = Path(f\"fold{fold_id}\")\n    best_model_path = exp_dir_path / f\"snapshot_epoch_{best_epoch}.pth\"\n    copy_to = f\"./best_model_fold{fold_id}.pth\"\n    shutil.copy(best_model_path, copy_to)\n    \n    for p in exp_dir_path.glob(\"*.pth\"):\n        p.unlink()","metadata":{"execution":{"iopub.status.busy":"2024-01-24T03:57:16.854193Z","iopub.status.idle":"2024-01-24T03:57:16.854550Z","shell.execute_reply.started":"2024-01-24T03:57:16.854385Z","shell.execute_reply":"2024-01-24T03:57:16.854401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference OOF","metadata":{}},{"cell_type":"code","source":"def run_inference_loop(model, loader, device):\n    model.to(device)\n    model.eval()\n    pred_list = []\n    with torch.no_grad():\n        for batch in tqdm(loader):\n            x = to_device(batch[\"data\"], device)\n            y = model(x)\n            pred_list.append(y.softmax(dim=1).detach().cpu().numpy())\n        \n    pred_arr = np.concatenate(pred_list)\n    del pred_list\n    return pred_arr","metadata":{"execution":{"iopub.status.busy":"2024-01-24T03:57:16.855704Z","iopub.status.idle":"2024-01-24T03:57:16.856018Z","shell.execute_reply.started":"2024-01-24T03:57:16.855856Z","shell.execute_reply":"2024-01-24T03:57:16.855877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_arr = train[CLASSES].values\noof_pred_arr = np.zeros((len(train), N_CLASSES))\nscore_list = []\n\nfor fold_id in range(N_FOLDS):\n    print(f\"\\n[fold {fold_id}]\")\n    device = torch.device(CFG.device)\n\n    # # get_dataloader\n    _, val_path_label, _, val_idx = get_path_label(fold_id, train)\n    _, val_transform = get_transforms(CFG)\n    val_dataset = HMSHBACSpecDataset(**val_path_label, transform=val_transform)\n    val_loader = torch.utils.data.DataLoader(\n        val_dataset, batch_size=CFG.batch_size, num_workers=4, shuffle=False, drop_last=False)\n    \n    # # get model\n    model_path = f\"./best_model_fold{fold_id}.pth\"\n    model = HMSHBACSpecModel(\n        model_name=CFG.model_name, pretrained=False, num_classes=6, in_channels=1)\n    model.load_state_dict(torch.load(model_path, map_location=device))\n    \n    # # inference\n    val_pred = run_inference_loop(model, val_loader, device)\n    oof_pred_arr[val_idx] = val_pred\n    \n    del val_idx, val_path_label\n    del model, val_loader\n    torch.cuda.empty_cache()\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-01-24T03:57:16.857667Z","iopub.status.idle":"2024-01-24T03:57:16.858003Z","shell.execute_reply.started":"2024-01-24T03:57:16.857833Z","shell.execute_reply":"2024-01-24T03:57:16.857849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculate OOF score","metadata":{}},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-01-24T03:57:16.859179Z","iopub.status.idle":"2024-01-24T03:57:16.859523Z","shell.execute_reply.started":"2024-01-24T03:57:16.859359Z","shell.execute_reply":"2024-01-24T03:57:16.859375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/kaggle-kl-div')\nfrom kaggle_kl_div import score\n\ntrue = train[[\"label_id\"] + CLASSES].copy()\n\noof = pd.DataFrame(oof_pred_arr, columns=CLASSES)\noof.insert(0, \"label_id\", train[\"label_id\"])\n\ncv_score = score(solution=true, submission=oof, row_id_column_name='label_id')\nprint('CV Score KL-Div for ResNet34d',cv_score)","metadata":{"execution":{"iopub.status.busy":"2024-01-24T03:57:16.859179Z","iopub.status.idle":"2024-01-24T03:57:16.859523Z","shell.execute_reply.started":"2024-01-24T03:57:16.859359Z","shell.execute_reply":"2024-01-24T03:57:16.859375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n\n\n\n\n# EOF","metadata":{}}]}