{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Data Loading","metadata":{}},{"cell_type":"code","source":"#amrica ka dalaal, baheenchod\nimport os\nimport pandas as pd\n\nOUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\nTRAIN_PATH = \"../input/decimal-damage-dataset/Damage_Detection/Train\"\nTEST_PATH = \"../input/decimal-damage-dataset/Damage_Detection/Test\"","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:36.753942Z","iopub.execute_input":"2022-08-22T19:08:36.754474Z","iopub.status.idle":"2022-08-22T19:08:36.760779Z","shell.execute_reply.started":"2022-08-22T19:08:36.754392Z","shell.execute_reply":"2022-08-22T19:08:36.759642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/decimal-damage-dataset/Damage_Detection/training_label.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:36.763407Z","iopub.execute_input":"2022-08-22T19:08:36.764071Z","iopub.status.idle":"2022-08-22T19:08:36.792860Z","shell.execute_reply.started":"2022-08-22T19:08:36.764034Z","shell.execute_reply":"2022-08-22T19:08:36.792123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_path(row):\n    return os.path.join(TRAIN_PATH, row['image'].split(\"/\")[1])\n\ntrain['path'] = train.apply(lambda row: get_path(row), axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:36.795336Z","iopub.execute_input":"2022-08-22T19:08:36.796014Z","iopub.status.idle":"2022-08-22T19:08:36.811675Z","shell.execute_reply.started":"2022-08-22T19:08:36.795973Z","shell.execute_reply":"2022-08-22T19:08:36.811019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:36.814352Z","iopub.execute_input":"2022-08-22T19:08:36.814795Z","iopub.status.idle":"2022-08-22T19:08:36.830406Z","shell.execute_reply.started":"2022-08-22T19:08:36.814746Z","shell.execute_reply":"2022-08-22T19:08:36.829689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CFG","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# CFG\n# ====================================================\nclass CFG:\n    debug=False\n    print_freq=500\n    num_workers=4\n    model_name='efficientnet_b4'\n    size=224\n    scheduler='CosineAnnealingWarmRestarts' # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts']\n    criterion='CrossEntropyLoss' # ['CrossEntropyLoss', LabelSmoothing', 'FocalLoss' 'FocalCosineLoss', 'SymmetricCrossEntropyLoss', 'BiTemperedLoss', 'TaylorCrossEntropyLoss']\n    epochs=80\n    #factor=0.2 # ReduceLROnPlateau\n    #patience=4 # ReduceLROnPlateau\n    #eps=1e-6 # ReduceLROnPlateau\n    #T_max=10 # CosineAnnealingLR\n    T_0=10 # CosineAnnealingWarmRestarts\n    lr=1e-4\n    min_lr=1e-6\n    batch_size=32\n    weight_decay=1e-6\n    gradient_accumulation_steps=1\n    max_grad_norm=1000\n    seed=42\n    target_size=4\n    target_col='extent_of_damage'\n    n_fold=5\n    trn_fold=[0,1,2,3,4] #[0,1]\n    train=True\n    smoothing=0.05\n    t1=0.3 # bi-tempered-loss https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/202017\n    t2=1.0 # bi-tempered-loss https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/202017\n    \nif CFG.debug:\n    CFG.epochs = 1\n    train = train.sample(n=1000, random_state=CFG.seed).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:36.833471Z","iopub.execute_input":"2022-08-22T19:08:36.833846Z","iopub.status.idle":"2022-08-22T19:08:36.842181Z","shell.execute_reply.started":"2022-08-22T19:08:36.833810Z","shell.execute_reply":"2022-08-22T19:08:36.841263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Library","metadata":{}},{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:36.843713Z","iopub.execute_input":"2022-08-22T19:08:36.844121Z","iopub.status.idle":"2022-08-22T19:08:43.090630Z","shell.execute_reply.started":"2022-08-22T19:08:36.844079Z","shell.execute_reply":"2022-08-22T19:08:43.089725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n\nimport os\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset, WeightedRandomSampler\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport timm\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:43.092402Z","iopub.execute_input":"2022-08-22T19:08:43.092760Z","iopub.status.idle":"2022-08-22T19:08:43.104358Z","shell.execute_reply.started":"2022-08-22T19:08:43.092727Z","shell.execute_reply":"2022-08-22T19:08:43.103281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n\ndef init_logger(log_file=OUTPUT_DIR+'train.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = init_logger()\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:43.107770Z","iopub.execute_input":"2022-08-22T19:08:43.108390Z","iopub.status.idle":"2022-08-22T19:08:43.121192Z","shell.execute_reply.started":"2022-08-22T19:08:43.108354Z","shell.execute_reply":"2022-08-22T19:08:43.120116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CV split","metadata":{}},{"cell_type":"code","source":"folds = train.copy()\nFold = StratifiedKFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds[CFG.target_col])):\n    folds.loc[val_index, 'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype(int)\nprint(folds.groupby(['fold', CFG.target_col]).size())","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:43.123798Z","iopub.execute_input":"2022-08-22T19:08:43.124383Z","iopub.status.idle":"2022-08-22T19:08:43.148081Z","shell.execute_reply.started":"2022-08-22T19:08:43.124347Z","shell.execute_reply":"2022-08-22T19:08:43.147176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\nclass TrainDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['path'].values\n        self.labels = df[CFG.target_col].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_path = self.file_names[idx]\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).long()\n        return image, label\n    \n\nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['path'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_path = self.file_names[idx]\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:43.149264Z","iopub.execute_input":"2022-08-22T19:08:43.149890Z","iopub.status.idle":"2022-08-22T19:08:43.161750Z","shell.execute_reply.started":"2022-08-22T19:08:43.149853Z","shell.execute_reply":"2022-08-22T19:08:43.160858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = TrainDataset(train, transform=None)\n\nfor i in range(2):\n    image, label = train_dataset[i]\n    plt.imshow(image)\n    plt.title(f'label: {label}')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:43.163125Z","iopub.execute_input":"2022-08-22T19:08:43.163787Z","iopub.status.idle":"2022-08-22T19:08:43.501392Z","shell.execute_reply.started":"2022-08-22T19:08:43.163752Z","shell.execute_reply":"2022-08-22T19:08:43.500449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transforms","metadata":{}},{"cell_type":"code","source":"# # ====================================================\n# # Transforms\n# # ====================================================\n# def get_transforms(*, data):\n    \n#     if data == 'train':\n#         return A.Compose([\n#             A.Resize(CFG.size, CFG.size),\n# #             A.RandomResizedCrop(CFG.size, CFG.size),\n# #             A.Transpose(p=0.5),\n#             A.HorizontalFlip(p=0.5),\n# #             A.VerticalFlip(p=0.5),\n#             A.ShiftScaleRotate(p=0.5),\n#             A.Normalize(\n#                 mean=[0.485, 0.456, 0.406],\n#                 std=[0.229, 0.224, 0.225],\n#             ),\n#             ToTensorV2(),\n#         ])\n\n#     elif data == 'valid':\n#         return A.Compose([\n#             A.Resize(CFG.size, CFG.size),\n#             A.Normalize(\n#                 mean=[0.485, 0.456, 0.406],\n#                 std=[0.229, 0.224, 0.225],\n#             ),\n#             ToTensorV2(),\n#         ])\n\ndef get_transforms(*, data):\n    \n    if data == 'train':\n        return A.Compose([\n            A.RandomCrop(200, 200),\n            A.Resize(CFG.size, CFG.size),\n#             A.Transpose(p=0.5),\n            A.HorizontalFlip(p=0.5),\n#             A.VerticalFlip(p=0.5),\n            A.ShiftScaleRotate(p=0.5, rotate_limit=45),\n            A.Cutout(num_holes=8, max_h_size=16, max_w_size=16),\n            A.OneOf([\n                  A.MotionBlur(p=1),\n                  A.OpticalDistortion(p=1),\n                  A.GaussNoise(p=1)                 \n            ], p=1),\n            \n            A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n\n    elif data == 'valid':\n        return A.Compose([\n            A.Resize(CFG.size, CFG.size),\n            A.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\ntrain_dataset = TrainDataset(train, transform=get_transforms(data='train'))\n\nfor i in range(2):\n    image, label = train_dataset[i]\n    plt.imshow(image[0])\n    plt.title(f'label: {label}')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:43.502920Z","iopub.execute_input":"2022-08-22T19:08:43.503263Z","iopub.status.idle":"2022-08-22T19:08:43.840109Z","shell.execute_reply.started":"2022-08-22T19:08:43.503227Z","shell.execute_reply":"2022-08-22T19:08:43.839257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = TrainDataset(train, transform=get_transforms(data='train'))\n\nfor i in range(2):\n    image, label = train_dataset[i]\n    plt.imshow(image[0])\n    plt.title(f'label: {label}')\n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:43.841257Z","iopub.execute_input":"2022-08-22T19:08:43.841723Z","iopub.status.idle":"2022-08-22T19:08:44.170782Z","shell.execute_reply.started":"2022-08-22T19:08:43.841688Z","shell.execute_reply":"2022-08-22T19:08:44.169928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# MODEL\n# ====================================================\nclass CustomEfficientNet(nn.Module):\n    def __init__(self, model_name=CFG.model_name, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(CFG.model_name, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \n\nclass CustomResNext(nn.Module):\n    def __init__(self, model_name='resnext50_32x4d', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.fc.in_features\n        self.model.fc = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:44.172182Z","iopub.execute_input":"2022-08-22T19:08:44.172747Z","iopub.status.idle":"2022-08-22T19:08:44.183696Z","shell.execute_reply.started":"2022-08-22T19:08:44.172705Z","shell.execute_reply":"2022-08-22T19:08:44.182818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CustomEfficientNet(model_name=CFG.model_name, pretrained=True)\ntrain_dataset = TrainDataset(train, transform=get_transforms(data='train'))\ntrain_loader = DataLoader(train_dataset, batch_size=4, shuffle=True,\n                          num_workers=4, pin_memory=True, drop_last=True)\n\n# for image, label in train_loader:\n#     output = model(image)\n#     print(output)\n#     break","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:44.185396Z","iopub.execute_input":"2022-08-22T19:08:44.185763Z","iopub.status.idle":"2022-08-22T19:08:44.743517Z","shell.execute_reply.started":"2022-08-22T19:08:44.185726Z","shell.execute_reply":"2022-08-22T19:08:44.742472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Criterion","metadata":{}},{"cell_type":"markdown","source":"## Label Smoothing","metadata":{"_kg_hide-input":true}},{"cell_type":"code","source":"# ====================================================\n# Label Smoothing\n# ====================================================\nclass LabelSmoothingLoss(nn.Module): \n    def __init__(self, classes=5, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-22T19:08:44.745796Z","iopub.execute_input":"2022-08-22T19:08:44.746113Z","iopub.status.idle":"2022-08-22T19:08:44.757495Z","shell.execute_reply.started":"2022-08-22T19:08:44.746083Z","shell.execute_reply":"2022-08-22T19:08:44.756556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Focal Loss","metadata":{}},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, reduce=True):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduce = reduce\n\n    def forward(self, inputs, targets):\n        BCE_loss = nn.CrossEntropyLoss()(inputs, targets)\n\n        pt = torch.exp(-BCE_loss)\n        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n\n        if self.reduce:\n            return torch.mean(F_loss)\n        else:\n            return F_loss","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:44.758906Z","iopub.execute_input":"2022-08-22T19:08:44.759248Z","iopub.status.idle":"2022-08-22T19:08:44.768953Z","shell.execute_reply.started":"2022-08-22T19:08:44.759212Z","shell.execute_reply":"2022-08-22T19:08:44.768158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper functions","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:44.770754Z","iopub.execute_input":"2022-08-22T19:08:44.771122Z","iopub.status.idle":"2022-08-22T19:08:44.778777Z","shell.execute_reply.started":"2022-08-22T19:08:44.771063Z","shell.execute_reply":"2022-08-22T19:08:44.778109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))\n\n\ndef train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to train mode\n    model.train()\n    start = end = time.time()\n    global_step = 0\n    for step, (images, labels) in tqdm(enumerate(train_loader), total = len(train_loader)):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        y_preds = model(images)\n        loss = criterion(y_preds, labels)\n        \n        losses.update(loss.item(), batch_size)\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps       \n        loss.backward()\n        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), CFG.max_grad_norm)\n        if (step + 1) % CFG.gradient_accumulation_steps == 0:\n            optimizer.step()\n            optimizer.zero_grad()\n            global_step += 1\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(train_loader)-1):\n            print('Epoch: [{0}][{1}/{2}] '\n                  'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  'Grad: {grad_norm:.4f}  '\n                  #'LR: {lr:.6f}  '\n                  .format(\n                   epoch+1, step, len(train_loader), batch_time=batch_time,\n                   data_time=data_time, loss=losses,\n                   remain=timeSince(start, float(step+1)/len(train_loader)),\n                   grad_norm=grad_norm,\n                   #lr=scheduler.get_lr()[0],\n                   ))\n    return losses.avg\n\n\ndef valid_fn(valid_loader, model, criterion, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to evaluation mode\n    model.eval()\n    preds = []\n    start = end = time.time()\n    for step, (images, labels) in enumerate(valid_loader):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        # compute loss\n        with torch.no_grad():\n            y_preds = model(images)\n        loss = criterion(y_preds, labels)\n        losses.update(loss.item(), batch_size)\n        # record accuracy\n        preds.append(y_preds.softmax(1).to('cpu').numpy())\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(valid_loader)-1):\n            print('EVAL: [{0}/{1}] '\n                  'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  .format(\n                   step, len(valid_loader), batch_time=batch_time,\n                   data_time=data_time, loss=losses,\n                   remain=timeSince(start, float(step+1)/len(valid_loader)),\n                   ))\n    predictions = np.concatenate(preds)\n    return losses.avg, predictions\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:44.780439Z","iopub.execute_input":"2022-08-22T19:08:44.780834Z","iopub.status.idle":"2022-08-22T19:08:44.811470Z","shell.execute_reply.started":"2022-08-22T19:08:44.780799Z","shell.execute_reply":"2022-08-22T19:08:44.810631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train loop","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Train loop\n# ====================================================\ndef train_loop(folds, fold):\n\n    LOGGER.info(f\"========== fold: {fold} training ==========\")\n\n    # ====================================================\n    # loader\n    # ====================================================\n    trn_idx = folds[folds['fold'] != fold].index\n    val_idx = folds[folds['fold'] == fold].index\n\n    train_folds = folds.loc[trn_idx].reset_index(drop=True)\n    valid_folds = folds.loc[val_idx].reset_index(drop=True)\n\n    train_dataset = TrainDataset(train_folds, \n                                 transform=get_transforms(data='train'))\n    valid_dataset = TrainDataset(valid_folds, \n                                 transform=get_transforms(data='valid'))\n\n    train_loader = DataLoader(train_dataset, \n                              batch_size=CFG.batch_size, \n                              shuffle=True, \n                              num_workers=CFG.num_workers, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, \n                              batch_size=CFG.batch_size, \n                              shuffle=False, \n                              num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n    \n    # ====================================================\n    # scheduler \n    # ====================================================\n    def get_scheduler(optimizer):\n        if CFG.scheduler=='ReduceLROnPlateau':\n            scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=CFG.factor, patience=CFG.patience, verbose=True, eps=CFG.eps)\n        elif CFG.scheduler=='CosineAnnealingLR':\n            scheduler = CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr, last_epoch=-1)\n        elif CFG.scheduler=='CosineAnnealingWarmRestarts':\n            scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr, last_epoch=-1)\n        return scheduler\n\n    # ====================================================\n    # model & optimizer\n    # ====================================================\n    #model = CustomResNext(CFG.model_name, pretrained=True)\n    model = CustomEfficientNet(CFG.model_name, pretrained=True)\n    \n    model.to(device)\n\n    optimizer = Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)\n    scheduler = get_scheduler(optimizer)\n       \n        \n    \n    # ====================================================\n    # Criterion - ['LabelSmoothing', 'FocalLoss' 'FocalCosineLoss', 'SymmetricCrossEntropyLoss', 'BiTemperedLoss', 'TaylorCrossEntropyLoss']\n    # ====================================================\n    \n    def get_criterion():\n        if CFG.criterion=='CrossEntropyLoss':\n            criterion = nn.CrossEntropyLoss()\n        elif CFG.criterion=='LabelSmoothing':\n            criterion = LabelSmoothingLoss(classes=CFG.target_size, smoothing=CFG.smoothing)\n        elif CFG.criterion=='FocalLoss':\n            criterion = FocalLoss().to(device)\n        return criterion\n\n\n    # ====================================================\n    # loop \n    # ====================================================\n    criterion = get_criterion()\n    LOGGER.info(f'Criterion: {criterion}')\n\n    best_score = 0.\n    best_loss = np.inf\n    \n    for epoch in range(CFG.epochs):\n        \n        start_time = time.time()\n        \n        # train\n        avg_loss = train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device)\n\n        # eval\n        avg_val_loss, preds = valid_fn(valid_loader, model, criterion, device)\n        valid_labels = valid_folds[CFG.target_col].values\n        \n        if isinstance(scheduler, ReduceLROnPlateau):\n            scheduler.step(avg_val_loss)\n        elif isinstance(scheduler, CosineAnnealingLR):\n            scheduler.step()\n        elif isinstance(scheduler, CosineAnnealingWarmRestarts):\n            scheduler.step()\n\n        # scoring\n        score = get_score(valid_labels, preds.argmax(1))\n\n        elapsed = time.time() - start_time\n\n        LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n        LOGGER.info(f'Epoch {epoch+1} - Accuracy: {score}')\n\n        if score > best_score:\n            print('YES')\n            best_score = score\n            LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n            torch.save({'model': model.state_dict(), \n                        'preds': preds,\n                        'optimizer': optimizer.state_dict(),\n                        'scheduler': scheduler.state_dict()},\n                        OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.pth')\n    \n    check_point = torch.load(OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.pth')\n    valid_folds[[str(c) for c in range(CFG.target_size)]] = check_point['preds']\n    valid_folds['preds'] = check_point['preds'].argmax(1)\n\n    return valid_folds","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:44.812803Z","iopub.execute_input":"2022-08-22T19:08:44.813336Z","iopub.status.idle":"2022-08-22T19:08:44.838391Z","shell.execute_reply.started":"2022-08-22T19:08:44.813260Z","shell.execute_reply":"2022-08-22T19:08:44.837608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# main\n# ====================================================\ndef main():\n    def get_result(result_df):\n        preds = result_df['preds'].values\n        labels = result_df[CFG.target_col].values\n        score = get_score(labels, preds)\n        LOGGER.info(f'Score: {score:<.5f}')\n    \n    if CFG.train: \n        # train \n        oof_df = pd.DataFrame()\n        for fold in range(CFG.n_fold):\n            if fold in CFG.trn_fold:\n                _oof_df = train_loop(folds, fold)\n                oof_df = pd.concat([oof_df, _oof_df])\n                LOGGER.info(f\"========== fold: {fold} result ==========\")\n                get_result(_oof_df)\n        # CV result\n        LOGGER.info(f\"========== CV ==========\")\n        get_result(oof_df)\n        # save result\n        oof_df.to_csv(OUTPUT_DIR+'oof_df.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:44.840253Z","iopub.execute_input":"2022-08-22T19:08:44.840721Z","iopub.status.idle":"2022-08-22T19:08:44.852183Z","shell.execute_reply.started":"2022-08-22T19:08:44.840685Z","shell.execute_reply":"2022-08-22T19:08:44.851514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    main()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:08:44.853929Z","iopub.execute_input":"2022-08-22T19:08:44.854322Z","iopub.status.idle":"2022-08-22T19:15:41.341468Z","shell.execute_reply.started":"2022-08-22T19:08:44.854237Z","shell.execute_reply":"2022-08-22T19:15:41.339645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from glob import glob\n# test_df = pd.DataFrame(columns=[\"path\"])\n# test_images = glob(\"../input/decimal-damage-dataset/Damage_Detection/Test/*\")\n# test_images[0]\n# test_df[\"path\"] = pd.Series(test_images)\n# image = cv2.imread(test_df.loc[0,\"path\"])\n# image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n# plt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:15:41.342718Z","iopub.status.idle":"2022-08-22T19:15:41.343139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:15:41.344386Z","iopub.status.idle":"2022-08-22T19:15:41.345086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_dataset = TestDataset(test_df, transform=get_transforms(data='valid'))\n# testloader=DataLoader(test_dataset,batch_size=32, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:15:41.346229Z","iopub.status.idle":"2022-08-22T19:15:41.346980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = CustomEfficientNet(CFG.model_name, pretrained=True)\n# model.to(device)\n# checkpoint = torch.load(\"./efficientnet_b3_fold0_best.pth\")\n# model.load_state_dict(checkpoint['model'])\n\n# model.eval()\n# preds = []\n\n# for step, (images) in enumerate(testloader):\n#     # measure data loading time\n#     images = images.to(device)\n#     batch_size = CFG.batch_size\n#     # compute loss\n#     with torch.no_grad():\n#         y_preds = model(images)\n#     # record accuracy\n#     preds.append(y_preds.softmax(1).to('cpu').numpy())\n#     # measure elapsed time","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:15:41.348121Z","iopub.status.idle":"2022-08-22T19:15:41.348734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions = np.concatenate(preds)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:15:41.349854Z","iopub.status.idle":"2022-08-22T19:15:41.350493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_df=pd.DataFrame(columns=[\"image\",\"class\",\"extent_of_damage\"])\n# for i,row in test_df.iterrows():\n#     img_path=row[\"path\"]\n#     pred=predictions[i]\n#     pred = pred.argmax()\n#     img_path=img_path.split(r\"/\")[-1]\n#     dic={\"image\":img_path,\"class\":\"\",\"extent_of_damage\":pred}\n#     pred_df = pred_df.append(dic, ignore_index = True)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:15:41.351948Z","iopub.status.idle":"2022-08-22T19:15:41.352707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:15:41.353919Z","iopub.status.idle":"2022-08-22T19:15:41.354584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_df.to_csv('extent_pred.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-22T19:15:41.355910Z","iopub.status.idle":"2022-08-22T19:15:41.356780Z"},"trusted":true},"execution_count":null,"outputs":[]}]}