{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path.append('../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import glob\nimport math\nimport time\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport PIL.Image\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom tqdm import tqdm_notebook as tqdm\nimport sklearn.metrics\nfrom sklearn.model_selection import StratifiedKFold\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data.sampler import RandomSampler\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\nimport albumentations\nfrom efficientnet_pytorch import model as enet\\\n\ndevice = torch.device('cuda')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"kernel_type = 'effnet-b1-unseen'\nenet_type = 'efficientnet-b1'\nfold = 0\nHEIGHT = 137\nWIDTH = 236\nimage_size = 128\ncut_size = int(image_size * 0.85)\ndata_dir = '../input/bengaliaicv19feather'\nbatch_size = 128\nnum_workers = 4\ninit_lr = 0.001\nc0_dim = 1295\nc1_dim = 168\nc2_dim = 11\nc3_dim = 7\nout_dim = c0_dim + c1_dim + c2_dim + c3_dim\nloss_weight = [4, 2, 1, 1]\nn_epochs = 30\nuse_amp = False\n\nfiles_train = [f'train_image_data_{fid}.feather' for fid in range(4)]\nfiles_test = [f'test_image_data_{fid}.feather' for fid in range(4)]\n\ndf_train = pd.read_csv(os.path.join('../input/bengaliai-cv19', f'train.csv'))\ndf_class_map = pd.read_csv(os.path.join('../input/bengaliai-cv19', f'class_map.csv'))\n\nid2grapheme = {i: grapheme for i, grapheme in enumerate(df_train.grapheme.unique())}\ngrapheme2id = {grapheme: i for i, grapheme in enumerate(df_train.grapheme.unique())}\ndf_train['grapheme_id'] = df_train['grapheme'].map(grapheme2id)\n\nn_fold = 5\nskf = StratifiedKFold(n_fold, random_state=42)\nfor i_fold, (train_idx, val_idx) in enumerate(skf.split(df_train, df_train.grapheme)):\n    df_train.loc[val_idx, 'fold'] = i_fold\ndf_train['fold'] = df_train['fold'].astype(int)\n\ndf_label_map = []\nfor i, df in tqdm(df_train.groupby('grapheme_id')):\n    df_label_map.append(df.iloc[:, 1:6].drop_duplicates())\ndf_label_map = pd.concat(df_label_map).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data & Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_data(files):\n    tmp = []\n    for f in files:\n        F = os.path.join(data_dir, f)\n        data = pd.read_feather(F)\n        res = data.iloc[:, 1:].values\n        imgs = []\n        for i in tqdm(range(res.shape[0])):\n            img = res[i].squeeze().reshape(HEIGHT, WIDTH)\n            img = cv2.resize(img, (128, 128))\n            imgs.append(img)\n        imgs = np.asarray(imgs)\n        \n        tmp.append(imgs)\n    tmp = np.concatenate(tmp, 0)\n    return tmp\n\n\nclass BengaliDataset(Dataset):\n    def __init__(self, csv, data, idx, split, mode, transform=None):\n\n        self.csv = csv.reset_index()\n        self.data = data\n        self.idx = np.asarray(idx)\n        self.split = split\n        self.mode = mode\n        self.transform = transform\n\n    def __len__(self):\n        return self.idx.shape[0]\n\n    def __getitem__(self, index):\n        index = self.idx[index]\n        this_img_id = self.csv.iloc[index].image_id\n        \n        image = self.data[index]\n        image = 255 - image\n\n        if self.transform is not None:\n            image_origin = image.astype(np.float32).copy()\n            res = self.transform(image=image)\n            image = res['image'].astype(np.float32)\n        else:\n            image_origin = image.astype(np.float32).copy()\n            image = image.astype(np.float32)\n\n        image /= 255\n        image = image[np.newaxis, :, :]\n        image = np.repeat(image, 3, 0)  # 1ch to 3ch\n        ###\n        image_origin /= 255\n        image_origin = image_origin[np.newaxis, :, :]\n        image_origin = np.repeat(image_origin, 3, 0)  # 1ch to 3ch\n        ###\n\n        if self.mode == 'test':\n            return torch.tensor(image)\n        else:\n            label_0 = self.csv.iloc[index].grapheme_id\n            label_1 = self.csv.iloc[index].grapheme_root\n            label_2 = self.csv.iloc[index].vowel_diacritic\n            label_3 = self.csv.iloc[index].consonant_diacritic\n            label = [label_0, label_1, label_2, label_3]\n            return torch.tensor(image), torch.tensor(image_origin), torch.tensor(label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = read_data(files_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_train = albumentations.Compose([\n    albumentations.RandomBrightness(limit=0.2, p=0.75),\n    albumentations.OneOf([\n        albumentations.MotionBlur(blur_limit=5),\n        albumentations.MedianBlur(blur_limit=5),\n        albumentations.GaussianBlur(blur_limit=5),\n        albumentations.GaussNoise(var_limit=(5.0, 30.0)),\n    ], p=0.7),\n    albumentations.OneOf([\n        albumentations.OpticalDistortion(distort_limit=1.0),\n        albumentations.GridDistortion(num_steps=5, distort_limit=1.),\n        albumentations.ElasticTransform(alpha=3),\n    ], p=0.7),\n    albumentations.ShiftScaleRotate(shift_limit=0.5, scale_limit=0.15, rotate_limit=15, border_mode=0, p=0.75),\n    albumentations.Cutout(max_h_size=cut_size, max_w_size=cut_size, num_holes=1, p=0.7),\n\n])\ntransforms_val = albumentations.Compose([\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_show = df_train.iloc[:1000]\ndataset_show = BengaliDataset(df_show, data, list(range(df_show.shape[0])), 'train', 'train', transform=transforms_train)\n\nfrom pylab import rcParams\nrcParams['figure.figsize'] = 20,10\nfor i in range(2):\n    f, axarr = plt.subplots(1,5)\n    for p in range(5):\n        idx = np.random.randint(0, len(dataset_show))\n        img, img_org, label = dataset_show[idx]\n        axarr[p].imshow(img.transpose(0, 1).transpose(1,2).squeeze())\n        axarr[p].set_title(idx)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"pretrained_dict = {f'efficientnet-b{i}': f for i, f in enumerate(sorted(glob.glob('../input/efficientnet-pytorch/*pth')))}\n\n\nsigmoid = torch.nn.Sigmoid()\nclass Swish(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, i):\n        result = i * sigmoid(i)\n        ctx.save_for_backward(i)\n        return result\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        i = ctx.saved_variables[0]\n        sigmoid_i = sigmoid(i)\n        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))\n\nswish = Swish.apply\n\nclass Swish_module(nn.Module):\n    def forward(self, x):\n        return swish(x)\n\nswish_layer = Swish_module()\n\ndef relu_fn(x):\n    \"\"\" Swish activation function \"\"\"\n    return swish_layer(x)\n\n\nclass enet_3cg(nn.Module):\n\n    def __init__(self, backbone, out_dim_1, out_dim_2):\n        super(enet_3cg, self).__init__()\n        self.enet = enet.EfficientNet.from_name(backbone)\n        self.enet.load_state_dict(torch.load(pretrained_dict[backbone]), strict=True)\n        self.dropouts = nn.ModuleList([\n            nn.Dropout(0.5) for _ in range(5)\n        ])\n        self.myfc_1 = nn.Linear(self.enet._fc.in_features, out_dim_2)\n        self.activate = Swish_module()\n        self.myfc_2 = nn.Linear(out_dim_2, out_dim_1)\n        self.enet._fc = nn.Identity()\n\n    def extract(self, x):\n        return self.enet(x)\n\n    def forward(self, x):\n        x = self.extract(x)\n        for i, dropout in enumerate(self.dropouts):\n            if i == 0:\n                out_2 = self.myfc_1(dropout(x))\n            else:\n                out_2 += self.myfc_1(dropout(x))\n        out_2 /= len(self.dropouts)\n        out_1 = self.myfc_2(self.activate(out_2))\n        return out_1, out_2\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def criterion(logits_1, logits_2, target, loss_weight=loss_weight, is_val=False):\n    loss_1 = nn.CrossEntropyLoss()(logits_2[:, :c1_dim], target[:, 1]) * loss_weight[1]\n    loss_2 = nn.CrossEntropyLoss()(logits_2[:, c1_dim:c1_dim+c2_dim], target[:, 2]) * loss_weight[2]\n    loss_3 = nn.CrossEntropyLoss()(logits_2[:, c1_dim+c2_dim:], target[:, 3]) * loss_weight[3]\n    if is_val:\n        loss = (loss_1 + loss_2 + loss_3) / sum(loss_weight[1:])\n    else:\n        loss_0 = nn.CrossEntropyLoss()(logits_1, target[:, 0]) * loss_weight[0]\n        loss = (loss_0 + loss_1 + loss_2 + loss_3) / sum(loss_weight)\n    return loss\n\nCE = nn.CrossEntropyLoss()\ndef criterion_mix(logits_1, logits_2, target, loss_weight=loss_weight):\n    target, shuffled_target, lam = target\n\n    loss_0 = nn.CrossEntropyLoss()(logits_1, target[:, 0]) * loss_weight[0]\n    loss_1 = nn.CrossEntropyLoss()(logits_2[:, :c1_dim], target[:, 1]) * loss_weight[1]\n    loss_2 = nn.CrossEntropyLoss()(logits_2[:, c1_dim:c1_dim+c2_dim], target[:, 2]) * loss_weight[2]\n    loss_3 = nn.CrossEntropyLoss()(logits_2[:, c1_dim+c2_dim:], target[:, 3]) * loss_weight[3]\n\n    loss = (loss_0 + loss_1 + loss_2 + loss_3) / sum(loss_weight)\n    \n    loss_0_mix = nn.CrossEntropyLoss()(logits_1, shuffled_target[:, 0]) * loss_weight[0]\n    loss_1_mix = nn.CrossEntropyLoss()(logits_2[:, :c1_dim], shuffled_target[:, 1]) * loss_weight[1]\n    loss_2_mix = nn.CrossEntropyLoss()(logits_2[:, c1_dim:c1_dim+c2_dim], shuffled_target[:, 2]) * loss_weight[2]\n    loss_3_mix = nn.CrossEntropyLoss()(logits_2[:, c1_dim+c2_dim:], shuffled_target[:, 3]) * loss_weight[3]\n\n    loss_mix = (loss_0_mix + loss_1_mix + loss_2_mix + loss_3_mix) / sum(loss_weight)\n\n    return lam * loss + (1 - lam) * loss_mix\n\n\ndef rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = np.int(W * cut_rat)\n    cut_h = np.int(H * cut_rat)\n\n    # uniform\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n    return bbx1, bby1, bbx2, bby2\n\n    \ndef cutmix(data, target, alpha, clip=[0.3, 0.7]):\n    indices = torch.randperm(data.size(0))\n    shuffled_data = data[indices]\n    shuffled_target = target[indices]\n\n    lam = np.clip(np.random.beta(alpha, alpha), clip[0], clip[1])\n    bbx1, bby1, bbx2, bby2 = rand_bbox(data.size(), lam)\n    data[:, :, bbx1:bbx2, bby1:bby2] = data[indices, :, bbx1:bbx2, bby1:bby2]\n    # adjust lambda to exactly match pixel ratio\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (data.size()[-1] * data.size()[-2]))\n    targets = (target, shuffled_target, lam)\n\n    return data, targets","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train & Val"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_score(solution, submission):\n\n    scores = []\n    for component in ['grapheme_root', 'consonant_diacritic', 'vowel_diacritic']:\n        y_true_subset = solution[component].values\n        y_pred_subset = submission[component].values\n        scores.append(sklearn.metrics.recall_score(\n            y_true_subset, y_pred_subset, average='macro'))\n    final_score = np.average(scores, weights=[2,1,1])\n    return final_score\n\n\ndef train_epoch(loader, optimizer):\n\n    model.train()\n    train_loss = []\n    bar = tqdm(loader)\n    for (data, data_org, target) in bar:\n\n        data, data_org, target = data.to(device), data_org.to(device), target.to(device)\n        ### mixup & cutmix & cutout\n        rand_p = np.random.rand()\n        if rand_p <= 0.0:\n            data, target = mixup(data_org, target, 1.)  # process from origin\n            loss_func = criterion_mix\n        elif 0.0 < rand_p <= 0.2:\n            data, target = cutmix(data_org, target, 1.)  # process from origin\n            loss_func = criterion_mix\n        else:\n            loss_func = criterion\n        ###\n        optimizer.zero_grad()\n        logits_1, logits_2 = model(data)\n        loss = loss_func(logits_1, logits_2, target)\n\n        if not use_amp:\n            loss.backward()\n        else:\n            with amp.scale_loss(loss, optimizer) as scaled_loss:\n                scaled_loss.backward()\n\n#         torch.nn.utils.clip_grad_norm_(model.parameters(), clip_grad)\n        optimizer.step()\n\n        loss_np = loss.detach().cpu().numpy()\n        train_loss.append(loss_np)\n        smooth_loss = sum(train_loss[-20:]) / min(len(train_loss), 20)\n        bar.set_description('loss: %.5f, smth: %.5f' % (loss_np, smooth_loss))\n    return train_loss\n\n\ndef val_epoch(loader, get_output=False):\n\n    model.eval()\n    val_loss = []\n    outputs = []\n    LOGITS_1, LOGITS_2, LOGITS_M = [], [], []\n    p1, p2, p3 = [], [], []\n    masks = []\n    acc, acc1, acc2, acc3 = 0.0,0.0,0.0,0.0\n\n    with torch.no_grad():\n        for (data, data_org, target) in tqdm(loader):\n            data, target = data.to(device), target.to(device)\n            logits_1, logits_2 = model(data)\n\n            loss = criterion(logits_1, logits_2, target, is_val=True)\n\n            pred = logits_1.argmax(1).detach()\n            pred1 = logits_2[:, :c1_dim].argmax(1).detach()\n            pred2 = logits_2[:, c1_dim:c1_dim+c2_dim].argmax(1).detach()\n            pred3 = logits_2[:, c1_dim+c2_dim:].argmax(1).detach()\n            outputs.append(pred)\n            p1.append(pred1)\n            p2.append(pred2)\n            p3.append(pred3)\n\n            acc += (target[:, 0] == pred).sum().cpu().numpy()\n            acc1 += (target[:, 1] == pred1).sum().cpu().numpy()\n            acc2 += (target[:, 2] == pred2).sum().cpu().numpy()\n            acc3 += (target[:, 3] == pred3).sum().cpu().numpy()\n            \n            if get_output:\n                LOGITS_1.append(logits_1)\n                LOGITS_2.append(logits_2)\n\n            val_loss.append(loss.detach().cpu().numpy())\n\n        val_loss = np.mean(val_loss)\n        acc = acc / len(dataset_valid) * 100\n        acc1 = acc1 / len(dataset_valid) * 100\n        acc2 = acc2 / len(dataset_valid) * 100\n        acc3 = acc3 / len(dataset_valid) * 100\n\n    preds = torch.cat(outputs).cpu().numpy()\n    solution = df_train.iloc[valid_idx]\n    submission1 = df_label_map.iloc[preds]\n    score1 = get_score(solution, submission1)\n    \n    submission2 = pd.DataFrame({\n        'grapheme_root': torch.cat(p1).cpu().numpy(),\n        'vowel_diacritic': torch.cat(p2).cpu().numpy(),\n        'consonant_diacritic': torch.cat(p3).cpu().numpy(),\n    })\n    score2 = get_score(solution, submission2)\n\n    if get_output:\n        LOGITS_1 = torch.cat(LOGITS_1).cpu().numpy()\n        LOGITS_2 = torch.cat(LOGITS_2).cpu().numpy()\n        return LOGITS_1, LOGITS_2\n    else:\n        return val_loss, acc, acc1, acc2, acc3, score1, score2\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Main Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"n_fold = 5\nrecord = [{'train_loss': [], 'val_loss': [], 'score1': [], 'score2': []} for x in range(n_fold)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i_fold = fold\ntrain_idx, valid_idx = np.where((df_train['fold'] != i_fold))[0], np.where((df_train['fold'] == i_fold))[0]\n\ndataset_train = BengaliDataset(df_train, data, train_idx, 'train', 'train', transform=transforms_train)\ndataset_valid = BengaliDataset(df_train, data, valid_idx, 'train', 'val', transform=transforms_val)\ntrain_loader = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size, sampler=RandomSampler(dataset_train), num_workers=num_workers)\nvalid_loader = torch.utils.data.DataLoader(dataset_valid, batch_size=batch_size, sampler=None, num_workers=num_workers)\n\ntargets0 = df_train.loc[valid_idx]['grapheme_id'].values\n\nmodel = enet_3cg(enet_type, out_dim_1=c0_dim, out_dim_2=c1_dim+c2_dim+c3_dim)\nmodel = model.to(device)\n\nmax_score = 0\nmodel_file = f'{kernel_type}_best_fold{i_fold}.pth'\n\nprint('Training All Layers...')\noptimizer = optim.Adam(model.parameters(), lr=init_lr)\nif use_amp:\n    model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\")\nscheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, n_epochs)\nfor epoch in range(1, n_epochs+1):\n    print(time.ctime(), 'Epoch:', epoch)\n    scheduler_cosine.step(epoch-1)\n\n    train_loss = train_epoch(train_loader, optimizer)\n    val_loss, acc, acc1, acc2, acc3, score1, score2 = val_epoch(valid_loader)\n\n    content = time.ctime() + ' ' + f'Epoch {epoch}, lr: {optimizer.param_groups[0][\"lr\"]:.7f}, train loss: {np.mean(train_loss):.5f}, val loss: {np.mean(val_loss):.5f}, acc: {(acc):.5f}, acc1: {(acc1):.5f}, acc2: {(acc2):.5f}, acc3: {(acc3):.5f}, score1: {(score1):.6f}, score2: {(score2):.6f}'\n    print(content)\n    with open(f'log_{kernel_type}.txt', 'a') as appender:\n        appender.write(content + '\\n')\n\n    if score1 >= max_score:\n        print('score2 ({:.6f} --> {:.6f}).  Saving model ...'.format(max_score, score1))\n        torch.save(model.state_dict(), model_file)\n        max_score = score1\n\n    record[i_fold]['train_loss'].append(np.mean(train_loss))\n    record[i_fold]['val_loss'].append(val_loss)\n    record[i_fold]['score1'].append(np.mean(score1))\n    record[i_fold]['score2'].append(score2)\n\ntorch.save(model.state_dict(), os.path.join(f'{kernel_type}_model_fold{i_fold}.pth'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}