{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image\nimport math\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import OneCycleLR\nfrom torch.utils.data import Dataset, DataLoader, Subset, WeightedRandomSampler, TensorDataset\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import notebook\nimport torchvision\nfrom torchvision import transforms\nfrom torchvision import models\nfrom collections import Counter\nfrom pathlib import Path\nfrom sklearn.metrics import confusion_matrix, recall_score\nimport seaborn as sn\n!pip install torchsummary \nimport torchsummary\n!pip install torch-lr-finder\nfrom torch_lr_finder import LRFinder\nimport copy\nimport math\nimport random\nfrom PIL.Image import BICUBIC\nimport json","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(*torch.__config__.show().split(\"\\n\"), sep=\"\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.get_num_threads()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.set_num_threads(2 if torch.cuda.is_available() else 4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.manual_seed(0)\nnp.random.seed(0)\nrandom.seed(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/bengaliai-cv19/train.csv')\ndf_test = pd.read_csv('/kaggle/input/bengaliai-cv19/test.csv')\ndf_class = pd.read_csv('/kaggle/input/bengaliai-cv19/class_map.csv')\ndf_submission = pd.read_csv('/kaggle/input/bengaliai-cv19/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_tensordataset_from_npys(npy_locs, ids_file, label_loc=None):\n    with open(ids_file) as f:\n        ids = json.load(f)\n    X = []\n    npy_locs = notebook.tqdm(npy_locs)\n    for npy_loc in npy_locs:\n        x = np.load(npy_loc)\n        X.append(x)\n    X = np.vstack(X)\n    X = X.reshape(-1, 1, 137, 236)\n    X = torch.from_numpy(X)\n    ids = dict((s,i) for (i,s) in enumerate(ids))\n    if label_loc is None:\n        return TensorDataset(X)\n    else:\n        graphemes = torch.zeros(X.shape[0], dtype=torch.long)\n        vowel_diacs = torch.zeros(X.shape[0], dtype=torch.long)\n        consonant_diacs = torch.zeros(X.shape[0], dtype=torch.long)\n        lbl_df = pd.read_csv(label_loc)\n        for row in lbl_df.itertuples():\n            if row.image_id not in ids:\n                continue\n            idx = ids[row.image_id]\n            graphemes[idx] = row.grapheme_root\n            vowel_diacs[idx] = row.vowel_diacritic\n            consonant_diacs[idx] = row.consonant_diacritic\n        return TensorDataset(X, graphemes, vowel_diacs, consonant_diacs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_tensordataset_from_dfs(parquet_locs, label_loc=None):\n    ids = []\n    X = []\n    parquet_locs = notebook.tqdm(parquet_locs)\n    i = 0\n    for parquet_loc in parquet_locs:\n        i += 1\n        df = pd.read_parquet(parquet_loc)\n        ids.extend(df.image_id.tolist())\n        x = df.iloc[:, 1:].to_numpy(dtype=np.uint8)\n        del df\n        X.append(x)\n    X = np.vstack(X)\n    X = X.reshape(-1, 1, 137, 236)\n    X = torch.from_numpy(X)\n    ids = dict((s,i) for (i,s) in enumerate(ids))\n    if label_loc is None:\n        return TensorDataset(X)\n    else:\n        graphemes = torch.zeros(X.shape[0], dtype=torch.long)\n        vowel_diacs = torch.zeros(X.shape[0], dtype=torch.long)\n        consonant_diacs = torch.zeros(X.shape[0], dtype=torch.long)\n        lbl_df = pd.read_csv(label_loc)\n        for row in lbl_df.itertuples():\n            if row.image_id not in ids:\n                continue\n            idx = ids[row.image_id]\n            graphemes[idx] = row.grapheme_root\n            vowel_diacs[idx] = row.vowel_diacritic\n            consonant_diacs[idx] = row.consonant_diacritic\n        return TensorDataset(X, graphemes, vowel_diacs, consonant_diacs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ds = make_tensordataset_from_dfs(\n#     ['/kaggle/input/bengaliai-cv19/train_image_data_{}.parquet'.format(i) for i in range(4)], \n#     '/kaggle/input/bengaliai-cv19/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = make_tensordataset_from_npys(\n    ['/kaggle/input/bangla-grapheme-npy/tr-ds-{}.npy'.format(i) for i in range(1, 5)],\n    '/kaggle/input/bangla-grapheme-npy/tr-ds-ids.json',\n    '/kaggle/input/bengaliai-cv19/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = random.randrange(len(ds))\nplt.imshow(ds[idx][0].permute(1, 2, 0).reshape(137, 236), cmap='gray', vmin=0, vmax=255)\nds[idx][1], ds[idx][2], ds[idx][3], ds[idx][0].max(), ds[idx][0].min(), idx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tmp = torch.nonzero(ds[100000][0] < 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tmp[:, 1].max(), tmp[:, 1].min(), tmp[:, 2].max(), tmp[:, 2].min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_indices, va_indices = train_test_split(\n    list(range(len(ds))), \n    test_size=.1, \n    train_size=.9, \n    random_state=42\n)#, stratify=ds.tensors[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TensorWithImageTransforms(Dataset):\n    def __init__(self, tensor_dataset, transforms, p=0.0):\n        super(TensorWithImageTransforms, self).__init__()\n        self.ds = tensor_dataset\n        self.tr = transforms\n        self.nt = len(self.ds[0])\n        self.ln = len(self.ds)\n        self.p = p\n        \n    def __getitem__(self, index):\n        img = self.ds[index][0]\n        if random.random() > self.p:\n            img = self.tr(img)\n            img *= 255.0\n        else:\n            img = img.float()\n        return (img,) + self.ds[index][1:]\n    \n    def __len__(self):\n        return self.ln","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_ds = Subset(ds, tr_indices)\nva_ds = Subset(ds, va_indices)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfms = transforms.Compose([\n    transforms.ToPILImage(mode='L'),\n    transforms.Pad((64, 13), padding_mode='reflect'),\n    transforms.RandomAffine(degrees=10.0, translate=(0.15, 0.05), scale=(0.90, 1.05), resample=BICUBIC, fillcolor=255),\n    transforms.CenterCrop((137, 236)),\n    transforms.ToTensor(),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_ds_tfms = TensorWithImageTransforms(tr_ds, tfms, p=0.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = random.randrange(len(tr_ds_tfms))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"im = tr_ds_tfms[idx][0].permute(1, 2, 0).reshape(137, 236)\nplt.imshow(im, cmap='gray', vmin=0., vmax=255.)\ng, v, c = map(lambda t: t.item(), tr_ds_tfms[idx][1:])\nprint(df_class[(df_class['label'] == g) & (df_class['component_type'] == 'grapheme_root')]['component'])\nprint(df_class[(df_class['label'] == v) & (df_class['component_type'] == 'vowel_diacritic')]['component'])\nprint(df_class[(df_class['label'] == c) & (df_class['component_type'] == 'consonant_diacritic')]['component'])\nim.max(), im.min(), idx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_ds = tr_ds_tfms","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(ds), len(tr_ds), len(va_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_graphemes, n_vowel_diacs, n_consonant_diacs = len(set(df_train['grapheme_root'])), len(set(df_train['vowel_diacritic'])), len(set(df_train['consonant_diacritic']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models.resnet152()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torchsummary.summary(models.resnet152().to(device), input_size=(3,137,236), batch_size=32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_linear_block(in_size, out_size):\n    block = nn.Sequential(\n        nn.Linear(in_size, out_size), \n        nn.ReLU(), \n        nn.BatchNorm1d(num_features=out_size),\n    )\n    nn.init.xavier_normal_(block[0].weight.data)\n    nn.init.zeros_(block[0].bias.data)\n    return block\n\ndef make_ff_predictor(in_size, intermediate_size, out_size, layer_count):\n    layers = [make_linear_block(in_size, intermediate_size)]\n    for i in range(layer_count):\n        layers.append(make_linear_block(intermediate_size, intermediate_size))\n    layers.append(make_linear_block(intermediate_size, out_size))\n    layers = nn.Sequential(*layers)\n    return layers\n\ndef make_squeeze_predictor(in_size, out_size):\n    return nn.Sequential(\n            nn.Dropout(p=0.5),\n            nn.Conv2d(in_size, out_size, kernel_size=(1, 1)),\n            nn.ReLU(),\n            nn.AdaptiveAvgPool2d(output_size=(1,1)),\n            nn.Flatten()\n        )\n\nclass BanglaHandwrittenGraphemeNN(nn.Module):\n    def __init__(self):\n        super(BanglaHandwrittenGraphemeNN, self).__init__()\n#         base = models.squeezenet1_0(pretrained=True).features\n        base = models.resnet152(pretrained=True)\n        base.fc = nn.Identity()\n#         base = models.densenet121(pretrained=True)\n#         base.classifier = nn.Identity()\n        self.base = base\n        feature_size = 2048\n        self.grapheme_predictor = make_ff_predictor(feature_size, 512, n_graphemes, 2)\n        self.vowel_diac_predictor = make_ff_predictor(feature_size, 512, n_vowel_diacs, 1)\n        self.consonant_diacs = make_ff_predictor(feature_size, 512, n_consonant_diacs, 1)\n\n    def convert_to_grayscale(self):\n        with torch.no_grad():\n            conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n            conv1.weight.data = torch.sum(self.base.conv1.weight.data, dim=1, keepdim=True)\n            self.base.conv1 = conv1\n#             conv1.weight.data = torch.sum(self.base.features.conv0.weight.data, dim=1, keepdim=True)\n#             self.base.features.conv0 = conv1\n            \n    def freeze(self):\n        for p in self.base.parameters():\n            p.requires_grad = False\n        \n    def unfreeze(self):\n        for p in self.base.parameters():\n            p.requires_grad = True\n        \n    def forward(self, x):\n        features = self.base(x)\n        g_pred = self.grapheme_predictor(features)\n        v_pred = self.vowel_diac_predictor(features)\n        c_pred = self.consonant_diacs(features)\n        return g_pred, v_pred, c_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, alpha=1., gamma=1.):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n    def forward(self, inputs, targets, **kwargs):\n        logits = nn.functional.log_softmax(inputs)\n        CE_loss = nn.functional.nll_loss(logits, targets, reduction='none')\n        pt = torch.exp(-CE_loss)\n        F_loss = self.alpha * ((1-pt)**self.gamma) * CE_loss\n        return F_loss.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MultiTaskLoss(nn.Module):\n    def __init__(self, num_tasks, init_weight=None):\n        super(MultiTaskLoss, self).__init__()\n        self.n = num_tasks\n        if init_weight is None:\n            self.w = nn.Parameter(torch.zeros(self.n))\n        else:\n            self.w = nn.Parameter(torch.tensor(init_weight))\n            \n    def freeze(self):\n        for p in self.parameters():\n            p.requires_grad = False\n\n    def forward(self, losses):\n        return torch.sum(torch.exp(-2.0 * self.w) * losses) + torch.sum(self.w)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_multi_task_batch(model, optimizer, criterions, mtl_criterion, device, batch):\n    img, g, v, c = batch\n    img, g, v, c = img.to(device), g.to(device), v.to(device), c.to(device)\n    img = img / 255.0\n    g_criterion, v_criterion, c_criterion = criterions\n    optimizer.zero_grad()\n    g_pred, v_pred, c_pred = model(img)\n    g_loss = g_criterion(g_pred, g)\n    v_loss = v_criterion(v_pred, v)\n    c_loss = c_criterion(c_pred, c)\n    loss = mtl_criterion(torch.stack((g_loss, v_loss, c_loss)))\n    loss.backward()\n    optimizer.step()\n    return loss.item(), g_loss.item(), v_loss.item(), c_loss.item()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def validate_multi_task_batch(model, criterions, mtl_criterion, device, batch):\n    with torch.no_grad():\n        img, g, v, c = batch\n        img, g, v, c = img.to(device), g.to(device), v.to(device), c.to(device)\n        img = img / 255.0\n        g_pred, v_pred, c_pred = model(img)\n        g_criterion, v_criterion, c_criterion = criterions\n        g_loss = g_criterion(g_pred, g)\n        v_loss = v_criterion(v_pred, v)\n        c_loss = c_criterion(c_pred, c)\n        loss = mtl_criterion(torch.stack((g_loss, v_loss, c_loss)))\n        losses = (loss.item(), g_loss.item(), v_loss.item(), c_loss.item())\n        preds = (g_pred.argmax(1).tolist(), v_pred.argmax(1).tolist(), c_pred.argmax(1).tolist())\n        trues = (g.tolist(), v.tolist(), c.tolist())\n        return losses, preds, trues","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# LRFinder\ndef lr_finder(model, optimizer, criterions, mtl_criterion, device, dl, num_iter=10, start_lr=1e-4, end_lr=1.0):\n    model_state =  copy.deepcopy(model.state_dict())\n    optim_state =  copy.deepcopy(optimizer.state_dict())\n    mtl_criterion_state = copy.deepcopy(mtl_criterion.state_dict())\n    for param in optimizer.param_groups:\n        param['lr'] = start_lr\n    gamma = (end_lr / start_lr) ** (1 / num_iter)\n    print(gamma, start_lr, end_lr)\n    lrf_sched = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma)\n    count = 0\n    lrf_losses = []\n    lrs = []\n    min_loss = math.inf\n    done = False\n    while not done:\n        batches = dl\n        batches = notebook.tqdm(tr_dl)\n        for batch in batches:\n            count += 1\n            losses = train_multi_task_batch(model, optimizer, criterions, mtl_criterion, device, batch)\n            print(losses)\n            min_loss = min(min_loss, losses[0])\n            lrf_losses.append(losses)\n            lrf_sched.step()\n            lrs.append([pg['lr'] for pg in optimizer.param_groups])\n            if num_iter == count:# or losses[0] /10.0 > min_loss:\n                done = True\n                print(min_loss, losses[0] / 10.0)\n                break\n    model.load_state_dict(model_state)\n    optimizer.load_state_dict(optim_state)\n    mtl_criterion.load_state_dict(mtl_criterion_state)\n    return lrf_losses, lrs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = BanglaHandwrittenGraphemeNN().to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.convert_to_grayscale()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_state_dict(torch.load('/kaggle/input/bangla-handwritten-grapheme-uncertainty-weighted/model.pth', map_location=device))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.freeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"g_criterion = nn.CrossEntropyLoss()#FocalLoss(gamma=1.0)\nv_criterion = nn.CrossEntropyLoss()#FocalLoss(gamma=1.0)\nc_criterion = nn.CrossEntropyLoss()#FocalLoss(gamma=1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterions = (g_criterion, v_criterion, c_criterion)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mtl_criterion = MultiTaskLoss(3).to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mtl_criterion.load_state_dict(torch.load('/kaggle/input/bangla-handwritten-grapheme-uncertainty-weighted/mtlc.pth', map_location=device))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# mtl_criterion.freeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = optim.Adam([{'params': model.parameters()}, {'params': mtl_criterion.parameters()}])\n# optimizer = optim.Adam(model.parameters(), lr=1e-3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer.load_state_dict(torch.load('/kaggle/input/bangla-handwritten-grapheme-uncertainty-weighted/optim.pth', map_location=device))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean = lambda l: sum(l) / len(l)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_dl = DataLoader(tr_ds, batch_size=32, num_workers=2, pin_memory=True, shuffle=True, drop_last=True)\nva_dl = DataLoader(va_ds, batch_size=32, num_workers=2, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# lrf_losses, lrs = lr_finder(model, optimizer, criterions, mtl_criterion, device, tr_dl, num_iter=100, start_lr=1e-5, end_lr=1e0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# skip_first = 1\n# skip_last = 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plt.plot([t[0] for t in lrs[skip_first:-skip_last]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plt.plot([t[0] for t in lrs[skip_first:-skip_last]], [t[0] for t in lrf_losses][skip_first:-skip_last])\n# plt.plot([t[0] for t in lrs[skip_first:-skip_last]], [t[1] for t in lrf_losses][skip_first:-skip_last])\n# plt.plot([t[0] for t in lrs[skip_first:-skip_last]], [t[2] for t in lrf_losses][skip_first:-skip_last])\n# plt.plot([t[0] for t in lrs[skip_first:-skip_last]], [t[3] for t in lrf_losses][skip_first:-skip_last])\n# plt.xscale('log')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# lr = lrs[32][0]\n# lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = 1e-4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for param in optimizer.param_groups:\n#         param['lr'] = lr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_losses = []\nva_losses = []\nva_scores = []\nmtl_weights = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_epochs = 5\nsteps_per_epoch = len(tr_dl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# scheduler = OneCycleLR(optimizer, lr, epochs=num_epochs, steps_per_epoch=steps_per_epoch)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = range(num_epochs)\n# epochs = notebook.tqdm(range(num_epochs))\nfor epoch in epochs:\n    model.train()\n    count = 0\n    batches = tr_dl\n#     batches = notebook.tqdm(tr_dl)\n    for batch in batches:\n        losses = train_multi_task_batch(model, optimizer, criterions, mtl_criterion, device, batch)\n#         batches.set_description(\"{:0.4f} {:0.4f} {:0.4f} {:0.4f}\".format(*losses))\n#         scheduler.step()\n        tr_losses.append(losses)\n        mtl_weights.append(mtl_criterion.w.tolist())\n    print(mean([t[0] for t in tr_losses]), \n          mean([t[1] for t in tr_losses]), \n          mean([t[2] for t in tr_losses]), \n          mean([t[3] for t in tr_losses]))\n    \n    model.eval()\n    va_batch_losses = []\n    \n    va_g_preds = []\n    va_v_preds = []\n    va_c_preds = []\n    \n    va_g_trues = []\n    va_v_trues = []\n    va_c_trues = []\n    batches = va_dl\n#     batches = notebook.tqdm(va_dl)\n    for batch in batches:\n        losses, preds, trues = validate_multi_task_batch(model, criterions, mtl_criterion, device, batch)\n        va_batch_losses.append(losses)\n\n        g_pred, v_pred, c_pred = preds\n        g_true, v_true, c_true = trues\n\n        va_g_trues.extend(g_true)\n        va_v_trues.extend(v_true)\n        va_c_trues.extend(c_true)\n\n        va_g_preds.extend(g_pred)\n        va_v_preds.extend(v_pred)\n        va_c_preds.extend(c_pred)\n\n    avg_loss = mean([t[0] for t in va_batch_losses])\n    avg_g_loss = mean([t[1] for t in va_batch_losses])\n    avg_v_loss = mean([t[2] for t in va_batch_losses])\n    avg_c_loss = mean([t[3] for t in va_batch_losses])\n    va_losses.append((avg_loss, avg_g_loss, avg_v_loss, avg_c_loss))\n    g_rec = recall_score(va_g_trues, va_g_preds, average='macro')\n    v_rec = recall_score(va_v_trues, va_v_preds, average='macro')\n    c_rec = recall_score(va_c_trues, va_c_preds, average='macro')\n    score = 0.5 * g_rec + 0.25 * v_rec + 0.25 * c_rec\n    va_scores.append((score, g_rec, v_rec, c_rec))\n    print(va_losses[-1])\n    print(va_scores[-1])\n    print(confusion_matrix(va_v_trues, va_v_preds))\n    print(confusion_matrix(va_c_trues, va_c_preds))\n    plt.figure(figsize = (20, 20))\n    sn.heatmap(np.log1p(confusion_matrix(va_g_trues, va_g_preds)))\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([t[0] for t in tr_losses])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([t[1] for t in tr_losses])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([t[2] for t in tr_losses])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([t[3] for t in tr_losses])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mtl_weights[-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(mtl_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([t[0] for t in mtl_weights])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([t[1] for t in mtl_weights])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([t[2] for t in mtl_weights])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(va_scores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([t[0] for t in va_losses])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([t[1] for t in va_losses])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([t[2] for t in va_losses])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([t[3] for t in va_losses])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save(model.state_dict(), 'model.pth')\ntorch.save(optimizer.state_dict(), 'optim.pth')\ntorch.save(mtl_criterion.state_dict(), 'mtlc.pth')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}