{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import Subset\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import WeightedRandomSampler\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam\nfrom torch.nn import CosineSimilarity\nfrom torchvision.transforms import ToTensor\nfrom torchvision import transforms\nimport torchvision.models as models\nfrom pathlib import Path\nimport PIL.Image\nimport random\nimport math\nfrom tqdm import tqdm_notebook as tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom torch.optim.lr_scheduler import ExponentialLR\nimport os\nimport glob\nimport cv2\nimport numpy as np\nimport PIL\nfrom sklearn.metrics import cohen_kappa_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Copy pretrained models"},{"metadata":{"trusted":true},"cell_type":"code","source":"# copy pretrained weights for resnet50 to the folder fastai will search by default\nPath('/tmp/.cache/torch/checkpoints/').mkdir(exist_ok=True, parents=True)\n!cp /kaggle/input/fastai-pretrained-models/* /tmp/.cache/torch/checkpoints/\n!cp /kaggle/input/pretrained-pytorch-models/* /tmp/.cache/torch/checkpoints/\n!cp /kaggle/input/resnet152/* /tmp/.cache/torch/checkpoints/\n!mv /tmp/.cache/torch/checkpoints/resnet152.pth /tmp/.cache/torch/checkpoints/resnet152-b121ed2d.pth\n!ls /tmp/.cache/torch/checkpoints/","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Set fixed seed"},{"metadata":{"trusted":true},"cell_type":"code","source":"random.seed(42)\ntorch.manual_seed(42)\nnp.random.seed(42)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data from 2015"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('../input/resized-2015-2019-blindness-detection-images/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_paths = []\nimg_diags = []\nimg_ids = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(path/'labels/testLabels15.csv')\ndf['path'] = df.image.apply(lambda p: path/'resized test 15'/(p + '.jpg'))\nimg_paths.extend(df['path'].tolist())\nimg_diags.extend(df['level'].tolist())\nimg_ids.extend(df['image'].tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(path/'labels/trainLabels15.csv')\ndf['path'] = df.image.apply(lambda p: path/'resized train 15'/(p + '.jpg'))\nimg_paths.extend(df['path'].tolist())\nimg_diags.extend(df['level'].tolist())\nimg_ids.extend(df['image'].tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_df = pd.DataFrame({'id_code':img_ids, 'diagnosis': img_diags, 'path':img_paths})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_df, tr_va_df = train_test_split(tr_df, test_size=0.05, random_state=42, stratify=tr_df.diagnosis)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data from 2019"},{"metadata":{"trusted":true},"cell_type":"code","source":"# path = Path('../input/aptos2019-blindness-detection')\npath = Path('../input/resized-2015-2019-blindness-detection-images/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(path/'labels/trainLabels19.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_paths = []\nimg_diags = []\nimg_ids = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['path'] = df.id_code.apply(lambda p: path/'resized train 19'/(p + '.jpg'))\nimg_paths.extend(df['path'].tolist())\nimg_diags.extend(df['diagnosis'].tolist())\nimg_ids.extend(df['id_code'].tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"te_va_df = pd.DataFrame({'id_code':img_ids, 'diagnosis': img_diags, 'path':img_paths})\ntr_df = pd.DataFrame({'id_code':img_ids, 'diagnosis': img_diags, 'path':img_paths})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_df, tr_va_df = train_test_split(tr_df, test_size=0.20, random_state=42, stratify=tr_df.diagnosis)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Circle crop"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CircleCrop:\n    def __init__(self, radius):\n        self.radius = radius\n        circle_img = np.zeros((2 * radius, 2 * radius), np.uint8)\n        cv2.circle(circle_img, (radius, radius), int(radius), 1, thickness=-1)\n        self.mask = torch.tensor(circle_img, dtype=torch.float32)\n        \n    def __call__(self, img):\n        return torch.mul(img, self.mask)\n    \n    def __repr__(self):\n        return self.__class__.__name__ + '(radius={0})'.format(self.radius)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# APTOS Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class APTOSDataset(Dataset):\n    def __init__(self, df, labels=None, augmentation=False, img_size=480):\n        self.len = len(df)\n        self.labels = labels\n        self.images = df.path.tolist()\n        self.trfms = transforms.Compose([\n            transforms.Resize(img_size),\n            transforms.CenterCrop(img_size),\n            transforms.RandomApply((\n                transforms.RandomHorizontalFlip(p=0.5),\n                transforms.RandomVerticalFlip(p=0.5),\n                transforms.RandomAffine(degrees=180, scale=(1.0, 1.5), resample=PIL.Image.BICUBIC),\n                ), p = 1 if augmentation else 0  \n            ),\n            transforms.ColorJitter(\n                saturation=(1.0, 1.0),\n                brightness=(1.0, 1.5), \n                contrast=(1.5, 2.5),\n            ),\n            transforms.ToTensor(),\n#             transforms.Normalize(mean=[0.485, 0.456, 0.406],\n#                                  std=[0.229, 0.224, 0.225]),\n            CircleCrop(img_size // 2),\n            ])\n        \n    def __getitem__(self, index):\n        if self.labels is not None:\n            label = self.labels[index]\n        else:\n            label = 0\n        path = self.images[index]\n        img = PIL.Image.open(path)\n        img = self.trfms(img)\n        return img, label\n    \n    def __len__(self):\n        return self.len","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('../input')\nimg_size = 256","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_ds = APTOSDataset(tr_df, tr_df['diagnosis'].values, augmentation=True, img_size=img_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_va_ds = APTOSDataset(tr_va_df, tr_va_df['diagnosis'].values, augmentation=False, img_size=img_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"te_va_ds = APTOSDataset(te_va_df, te_va_df['diagnosis'].values, augmentation=False, img_size=img_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(tr_ds), len(tr_va_ds), len(te_va_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img, label = tr_ds[random.randint(0, len(tr_ds) - 1)]\nplt.imshow(transforms.ToPILImage()(img))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img, label = tr_va_ds[random.randint(0, len(tr_va_ds) - 1)]\nplt.imshow(transforms.ToPILImage()(img))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img, label = te_va_ds[random.randint(0, len(te_va_ds) - 1)]\nplt.imshow(transforms.ToPILImage()(img))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataloaders"},{"metadata":{"trusted":true},"cell_type":"code","source":"bs = 48\nnw = 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class_weights = torch.tensor([0.05,0.20,0.15,0.30,0.30])\n# tr_labels = tr_ds.labels\n# sample_weights = class_weights[tr_labels]\n# sampler = WeightedRandomSampler(sample_weights, num_samples=len(tr_ds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_dl = DataLoader(tr_ds, batch_size=bs, num_workers=nw, drop_last=True, pin_memory=True, shuffle=True)\n# tr_dl = DataLoader(tr_ds, batch_size=bs, num_workers=nw, drop_last=False, pin_memory=True, sampler=sampler)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_va_dl = DataLoader(tr_va_ds, batch_size=bs, num_workers=nw, drop_last=False, pin_memory=True, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"te_va_dl = DataLoader(te_va_ds, batch_size=bs, num_workers=nw, drop_last=False, pin_memory=True, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Flatten(nn.Module):\n    def forward(self, x):\n        return x.view(x.size()[0], -1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Bias(nn.Module):\n    def __init__(self, num_out):\n        super(Bias, self).__init__()\n        self.bias = nn.Parameter(torch.zeros(num_out).float())\n        \n    def forward(self, x):\n        return x + self.bias","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_classes = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Linear classifier\n# model = nn.Sequential(\n# #     nn.AvgPool2d(kernel_size=2),\n#     Flatten(),\n#     nn.Linear(in_features=480 * 480 * 3,\n#              out_features=num_classes)\n# ).cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = models.densenet201(pretrained=True)\nfor param in model.parameters():\n    param.requires_grad = False\nmodel.classifier = nn.Sequential(\n    nn.BatchNorm1d(1920),\n    nn.Dropout(p=0.5, inplace=False),\n    nn.Linear(in_features=1920, out_features=512),\n    nn.ReLU(inplace=True),\n    nn.BatchNorm1d(512),\n    nn.Dropout(p=0.5, inplace=False),\n    nn.Linear(in_features=512, out_features=num_classes),\n    nn.Linear(in_features=num_classes, out_features=1, bias=False),\n    Bias(num_classes - 1)\n#     nn.Hardtanh(min_val=0.0, max_val=4.0)\n)\n# model.fc = nn.Linear(in_features=2048, out_features=num_classes)\n# model.classifier.requires_grad = True\nmodel = model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for name, param in model.named_parameters():\n# # for layer in model.features:\n#     if 'norm' in name:\n# #     if isinstance(layer, nn.BatchNorm2d):\n# #         for param in layer.parameters():\n#         param.requires_grad = True\n#     else:\n# #         for param in layer.parameters():\n#         param.requires_grad = False\n\nfor param in model.classifier.parameters():\n    param.requires_grad = False\n    \n# for layer in model.modules():\n#     if isinstance(layer, nn.BatchNorm1d):\n#         for param in layer.parameters():\n#             param.requires_grad = False\n    \n# for param in model.features.transition1.parameters():\n#     param.requires_grad = True\n    \n# for param in model.features.transition2.parameters():\n#     param.requires_grad = True\n    \n# for param in model.features.transition3.parameters():\n#     param.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for name, param in model.named_parameters():\n    param.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.classifier[8].bias.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for name, param in model.named_parameters():\n    print('{:<48}\\t{}\\t{}'.format(name, param.requires_grad, torch.numel(param)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = Adam(model.parameters(), lr=3e-5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint = torch.load('../input/aptos-pytorch/checkpoint.tar')\nmodel.load_state_dict(checkpoint['model'], strict=False)\noptimizer.load_state_dict(checkpoint['optimizer'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for p in model.layer4.parameters():\n#     p.requires_grad = True\n# for p in model.layer3.parameters():\n#     p.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer.param_groups[0]['lr'] = 3e-6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# optimizer.param_groups[0]['betas'] = (0.999, 0.999)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CumulativeProbabilityOrdinalLoss(nn.Module):\n    def __init__(self):\n        super(CumulativeProbabilityOrdinalLoss, self).__init__()\n        \n    def forward(self, output, label):\n        prob = F.softmax(output, dim=1)\n        cdf = torch.cumsum(prob, dim=1)\n        one_hot = torch.zeros_like(prob)\n        one_hot.scatter_(1, label.view(-1, 1), 1)\n        label_cdf = torch.cumsum(one_hot, dim=1)\n        return F.mse_loss(cdf, label_cdf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CoralLoss(nn.Module):\n    def __init__(self):\n        super(CoralLoss, self).__init__()\n        self.exceed_rank = torch.tensor([[0, 0, 0, 0],\n                            [1, 0, 0, 0],\n                            [1, 1, 0, 0],\n                            [1, 1, 1, 0],\n                            [1, 1, 1, 1],\n                           ]).float().cuda()\n        \n    def forward(self, output, label):\n        return F.binary_cross_entropy_with_logits(output, self.exceed_rank[label])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class SquareWeightedMSEOnProbabilityLoss(nn.Module):\n    def __init__(self):\n        super(SquareWeightedMSEOnProbabilityLoss, self).__init__()\n        self.weights = torch.tensor([\n            [0, 1, 4, 9, 16], \n            [1, 0, 1, 4, 9], \n            [4, 1, 0, 1, 4], \n            [9, 4, 1, 0, 1], \n            [16, 9, 4, 1, 0]], dtype=torch.float32).cuda()\n        \n    def forward(self, output, label):\n        prob = F.softmax(output, dim=1)\n        return torch.mean(torch.pow(torch.mul(self.weights[label], prob), 2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LabelSmoothingLoss(nn.Module):\n    def __init__(self, classes, smoothing=0.0, dim=-1):\n        super(LabelSmoothingLoss, self).__init__()\n        self.confidence = 1.0 - smoothing\n        self.smoothing = smoothing\n        self.cls = classes\n        self.dim = dim\n\n    def forward(self, pred, target):\n        pred = pred.log_softmax(dim=self.dim)\n        with torch.no_grad():\n            # true_dist = pred.data.clone()\n            true_dist = torch.zeros_like(pred)\n            true_dist.fill_(self.smoothing / (self.cls - 1))\n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)\n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, gamma=3., reduction='mean', weight=None):\n        super().__init__()\n        self.gamma = gamma\n        self.reduction = reduction\n        self.weight = weight\n\n    def forward(self, inputs, targets):\n        CE_loss = nn.CrossEntropyLoss(reduction='none', weight=self.weight)(inputs, targets)\n        pt = torch.exp(-CE_loss)\n        F_loss = ((1 - pt)**self.gamma) * CE_loss\n        if self.reduction == 'sum':\n            return F_loss.sum()\n        elif self.reduction == 'mean':\n            return F_loss.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# criterion = nn.CrossEntropyLoss()\n# criterion = LabelSmoothingLoss(0.2)\n# ce_loss_weights = torch.tensor([1.0, 15.0, 5.0, 10.0, 10.0])\n# ce_loss_weights /= ce_loss_weights.sum()\n# criterion = FocalLoss()\n# criterion = nn.MSELoss()\ncriterion = CoralLoss()\n# criterion = CumulativeProbabilityOrdinalLoss()\n# criterion = SquareWeightedMSEOnProbabilityLoss()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"def reg_out_to_class(out):\n    return torch.round(out).int()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def coral_out_to_class(out):\n    out = torch.sigmoid(out)\n    out = out > 0.5\n    return torch.sum(out, dim=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_one_epoch(model, optimizer, criterion, dl, batch_count=None):\n    if batch_count is None:\n        batch_count = len(dl)\n    metrics = []\n#     count = 0\n    for inputs, targets in tqdm(dl):\n        model.train()\n        inputs, targets = inputs.cuda(), targets.cuda() #targets.reshape((-1,1)).float().cuda()\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        optimizer.step()\n        \n        with torch.no_grad():\n            model.eval()\n#             preds = torch.argmax(outputs, dim=1)\n#             preds = reg_out_to_class(outputs)\n            preds = coral_out_to_class(outputs)\n#             print(preds.dtype)\n#             targets = targets.int()\n            accu = (preds == targets).sum().item() / float(inputs.shape[0])\n            metrics.append((loss.item(), accu))\n#         count += 1\n#         if count >= batch_count:\n#             break\n    \n    return metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def validate_model_on(model, dl, criterion):\n    with torch.no_grad():\n        model.eval()\n        conf_matrix = [[0 for i in range(num_classes)] for j in range(num_classes)]\n        loss = 0.0\n        accu = 0.0\n        y1, y2 = [], []\n        for inputs, targets in tqdm(dl):\n            inputs, targets = inputs.cuda(), targets.cuda() #targets.reshape((-1,1)).float().cuda()\n            outputs = model(inputs)\n#             preds = torch.argmax(outputs, dim=1)\n#             preds = reg_out_to_class(outputs)\n            preds = coral_out_to_class(outputs)\n            y1.extend(targets.tolist())\n            y2.extend(preds.tolist())\n            loss += criterion(outputs, targets).item()\n#             targets = targets.int()\n#             print(preds, targets)\n            accu += (preds == targets).sum().item() / float(inputs.shape[0])\n            for i in range(len(preds)):\n                conf_matrix[targets[i]][preds[i]] += 1\n        loss /= len(dl)\n        accu /= len(dl)\n        kappa = cohen_kappa_score(y1, y2, weights='quadratic')\n    return loss, accu, kappa, conf_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_epochs = 40","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_metrics = []\ntr_va_metrics = []\nte_va_metrics = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for epoch in tqdm(range(num_epochs)):                        \n    print('Epoch: {}'.format(epoch))\n    metrics = train_one_epoch(model, optimizer, criterion, tr_dl)\n    mean = lambda l: sum(l) / len(l)\n    tr_loss = mean(list(map(lambda m:m[0], metrics)))\n    tr_accu = mean(list(map(lambda m:m[1], metrics)))\n    tr_metrics.extend(metrics)\n    print('Train Loss: {} Acc: {}'.format(tr_loss, tr_accu))\n\n    tr_va_loss, tr_va_accu, tr_va_kappa, tr_va_conf_matrix = validate_model_on(model, tr_va_dl, criterion)\n    print('Train Val Loss: {} Acc: {} Kappa: {}'.format(tr_va_loss, tr_va_accu, tr_va_kappa))\n    for row in tr_va_conf_matrix:\n        print(row)\n    tr_va_metrics.append((tr_va_loss, tr_va_accu, tr_va_kappa))\n    \n#     te_va_loss, te_va_accu, te_va_kappa, te_va_conf_matrix = validate_model_on(model, te_va_dl, criterion)\n#     print('Test Val Loss: {} Acc: {} Kappa: {}'.format(te_va_loss, te_va_accu, te_va_kappa))\n#     for row in te_va_conf_matrix:\n#         print(row)\n#     te_va_metrics.append((te_va_loss, te_va_accu, te_va_kappa))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# del model\n# import gc\n# gc.collect()\n# torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!nvidia-smi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(list(map(lambda t:t[0], tr_metrics)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(list(map(lambda t:t[1], tr_metrics)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(list(map(lambda t:t[0], tr_va_metrics)))\nplt.plot(list(map(lambda t:t[0], te_va_metrics)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(list(map(lambda t:t[1], tr_va_metrics)))\nplt.plot(list(map(lambda t:t[1], te_va_metrics)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(list(map(lambda t:t[2], tr_va_metrics)))\nplt.plot(list(map(lambda t:t[2], te_va_metrics)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save({'model': model.state_dict(),\n           'optimizer': optimizer.state_dict()}, '/kaggle/working/checkpoint.tar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}