{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install git+https://github.com/Cadene/pretrained-models.pytorch","metadata":{"_uuid":"778959bdbda9432b2ba41c1b549d0225d22cc958","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport cv2\nimport torch\nimport torch.nn as nn \nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import transforms, Compose\nimport torchvision.models as models\nimport pretrainedmodels\nfrom sklearn.model_selection import train_test_split\nfrom PIL import Image\nfrom tqdm import tqdm_notebook, tqdm\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH = '../input/'\nTRAIN = os.path.join(PATH, 'train')\nTEST = os.path.join(PATH, 'test')\n\ntrain_df = pd.read_csv(os.path.join(PATH, 'train.csv'))\n# Try known only \n# train_df = train_df[train_df['Id'] != 'new_whale']\nNCATS = train_df['Id'].nunique()\ncatlist = train_df['Id'].unique()\nprint(\"Number of train images: {}\".format(len(os.listdir(\"../input/train\"))))\nprint(\"Number of test images: {}\".format(len(os.listdir(\"../input/test\"))))","metadata":{"_uuid":"d5af3fe4b389edc50f24681e9267dcd4796b3d11","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Helpers\n\ndef label2sparse(label):\n    vec = np.zeros((NCATS, 1))\n    vec[np.ravel(np.where(catlist == label))[0]] = 1\n    return torch.Tensor(vec.squeeze())\n\ndef label2ix(label):\n    return np.ravel(np.where(catlist == label)).squeeze()\n\ndef ix2label(ix):\n    return catlist[ix]\n\ndef sparse2label(sparse):\n    return catlist[np.argmax(sparse.squeeze(), axis=-1)]","metadata":{"_uuid":"d9f6a2dfb3b7aed5c7472a7b4596ef30290acb84","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pretrainedmodels.model_names","metadata":{"_uuid":"ae571ea55570dd3c8721f2431294b532df02a1cb","_kg_hide-output":true,"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name = 'se_resnext50_32x4d'\npretrainedmodels.pretrained_settings[model_name]\nmodel = pretrainedmodels.__dict__[model_name](num_classes=1000)","metadata":{"_uuid":"eb7516bf688cb943bf979ae68990909b566ae4ba","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.last_linear = nn.Conv2d(2688, NCATS, kernel_size=(1, 1), stride=(1, 1))\nmodel.last_linear = nn.Linear(in_features=2048, out_features=NCATS, bias=True)","metadata":{"_uuid":"a45f0fe45a9dc84561e9e08538af563cc9853da9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = Compose([transforms.ToPILImage(), \n                     transforms.RandomHorizontalFlip(0.5),\n                     transforms.Resize((224,224)),\n                     transforms.RandomAffine(degrees=15),\n                     transforms.ToTensor(),\n                     transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                        std=[0.229, 0.224, 0.225])])\n\ntest_transform = Compose([transforms.ToPILImage(), \n                     transforms.Resize((224,224)),\n                     transforms.ToTensor(),\n                    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                    std=[0.229, 0.224, 0.225]),])","metadata":{"_uuid":"630b9d7d8c918f6b6997f12408a000042eb90b40","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def accuracy(output, target, topk=(5,)):\n    \"\"\"Computes the accuracy over the k top predictions for the specified values of k\"\"\"\n    with torch.no_grad():\n        maxk = max(topk)\n        batch_size = target.size(0)\n\n        _, pred = output.topk(maxk, 1, True, True)\n        pred = pred.t()\n        correct = pred.eq(target.view(1, -1).expand_as(pred))\n\n        res = []\n        for k in topk:\n            correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)\n            res.append(correct_k.mul_(100.0 / batch_size))\n        return res\n    \ndef mapk(output, target, k=5):\n    \"\"\"\n    Computes the mean average precision at k.\n    \n    Parameters\n    ----------\n    output (torch.Tensor): A Tensor of predicted elements.\n                           Shape: (N,C)  where C = number of classes, N = batch size\n    target (torch.int): A Tensor of elements that are to be predicted. \n                        Shape: (N) where each value is  0≤targets[i]≤C−1\n    k (int, optional): The maximum number of predicted elements\n    \n    Returns\n    -------\n    score (torch.float):  The mean average precision at k over the output\n    \"\"\"\n    with torch.no_grad():\n        batch_size = target.size(0)\n\n        _, pred = output.topk(k, 1, True, True)\n        pred = pred.t()\n        correct = pred.eq(target.view(1, -1).expand_as(pred))\n\n        for i in range(k):\n            correct[i] = correct[i]*(k-i)\n            \n        score = correct[:k].view(-1).float().sum(0, keepdim=True)\n        score.mul_(1.0 / (k * batch_size))\n        return score","metadata":{"_uuid":"f193f925f1e58e6832d3388f6112b363a3e02a69","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs= train_df['Image'].values\nlabels = train_df['Id'].values","metadata":{"_uuid":"516a54baf99f0fb92e415518bd4c46833417a7f9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(imgs, labels, test_size=0.05)","metadata":{"_uuid":"28d3c622cc3bd9e4fec94a9e9b3c6b8a63641a85","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class WhaleDataset(Dataset):\n    def __init__(self, img_list, label_list=None, transforms=None, train=True):\n        super(WhaleDataset, self).__init__()\n        self.img_list = img_list\n        self.label_list = label_list\n        self.transforms = transforms\n        self.train = train\n        \n    def __getitem__(self, idx):\n        img = cv2.imread(os.path.join(TRAIN, self.img_list[idx]))\n        if self.transforms:\n            img = self.transforms(img)\n        if self.train:\n            y = label2ix(self.label_list[idx])\n            return img, y\n        return img\n    \n    def __len__(self):\n        return len(self.img_list)","metadata":{"_uuid":"03cc93a576b4925d397b44a2ab0c328a65d12809","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = WhaleDataset(img_list=X_train, label_list=y_train, transforms=train_transform, train=True)\nvalid_data = WhaleDataset(img_list=X_val, label_list=y_val, transforms=test_transform, train=True)","metadata":{"_uuid":"53cb0de594995ee860299041169dcbaa6a0de97a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(train_data, shuffle=True, batch_size=32)\nvalid_loader = DataLoader(valid_data, shuffle=False, batch_size=32)","metadata":{"_uuid":"f3c56262b6bc3cffe5f35da9af9eb4047864fa0c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check = next(iter(train_loader))\ncheck_ims = check[0]\ncheck_labels = check[1]","metadata":{"_uuid":"b7a4d44c741a45cd6d1bb8384fc6cecfad7ecc20","scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.cpu()\nt = model(check_ims)","metadata":{"_uuid":"2da388405bce8a5e9941ca46a246707874f5bbec","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, axs = plt.subplots(1, min(5,len(check_ims)), figsize=(20,5))\nfor i in range(min(5, len(check_ims))):\n    axs[i].imshow(check_ims[i].permute(1,2,0))\n    axs[i].set_title(ix2label(check_labels[i]))","metadata":{"_uuid":"50c7fea0b56e822be078a46bd2d70884fbed8a21","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = torch.optim.Adam(model.parameters(), lr=0.002)\ncriterion = nn.CrossEntropyLoss()","metadata":{"_uuid":"a61e25648038570a8a3c9993d9c9d9cee5fb64a7","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%timeit\n\nepoch = 1\nmodel.cuda()\nmodel.train()\nbest_map5 = 0.\npbar = tqdm_notebook(train_loader)\nfor ep in range(epoch):\n    tr_loss, tr_score = 0., 0.\n    for batch_idx, (x, y) in enumerate(pbar):\n        pbar.set_postfix({\"Train loss\": tr_loss / (batch_idx + 1), \"Train Map@5\": tr_score / (batch_idx + 1)})\n        x, y = x.cuda(), y.cuda()\n        outputs = model(x)\n        loss = criterion(outputs, y)\n        tr_loss += loss.item()\n        tr_score += mapk(outputs, y).item()\n        loss.backward()\n        optimizer.step()\n        if not (batch_idx + 1) % 50:\n#             print(f\"Train loss: {tr_loss / (batch_idx + 1)}\")\n            print(f\"Train loss {tr_loss / (batch_idx + 1)} Train Map@5 {tr_score / (batch_idx + 1)}\")\n        \n        if not (batch_idx + 1) % 100:\n            model.eval()\n            val_loss, val_score = 0., 0.\n            for val_idx, (x, y) in enumerate(valid_loader):\n                x, y = x.cuda(), y.cuda()\n                outputs = model(x)\n                loss = criterion(outputs, y)\n                val_loss += loss.item()\n                val_score += mapk(outputs, y).item()\n            full_val_loss = val_loss / len(valid_loader)\n            full_val_score = val_score / len(valid_loader)\n            print(f\"Validation loss {full_val_loss:.5f}, Validation Map@5 {full_val_score:.5f}\")\n            if full_val_score > best_map5:\n                print(f'Validation Map@5 increased from {best_map5:.5f} to {full_val_score:.5f}')\n                best_map5 = full_val_score\n                torch.save({'epoch': epoch,\n                           'model_state_dict': model.state_dict(),\n                           'optimizer_state_dict': optimizer.state_dict(),\n                           'loss': full_val_loss,\n                           'map5': full_val_score}, f'{model_name}_run1.pth')\n\n            model.train()","metadata":{"_uuid":"b278e2512985c11222b85a67053ce7f47e62b9a5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"We are done\")","metadata":{"_uuid":"8ec76a6b49ab9f070e7ffb19c5bf0844cc723924","trusted":true},"execution_count":null,"outputs":[]}]}