{"cells":[{"metadata":{"_uuid":"65cd3cf5ca1ec121afb3076284db222d43c31332"},"cell_type":"markdown","source":"## General information\n\nIn this kernel I'll build a CNN in Pytorch to identify Whales. The first attempt was done by using a simple CNN from scratch, but it didn't work well, so I'll use pre-trained nets.\n\n![](https://s23444.pcdn.co/wp-content/uploads/2014/04/Gold-Coast-whale.jpg.optimal.jpg)"},{"metadata":{"_uuid":"dc446c45b410c4576fd2a880cf38c1a1e3f16c3b"},"cell_type":"markdown","source":"### Versions\nHere I'll list significant changes in the notebook.\n* v10 - added augmentations and scheduler\n* v18 - changed augmentations to albumentations"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mplimg\nfrom matplotlib.pyplot import imshow\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport time \nimport tqdm\nfrom PIL import Image\ntrain_on_gpu = True\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\n\nimport warnings\nwarnings.simplefilter(\"ignore\", category=DeprecationWarning)\n\nfrom collections import OrderedDict\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"90e2be0bcfc3a6cc18ffc50e0d3ed670ac1d6f93"},"cell_type":"code","source":"!pip install albumentations > /dev/null 2>&1","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"d16a5bfa1ac429a45db1520855cc1dc07615db58"},"cell_type":"code","source":"!pip install pretrainedmodels > /dev/null 2>&1","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"e8621320581aa17d3057c3067609e62486e72750"},"cell_type":"code","source":"import albumentations\nfrom albumentations import torch as AT\nimport pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"79ca765cb9e9c1dcfb69300bba292a3a789ea71b"},"cell_type":"markdown","source":"### Data overview"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d607af1c2739c6a7cebc4fe4362870b59f2bce02"},"cell_type":"markdown","source":"For train data we have a DataFrame with image names and ids. And of course for train and test we have images in separate folders."},{"metadata":{"trusted":true,"_uuid":"0604444c22d6783a808475c80b346ae368f85ad9"},"cell_type":"code","source":"print(f\"There are {len(os.listdir('../input/train'))} images in train dataset with {train_df.Id.nunique()} unique classes.\")\nprint(f\"There are {len(os.listdir('../input/test'))} images in test dataset.\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5828acc2518e76b64f379e6ffec0440301ce3765"},"cell_type":"markdown","source":"25k images in train and 5k different whales!\nLet's have a look at them"},{"metadata":{"trusted":true,"_uuid":"51aec6cb3e890054604df1dc346b1cc2cd0c38c9","_kg_hide-input":true},"cell_type":"code","source":"fig = plt.figure(figsize=(25, 4))\ntrain_imgs = os.listdir(\"../input/train\")\nfor idx, img in enumerate(np.random.choice(train_imgs, 20)):\n    ax = fig.add_subplot(2, 20//2, idx+1, xticks=[], yticks=[])\n    im = Image.open(\"../input/train/\" + img)\n    plt.imshow(im)\n    lab = train_df.loc[train_df.Image == img, 'Id'].values[0]\n    ax.set_title(f'Label: {lab}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"70da229d34b5d4890e6901b7608bf65ee996ff09"},"cell_type":"code","source":"train_df.Id.value_counts().head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ef5018c01bffd05df3e0ffc1eef5f7d4c9c089f"},"cell_type":"code","source":"for i in range(1, 4):\n    print(f'There are {train_df.Id.value_counts()[train_df.Id.value_counts().values==i].shape[0]} classes with {i} samples in train data.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b64aaf68584a68f50eee0594f98389d952a62820"},"cell_type":"code","source":"plt.title('Distribution of classes excluding new_whale');\ntrain_df.Id.value_counts()[1:].plot(kind='hist');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0efdb1c87f2a4fee0329a6db629b3c382851b01c"},"cell_type":"markdown","source":"We can see that there is a huge disbalance in the data. There are many classes with only one or several samples , some classes have 50+ samples and \"default\" class has almost 10k samples."},{"metadata":{"trusted":true,"_uuid":"3a5ae37aeb6620e88e9dc5a757db1ade119e8719"},"cell_type":"code","source":"np.array(im).shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c427a42847b93030081ca999760b02eb32bcc1d5"},"cell_type":"markdown","source":"At least some images are quite big"},{"metadata":{"_uuid":"8d91957c713d868233f89c6d5366b35484e5e0e9"},"cell_type":"markdown","source":"### Preparing data for Pytorch\nData for Pytorch needs to be prepared:\n* we need to define transformations;\n* then we need to initialize a dataset class;\n* then we need to create dataloaders which will be used by the model;"},{"metadata":{"_uuid":"bba74501505504f3ed808e6fdb143c9902896702"},"cell_type":"markdown","source":"#### Transformations\n\nBasic transformations include only resizing the image to the necessary size, converting to Pytorch tensor and normalizing"},{"metadata":{"trusted":true,"_uuid":"926bc87c2e62085a531d44dab7dc34275e4ac1d9"},"cell_type":"code","source":"data_transforms = transforms.Compose([\n                                      transforms.Resize((100, 100)),\n                                      transforms.ToTensor(),\n                                      transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                             std=[0.229, 0.224, 0.225])\n    ])\ndata_transforms_test = transforms.Compose([\n                                           transforms.Resize((100, 100)),\n                                           transforms.ToTensor(),\n                                           transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                                 std=[0.229, 0.224, 0.225])\n])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3d7421376b463b9482df4478f654423007508e2c"},"cell_type":"markdown","source":"#### Encoding labels\nLabels need to be one-hot encoded."},{"metadata":{"trusted":true,"_uuid":"ada2c43ff6f77b5a9690e8c3479566a354691f57","_kg_hide-input":true},"cell_type":"code","source":"def prepare_labels(y):\n    # From here: https://www.kaggle.com/pestipeti/keras-cnn-starter\n    values = np.array(y)\n    label_encoder = LabelEncoder()\n    integer_encoded = label_encoder.fit_transform(values)\n\n    onehot_encoder = OneHotEncoder(sparse=False)\n    integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n    onehot_encoded = onehot_encoder.fit_transform(integer_encoded)\n\n    y = onehot_encoded\n    return y, label_encoder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b687499b0db7416a0057dbca5ba45e66e3eb2d7e"},"cell_type":"code","source":"y, le = prepare_labels(train_df['Id'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3b60f6f716a8f1f2a8597a420e144e7c0f7d6cc7"},"cell_type":"markdown","source":"#### Dataset\n\nNow we need to create a dataset. Sadly, default version won't work, as images for each class are supposed to be in separate folders. So I write a custom WhaleDataset."},{"metadata":{"trusted":true,"_uuid":"755513450736a1351b5bdf113ed790be93bb770f"},"cell_type":"code","source":"class WhaleDataset(Dataset):\n    def __init__(self, datafolder, datatype='train', df=None, transform = transforms.Compose([transforms.ToTensor()]), y=None):\n        self.datafolder = datafolder\n        self.datatype = datatype\n        self.y = y\n        if self.datatype == 'train':\n            self.df = df.values\n        self.image_files_list = [s for s in os.listdir(datafolder)]\n        self.transform = transform\n\n\n    def __len__(self):\n        return len(self.image_files_list)\n    \n    def __getitem__(self, idx):\n        if self.datatype == 'train':\n            img_name = os.path.join(self.datafolder, self.df[idx][0])\n            label = self.y[idx]\n            \n        elif self.datatype == 'test':\n            img_name = os.path.join(self.datafolder, self.image_files_list[idx])\n            label = np.zeros((5005,))\n\n        image = Image.open(img_name).convert('RGB')\n        image = self.transform(image)\n        if self.datatype == 'train':\n            return image, label\n        elif self.datatype == 'test':\n            # so that the images will be in a correct order\n            return image, label, self.image_files_list[idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25eef61b519adf930a0186db3206ca4f354a92e5"},"cell_type":"code","source":"train_dataset = WhaleDataset(datafolder='../input/train/', datatype='train', df=train_df, transform=data_transforms, y=y)\ntest_set = WhaleDataset(datafolder='../input/test/', datatype='test', transform=data_transforms_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"13105616c3e0b05f0a5b5a31466bc6c665306a39"},"cell_type":"markdown","source":"#### Loaders\n\nNow we create loaders. Here we define which images will be used, batch size and other things."},{"metadata":{"trusted":true,"_uuid":"4c310efe7efd3b8b952ad7ecdaa2281818fc2758"},"cell_type":"code","source":"train_sampler = SubsetRandomSampler(list(range(len(os.listdir('../input/train')))))\nvalid_sampler = SubsetRandomSampler(list(range(len(os.listdir('../input/test')))))\nbatch_size = 512\nnum_workers = 0\n\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers)\n# less size for test loader.\ntest_loader = torch.utils.data.DataLoader(test_set, batch_size=32, num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"42b9ce37d9bdd972279fd1d59a4921f4fda43fc9"},"cell_type":"markdown","source":"### Basic CNN\n\nNow we can define the model. For now I'll use a simple architecture with two convolutional layers."},{"metadata":{"trusted":true,"_uuid":"34b0f5d1da15f33338a2d6829e9b630537f03a26"},"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self):\n        super(Net, self).__init__()\n        self.conv1 = nn.Conv2d(3, 32, 7, padding=1)\n        self.conv2_bn = nn.BatchNorm2d(32)\n        self.pool = nn.MaxPool2d(2, 2)\n        \n        self.conv2 = nn.Conv2d(32, 64, 3, padding=1)        \n        self.pool2 = nn.AvgPool2d(3, 3)\n        \n        self.fc1 = nn.Linear(64 * 4 * 4 * 16, 1024)\n        self.fc2 = nn.Linear(1024, 5005)\n\n        self.dropout = nn.Dropout(0.5)        \n\n    def forward(self, x):\n        x = self.pool(F.relu(self.conv2_bn(self.conv1(x))))\n        x = self.pool2(F.relu(self.conv2(x)))\n        x = x.view(-1, 64 * 4 * 4 * 16)\n        x = F.relu(self.fc1(x))\n        x = self.dropout(x)\n        x = self.fc2(x)\n\n        return x","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e3b34b9c419e87e383e5b3c49b1d9cbf20a89b75"},"cell_type":"markdown","source":"#### Initializing model\n\nWe need to define model, loss, oprimizer and possibly a scheduler."},{"metadata":{"trusted":true,"_uuid":"02d768d7a3c711800b8cd739c2414bbd210135f9"},"cell_type":"code","source":"model_conv = Net()\n\ncriterion = nn.BCEWithLogitsLoss()\n\noptimizer = optim.Adam(model_conv.parameters(), lr=0.01)\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8340f09e0b7b634b9b78d987d7bf0f4eb20bb327"},"cell_type":"markdown","source":"But this was an example. The basic model isn't really good. You can try it, but it is much better to use pre-trained models."},{"metadata":{"trusted":true,"_uuid":"7b0343907e89cea54c34dbdb1d07f19653d598d5","scrolled":true},"cell_type":"code","source":"# model_conv.cuda()\n# n_epochs = 10\n# for epoch in range(1, n_epochs+1):\n#     print(time.ctime(), 'Epoch:', epoch)\n\n#     train_loss = []\n#     exp_lr_scheduler.step()\n\n#     for batch_i, (data, target) in enumerate(train_loader):\n#         #print(batch_i)\n#         data, target = data.cuda(), target.cuda()\n\n#         optimizer.zero_grad()\n#         output = model_conv(data)\n#         loss = criterion(output, target.float())\n#         train_loss.append(loss.item())\n\n#         loss.backward()\n#         optimizer.step()\n\n#     print(f'Epoch {epoch}, train loss: {np.mean(train_loss):.4f}')\n\n# sub = pd.read_csv('../input/sample_submission.csv')\n\n# model_conv.eval()\n# for (data, target, name) in test_loader:\n#     data = data.cuda()\n#     output = model_conv(data)\n#     output = output.cpu().detach().numpy()\n#     for i, (e, n) in enumerate(list(zip(output, name))):\n#         sub.loc[sub['Image'] == n, 'Id'] = ' '.join(le.inverse_transform(e.argsort()[-5:][::-1]))\n        \n# sub.to_csv('basic_model.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d1b1257e94f3e8fe71c2c5d90751068cc6d0c130"},"cell_type":"markdown","source":"### Pretrained model\n\nNow I'll use a pretrained model. First this is changing transformations: I use a bigger image size and add augmentations."},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"1436925b67b037f54f6a61d19da193bffcab5291"},"cell_type":"code","source":"class WhaleDataset(Dataset):\n    def __init__(self, datafolder, datatype='train', df=None, transform = transforms.Compose([transforms.ToTensor()]), y=None\n                ):\n        self.datafolder = datafolder\n        self.datatype = datatype\n        self.y = y\n        if self.datatype == 'train':\n            self.df = df.values\n        self.image_files_list = [s for s in os.listdir(datafolder)]\n        self.transform = transform\n\n\n    def __len__(self):\n        return len(self.image_files_list)\n    \n    def __getitem__(self, idx):\n        if self.datatype == 'train':\n            img_name = os.path.join(self.datafolder, self.df[idx][0])\n            label = self.y[idx]\n            \n        elif self.datatype == 'test':\n            img_name = os.path.join(self.datafolder, self.image_files_list[idx])\n            label = np.zeros((5005,))\n\n        img = cv2.imread(img_name)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        image = self.transform(image=img)\n        image = image['image']\n        if self.datatype == 'train':\n            return image, label\n        elif self.datatype == 'test':\n            # so that the images will be in a correct order\n            return image, label, self.image_files_list[idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"571a135e2466c86498f1da8e3297afc8f1709143"},"cell_type":"code","source":"data_transforms = albumentations.Compose([\n    albumentations.Resize(160, 320),\n    albumentations.HorizontalFlip(),\n    albumentations.RandomBrightness(),\n    albumentations.ShiftScaleRotate(rotate_limit=15, scale_limit=0.10),\n    albumentations.JpegCompression(80),\n    albumentations.HueSaturationValue(),\n    albumentations.Normalize(),\n    AT.ToTensor()\n    ])\ndata_transforms_test = albumentations.Compose([\n    albumentations.Resize(160, 320),\n    albumentations.Normalize(),\n    AT.ToTensor()\n    ])\n\ntrain_dataset = WhaleDataset(datafolder='../input/train/', datatype='train', df=train_df, transform=data_transforms, y=y)\ntest_set = WhaleDataset(datafolder='../input/test/', datatype='test', transform=data_transforms_test)\n\ntrain_sampler = SubsetRandomSampler(list(range(len(os.listdir('../input/train')))))\nvalid_sampler = SubsetRandomSampler(list(range(len(os.listdir('../input/test')))))\nbatch_size = 10\nnum_workers = 2\n# prepare data loaders (combine dataset and sampler)\ntrain_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers)\n#valid_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers)\ntest_loader = torch.utils.data.DataLoader(test_set, batch_size=10, num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e87ced704a1a423093683f37062b7073d8b3521e"},"cell_type":"code","source":"# model_conv = torchvision.models.resnet101(pretrained=True)\n# for i, param in model_conv.named_parameters():\n#     param.requires_grad = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7fd8661be7755e5149ca3f2b24e4a09ce386b5f0"},"cell_type":"code","source":"# final_layer = nn.Sequential(OrderedDict([\n#                           ('fc1', nn.Linear(2048, 1024)),\n#                           ('relu', nn.ReLU()),\n#                           ('dropout', nn.Dropout(0.1)),\n#                           ('fc2', nn.Linear(1024, 5005)),\n#                           ]))\n# model_conv.fc = final_layer\n#model_conv.fc = nn.Linear(2048, 5005)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd0755f60202ac421d7be7700ed6662e01f72ac9"},"cell_type":"code","source":"model_conv = pretrainedmodels.resnext101_64x4d()\nmodel_conv.avg_pool = nn.AvgPool2d((5,10))\nmodel_conv.last_linear = nn.Linear(model_conv.last_linear.in_features, 5005)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3817adbcf500a7490a77da87a5fb8d37b2fa51b"},"cell_type":"code","source":"model_conv.cuda()\ncriterion = nn.BCEWithLogitsLoss()\n\noptimizer = optim.Adam(model_conv.parameters(), lr=0.01)\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"545a889a1ee511e173f4e9d8e8b1c03285276595"},"cell_type":"code","source":"n_epochs = 4\nfor epoch in range(1, n_epochs+1):\n    print(time.ctime(), 'Epoch:', epoch)\n\n    train_loss = []\n    \n\n    for batch_i, (data, target) in enumerate(train_loader):\n        # print(f'Batch {batch_i} of 50')\n        data, target = data.cuda(), target.cuda()\n\n        optimizer.zero_grad()\n        output = model_conv(data)\n        loss = criterion(output, target.float())\n        train_loss.append(loss.item())\n\n        loss.backward()\n        optimizer.step()\n    exp_lr_scheduler.step()\n\n    print(f'Epoch {epoch}, train loss: {np.mean(train_loss):.4f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"f006be49007b9d62e26b97f8777c6116a3965567"},"cell_type":"code","source":"sub = pd.read_csv('../input/sample_submission.csv')\n\nmodel_conv.eval()\nfor (data, target, name) in test_loader:\n    data = data.cuda()\n    output = model_conv(data)\n    output = output.cpu().detach().numpy()\n    for i, (e, n) in enumerate(list(zip(output, name))):\n        sub.loc[sub['Image'] == n, 'Id'] = ' '.join(le.inverse_transform(e.argsort()[-5:][::-1]))\n        \nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}