{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\n\nimport os\nimport warnings\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nfrom matplotlib import pyplot as plt\n\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torchvision import transforms\n\nfrom PIL import Image\nimport cv2\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder\n\n\nwarnings.filterwarnings(\"ignore\",category=DeprecationWarning)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d98f04ea4f5039cd37669243369c9662e850d6c7"},"cell_type":"code","source":"os.listdir('../input')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"84ec3d84a106b4c143782fd1b0d3a16d98c60e6f"},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\nTRN_IMGS_DIR = '../input/train/'\nTST_IMGS_DIR = '../input/test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0dfbfeb062c033d85333a03a0fef04440ff18815"},"cell_type":"code","source":"NCLASSES = len(train_df['Id'].unique())\nprint(NCLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da7963e7dcead8f12a5efe95ed3734a2a0da899c"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5871eafdc8c0b05b9d033a8f04995b2e3af674e1"},"cell_type":"markdown","source":"## Creating Labels"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def prepare_labels(y):\n    # From here: https://www.kaggle.com/pestipeti/keras-cnn-starter\n    values = np.array(y)\n    label_encoder = LabelEncoder()\n    integer_encoded = label_encoder.fit_transform(values)\n\n    onehot_encoder = OneHotEncoder(sparse=False)\n    integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n    onehot_encoded = onehot_encoder.fit_transform(integer_encoded)\n\n    y = onehot_encoded\n    return y, label_encoder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5227809fcfa52e71875e3fd55a334476719c8de1"},"cell_type":"code","source":"y, label_encoder = prepare_labels(train_df['Id'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"78f8034ba35daeab88359032ec680d776f91890f"},"cell_type":"markdown","source":"## Creating Augmentations"},{"metadata":{"trusted":true,"_uuid":"60bf2d2d91da443875bc3186afb16959cc83c915"},"cell_type":"code","source":"trn_trnsfms = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.RandomAffine(degrees=30),\n    transforms.ToTensor(),\n    transforms.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225]\n    )\n])\n\ntst_trnsfms = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225]\n    )\n])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"87e029078dc4c24779bb969cf78885e0955627d3"},"cell_type":"markdown","source":"## Creating PyTorch Dataloader"},{"metadata":{"trusted":true,"_uuid":"03290de1b3022f4f76c33ce4874502e60ceb4024"},"cell_type":"code","source":"class WhaleDataLoader(Dataset):\n    def __init__(self, image_folder, process='train', df=None, transform=None, y=None):\n        self.image_folder = image_folder\n        self.imgs_list = [img for img in os.listdir(image_folder)]\n        self.process = process\n        self.transform = transform\n        self.y = y\n        if self.process == 'train':\n            self.df = df.values\n    \n    def __len__(self):\n        return len(self.imgs_list)\n    \n    def __getitem__(self, idx):\n        if self.process == 'train':\n            img_name = os.path.join(self.image_folder, self.df[idx][0])\n            label = self.y[idx]\n        \n        elif self.process == 'test':\n            img_name = os.path.join(self.image_folder, self.imgs_list[idx])\n            label = np.zeros((NCLASSES,))\n        \n        img = Image.open(img_name).convert('RGB')\n        img = self.transform(img)\n        if self.process == 'train':\n            return img, label\n        elif self.process == 'test':\n            return img, label, self.imgs_list[idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca1c61cb7a9952d486e5771b1a280ff1cadbda0b"},"cell_type":"code","source":"train_dataloader = WhaleDataLoader(image_folder = TRN_IMGS_DIR, process='train', df=train_df, transform=trn_trnsfms, y=y)\ntest_dataloader = WhaleDataLoader(image_folder = TST_IMGS_DIR, process='test', transform=tst_trnsfms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3c023aecff72dc6635160f8aed33d7b9cdaaf33"},"cell_type":"code","source":"batch_size = 10\nnum_workers = 4\n\ntrain_loader = DataLoader(train_dataloader, batch_size=batch_size, num_workers=num_workers, pin_memory=True, shuffle=True)\ntest_loader = DataLoader(test_dataloader, batch_size=batch_size, num_workers=num_workers, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e39f3a0dccccd39c4f0e8683158588d86736debe"},"cell_type":"markdown","source":"# Creating SeResnet Model\n\nhttps://github.com/KaiyangZhou/deep-person-reid/blob/master/torchreid/models/senet.py"},{"metadata":{"trusted":true,"_uuid":"e39504c4444845ed640508704eba81f4408ff190"},"cell_type":"code","source":"\"\"\"\nResNet code gently borrowed from\nhttps://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py\n\"\"\"\nfrom __future__ import print_function, division, absolute_import\nfrom collections import OrderedDict\nimport math\n\nimport torch.nn as nn\nfrom torch.utils import model_zoo\n\npretrained_settings = {\n    'se_resnext101_32x4d': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],\n            'num_classes': 1000\n        }\n    },\n}\n\n\nclass SEModule(nn.Module):\n\n    def __init__(self, channels, reduction):\n        super(SEModule, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,\n                             padding=0)\n        self.relu = nn.ReLU(inplace=True)\n        self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,\n                             padding=0)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        module_input = x\n        x = self.avg_pool(x)\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.fc2(x)\n        x = self.sigmoid(x)\n        return module_input * x\n\n\nclass Bottleneck(nn.Module):\n    \"\"\"\n    Base class for bottlenecks that implements `forward()` method.\n    \"\"\"\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out = self.se_module(out) + residual\n        out = self.relu(out)\n\n        return out\n\n\nclass SEBottleneck(Bottleneck):\n    \"\"\"\n    Bottleneck for SENet154.\n    \"\"\"\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None):\n        super(SEBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes * 2)\n        self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,\n                               stride=stride, padding=1, groups=groups,\n                               bias=False)\n        self.bn2 = nn.BatchNorm2d(planes * 4)\n        self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,\n                               bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SEResNetBottleneck(Bottleneck):\n    \"\"\"\n    ResNet bottleneck with a Squeeze-and-Excitation module. It follows Caffe\n    implementation and uses `stride=stride` in `conv1` and not in `conv2`\n    (the latter is used in the torchvision implementation of ResNet).\n    \"\"\"\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None):\n        super(SEResNetBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,\n                               stride=stride)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,\n                               groups=groups, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SEResNeXtBottleneck(Bottleneck):\n    \"\"\"\n    ResNeXt bottleneck type C with a Squeeze-and-Excitation module.\n    \"\"\"\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None, base_width=4):\n        super(SEResNeXtBottleneck, self).__init__()\n        width = math.floor(planes * (base_width / 64)) * groups\n        self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,\n                               stride=1)\n        self.bn1 = nn.BatchNorm2d(width)\n        self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,\n                               padding=1, groups=groups, bias=False)\n        self.bn2 = nn.BatchNorm2d(width)\n        self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SENet(nn.Module):\n\n    def __init__(self, block, layers, groups, reduction, dropout_p=0.2,\n                 inplanes=128, input_3x3=True, downsample_kernel_size=3,\n                 downsample_padding=1, num_classes=1000):\n        \"\"\"\n        Parameters\n        ----------\n        block (nn.Module): Bottleneck class.\n            - For SENet154: SEBottleneck\n            - For SE-ResNet models: SEResNetBottleneck\n            - For SE-ResNeXt models:  SEResNeXtBottleneck\n        layers (list of ints): Number of residual blocks for 4 layers of the\n            network (layer1...layer4).\n        groups (int): Number of groups for the 3x3 convolution in each\n            bottleneck block.\n            - For SENet154: 64\n            - For SE-ResNet models: 1\n            - For SE-ResNeXt models:  32\n        reduction (int): Reduction ratio for Squeeze-and-Excitation modules.\n            - For all models: 16\n        dropout_p (float or None): Drop probability for the Dropout layer.\n            If `None` the Dropout layer is not used.\n            - For SENet154: 0.2\n            - For SE-ResNet models: None\n            - For SE-ResNeXt models: None\n        inplanes (int):  Number of input channels for layer1.\n            - For SENet154: 128\n            - For SE-ResNet models: 64\n            - For SE-ResNeXt models: 64\n        input_3x3 (bool): If `True`, use three 3x3 convolutions instead of\n            a single 7x7 convolution in layer0.\n            - For SENet154: True\n            - For SE-ResNet models: False\n            - For SE-ResNeXt models: False\n        downsample_kernel_size (int): Kernel size for downsampling convolutions\n            in layer2, layer3 and layer4.\n            - For SENet154: 3\n            - For SE-ResNet models: 1\n            - For SE-ResNeXt models: 1\n        downsample_padding (int): Padding for downsampling convolutions in\n            layer2, layer3 and layer4.\n            - For SENet154: 1\n            - For SE-ResNet models: 0\n            - For SE-ResNeXt models: 0\n        num_classes (int): Number of outputs in `last_linear` layer.\n            - For all models: 1000\n        \"\"\"\n        super(SENet, self).__init__()\n        self.inplanes = inplanes\n        if input_3x3:\n            layer0_modules = [\n                ('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,\n                                    bias=False)),\n                ('bn1', nn.BatchNorm2d(64)),\n                ('relu1', nn.ReLU(inplace=True)),\n                ('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,\n                                    bias=False)),\n                ('bn2', nn.BatchNorm2d(64)),\n                ('relu2', nn.ReLU(inplace=True)),\n                ('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,\n                                    bias=False)),\n                ('bn3', nn.BatchNorm2d(inplanes)),\n                ('relu3', nn.ReLU(inplace=True)),\n            ]\n        else:\n            layer0_modules = [\n                ('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,\n                                    padding=3, bias=False)),\n                ('bn1', nn.BatchNorm2d(inplanes)),\n                ('relu1', nn.ReLU(inplace=True)),\n            ]\n        # To preserve compatibility with Caffe weights `ceil_mode=True`\n        # is used instead of `padding=1`.\n        layer0_modules.append(('pool', nn.MaxPool2d(3, stride=2,\n                                                    ceil_mode=True)))\n        self.layer0 = nn.Sequential(OrderedDict(layer0_modules))\n        self.layer1 = self._make_layer(\n            block,\n            planes=64,\n            blocks=layers[0],\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=1,\n            downsample_padding=0\n        )\n        self.layer2 = self._make_layer(\n            block,\n            planes=128,\n            blocks=layers[1],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.layer3 = self._make_layer(\n            block,\n            planes=256,\n            blocks=layers[2],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.layer4 = self._make_layer(\n            block,\n            planes=512,\n            blocks=layers[3],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.avg_pool = nn.AvgPool2d(7, stride=1)\n        self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None\n        self.last_linear = nn.Linear(512 * block.expansion, num_classes)\n\n    def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,\n                    downsample_kernel_size=1, downsample_padding=0):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes * block.expansion,\n                          kernel_size=downsample_kernel_size, stride=stride,\n                          padding=downsample_padding, bias=False),\n                nn.BatchNorm2d(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, groups, reduction, stride,\n                            downsample))\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups, reduction))\n\n        return nn.Sequential(*layers)\n\n    def features(self, x):\n        x = self.layer0(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        return x\n\n    def logits(self, x):\n        x = self.avg_pool(x)\n        if self.dropout is not None:\n            x = self.dropout(x)\n        x = x.view(x.size(0), -1)\n        x = self.last_linear(x)\n        return x\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.logits(x)\n        return x\n\n\ndef initialize_pretrained_model(model, num_classes, settings):\n    assert num_classes == settings['num_classes'], \\\n        'num_classes should be {}, but is {}'.format(\n            settings['num_classes'], num_classes)\n    model.load_state_dict(model_zoo.load_url(settings['url']))\n    model.input_space = settings['input_space']\n    model.input_size = settings['input_size']\n    model.input_range = settings['input_range']\n    model.mean = settings['mean']\n    model.std = settings['std']\n\ndef se_resnext101(num_classes=1000, pretrained='imagenet'):\n    model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=num_classes)\n    if pretrained is not None:\n        settings = pretrained_settings['se_resnext101_32x4d'][pretrained]\n        initialize_pretrained_model(model, num_classes, settings)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"213079b6db1884ee6f0dcefc02722f23cc451d6b"},"cell_type":"code","source":"model = se_resnext101()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4859a01200eef2121283d91f030fab273f12f9b7"},"cell_type":"code","source":"# model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d78d51ac411688f7d6db7ef6978b8f4bb664900"},"cell_type":"code","source":"model.last_linear = nn.Linear(model.last_linear.in_features, NCLASSES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ded052b6531b92c68d11b95747f62c1849335e4"},"cell_type":"code","source":"model = model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d99a972e9248ddab7d672d735492552b4a1c246d"},"cell_type":"code","source":"criterion = nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.0005)\nscheduler = lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cdca1669d22968e9e0ea7357ffbbd0f3077d106e"},"cell_type":"code","source":"n_epochs = 13\nfor epoch in range(1, n_epochs+1):\n    train_loss = []\n    \n    for batch_i, (data, target) in tqdm(enumerate(train_loader), total = len(train_loader)):\n        data, target = data.cuda(), target.cuda()\n\n        optimizer.zero_grad()\n        output = model(data)\n        loss = criterion(output, target.float())\n        train_loss.append(loss.item())\n\n        loss.backward()\n        optimizer.step()\n    \n    scheduler.step()\n    \n    print(f'Epoch {epoch}, train loss: {np.mean(train_loss):.4f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c20dabccd9f181591155acca0bc482cf01e3cfff"},"cell_type":"code","source":"sub = pd.read_csv('../input/sample_submission.csv')\n\nmodel.eval()\nfor (data, target, name) in tqdm(test_loader):\n    data = data.cuda()\n    output = model(data)\n    output = output.cpu().detach().numpy()\n    for i, (e, n) in enumerate(list(zip(output, name))):\n        sub.loc[sub['Image'] == n, 'Id'] = ' '.join(label_encoder.inverse_transform(e.argsort()[-5:][::-1]))\n        \nsub.to_csv('submission.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef9d236467a63d86dd294e126957bd765bb648ac"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}