{"cells":[{"metadata":{},"cell_type":"markdown","source":"**Credits**\n\nCredit goes to Konstantin Lopuhin for his kernel, https://www.kaggle.com/lopuhin/imet-2019-submission, parts of which are used here,\nSeResnet models from torchvision library https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py\n\nand to my husband for his patience and support :)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \nimport argparse\nfrom pathlib import Path\nfrom typing import Callable, List\nimport numpy as np\nimport pandas as pd\nimport tqdm\nfrom multiprocessing.pool import ThreadPool\n\nimport torch\nfrom torch import nn, cuda\nfrom torch.nn import functional as F\nimport torchvision.models as M\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import Dataset \n    \nimport cv2\nfrom PIL import Image\nfrom torchvision.transforms import (\n    ToTensor, Normalize, Compose, Resize, CenterCrop, RandomCrop,\n    RandomHorizontalFlip, RandomGrayscale)\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"ON_KAGGLE = True\nN_CLASSES = 1103\nDATA_ROOT = Path('../input/imet-2019-fgvc6' if ON_KAGGLE else '../data')\nRUN_ROOT = '../input/seresnext101-folds/' if ON_KAGGLE else '../data/results/'\nuse_sample = False\nuse_cuda = cuda.is_available()\nSIZE = 352\n\ntrain_root = DATA_ROOT / 'train'\ntest_root = DATA_ROOT / 'test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nprint('Files present in this directory', os.listdir(RUN_ROOT))\nprint('Files present in this directory', os.listdir(DATA_ROOT))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\n\nSeResnet models\n\nhttps://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py\n\"\"\"\n\nfrom collections import OrderedDict\nimport math\nimport torch.nn as nn\nfrom torch.utils import model_zoo\n\nclass SEModule(nn.Module): \n\n    def __init__(self, channels, reduction):\n        super(SEModule, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,\n                             padding=0)\n        self.relu = nn.ReLU(inplace=True)\n        self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,\n                             padding=0)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        module_input = x\n        x = self.avg_pool(x)\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.fc2(x)\n        x = self.sigmoid(x)\n        return module_input * x\n\n\nclass Bottleneck(nn.Module):\n    \"\"\"\n    Base class for bottlenecks that implements `forward()` method.\n    \"\"\"\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out = self.se_module(out) + residual\n        out = self.relu(out)\n\n        return out\n\n\nclass SEBottleneck(Bottleneck):\n    \"\"\"\n    Bottleneck for SENet154.\n    \"\"\"\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None):\n        super(SEBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes * 2)\n        self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,\n                               stride=stride, padding=1, groups=groups,\n                               bias=False)\n        self.bn2 = nn.BatchNorm2d(planes * 4)\n        self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,\n                               bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SEResNetBottleneck(Bottleneck):\n    \"\"\"\n    ResNet bottleneck with a Squeeze-and-Excitation module. It follows Caffe\n    implementation and uses `stride=stride` in `conv1` and not in `conv2`\n    (the latter is used in the torchvision implementation of ResNet).\n    \"\"\"\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None):\n        super(SEResNetBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,\n                               stride=stride)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,\n                               groups=groups, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SEResNeXtBottleneck(Bottleneck):\n    \"\"\"\n    ResNeXt bottleneck type C with a Squeeze-and-Excitation module.\n    \"\"\"\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None, base_width=4):\n        super(SEResNeXtBottleneck, self).__init__()\n        width = math.floor(planes * (base_width / 64)) * groups\n        self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,\n                               stride=1)\n        self.bn1 = nn.BatchNorm2d(width)\n        self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,\n                               padding=1, groups=groups, bias=False)\n        self.bn2 = nn.BatchNorm2d(width)\n        self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SENet(nn.Module):\n\n    def __init__(self, block, layers, groups, reduction, dropout_p=0.2,\n                 inplanes=128, input_3x3=True, downsample_kernel_size=3,\n                 downsample_padding=1, num_classes=1000):\n        \"\"\"\n        Parameters\n        ----------\n        block (nn.Module): Bottleneck class.\n            - For SENet154: SEBottleneck\n            - For SE-ResNet models: SEResNetBottleneck\n            - For SE-ResNeXt models:  SEResNeXtBottleneck\n        layers (list of ints): Number of residual blocks for 4 layers of the\n            network (layer1...layer4).\n        groups (int): Number of groups for the 3x3 convolution in each\n            bottleneck block.\n            - For SENet154: 64\n            - For SE-ResNet models: 1\n            - For SE-ResNeXt models:  32\n        reduction (int): Reduction ratio for Squeeze-and-Excitation modules.\n            - For all models: 16\n        dropout_p (float or None): Drop probability for the Dropout layer.\n            If `None` the Dropout layer is not used.\n            - For SENet154: 0.2\n            - For SE-ResNet models: None\n            - For SE-ResNeXt models: None\n        inplanes (int):  Number of input channels for layer1.\n            - For SENet154: 128\n            - For SE-ResNet models: 64\n            - For SE-ResNeXt models: 64\n        input_3x3 (bool): If `True`, use three 3x3 convolutions instead of\n            a single 7x7 convolution in layer0.\n            - For SENet154: True\n            - For SE-ResNet models: False\n            - For SE-ResNeXt models: False\n        downsample_kernel_size (int): Kernel size for downsampling convolutions\n            in layer2, layer3 and layer4.\n            - For SENet154: 3\n            - For SE-ResNet models: 1\n            - For SE-ResNeXt models: 1\n        downsample_padding (int): Padding for downsampling convolutions in\n            layer2, layer3 and layer4.\n            - For SENet154: 1\n            - For SE-ResNet models: 0\n            - For SE-ResNeXt models: 0\n        num_classes (int): Number of outputs in `last_linear` layer.\n            - For all models: 1000\n        \"\"\"\n        super(SENet, self).__init__()\n        self.inplanes = inplanes\n        if input_3x3:\n            layer0_modules = [\n                ('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,\n                                    bias=False)),\n                ('bn1', nn.BatchNorm2d(64)),\n                ('relu1', nn.ReLU(inplace=True)),\n                ('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,\n                                    bias=False)),\n                ('bn2', nn.BatchNorm2d(64)),\n                ('relu2', nn.ReLU(inplace=True)),\n                ('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,\n                                    bias=False)),\n                ('bn3', nn.BatchNorm2d(inplanes)),\n                ('relu3', nn.ReLU(inplace=True)),\n            ]\n        else:\n            layer0_modules = [\n                ('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,\n                                    padding=3, bias=False)),\n                ('bn1', nn.BatchNorm2d(inplanes)),\n                ('relu1', nn.ReLU(inplace=True)),\n            ]\n        # To preserve compatibility with Caffe weights `ceil_mode=True`\n        # is used instead of `padding=1`.\n        layer0_modules.append(('pool', nn.MaxPool2d(3, stride=2,\n                                                    ceil_mode=True)))\n        self.layer0 = nn.Sequential(OrderedDict(layer0_modules))\n        self.layer1 = self._make_layer(\n            block,\n            planes=64,\n            blocks=layers[0],\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=1,\n            downsample_padding=0\n        )\n        self.layer2 = self._make_layer(\n            block,\n            planes=128,\n            blocks=layers[1],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.layer3 = self._make_layer(\n            block,\n            planes=256,\n            blocks=layers[2],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.layer4 = self._make_layer(\n            block,\n            planes=512,\n            blocks=layers[3],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        #self.avg_pool = nn.AvgPool2d(7, stride=1) \n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n\n        self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None\n        self.last_linear = nn.Linear(512 * block.expansion, num_classes)\n\n    def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,\n                    downsample_kernel_size=1, downsample_padding=0):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes * block.expansion,\n                          kernel_size=downsample_kernel_size, stride=stride,\n                          padding=downsample_padding, bias=False),\n                nn.BatchNorm2d(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, groups, reduction, stride,\n                            downsample))\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups, reduction))\n\n        return nn.Sequential(*layers)\n\n    def features(self, x):\n        x = self.layer0(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        return x\n\n    def logits(self, x):\n        x = self.avg_pool(x)\n        if self.dropout is not None:\n            x = self.dropout(x)\n        x = x.view(x.size(0), -1)\n        x = self.last_linear(x)\n        return x\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.logits(x)\n        return x\n\n\ndef initialize_pretrained_model(model, num_classes, settings):\n    assert num_classes == settings['num_classes'], \\\n        'num_classes should be {}, but is {}'.format(\n            settings['num_classes'], num_classes)\n    model.load_state_dict(model_zoo.load_url(settings['url']))\n    model.input_space = settings['input_space']\n    model.input_size = settings['input_size']\n    model.input_range = settings['input_range']\n    model.mean = settings['mean']\n    model.std = settings['std']\n\ndef se_resnext101(num_classes=1000, pretrained=None):\n    model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,\n                  dropout_p=0.5, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=num_classes)    \n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = se_resnext101()\nmodel.last_linear = nn.Linear(model.last_linear.in_features, N_CLASSES)\n\nall_params = list(model.parameters())\nuse_cuda = cuda.is_available()\nif use_cuda:\n    model = model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_model(model, root: str, fold: int, use_cuda: bool):\n    \"\"\"Loads model checkpoints\n       Choose evaluation mode\n    \"\"\"\n    best_model_path = root + 'best-metric_fold'+str(fold)+'.pt'\n    if use_cuda:\n        state_dict = torch.load(best_model_path)\n    else:\n        state_dict = torch.load(best_model_path, map_location='cpu')        \n    # model’s parameters\n    model.load_state_dict(state_dict['model'])\n    print('Loaded model from epoch {epoch}, step {step:,}'.format(**state_dict))\n    model.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_transform = Compose([\n    Resize((SIZE, SIZE)),\n    RandomHorizontalFlip(0.5),\n  #  CenterCrop(SIZE),\n  #  RandomCrop(SIZE),\n  #  ColorJitter(brightness=(0.6, 1.4), contrast=0, saturation=0, hue=0),\n  #  ColorJitter(brightness=0, contrast=(0.6, 1.4), saturation=0, hue=0),\n  #  ColorJitter(brightness=0, contrast=0, saturation=(0.6, 1.4), hue=0),  \n  #  ColorJitter(brightness=0, contrast=0, saturation=0, hue=(-0.2, 0.2)), \n  #  RandomGrayscale(p=0.2),    \n])    \n\ntensor_transform = Compose([\n    ToTensor(),\n    Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image(item, root: Path) -> Image.Image:\n    image = cv2.imread(str(root / f'{item.id}.png'))\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    return Image.fromarray(image)\n\ndef get_ids(root: Path) -> List[str]:\n    return sorted({p.name.split('_')[0] for p in root.glob('*.png')})\n\ndef mean_df(df: pd.DataFrame) -> pd.DataFrame:\n    return df.groupby(level=0).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nDatasets\n\"\"\" \nclass TTADataset(Dataset):\n    def __init__(self, root: Path, df: pd.DataFrame,\n                 image_transform: Callable, tta: int):\n        super().__init__()\n        self._root = root\n        self._df = df\n        self._image_transform = image_transform\n        self._tta = tta       \n\n    def __len__(self):\n        return len(self._df) * self._tta\n\n    def __getitem__(self, idx: int):\n        item = self._df.iloc[idx % len(self._df)]\n        image = load_image(item, self._root)  \n        \n        width, height = image.size             \n        if height < 320:\n            ratio = 400/height\n            image = image.resize((int(width * ratio), int(height * ratio)), Image.ANTIALIAS)            \n        if width < 320:\n            ratio = 400/width\n            image = image.resize((int(width * ratio), int(height * ratio)), Image.ANTIALIAS)     \n            \n        image = self._image_transform(image)\n        image = tensor_transform(image)    \n        return image, item.id   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(model, root: Path, predict_df: pd.DataFrame, save_root: str,\n            image_transform, batch_size: int, tta: int, workers: int, use_cuda: bool):\n    \"\"\"\n    Make and save preditions\n    \"\"\"    \n    valid_loader = DataLoader(\n                TTADataset(root, predict_df, image_transform, tta),\n                shuffle=False,\n                batch_size=batch_size,\n                num_workers=workers,\n                )\n    print(f'{len(valid_loader.dataset):,} in valid')\n    \n    all_outputs, all_ids = [], []        \n    with torch.no_grad():\n        for inputs, ids in tqdm.tqdm(valid_loader, desc='Predict'):\n            if use_cuda:\n                inputs = inputs.cuda()\n            outputs = torch.sigmoid(model(inputs))\n            all_outputs.append(outputs.data.cpu().numpy())\n            all_ids.extend(ids)    \n    # print(all_outputs) \n    df = pd.DataFrame(\n            data=np.concatenate(all_outputs),\n            index=all_ids,\n            columns=map(str, range(N_CLASSES)))\n    df = mean_df(df)\n    print('probs: ', df.head(10))    \n    #df.to_hdf(save_root, 'prob', index_label='id')\n    #print(f'Saved predictions to {out_path}')\n    return df\n       \n        \ndef binarize_prediction(probabilities, threshold: float, argsorted=None,\n                        min_labels=1, max_labels=10):\n    \"\"\" Return matrix of 0/1 predictions, same shape as probabilities.\n    \"\"\"\n    assert probabilities.shape[1] == N_CLASSES\n    if argsorted is None:\n        argsorted = probabilities.argsort(axis=1)\n    max_mask = _make_mask(argsorted, max_labels)\n    min_mask = _make_mask(argsorted, min_labels)\n    prob_mask = probabilities > threshold\n    return (max_mask & prob_mask) | min_mask\n\n\ndef _make_mask(argsorted, top_n: int):\n    mask = np.zeros_like(argsorted, dtype=np.uint8)\n    col_indices = argsorted[:, -top_n:].reshape(-1)\n    row_indices = [i // top_n for i in range(len(col_indices))]\n    mask[row_indices, col_indices] = 1\n    return mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_kwargs = dict(\n        image_transform = test_transform,\n        batch_size=16,\n        tta=2,\n        workers=0,\n        use_cuda=use_cuda\n        )    \n\nss = pd.read_csv(DATA_ROOT/'sample_submission.csv')\nif use_sample:\n    ss = ss.head(100)     \nprint(ss.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(\n        DATA_ROOT / 'sample_submission.csv', index_col='id')\n\ndef get_classes(item):\n    return ' '.join(cls for cls, is_present in item.items() if is_present)\n\ndfs = []\npredictions = []\nfolds = [0, 11, 2, 3, 4]\nfor fold in folds:\n    load_model(model, RUN_ROOT, fold, use_cuda)\n    out_path='test_'+str(fold)+'.h5'  \n    df = predict(model, test_root, ss, out_path, **predict_kwargs)\n    df = df.reindex(sample_submission.index)\n    dfs.append(df)\n    print(dfs)\n        \ndf = pd.concat(dfs)\nprint(df.head())\n# average 5 folds\ndf = mean_df(df)\ndf[:] = binarize_prediction(df.values, threshold=0.11)\ndf = df.apply(get_classes, axis=1)\ndf.name = 'attribute_ids'\ndf.to_csv('submission.csv', header=True)\nprint(df.head()) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Done!')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}