{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"import os, sys, math, time\nimport pandas as pd\nimport numpy as np\nfrom glob import glob\n\nfrom PIL import Image\nimport cv2\nimport imgaug as ia\nfrom imgaug import augmenters as iaa\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook, tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_SIZE = (300,300)\ndata = pd.read_csv('../input/imet-2019-fgvc6/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# unique labels - for sigmoid predict (1103 class)\nlabels_count = pd.Series(data['attribute_ids'].str.split(' ').sum()).value_counts()\npd.DataFrame(\n    data=np.array([labels_count.keys().values, labels_count.values]).T,\n    columns=['label', 'count'])[:3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# unique labels group - for softmax predict (9280 class)\ncount_groups = data['attribute_ids'].value_counts()\npd.DataFrame(\n    data=np.array([count_groups.keys().values, count_groups.values]).T,\n    columns=['group labels', 'count'])[:3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\ntorch.manual_seed(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class train_dataset(Dataset):\n    def __init__(self, df, path_to_images, shape=INPUT_SIZE, augmentation=True):\n        self.df = df\n        self.shape = shape\n        self.augmentation = augmentation\n        self.path_to_images = path_to_images\n        self.labels_group_encoder = {labels:0 \n                                     for labels in df['attribute_ids'].unique()}\n        labels_groups_mask = df['attribute_ids'].value_counts() > 1\n        labels_group = df['attribute_ids'].value_counts()[labels_groups_mask]\n        for idx, labels_group in enumerate(labels_group.index):\n            self.labels_group_encoder[labels_group] = idx\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, idx):\n        path_to_image = os.path.join(self.path_to_images, \n                                     self.df.iloc[idx]['id'] + '.png')\n        image = self._load_image(path_to_image)\n        \n        # create labels for sigmoid output\n        raw_sigmoid_label = self.df.iloc[idx]['attribute_ids'].strip().split(' ')\n        sigmoid_labels = torch.zeros(1103)\n        sigmoid_labels[[int(e) for e in raw_sigmoid_label]] = 1\n        \n        # create labels for softmax output\n        class_number = self.labels_group_encoder[self.df.iloc[idx]['attribute_ids']]\n        softmax_label = torch.tensor(class_number).long()\n        return (image, sigmoid_labels, softmax_label)\n    \n    def _load_image(self, path_to_image):\n        image = np.array(Image.open(path_to_image))\n        if self.augmentation:\n            image = self._augumentation_image(image)\n        image = cv2.resize(image, (self.shape[0], self.shape[1]))\n        image = np.divide(image, 255)\n        image = image.transpose(2,0,1)\n        return torch.tensor(image, dtype=torch.float)\n    \n    def _augumentation_image(self, image): \n        sometimes = lambda aug: iaa.Sometimes(0.1, aug)\n        \n        angle = np.random.randint(360)\n        augment_img = iaa.Sequential([\n            iaa.Fliplr(0.5),\n            iaa.Crop(px=(0, 50)),\n            sometimes(\n                iaa.Affine(\n                    scale={\"x\": (0.8, 1.2), \"y\": (0.8, 1.2)},\n                    translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)},\n                    rotate=(-25, 25),\n                    shear=(-16, 16),\n                    order=[0, 1],\n                    cval=(0, 255),\n                    mode=ia.ALL)),\n            sometimes(iaa.GaussianBlur(sigma=(0, 2.0))),\n            sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25))\n        ], random_order=True)\n        return augment_img.augment_image(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision.models.resnet import conv3x3\nfrom torchvision.models.resnet import conv1x1\n\nclass BasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, \n                 base_width=64, norm_layer=None):\n        super(BasicBlock, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        if groups != 1 or base_width != 64:\n            raise ValueError('BasicBlock only supports groups=1 and base_width=64')\n        self.conv1 = conv3x3(inplanes, planes, stride)\n        self.bn1 = norm_layer(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = conv3x3(planes, planes)\n        self.bn2 = norm_layer(planes)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        identity = x\n        \n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass ResNet(nn.Module):\n\n    def __init__(self, block, layers, sigmoid_num_class=1103, softmax_num_class=9280, \n                 zero_init_residual=False, groups=1, width_per_group=64, norm_layer=None):\n        super(ResNet, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n\n        self.inplanes = 64\n        self.groups = groups\n        self.base_width = width_per_group\n        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        self.bn1 = norm_layer(self.inplanes)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0], \n                                       norm_layer=norm_layer)\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2, \n                                       norm_layer=norm_layer)\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2, \n                                       norm_layer=norm_layer)\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2, \n                                       norm_layer=norm_layer)\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.sigmoid_fc = nn.Linear(512 * block.expansion, sigmoid_num_class)\n        self.softmax_fc = nn.Linear(512 * block.expansion, softmax_num_class)\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n        if zero_init_residual:\n            for m in self.modules():\n                if isinstance(m, Bottleneck):\n                    nn.init.constant_(m.bn3.weight, 0)\n                elif isinstance(m, BasicBlock):\n                    nn.init.constant_(m.bn2.weight, 0)\n\n    def _make_layer(self, block, planes, blocks, stride=1, norm_layer=None):\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                conv1x1(self.inplanes, planes * block.expansion, stride),\n                norm_layer(planes * block.expansion))\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride, downsample, self.groups, \n                            self.base_width, norm_layer))\n        self.inplanes = planes * block.expansion\n        for _ in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups=self.groups,\n                                base_width=self.base_width, norm_layer=norm_layer))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        x = x.view(x.size(0), -1)\n        \n        sigmoid_output = self.sigmoid_fc(x) # sigmoid output branch\n        softmax_output = self.softmax_fc(x) # softmax output branch\n        \n        return sigmoid_output, softmax_output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fbeta_score(prob, label, threshold=0.5, beta=2):\n    prob = prob > threshold\n    label = label > threshold\n\n    TP = (prob & label).sum(1).float()\n    TN = ((~prob) & (~label)).sum(1).float()\n    FP = (prob & (~label)).sum(1).float()\n    FN = ((~prob) & label).sum(1).float()\n\n    precision = TP / (TP + FP + 1e-12)\n    recall = TP / (TP + FN + 1e-12)\n    fscore = (1+beta**2)*precision*recall/(beta**2*precision+recall+1e-12)\n    return fscore.mean(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_epoch(model, train_loader, sigmoid_criterion, softmax_criterion, loss_blender, optimizer):\n    model.train()\n    \n    history = []\n    with tqdm(total=len(train_loader), file=sys.stdout) as pbar:\n        for step, (features, sigmoid_labels, softmax_labels) in enumerate(train_loader):\n            features, sigmoid_labels, softmax_labels = \\\n            features.cuda(), sigmoid_labels.cuda(), softmax_labels.cuda()\n            optimizer.zero_grad()\n            \n            sigmoid_logits, softmax_logits = model(features)\n            sigmoid_loss = sigmoid_criterion(sigmoid_logits, sigmoid_labels)\n            softmax_loss = softmax_criterion(softmax_logits, softmax_labels)\n            total_loss = loss_blender(sigmoid_loss, softmax_loss)\n            \n            total_loss.backward()\n            optimizer.step()\n\n            if step % 100 == 0:\n                f2 = fbeta_score(sigmoid_labels, torch.sigmoid(sigmoid_logits))\n                history.append({\n                    'f2_socre': f2,\n                    'total_loss':total_loss.item(),\n                    'softmax_loss':softmax_loss.item(),\n                    'sigmoid_loss':sigmoid_loss.item()})\n                pbar.set_description(\n                    'f2 (th=0.5): {0:.4}, total_loss {1:.4}, softmax_loss {2:.4}, sigmoid_loss {3:.4}'.format(\n                        f2.item(), total_loss.item(), softmax_loss.item(), sigmoid_loss.item()))\n                pbar.update(100)\n        pbar.close()\n        return history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_eval(model, valid_loader, sigmoid_criterion, softmax_criterion, loss_blender):\n    with torch.no_grad():\n        model.eval()\n\n        total_eval = 0; total_sigmoid_loss = 0; total_softmax_loss = 0; total_loss = 0\n        for step, (features, sigmoid_labels, softmax_labels) in enumerate(valid_loader):\n            features, sigmoid_labels, softmax_labels = \\\n            features.cuda(), sigmoid_labels.cuda(), softmax_labels.cuda()\n            \n            sigmoid_logits, softmax_logits = model(features)\n            \n            sigmoid_loss = sigmoid_criterion(sigmoid_logits, sigmoid_labels)\n            softmax_loss = softmax_criterion(softmax_logits, softmax_labels)\n            \n            total_loss += loss_blender(sigmoid_loss, softmax_loss)\n            total_eval += fbeta_score(sigmoid_labels, torch.sigmoid(sigmoid_logits))\n            \n            total_sigmoid_loss += sigmoid_loss.item()\n            total_softmax_loss += softmax_loss.item()\n\n        return total_loss/len(valid_loader), total_sigmoid_loss/len(valid_loader), softmax_loss/len(valid_loader), total_eval/len(valid_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_train(model, train_loader, valid_loader, sigmoid_criterion, softmax_criterion, loss_blender,\n                optimizer, n_epoch, checkpoint_path='./model.pth'):\n    \n    train_history = []; best_score = 0\n    for epoch in range(n_epoch):\n        train_history += train_epoch(\n            model, train_loader, sigmoid_criterion, softmax_criterion, loss_blender, optimizer)\n        val_total_loss, val_sigmoid_loss, val_softmax_loss, valid_score = model_eval(\n            model, valid_loader, sigmoid_criterion, softmax_criterion, loss_blender)\n        \n        print('val loss: {0:.4}, val f2 score: {1:.4}'.format(\n            val_total_loss, valid_score))\n        if valid_score > best_score:\n            print('save')\n            torch.save(model.state_dict(), checkpoint_path)\n            best_score = valid_score\n\n    return train_history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_history(history, figsize=(10,4)):\n    plt.figure(figsize=figsize)\n    total_loss = plt.plot([e['total_loss'] for e in history])\n    softmax_loss = plt.plot([e['softmax_loss'] for e in history])\n    sigmoid_loss = plt.plot([e['sigmoid_loss'] for e in history])\n    plt.legend((total_loss[0], softmax_loss[0], sigmoid_loss[0]), \n               ('total_loss', 'softmax_loss', 'sigmoid_loss'), fontsize=12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_idx, valid_idx, train_targets, valid_target = train_test_split(\n    np.arange(data.shape[0]), data['attribute_ids'], test_size=0.1, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainloader = DataLoader(\n    train_dataset(data.iloc[train_idx], '../input/imet-2019-fgvc6/train/', augmentation=True), \n    batch_size=64, shuffle=True, num_workers=8, pin_memory=True)\n\nvalidloader = DataLoader(\n    train_dataset(data.iloc[valid_idx], '../input/imet-2019-fgvc6/train/', augmentation=False), \n    batch_size=32, shuffle=False, num_workers=4, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"donor = models.resnet18()\ndonor.load_state_dict(torch.load('../input/resnet18/resnet18.pth'))\n\n# create empty model-acceptor\nmodel = ResNet(BasicBlock, [2,2,2,2])\n\n# transfer weights\nmodel.layer1.load_state_dict(donor.layer1.state_dict())\nmodel.layer2.load_state_dict(donor.layer2.state_dict())\nmodel.layer3.load_state_dict(donor.layer3.state_dict())\nmodel.layer4.load_state_dict(donor.layer4.state_dict())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_train(\n    model.cuda(), trainloader, validloader, \n    sigmoid_criterion=nn.BCEWithLogitsLoss(), \n    softmax_criterion=nn.CrossEntropyLoss(),\n    loss_blender=lambda sigmoid_loss, softmax_loss: 0.99*sigmoid_loss + 0.01*softmax_loss,\n    optimizer=torch.optim.Adam(model.parameters(), lr=0.0002),\n    n_epoch=25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_history(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scores = []\nwith torch.no_grad():\n    model.eval()\n    for step, (features, sigmoid_labels, softmax_labels) in enumerate(validloader):\n        features, sigmoid_labels, softmax_labels =\\\n        features.cuda(), sigmoid_labels.cuda(), softmax_labels.cuda()\n        \n        sigmoid_logits, softmax_logits = model(features)\n        batch_scores = np.array([fbeta_score(torch.sigmoid(sigmoid_logits), sigmoid_labels, threshold=conf) \n                                 for conf in np.arange(0,1,0.01)])\n        scores.append(batch_scores)\n        \nmean_scores = torch.FloatTensor(scores).cpu().numpy()\n\nplt.plot(np.arange(0,1,0.01), mean_scores.mean(axis=0), linewidth=2, color='r')\nsigmoid_best_score = mean_scores.mean(axis=0)[mean_scores.mean(axis=0).argmax()]\nsigmoid_best_conf = np.arange(0,1,0.01)[mean_scores.mean(axis=0).argmax()]\nplt.plot(sigmoid_best_conf, sigmoid_best_score, color='g', marker='x')\nprint('best score', sigmoid_best_score)\nprint('best conf', sigmoid_best_conf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"td = train_dataset(data.iloc[valid_idx], '../input/imet-2019-fgvc6/train/')\nsoftmax_label_decoder = {v:k for (k,v) in zip(\n    td.labels_group_encoder.keys(), td.labels_group_encoder.values())}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.read_csv('../input/imet-2019-fgvc6/sample_submission.csv')\n\npredicts = []; softmax_min_conf = 0.99\nwith torch.no_grad():\n    for fpath in tqdm_notebook(submit['id']):\n        image = np.array(Image.open('../input/imet-2019-fgvc6/test/'+fpath+'.png'))\n        image = cv2.resize(image, INPUT_SIZE)\n        image = np.divide(image, 255)\n        image = image.transpose(2,0,1)\n        image = torch.tensor(image[np.newaxis], dtype=torch.float)\n        image = image.cuda()\n        sigmoid_logits, softmax_logits = model(image)\n        \n        softmax_predicts = torch.softmax(softmax_logits, dim=1)[0]\n        idx = softmax_predicts.argmax()\n        softmax_class_number = int(idx.cpu().numpy())\n        \n        # if softmax have hight conf - softmax output predict, else sigmoid\n        if softmax_predicts[idx] > softmax_min_conf and softmax_class_number in softmax_label_decoder.keys():\n            predicts.append(softmax_label_decoder[softmax_class_number])\n        else:\n            sigmoid_predict = torch.sigmoid(sigmoid_logits).cpu().numpy()[0]\n            sigmoid_predict = np.arange(len(sigmoid_predict))[sigmoid_predict > sigmoid_best_conf]\n            predicts.append(str.join(' ', [str(e) for e in sigmoid_predict]))\n        \nsubmit['attribute_ids'] = predicts\nsubmit.to_csv('./submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}