{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import fastai\nfrom fastai.vision import *\nfastai.__version__","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Model: densenet201/se_resnext101_32x4d\n* Loss: Focal loss\n* Metric: $F_{2}$ score"},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 数据"},{"metadata":{"trusted":true},"cell_type":"code","source":"d_path = Path('../input/imet-2019-fgvc6')\nm_path = Path('../input/pytorch-model-zoo')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 训练数据"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(d_path/'train.csv')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 标签"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels_df = pd.read_csv(d_path/'labels.csv')\nlabels_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(d_path/'sample_submission.csv')\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 数据块"},{"metadata":{"trusted":true},"cell_type":"code","source":"# SIZE = 224\n# BATCH = 64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfms = get_transforms(max_lighting=0.1, max_zoom=1.05, max_warp=0.1, xtra_tfms=[(symmetric_warp(magnitude=(-0,0), p=0)),])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = ImageList.from_df(train_df, path=d_path, cols='id', folder='train', suffix='.png') \ntest = ImageList.from_df(test_df, path=d_path, cols='id', folder='test', suffix='.png') \n# data = (train.split_by_rand_pct(0.2, seed=1234)\n#         .label_from_df(cols='attribute_ids', label_delim=' ')\n#         .add_test(test)\n#         .transform(tfms, size=SIZE, resize_method=ResizeMethod.PAD, padding_mode='border')\n#         .databunch(path=Path('.'), bs=BATCH, device=torch.device('cuda:0')).normalize(imagenet_stats))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data.show_batch(rows=3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### FocalLoss"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Source: https://www.kaggle.com/c/human-protein-atlas-image-classification/discussion/78109\nclass FocalLoss(nn.Module):\n    def __init__(self, gamma=2):\n        super().__init__()\n        self.gamma = gamma\n\n    def forward(self, logit, target):\n        target = target.float()\n        max_val = (-logit).clamp(min=0)\n        loss = logit - logit * target + max_val + \\\n               ((-max_val).exp() + (-logit - max_val).exp()).log()\n\n        invprobs = F.logsigmoid(-logit * (target * 2.0 - 1.0))\n        loss = (invprobs * self.gamma).exp() * loss\n        if len(loss.size())==2:\n            loss = loss.sum(dim=1)\n        return loss.mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 学习器"},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import OrderedDict\nimport math\nimport torch.nn as nn\nfrom torch.utils import model_zoo\n\npretrained_settings = {\n    'senet154': {\n        'url': m_path/'senet154-c7b49a05.pth'\n    },\n    'se_resnet152': {\n        'url': m_path/'se_resnet152-d17c99b7.pth'\n    },\n    'se_resnext101_32x4d': {\n        'url': m_path/'se_resnext101_32x4d-3b2fe3d8.pth'\n    }\n}\n\nclass SEModule(nn.Module):\n\n    def __init__(self, channels, reduction):\n        super(SEModule, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1, padding=0)\n        self.relu = nn.ReLU(inplace=True)\n        self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1, padding=0)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        module_input = x\n        x = self.avg_pool(x)\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.fc2(x)\n        x = self.sigmoid(x)\n        return module_input * x\n\nclass Bottleneck(nn.Module):\n    def forward(self, x):\n        residual = x\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n        out = self.conv3(out)\n        out = self.bn3(out)\n        if self.downsample is not None:\n            residual = self.downsample(x)\n        out = self.se_module(out) + residual\n        out = self.relu(out)\n        return out\n\nclass SEBottleneck(Bottleneck):\n    expansion = 4\n    def __init__(self, inplanes, planes, groups, reduction, stride=1, downsample=None):\n        super(SEBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes * 2)\n        self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3, stride=stride, padding=1, groups=groups, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes * 4)\n        self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SEResNetBottleneck(Bottleneck):\n    expansion = 4\n    def __init__(self, inplanes, planes, groups, reduction, stride=1, downsample=None):\n        super(SEResNetBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False, stride=stride)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1, groups=groups, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SEResNeXtBottleneck(Bottleneck):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1, downsample=None, base_width=4):\n        super(SEResNeXtBottleneck, self).__init__()\n        width = math.floor(planes * (base_width / 64)) * groups\n        self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False, stride=1)\n        self.bn1 = nn.BatchNorm2d(width)\n        self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride, padding=1, groups=groups, bias=False)\n        self.bn2 = nn.BatchNorm2d(width)\n        self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SENet(nn.Module):\n\n    def __init__(self, block, layers, groups, reduction, dropout_p=0.2,\n                 inplanes=128, input_3x3=True, downsample_kernel_size=3,\n                 downsample_padding=1, num_classes=1000):\n        super(SENet, self).__init__()\n        self.inplanes = inplanes\n        if input_3x3:\n            layer0_modules = [\n                ('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1, bias=False)),\n                ('bn1', nn.BatchNorm2d(64)),\n                ('relu1', nn.ReLU(inplace=True)),\n                ('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1, bias=False)),\n                ('bn2', nn.BatchNorm2d(64)),\n                ('relu2', nn.ReLU(inplace=True)),\n                ('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1, bias=False)),\n                ('bn3', nn.BatchNorm2d(inplanes)),\n                ('relu3', nn.ReLU(inplace=True)),\n            ]\n        else:\n            layer0_modules = [\n                ('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2, padding=3, bias=False)),\n                ('bn1', nn.BatchNorm2d(inplanes)),\n                ('relu1', nn.ReLU(inplace=True)),\n            ]\n        layer0_modules.append(('pool', nn.MaxPool2d(3, stride=2, ceil_mode=True)))\n        self.layer0 = nn.Sequential(OrderedDict(layer0_modules))\n        self.layer1 = self._make_layer(\n            block,\n            planes=64,\n            blocks=layers[0],\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=1,\n            downsample_padding=0\n        )\n        self.layer2 = self._make_layer(\n            block,\n            planes=128,\n            blocks=layers[1],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.layer3 = self._make_layer(\n            block,\n            planes=256,\n            blocks=layers[2],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.layer4 = self._make_layer(\n            block,\n            planes=512,\n            blocks=layers[3],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.avg_pool = nn.AvgPool2d(7, stride=1)\n        self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None\n        self.last_linear = nn.Linear(512 * block.expansion, num_classes)\n\n    def _make_layer(self, block, planes, blocks, groups, reduction, stride=1, downsample_kernel_size=1, downsample_padding=0):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes * block.expansion, kernel_size=downsample_kernel_size, stride=stride, padding=downsample_padding, bias=False),\n                nn.BatchNorm2d(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, groups, reduction, stride, downsample))\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups, reduction))\n\n        return nn.Sequential(*layers)\n\n    def features(self, x):\n        x = self.layer0(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        return x\n\n    def logits(self, x):\n        x = self.avg_pool(x)\n        if self.dropout is not None:\n            x = self.dropout(x)\n        x = x.view(x.size(0), -1)\n        x = self.last_linear(x)\n        return x\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.logits(x)\n        return x\n\ndef senet154(pretrained=False):\n    model = SENet(SEBottleneck, [3, 8, 36, 3], groups=64, reduction=16, dropout_p=0.2, num_classes=1000)\n    if pretrained:\n        model.load_state_dict(torch.load(pretrained_settings['senet154']['url']))\n    return model\n\n\ndef se_resnet152(pretrained=False):\n    model = SENet(SEResNetBottleneck, [3, 8, 36, 3], groups=1, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=1000)\n    if pretrained:\n        model.load_state_dict(torch.load(pretrained_settings['se_resnet152']['url']))\n    return model\n\n\ndef se_resnext101_32x4d(pretrained=False):\n    model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=1000)\n    if pretrained:\n        model.load_state_dict(torch.load(pretrained_settings['se_resnext101_32x4d']['url']))\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn = cnn_learner(data, base_arch=models.densenet169, loss_func=FocalLoss(), metrics=fbeta).mixup()\n# learn = cnn_learner(data, se_resnext101_32x4d, loss_func=FocalLoss(), metrics=fbeta)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 训练模型"},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.lr_find()\n# learn.recorder.plot(suggestion=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# lr = 1e-2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.fit_one_cycle(5, slice(lr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.fit_one_cycle(3, slice(lr/1e4, lr/100))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.save('s1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### 更改图片大小继续训练"},{"metadata":{"trusted":true},"cell_type":"code","source":"SIZE = 224\nBATCH=32","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn = None\n# gc.collect()\n# learn = cnn_learner(data, se_resnext101_32x4d, loss_func=FocalLoss(), metrics=fbeta).load('s1')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data1 = (train.split_by_rand_pct(0.2, seed=1234)\n        .label_from_df(cols='attribute_ids', label_delim=' ')\n        .add_test(test)\n        .transform(tfms, size=SIZE)\n        .databunch(path=Path('.'), bs=BATCH, device=torch.device('cuda:0')).normalize(imagenet_stats))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = cnn_learner(data1, se_resnext101_32x4d, loss_func=FocalLoss(), metrics=fbeta)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data1.show_batch(rows=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.data=data1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.freeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot(suggestion=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr2 = 1e-6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(5, slice(lr2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(3, slice(lr2/2.6**3, lr2/5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot_losses()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 预测"},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_best_fixed_threshold(preds, targs, do_plot=True):\n    score = []\n    thrs = np.arange(0, 0.5, 0.01)\n    for thr in progress_bar(thrs):\n        score.append(fbeta(valid_preds[0],valid_preds[1], thresh=thr))\n    score = np.array(score)\n    pm = score.argmax()\n    best_thr, best_score = thrs[pm], score[pm].item()\n    print(f'thr={best_thr:.3f}', f'F2={best_score:.3f}')\n    if do_plot:\n        plt.plot(thrs, score)\n        plt.vlines(x=best_thr, ymin=score.min(), ymax=score.max())\n        plt.text(best_thr+0.03, best_score-0.01, f'$F_{2}=${best_score:.3f}', fontsize=14);\n        plt.show()\n    return best_thr\n\ni2c = np.array([[i, c] for c, i in learn.data.train_ds.y.c2i.items()]).astype(int) # indices to class number correspondence\n\ndef join_preds(preds, thr):\n    return [' '.join(i2c[np.where(t==1)[0],1].astype(str)) for t in (preds[0].sigmoid()>thr).long()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_preds = learn.TTA(ds_type=DatasetType.Valid)\nbest_thr = find_best_fixed_threshold(*valid_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = learn.TTA(ds_type=DatasetType.Test)\ntest_df.attribute_ids = join_preds(test_preds, best_thr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}