{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\n%reload_ext autoreload\n%autoreload 2\n# 多行输出\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\" ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1 数据准备"},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.vision import *\n# import pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 设置数据路径\npath = Path('../input/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d_path = path/'iwildcam-2019-fgvc6'\nm_path = path/'pytorch-model-zoo'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 这里每个图片都会有多个标签"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(d_path/'train.csv')\ntrain_df = pd.concat([train_df['id'],train_df['category_id']],axis=1,keys=['id','category_id'])\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(d_path/'test.csv')\ntest_df = pd.DataFrame(test_df['id'])\ntest_df['predicted'] = 0\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 图片变换\ntfms = get_transforms(do_flip=True, max_rotate=20, max_zoom=1.3, max_lighting=0.4,\n                      max_warp=0.4, p_affine=1., p_lighting=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_set = ImageList.from_df(test_df, path=d_path, cols='id', folder='test_images', suffix='.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 构建数据集\nnp.random.seed(42)\n# 使用 ImageList 是因为图像是多标签的\nsrc = (ImageList.from_df(train_df, path=d_path, folder='train_images', cols='id', suffix='.jpg')\n       .split_by_rand_pct(0.1)\n       .label_from_df(cols='category_id')\n       .add_test(test_set)\n      )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"[](http://)- 图像大小设置为 128"},{"metadata":{"trusted":true},"cell_type":"code","source":"# img_size=128\nimg_size=224\nbs=32","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = (src.transform(tfms, size=img_size)\n        .databunch(path='.', bs=bs, device= torch.device('cuda:0')).normalize(imagenet_stats))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 查看部分数据\ndata.show_batch(rows=3, figsize=(12,9))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 使用 densenet169/se_resnet152 作为模型"},{"metadata":{"trusted":true},"cell_type":"code","source":"f1 = partial(fbeta, beta=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 预训练模型加载"},{"metadata":{},"cell_type":"markdown","source":"- 从本地加载预训练模型"},{"metadata":{},"cell_type":"markdown","source":"https://github.com/Cadene/pretrained-models.pytorch/blob/master/pretrainedmodels/models/senet.py"},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import OrderedDict\nimport math\n\nimport torch.nn as nn\nfrom torch.utils import model_zoo\n\npretrained_settings = {\n    'senet154': {\n        'url': m_path/'senet154-c7b49a05.pth'\n    },\n    'se_resnet152': {\n        'url': m_path/'se_resnet152-d17c99b7.pth'\n    },\n    'se_resnext101_32x4d': {\n        'url': m_path/'se_resnext101_32x4d-3b2fe3d8.pth'\n    }\n}\n\n\nclass SEModule(nn.Module):\n\n    def __init__(self, channels, reduction):\n        super(SEModule, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,\n                             padding=0)\n        self.relu = nn.ReLU(inplace=True)\n        self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,\n                             padding=0)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        module_input = x\n        x = self.avg_pool(x)\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.fc2(x)\n        x = self.sigmoid(x)\n        return module_input * x\n\n\nclass Bottleneck(nn.Module):\n    def forward(self, x):\n        residual = x\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n        out = self.conv3(out)\n        out = self.bn3(out)\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out = self.se_module(out) + residual\n        out = self.relu(out)\n        return out\n\n\nclass SEBottleneck(Bottleneck):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None):\n        super(SEBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes * 2)\n        self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,\n                               stride=stride, padding=1, groups=groups,\n                               bias=False)\n        self.bn2 = nn.BatchNorm2d(planes * 4)\n        self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,\n                               bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SEResNetBottleneck(Bottleneck):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None):\n        super(SEResNetBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,\n                               stride=stride)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,\n                               groups=groups, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SEResNeXtBottleneck(Bottleneck):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None, base_width=4):\n        super(SEResNeXtBottleneck, self).__init__()\n        width = math.floor(planes * (base_width / 64)) * groups\n        self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,\n                               stride=1)\n        self.bn1 = nn.BatchNorm2d(width)\n        self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,\n                               padding=1, groups=groups, bias=False)\n        self.bn2 = nn.BatchNorm2d(width)\n        self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SENet(nn.Module):\n\n    def __init__(self, block, layers, groups, reduction, dropout_p=0.2,\n                 inplanes=128, input_3x3=True, downsample_kernel_size=3,\n                 downsample_padding=1, num_classes=1000):\n        super(SENet, self).__init__()\n        self.inplanes = inplanes\n        if input_3x3:\n            layer0_modules = [\n                ('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,\n                                    bias=False)),\n                ('bn1', nn.BatchNorm2d(64)),\n                ('relu1', nn.ReLU(inplace=True)),\n                ('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,\n                                    bias=False)),\n                ('bn2', nn.BatchNorm2d(64)),\n                ('relu2', nn.ReLU(inplace=True)),\n                ('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,\n                                    bias=False)),\n                ('bn3', nn.BatchNorm2d(inplanes)),\n                ('relu3', nn.ReLU(inplace=True)),\n            ]\n        else:\n            layer0_modules = [\n                ('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,\n                                    padding=3, bias=False)),\n                ('bn1', nn.BatchNorm2d(inplanes)),\n                ('relu1', nn.ReLU(inplace=True)),\n            ]\n        # To preserve compatibility with Caffe weights `ceil_mode=True`\n        # is used instead of `padding=1`.\n        layer0_modules.append(('pool', nn.MaxPool2d(3, stride=2,\n                                                    ceil_mode=True)))\n        self.layer0 = nn.Sequential(OrderedDict(layer0_modules))\n        self.layer1 = self._make_layer(\n            block,\n            planes=64,\n            blocks=layers[0],\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=1,\n            downsample_padding=0\n        )\n        self.layer2 = self._make_layer(\n            block,\n            planes=128,\n            blocks=layers[1],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.layer3 = self._make_layer(\n            block,\n            planes=256,\n            blocks=layers[2],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.layer4 = self._make_layer(\n            block,\n            planes=512,\n            blocks=layers[3],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.avg_pool = nn.AvgPool2d(7, stride=1)\n        self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None\n        self.last_linear = nn.Linear(512 * block.expansion, num_classes)\n\n    def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,\n                    downsample_kernel_size=1, downsample_padding=0):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes * block.expansion,\n                          kernel_size=downsample_kernel_size, stride=stride,\n                          padding=downsample_padding, bias=False),\n                nn.BatchNorm2d(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, groups, reduction, stride,\n                            downsample))\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups, reduction))\n\n        return nn.Sequential(*layers)\n\n    def features(self, x):\n        x = self.layer0(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        return x\n\n    def logits(self, x):\n        x = self.avg_pool(x)\n        if self.dropout is not None:\n            x = self.dropout(x)\n        x = x.view(x.size(0), -1)\n        x = self.last_linear(x)\n        return x\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.logits(x)\n        return x\n\ndef senet154(pretrained=False):\n    model = SENet(SEBottleneck, [3, 8, 36, 3], groups=64, reduction=16,\n                  dropout_p=0.2, num_classes=1000)\n    if pretrained:\n        model.load_state_dict(torch.load(pretrained_settings['senet154']['url']))\n    return model\n\n\ndef se_resnet152(pretrained=False):\n    model = SENet(SEResNetBottleneck, [3, 8, 36, 3], groups=1, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=1000)\n    if pretrained:\n        model.load_state_dict(torch.load(pretrained_settings['se_resnet152']['url']))\n    return model\n\n\ndef se_resnext101_32x4d(pretrained=False):\n    model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=1000)\n    if pretrained:\n        model.load_state_dict(torch.load(pretrained_settings['se_resnext101_32x4d']['url']))\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = cnn_learner(data, base_arch=models.densenet169, metrics=[f1, accuracy])\n# learn = cnn_learner(data, se_resnet152, metrics=[f1, accuracy])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 寻找学习率"},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 查看学习率情况并作出选择"},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.recorder.plot(suggestion=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 根据上述图像，我们选择学习率为 0.01"},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = 0.01","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 训练模型"},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(30, slice(lr))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- unfreeze 并训练浅层网络的权重，微调过程"},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 再次查找最佳学习率"},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 根据上述图，选择 lr1=1e-4\n- 再次训练浅层网络并保存结果\n- 使用判别式学习率"},{"metadata":{"trusted":false},"cell_type":"code","source":"lr1 = 1e-4\nlearn.fit_one_cycle(15, slice(lr1/2.6**3, lr1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"learn.save('stage-2')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![](http://)## 图像大小设置为 224"},{"metadata":{},"cell_type":"markdown","source":"- 仍旧使用之前的学习器，只是简单地更换数据"},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn=None\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# learn = cnn_learner(data, base_arch=se_resnet152, metrics=[f1, accuracy]).load('stage-2');","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# bs=32\n# img_sz=224","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# data = (src.transform(tfms, size=img_sz)\n#         .databunch(path='.', bs=bs, device= torch.device('cuda:0')).normalize(imagenet_stats))\n\n# learn.data = data","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# data.show_batch(rows=3, figsize=(12,9))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 我们仍然使用之前定义的模型\n- 再次训练之前，我们需要将浅层结构进行 freeze，不训练浅层结构的权重"},{"metadata":{"trusted":false},"cell_type":"code","source":"# learn.freeze()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 寻找最佳学习率并再次训练"},{"metadata":{"trusted":false},"cell_type":"code","source":"# learn.lr_find()\n# learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# lr2=1e-4","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# learn.fit_one_cycle(3, slice(lr2))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- 继续进行第二阶段的微调"},{"metadata":{"trusted":false},"cell_type":"code","source":"# learn.unfreeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# learn.lr_find()\n# learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# learn.fit_one_cycle(1, slice(lr2/2.6**3, lr2/5))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 测试"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(d_path/'sample_submission.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = learn.get_preds(DatasetType.Test)\ndf['Predicted'] = test_preds[0].argmax(dim=1)\ndf.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 提交结果"},{"metadata":{"trusted":true},"cell_type":"code","source":"# !kaggle competitions submit -c iwildcam-2019-fgvc6 -f submission.csv -m \"submit\"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}