{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#modules\nimport os\nimport numpy as np\nimport cv2\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils import np_utils\nfrom keras.utils.vis_utils import plot_model\nimport math\nfrom collections import OrderedDict\nimport re\nfrom PIL import Image\n\n\nimport torch\nimport torch.nn as nn\nfrom torch.nn import init\nfrom torchvision import models, transforms, datasets\nimport torch.optim as optim\nfrom torch.autograd import Variable\nfrom torch.utils.data.sampler import SubsetRandomSampler\nimport torch.utils.model_zoo as model_zoo\nimport torch.nn.functional as F\n","metadata":{"execution":{"iopub.status.busy":"2021-06-14T04:28:56.526955Z","iopub.execute_input":"2021-06-14T04:28:56.527465Z","iopub.status.idle":"2021-06-14T04:29:05.426769Z","shell.execute_reply.started":"2021-06-14T04:28:56.527360Z","shell.execute_reply":"2021-06-14T04:29:05.425550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#hyper-parameter\nBATCH_size = 16\nIMG_size = 256\nLR=1e-4 \nWEIGHT_decay=5e-4\nMODEL_name = 'OUR_BEST_MODEL' #to stroe the hyperparameter\nJITTER_strength = 0.2 \nCROP_size = 224","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:41:24.145925Z","iopub.execute_input":"2021-06-13T13:41:24.146266Z","iopub.status.idle":"2021-06-13T13:41:24.150844Z","shell.execute_reply.started":"2021-06-13T13:41:24.146231Z","shell.execute_reply":"2021-06-13T13:41:24.149881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#splitting the data into ratio(0.2)\ndef split_ids(path, ratio):\n    ids = []\n    train_ids = []\n    val_ids = []\n    \n    lists = os.listdir(path)\n    lists.sort()\n    \n    for i, label in enumerate(lists):\n        ids.append([])\n        for img_name in os.listdir(path+'/'+label):\n            ids[i].append(label+'/'+img_name)\n\n    for i in range(10):\n\n        ids_array = np.array(ids[i])\n        perm = np.random.permutation(np.arange(len(ids_array)))\n        cut = int(ratio*len(ids_array))\n\n        train_ids.append(ids_array[perm][cut:])\n        val_ids.append(ids_array[perm][:cut])\n        \n    return train_ids, val_ids\n\n\ntrain_path = \"../input/state-farm-distracted-driver-detection/imgs/train\"\ntrain_ids, val_ids = split_ids(train_path, 0.2)\n\nfor i in range(10):\n    print('found {} train, {} validation images for class {}'.format(len(train_ids[i]), len(val_ids[i]), i))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:41:24.152967Z","iopub.execute_input":"2021-06-13T13:41:24.153483Z","iopub.status.idle":"2021-06-13T13:41:24.735206Z","shell.execute_reply.started":"2021-06-13T13:41:24.153448Z","shell.execute_reply":"2021-06-13T13:41:24.734422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Defininng loader\ndef one_image_loader(path):\n    return Image.open(path).convert('RGB')\n\nclass Loader(torch.utils.data.Dataset):\n    def __init__(self, rootdir, split_type, ids=None, transform=None):\n        self.impath = rootdir\n        self.transform = transform\n        self.loader = one_image_loader\n            \n\n        imnames = []\n        imclasses = []\n        \n        for i in range(10):\n            if(split_type == 'train'):\n                for j in train_ids[i]:\n                    imclasses.append(i)\n                    imnames.append(j)\n            else :\n                for j in val_ids[i]:\n                    imclasses.append(i)\n                    imnames.append(j)\n\n        self.imnames = imnames\n        self.imclasses = imclasses\n    \n    def __getitem__(self, index):\n        original_img = self.loader(os.path.join(self.impath, self.imnames[index]))\n        img = self.transform(original_img)\n        label = self.imclasses[index]\n        return img, label\n        \n    def __len__(self):\n        return len(self.imnames)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:41:24.736787Z","iopub.execute_input":"2021-06-13T13:41:24.737137Z","iopub.status.idle":"2021-06-13T13:41:24.746625Z","shell.execute_reply.started":"2021-06-13T13:41:24.737102Z","shell.execute_reply":"2021-06-13T13:41:24.744614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#loading\ntrain_loader = torch.utils.data.DataLoader(\n            Loader(train_path, 'train', train_ids,\n                              transform=transforms.Compose([\n                                  transforms.Resize((IMG_size,IMG_size)),\n                                  transforms.ToTensor(),\n                                                                            transforms.ColorJitter(\n                                                brightness=JITTER_strength,\n                                                contrast=JITTER_strength,\n                                                saturation=JITTER_strength,\n                                                hue = JITTER_strength),\n                                            transforms.RandomResizedCrop(CROP_size),\n                                  \n                                  transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),])),\n                                batch_size= BATCH_size, shuffle=True, num_workers=0, pin_memory=True, drop_last=True)\nprint('train_loader done')\nprint('loaded {} train images'.format(len(train_loader.dataset)))\n\nvalidation_loader = torch.utils.data.DataLoader(\n            Loader(train_path, 'val', val_ids,\n                               transform=transforms.Compose([\n                                   transforms.Resize((IMG_size,IMG_size)),\n                                   transforms.ToTensor(),\n                                   transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),])),\n                               batch_size=BATCH_size, shuffle=False, num_workers=0, pin_memory=True, drop_last=False)\nprint('validation_loader done')\nprint('loaded {} validation images'.format(len(validation_loader.dataset)))\n\ndata_dict = {\n    'train':train_loader,\n    'validation': validation_loader\n            }","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:41:24.748143Z","iopub.execute_input":"2021-06-13T13:41:24.748636Z","iopub.status.idle":"2021-06-13T13:41:24.773583Z","shell.execute_reply.started":"2021-06-13T13:41:24.748596Z","shell.execute_reply":"2021-06-13T13:41:24.772513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Test loader\nclass TestLoader(torch.utils.data.Dataset):\n    def __init__(self, rootdir, transform=None):\n        self.impath = rootdir\n        self.transform = transform\n        self.loader = one_image_loader\n        self.imnames = os.listdir(rootdir)\n\n    \n    def __getitem__(self, index):\n        original_img = self.loader(os.path.join(self.impath, self.imnames[index]))\n        img = self.transform(original_img)\n\n        return self.imnames[index],img\n        \n    def __len__(self):\n        return len(self.imnames)\n    \n#loading the test data\ntest_path = \"../input/state-farm-distracted-driver-detection/imgs/test\"\ntest_loader = torch.utils.data.DataLoader(\n             TestLoader(test_path,\n                    transform=transforms.Compose([\n                                   transforms.Resize((IMG_size,IMG_size)),\n                                   transforms.ToTensor(),\n                                   transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),])),\n                               batch_size=BATCH_size, shuffle=False, num_workers=0, pin_memory=True, drop_last=False)\nprint('test_loader done')\nprint('loaded {} test images'.format(len(test_loader.dataset)))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:41:24.774772Z","iopub.execute_input":"2021-06-13T13:41:24.775185Z","iopub.status.idle":"2021-06-13T13:41:25.763352Z","shell.execute_reply.started":"2021-06-13T13:41:24.775147Z","shell.execute_reply":"2021-06-13T13:41:25.762447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ndevice","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:41:25.765034Z","iopub.execute_input":"2021-06-13T13:41:25.765406Z","iopub.status.idle":"2021-06-13T13:41:25.815200Z","shell.execute_reply.started":"2021-06-13T13:41:25.765369Z","shell.execute_reply":"2021-06-13T13:41:25.814213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def conv3x3(in_planes, out_planes, stride=1):\n    \"\"\"3x3 convolution with padding\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False)\n\ndef conv1x1(in_planes, out_planes, stride=1):\n    \"\"\"1x1 convolution\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-06T09:21:58.678839Z","iopub.execute_input":"2021-06-06T09:21:58.679163Z","iopub.status.idle":"2021-06-06T09:21:58.686603Z","shell.execute_reply.started":"2021-06-06T09:21:58.679134Z","shell.execute_reply":"2021-06-06T09:21:58.685746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#weight initialization functions\ndef weights_init_kaiming(m):\n    classname = m.__class__.__name__\n    if classname.find('Conv') != -1:\n        init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')\n        if m.bias is not None:\n            init.constant_(m.bias.data, 0.0)        \n    elif classname.find('Linear') != -1:\n        init.kaiming_normal_(m.weight.data, a=0, mode='fan_out')\n        if m.bias is not None:\n            init.constant_(m.bias.data, 0.0)\n    elif classname.find('BatchNorm1d') != -1:\n        init.normal_(m.weight.data, 1.0, 0.02)\n        if m.bias is not None:        \n            init.constant_(m.bias.data, 0.0)\n    elif classname.find('BatchNorm2d') != -1:\n        init.normal_(m.weight.data, 1.0, 0.02)\n        if m.bias is not None:        \n            init.constant_(m.bias.data, 0.0)\n    elif classname.find('BatchNorm') != -1:\n        if m.affine is not None:\n            init.constant_(m.weight.data, 1.0)\n            init.constant_(m.bias.data, 0.0)        \n\ndef weights_init_classifier(m):\n    classname = m.__class__.__name__\n    if classname.find('Linear') != -1:\n        init.normal_(m.weight.data, std=0.001)\n        init.constant_(m.bias.data, 0.0)     \n","metadata":{"execution":{"iopub.status.busy":"2021-06-13T08:44:06.113540Z","iopub.execute_input":"2021-06-13T08:44:06.113877Z","iopub.status.idle":"2021-06-13T08:44:06.131182Z","shell.execute_reply.started":"2021-06-13T08:44:06.113846Z","shell.execute_reply":"2021-06-13T08:44:06.130302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#blocks implementation\nclass ClassBlock(nn.Module):\n    def __init__(self, input_dim, class_num, dropout=True, relu=True, num_bottleneck=512): \n        super(ClassBlock, self).__init__()\n        add_block = []\n        add_block += [nn.Linear(input_dim, num_bottleneck)] \n        add_block += [nn.BatchNorm1d(num_bottleneck)]\n        if relu: \n            add_block += [nn.ReLU()]\n        if dropout: \n            add_block += [nn.Dropout(p=0.5)] \n        add_block = nn.Sequential(*add_block)\n        add_block.apply(weights_init_kaiming)\n        classifier = []\n        classifier += [nn.Linear(num_bottleneck, class_num)]\n        classifier = nn.Sequential(*classifier)\n        classifier.apply(weights_init_classifier)\n        self.add_block = add_block\n        self.classifier = classifier\n    def forward(self, x):\n        x = self.add_block(x)\n        x = self.classifier(x)\n        return x    ","metadata":{"execution":{"iopub.status.busy":"2021-06-13T08:44:08.440539Z","iopub.execute_input":"2021-06-13T08:44:08.440877Z","iopub.status.idle":"2021-06-13T08:44:08.448197Z","shell.execute_reply.started":"2021-06-13T08:44:08.440845Z","shell.execute_reply":"2021-06-13T08:44:08.447330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#models(Resnet 18, 50, 101)\nclass Res18(nn.Module):\n    def __init__(self, class_num):\n        super(Res18, self).__init__()\n        fea_dim = IMG_size\n        model_ft = models.resnet18(pretrained=False)\n        model_ft.avgpool = nn.AdaptiveAvgPool2d((1,1))\n        model_ft.fc = nn.Sequential()\n        self.model = model_ft\n        self.fc_embed = nn.Linear(512, fea_dim)\n        self.fc_embed.apply(weights_init_classifier)\n        self.classifier = ClassBlock(512, class_num)\n        self.classifier.apply(weights_init_classifier)\n        \n    def forward(self, x):\n        x = self.model.conv1(x)\n        x = self.model.bn1(x)\n        x = self.model.relu(x)\n        x = self.model.maxpool(x)\n        x = self.model.layer1(x)\n        x = self.model.layer2(x)\n        x = self.model.layer3(x)\n        x = self.model.layer4(x)\n        x = self.model.avgpool(x)\n        fea =  x.view(x.size(0), -1)\n        pred = self.classifier(fea)\n        return pred\n\n        \nclass Res50(nn.Module):\n    def __init__(self, class_num):\n        super(Res50, self).__init__()\n        fea_dim = IMG_size       \n        model_ft = models.resnet50(pretrained=False)\n        model_ft.avgpool = nn.AdaptiveAvgPool2d((1,1))\n        model_ft.fc = nn.Sequential()        \n        self.model = model_ft\n        self.fc_embed = nn.Linear(2048, fea_dim)\n        self.fc_embed.apply(weights_init_classifier)\n        self.classifier = ClassBlock(2048, class_num)\n        self.classifier.apply(weights_init_classifier)        \n        \n    def forward(self, x):\n        x = self.model.conv1(x)\n        x = self.model.bn1(x)\n        x = self.model.relu(x)\n        x = self.model.maxpool(x)\n        x = self.model.layer1(x)\n        x = self.model.layer2(x)\n        x = self.model.layer3(x)\n        x = self.model.layer4(x)\n        x = self.model.avgpool(x)\n        fea =  x.view(x.size(0), -1)\n        pred = self.classifier(fea)  \n        return pred\n        \nclass Res101(nn.Module):\n    def __init__(self, class_num):\n        super(Res101, self).__init__()\n        fea_dim = IMG_size     \n        model_ft = models.resnet101(pretrained=False)\n        model_ft.avgpool = nn.AdaptiveAvgPool2d((1,1))\n        model_ft.fc = nn.Sequential()        \n        self.model = model_ft\n        self.fc_embed = nn.Linear(2048, fea_dim)\n        self.fc_embed.apply(weights_init_classifier)\n        self.classifier = ClassBlock(2048, class_num)\n        self.classifier.apply(weights_init_classifier)        \n        \n    def forward(self, x):\n        x = self.model.conv1(x)\n        x = self.model.bn1(x)\n        x = self.model.relu(x)\n        x = self.model.maxpool(x)\n        x = self.model.layer1(x)\n        x = self.model.layer2(x)\n        x = self.model.layer3(x)\n        x = self.model.layer4(x)\n        x = self.model.avgpool(x)\n        fea =  x.view(x.size(0), -1)\n        pred = self.classifier(fea)\n        return pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nfrom torch.hub import load_state_dict_from_url\nfrom torchvision.models import ResNet\n\nfrom torch import nn\n\n\nclass SELayer(nn.Module):\n    def __init__(self, channel, reduction=16):\n        super(SELayer, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Sequential(\n            nn.Linear(channel, channel // reduction, bias=False),\n            nn.ReLU(inplace=True),\n            nn.Linear(channel // reduction, channel, bias=False),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        b, c, _, _ = x.size()\n        y = self.avg_pool(x).view(b, c)\n        y = self.fc(y).view(b, c, 1, 1)\n        return x * y.expand_as(x)\n\n\ndef conv3x3(in_planes, out_planes, stride=1):\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False)\n\n\nclass SEBasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None,\n                 *, reduction=16):\n        super(SEBasicBlock, self).__init__()\n        self.conv1 = conv3x3(inplanes, planes, stride)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = conv3x3(planes, planes, 1)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.se = SELayer(planes, reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        residual = x\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.se(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n        out = self.relu(out)\n\n        return out\n\n\nclass SEBottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None,\n                 *, reduction=16):\n        super(SEBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,\n                               padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se = SELayer(planes * 4, reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n        out = self.se(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n        out = self.relu(out)\n\n        return out\n\n\ndef se_resnet18(num_classes=1_000):\n    \"\"\"Constructs a ResNet-18 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNet(SEBasicBlock, [2, 2, 2, 2], num_classes=num_classes)\n    model.avgpool = nn.AdaptiveAvgPool2d(1)\n    return model\n\n\ndef se_resnet34(num_classes=1_000):\n    \"\"\"Constructs a ResNet-34 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNet(SEBasicBlock, [3, 4, 6, 3], num_classes=num_classes)\n    model.avgpool = nn.AdaptiveAvgPool2d(1)\n    return model\n\n\ndef se_resnet50(num_classes=1_000, pretrained=False):\n    \"\"\"Constructs a ResNet-50 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNet(SEBottleneck, [3, 4, 6, 3], num_classes=num_classes)\n    model.avgpool = nn.AdaptiveAvgPool2d(1)\n    if pretrained:\n        model.load_state_dict(load_state_dict_from_url(\n            \"https://github.com/moskomule/senet.pytorch/releases/download/archive/seresnet50-60a8950a85b2b.pkl\"))\n    return model\n\n\ndef se_resnet101(num_classes=1_000):\n    \"\"\"Constructs a ResNet-101 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNet(SEBottleneck, [3, 4, 23, 3], num_classes=num_classes)\n    model.avgpool = nn.AdaptiveAvgPool2d(1)\n    return model\n\n\ndef se_resnet152(num_classes=1_000):\n    \"\"\"Constructs a ResNet-152 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNet(SEBottleneck, [3, 8, 36, 3], num_classes=num_classes)\n    model.avgpool = nn.AdaptiveAvgPool2d(1)\n    return model\n\n\nclass CifarSEBasicBlock(nn.Module):\n    def __init__(self, inplanes, planes, stride=1, reduction=16):\n        super(CifarSEBasicBlock, self).__init__()\n        self.conv1 = conv3x3(inplanes, planes, stride)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = conv3x3(planes, planes)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.se = SELayer(planes, reduction)\n        if inplanes != planes:\n            self.downsample = nn.Sequential(nn.Conv2d(inplanes, planes, kernel_size=1, stride=stride, bias=False),\n                                            nn.BatchNorm2d(planes))\n        else:\n            self.downsample = lambda x: x\n        self.stride = stride\n\n    def forward(self, x):\n        residual = self.downsample(x)\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.se(out)\n\n        out += residual\n        out = self.relu(out)\n\n        return out\n\n\nclass CifarSEResNet(nn.Module):\n    def __init__(self, block, n_size, num_classes=10, reduction=16):\n        super(CifarSEResNet, self).__init__()\n        self.inplane = 16\n        self.conv1 = nn.Conv2d(\n            3, self.inplane, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(self.inplane)\n        self.relu = nn.ReLU(inplace=True)\n        self.layer1 = self._make_layer(\n            block, 16, blocks=n_size, stride=1, reduction=reduction)\n        self.layer2 = self._make_layer(\n            block, 32, blocks=n_size, stride=2, reduction=reduction)\n        self.layer3 = self._make_layer(\n            block, 64, blocks=n_size, stride=2, reduction=reduction)\n        self.avgpool = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(64, num_classes)\n        self.initialize()\n\n    def initialize(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight)\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n    def _make_layer(self, block, planes, blocks, stride, reduction):\n        strides = [stride] + [1] * (blocks - 1)\n        layers = []\n        for stride in strides:\n            layers.append(block(self.inplane, planes, stride, reduction))\n            self.inplane = planes\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n\n        x = self.avgpool(x)\n        x = x.view(x.size(0), -1)\n        x = self.fc(x)\n\n        return x\n\n\nclass CifarSEPreActResNet(CifarSEResNet):\n    def __init__(self, block, n_size, num_classes=10, reduction=16):\n        super(CifarSEPreActResNet, self).__init__(\n            block, n_size, num_classes, reduction)\n        self.bn1 = nn.BatchNorm2d(self.inplane)\n        self.initialize()\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n\n        x = self.bn1(x)\n        x = self.relu(x)\n\n        x = self.avgpool(x)\n        x = x.view(x.size(0), -1)\n        x = self.fc(x)\n\n\ndef se_resnet20(**kwargs):\n    \"\"\"Constructs a ResNet-18 model.\n    \"\"\"\n    model = CifarSEResNet(CifarSEBasicBlock, 3, **kwargs)\n    return model\n\n\ndef se_resnet32(**kwargs):\n    \"\"\"Constructs a ResNet-34 model.\n    \"\"\"\n    model = CifarSEResNet(CifarSEBasicBlock, 5, **kwargs)\n    return model\n\n\ndef se_resnet56(**kwargs):\n    \"\"\"Constructs a ResNet-34 model.\n    \"\"\"\n    model = CifarSEResNet(CifarSEBasicBlock, 9, **kwargs)\n    return model\n\n\ndef se_preactresnet20(**kwargs):\n    \"\"\"Constructs a ResNet-18 model.\n    \"\"\"\n    model = CifarSEPreActResNet(CifarSEBasicBlock, 3, **kwargs)\n    return model\n\n\ndef se_preactresnet32(**kwargs):\n    \"\"\"Constructs a ResNet-34 model.\n    \"\"\"\n    model = CifarSEPreActResNet(CifarSEBasicBlock, 5, **kwargs)\n    return model\n\n\ndef se_preactresnet56(**kwargs):\n    \"\"\"Constructs a ResNet-34 model.\n    \"\"\"\n    model = CifarSEPreActResNet(CifarSEBasicBlock, 9, **kwargs)\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:41:30.466142Z","iopub.execute_input":"2021-06-13T13:41:30.466480Z","iopub.status.idle":"2021-06-13T13:41:30.515225Z","shell.execute_reply.started":"2021-06-13T13:41:30.466452Z","shell.execute_reply":"2021-06-13T13:41:30.514380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SAM(torch.optim.Optimizer): # SAM optimizer from https://github.com/davda54/sam\n    def __init__(self, params, base_optimizer, rho=0.05, adaptive=False, **kwargs):\n        assert rho >= 0.0, f\"Invalid rho, should be non-negative: {rho}\"\n\n        defaults = dict(rho=rho, adaptive=adaptive, **kwargs)\n        super(SAM, self).__init__(params, defaults)\n\n        self.base_optimizer = base_optimizer(self.param_groups, **kwargs)\n        self.param_groups = self.base_optimizer.param_groups\n\n    @torch.no_grad()\n    def first_step(self, zero_grad=False):\n        grad_norm = self._grad_norm()\n        for group in self.param_groups:\n            scale = group[\"rho\"] / (grad_norm + 1e-12)\n\n            for p in group[\"params\"]:\n                if p.grad is None: continue\n                e_w = (torch.pow(p, 2) if group[\"adaptive\"] else 1.0) * p.grad * scale.to(p)\n                p.add_(e_w)  # climb to the local maximum \"w + e(w)\"\n                self.state[p][\"e_w\"] = e_w\n\n        if zero_grad: self.zero_grad()\n\n    @torch.no_grad()\n    def second_step(self, zero_grad=False):\n        for group in self.param_groups:\n            for p in group[\"params\"]:\n                if p.grad is None: continue\n                p.sub_(self.state[p][\"e_w\"])  # get back to \"w\" from \"w + e(w)\"\n\n        self.base_optimizer.step()  # do the actual \"sharpness-aware\" update\n\n        if zero_grad: self.zero_grad()\n\n    @torch.no_grad()\n    def step(self, closure=None):\n        assert closure is not None, \"Sharpness Aware Minimization requires closure, but it was not provided\"\n        closure = torch.enable_grad()(closure)  # the closure should do a full forward-backward pass\n\n        self.first_step(zero_grad=True)\n        closure()\n        self.second_step()\n\n    def _grad_norm(self):\n        shared_device = self.param_groups[0][\"params\"][0].device  # put everything on the same device, in case of model parallelism\n        norm = torch.norm(\n                    torch.stack([\n                        ((torch.abs(p) if group[\"adaptive\"] else 1.0) * p.grad).norm(p=2).to(shared_device)\n                        for group in self.param_groups for p in group[\"params\"]\n                        if p.grad is not None\n                    ]),\n                    p=2\n               )\n        return norm","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:41:35.648924Z","iopub.execute_input":"2021-06-13T13:41:35.649276Z","iopub.status.idle":"2021-06-13T13:41:35.661191Z","shell.execute_reply.started":"2021-06-13T13:41:35.649244Z","shell.execute_reply":"2021-06-13T13:41:35.660307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nfrom torch.optim.lr_scheduler import _LRScheduler\n\nclass CosineAnnealingWarmUpRestarts(_LRScheduler):\n    def __init__(self, optimizer, T_0, T_mult=1, eta_max=0.1, T_up=0, gamma=1., last_epoch=-1):\n        if T_0 <= 0 or not isinstance(T_0, int):\n            raise ValueError(\"Expected positive integer T_0, but got {}\".format(T_0))\n        if T_mult < 1 or not isinstance(T_mult, int):\n            raise ValueError(\"Expected integer T_mult >= 1, but got {}\".format(T_mult))\n        if T_up < 0 or not isinstance(T_up, int):\n            raise ValueError(\"Expected positive integer T_up, but got {}\".format(T_up))\n        self.T_0 = T_0\n        self.T_mult = T_mult\n        self.base_eta_max = eta_max\n        self.eta_max = eta_max\n        self.T_up = T_up\n        self.T_i = T_0\n        self.gamma = gamma\n        self.cycle = 0\n        self.T_cur = last_epoch\n        super(CosineAnnealingWarmUpRestarts, self).__init__(optimizer, last_epoch)\n\n    \n    def get_lr(self):\n        if self.T_cur == -1:\n            return self.base_lrs\n        elif self.T_cur < self.T_up:\n            return [(self.eta_max - base_lr)*self.T_cur / self.T_up + base_lr for base_lr in self.base_lrs]\n        else:\n            return [base_lr + (self.eta_max - base_lr) * (1 + math.cos(math.pi * (self.T_cur-self.T_up) / (self.T_i - self.T_up))) / 2\n                    for base_lr in self.base_lrs]\n\n    def step(self, epoch=None):\n        if epoch is None:\n            epoch = self.last_epoch + 1\n            self.T_cur = self.T_cur + 1\n            if self.T_cur >= self.T_i:\n                self.cycle += 1\n                self.T_cur = self.T_cur - self.T_i\n                self.T_i = (self.T_i - self.T_up) * self.T_mult + self.T_up\n        else:\n            if epoch >= self.T_0:\n                if self.T_mult == 1:\n                    self.T_cur = epoch % self.T_0\n                    self.cycle = epoch // self.T_0\n                else:\n                    n = int(math.log((epoch / self.T_0 * (self.T_mult - 1) + 1), self.T_mult))\n                    self.cycle = n\n                    self.T_cur = epoch - self.T_0 * (self.T_mult ** n - 1) / (self.T_mult - 1)\n                    self.T_i = self.T_0 * self.T_mult ** (n)\n            else:\n                self.T_i = self.T_0\n                self.T_cur = epoch\n                \n        self.eta_max = self.base_eta_max * (self.gamma**self.cycle)\n        self.last_epoch = math.floor(epoch)\n        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n            param_group['lr'] = lr","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Res50(10) #num classes : 10, Resnet50\nmodel.eval()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:41:37.640742Z","iopub.execute_input":"2021-06-13T13:41:37.641091Z","iopub.status.idle":"2021-06-13T13:41:38.119465Z","shell.execute_reply.started":"2021-06-13T13:41:37.641040Z","shell.execute_reply":"2021-06-13T13:41:38.118666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parameters = filter(lambda p: p.requires_grad, model.parameters())\nn_parameters = sum([p.data.nelement() for p in model.parameters()])\nprint('  + Number of params: {}'.format(n_parameters))\n","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:41:42.016736Z","iopub.execute_input":"2021-06-13T13:41:42.017081Z","iopub.status.idle":"2021-06-13T13:41:42.024419Z","shell.execute_reply.started":"2021-06-13T13:41:42.017033Z","shell.execute_reply":"2021-06-13T13:41:42.023611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.cuda()\ncriterion = nn.CrossEntropyLoss()\nbase_optimizer = optim.Adam\noptimizer = SAM(model.parameters(), base_optimizer, lr=0, weight_decay=WEIGHT_decay)\nscheduler = CosineAnnealingWarmUpRestarts(optimizer, T_0=8, T_mult=2, eta_max=LR,  T_up=2, gamma=0.5)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:41:44.120336Z","iopub.execute_input":"2021-06-13T13:41:44.120698Z","iopub.status.idle":"2021-06-13T13:41:49.720977Z","shell.execute_reply.started":"2021-06-13T13:41:44.120667Z","shell.execute_reply":"2021-06-13T13:41:49.720169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_losses = []\ntraining_acc = []\nvalid_losses = []\nvalid_acc = []\n\ndef train_model(model, criterion, optimizer, num_epochs=3):\n    best_acc = -1.0\n    \n    for epoch in range(num_epochs):            \n        print('Epoch {}/{}'.format(epoch+1, num_epochs))\n        print('-' * 10)\n\n        for phase in ['train', 'validation']:\n            if phase == 'train':\n                model.train()\n            else:\n                model.eval()\n\n            running_loss = 0.0\n            running_corrects = 0\n            \n            if phase == 'train':\n                for inputs, labels in data_dict[phase]:\n                    \n                    inputs = inputs.cuda()\n                    labels = labels.cuda()\n    \n                    outputs = model(inputs)\n                    loss = criterion(outputs, labels)\n                \n                    optimizer.zero_grad()\n                    loss.backward()\n                    optimizer.step()\n\n                    _, preds = torch.max(outputs, 1)\n \n                    running_loss += loss.item() * inputs.shape[0]\n                    running_corrects += torch.sum(preds == labels.data)\n            else:\n                with torch.no_grad():\n                    for inputs, labels in data_dict[phase]:\n                        \n                        inputs = inputs.cuda()\n                        labels = labels.cuda()\n    \n                        outputs = model(inputs)\n                        loss = criterion(outputs, labels)\n            \n                        _, preds = torch.max(outputs, 1)\n \n                        running_loss += loss.item() *  inputs.shape[0]\n                        running_corrects += torch.sum(preds == labels.data)\n\n            epoch_loss = running_loss / (len(data_dict[phase].dataset))  \n            epoch_acc = running_corrects.double() / (len(data_dict[phase].dataset)) * 100 # 0 ~ 100 (%)\n            \n            if phase =='train':\n                training_losses.append(epoch_loss)\n                training_acc.append(epoch_acc)\n            else:\n                valid_losses.append(epoch_loss)\n                valid_acc.append(epoch_acc)\n                \n                if(epoch_acc > best_acc):\n                    best_acc = epoch_acc\n                    print('model achieved the best accuracy ({:.4f}%) - saving best checkpoint...'.format(best_acc))\n                    ckpt = { 'classifier': model.state_dict(),\n                             'optimizer' : optimizer.state_dict(),\n                             'best_acc' : best_acc }\n                    torch.save(ckpt, './'+MODEL_name+'_best')\n                                         \n            print('{} loss: {:.4f}, acc: {:.4f}'.format(phase, epoch_loss,   epoch_acc))\n        scheduler.step()    \n        if(valid_acc[len(valid_acc)-1] < best_acc - 5) :\n            print('It might seems that overfitting occurs. Stop training\\n')\n            break","metadata":{"execution":{"iopub.status.busy":"2021-06-06T06:40:31.12556Z","iopub.execute_input":"2021-06-06T06:40:31.126073Z","iopub.status.idle":"2021-06-06T06:40:31.14335Z","shell.execute_reply.started":"2021-06-06T06:40:31.126025Z","shell.execute_reply":"2021-06-06T06:40:31.142052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_losses = []\ntraining_acc = []\nvalid_losses = []\nvalid_acc = []\n\ndef train_model_SAM(model, criterion, optimizer, num_epochs=3):\n    best_acc = -1.0\n    \n    for epoch in range(num_epochs):            \n        print('Epoch {}/{}'.format(epoch+1, num_epochs))\n        print('-' * 10)\n\n        for phase in ['train', 'validation']:\n            if phase == 'train':\n                model.train()\n            else:\n                model.eval()\n\n            running_loss = 0.0\n            running_corrects = 0\n            \n            if phase == 'train':\n                for inputs, labels in data_dict[phase]:\n                    \n                    inputs = inputs.cuda()\n                    labels = labels.cuda()\n    \n                    outputs = model(inputs)\n                    loss = criterion(outputs, labels)\n                \n                    optimizer.zero_grad()\n                    loss.backward()\n                    optimizer.first_step(zero_grad=True)\n                    \n                    criterion(model(inputs), labels).backward()\n                    optimizer.second_step(zero_grad=True)\n\n                    _, preds = torch.max(outputs, 1)\n \n                    running_loss += loss.item() * inputs.shape[0]\n                    running_corrects += torch.sum(preds == labels.data)\n            else:\n                with torch.no_grad():\n                    for inputs, labels in data_dict[phase]:\n                        \n                        inputs = inputs.cuda()\n                        labels = labels.cuda()\n    \n                        outputs = model(inputs)\n                        loss = criterion(outputs, labels)\n            \n                        _, preds = torch.max(outputs, 1)\n \n                        running_loss += loss.item() *  inputs.shape[0]\n                        running_corrects += torch.sum(preds == labels.data)\n\n            epoch_loss = running_loss / (len(data_dict[phase].dataset))  \n            epoch_acc = running_corrects.double() / (len(data_dict[phase].dataset)) * 100 # 0 ~ 100 (%)\n            \n            if phase =='train':\n                training_losses.append(epoch_loss)\n                training_acc.append(epoch_acc)\n            else:\n                valid_losses.append(epoch_loss)\n                valid_acc.append(epoch_acc)\n                \n                if(epoch_acc > best_acc):\n                    best_acc = epoch_acc\n                    print('model achieved the best accuracy ({:.4f}%) - saving best checkpoint...'.format(best_acc))\n                    ckpt = { 'classifier': model.state_dict(),\n                             'optimizer' : optimizer.state_dict(),\n                             'best_acc' : best_acc }\n                    torch.save(ckpt, './'+MODEL_name+'_best')\n                                         \n            print('{} loss: {:.4f}, acc: {:.4f}'.format(phase, epoch_loss,   epoch_acc))\n        scheduler.step()    \n        if(valid_acc[len(valid_acc)-1] < best_acc - 5) :\n            print('It might seems that overfitting occurs. Stop training\\n')\n            break","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:41:49.722481Z","iopub.execute_input":"2021-06-13T13:41:49.722796Z","iopub.status.idle":"2021-06-13T13:41:49.737565Z","shell.execute_reply.started":"2021-06-13T13:41:49.722763Z","shell.execute_reply":"2021-06-13T13:41:49.736556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_model_SAM(model, criterion, optimizer, num_epochs=50)","metadata":{"execution":{"iopub.status.busy":"2021-06-13T13:41:50.495676Z","iopub.execute_input":"2021-06-13T13:41:50.495996Z","iopub.status.idle":"2021-06-13T16:23:13.222035Z","shell.execute_reply.started":"2021-06-13T13:41:50.495965Z","shell.execute_reply":"2021-06-13T16:23:13.221145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualize the results\nimport matplotlib.pyplot as plt\n\nplt.plot(training_losses, label='train loss')\nplt.plot(valid_losses, label='valid loss')\nplt.legend()\nplt.show()\n\nplt.plot(training_acc, label='train acc(%)')\nplt.plot(valid_acc, label='valid acc(%)')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2021-06-13T16:23:13.223646Z","iopub.execute_input":"2021-06-13T16:23:13.224162Z","iopub.status.idle":"2021-06-13T16:23:13.534754Z","shell.execute_reply.started":"2021-06-13T16:23:13.224119Z","shell.execute_reply":"2021-06-13T16:23:13.533968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#analyze the results\nckpt = torch.load('./'+MODEL_name+'_best')\n\n#1. load the trained model\nmodel.load_state_dict(ckpt['classifier'])\noptimizer.load_state_dict(ckpt['optimizer'])\nbest_acc = ckpt['best_acc']\nprint('checkpoint is loaded !')\nprint('loaded model''s best accuracy : %.4f' % best_acc)\n\n#2. Testing\nprint('testing the loaded model')\nmodel.eval()\nrunning_loss = 0.0\nrunning_corrects = 0\n\nwith torch.no_grad():\n    for inputs, labels in data_dict['validation']:\n        \n        inputs = inputs.cuda()\n        labels = labels.cuda()\n    \n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n            \n        _, preds = torch.max(outputs, 1)\n \n        running_loss += loss.item() * inputs.shape[0]\n        running_corrects += torch.sum(preds == labels.data)\n        \ntest_loss = running_loss / len(data_dict['validation'].dataset)  \ntest_acc = running_corrects.double() /  len(data_dict['validation'].dataset) * 100 # 0 ~ 100 (%)\nprint('Test_loss : {:.4f}, Test accuracy : {:.4f}'.format(test_loss, test_acc))","metadata":{"execution":{"iopub.status.busy":"2021-06-13T16:23:13.536332Z","iopub.execute_input":"2021-06-13T16:23:13.536663Z","iopub.status.idle":"2021-06-13T16:24:05.536367Z","shell.execute_reply.started":"2021-06-13T16:23:13.536629Z","shell.execute_reply":"2021-06-13T16:24:05.535400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#submission\nimport pandas as pd\n\nmodel.eval()\nnp.set_printoptions(precision=10, suppress=True)\n\n# (1) eval & write the test set's result\nprint('Writing start..')\n\nf = open('/kaggle/working/submission.csv', 'w')\nf.write('img,c0,c1,c2,c3,c4,c5,c6,c7,c8,c9\\n')\n\nwith torch.no_grad():\n    for i, data in enumerate(test_loader):\n        if not i%100 :\n            print(i*100.0/len(test_loader), \"% done.\")\n        name, img_data = data\n    \n        img_data = img_data.cuda()\n        img_data = model(img_data)\n\n        img_data = F.softmax(img_data, dim=1)\n        img_data = img_data.detach().cpu().numpy()\n\n        for j in range(img_data.shape[0]):\n            f.write(name[j])\n            for k in img_data[j]:\n                f.write(',')\n                f.write(str(k))\n            f.write('\\n')\nf.close()\n\n#(2) check the file\npd.read_csv('/kaggle/working/submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-13T16:24:05.537707Z","iopub.execute_input":"2021-06-13T16:24:05.538210Z","iopub.status.idle":"2021-06-13T16:46:26.954331Z","shell.execute_reply.started":"2021-06-13T16:24:05.538174Z","shell.execute_reply":"2021-06-13T16:46:26.953492Z"},"trusted":true},"execution_count":null,"outputs":[]}]}