{"cells":[{"metadata":{},"cell_type":"markdown","source":"Changelog - \n* pretrainedmodels by Cadene repo\n* seresnext50_32x4d\n* added custom head\n* added code for mish\n* adaptiveconcatpool2d/bn_drop\n* albumentations (SSR,normalize imagenet weights)\n* conv2d model 3channel\n* nn.SmoothL1Loss\n* sigmoid/relu - >mish (?)\n\n\nchanges to try/make -\n* over9k / ranger optim\n* augmix\n* 5-fold CV\n* GeM pooling for head\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport pickle\nimport pretrainedmodels\nimport time\nimport torchvision\nimport torch.nn as nn\nfrom tqdm import tqdm_notebook as tqdm\n\nfrom PIL import Image, ImageFile\nfrom torch.utils.data import Dataset\nimport torch\nimport torch.optim as optim\nfrom torchvision import transforms\nfrom torch.optim import lr_scheduler\nimport os\nfrom albumentations import (\n    PadIfNeeded,\n    HorizontalFlip,\n    VerticalFlip,    \n    CenterCrop,    \n    Crop,\n    Rotate,\n    Compose,\n    Transpose,\n    RandomRotate90,\n    ElasticTransform,\n    GridDistortion, \n    OpticalDistortion,\n    RandomSizedCrop,\n    OneOf,\n    #MultiplicativeNoise,\n    #CoarseDropout,\n    CLAHE,\n    RandomBrightnessContrast,    \n    Cutout,\n    RandomGamma,\n    Rotate,\n    ShiftScaleRotate ,\n    GaussNoise,\n    Blur,\n    #GlassBlur,\n    Normalize,\n    MotionBlur,\n    MedianBlur,   \n    IAAPiecewiseAffine,\n    GaussianBlur\n)\n\ndevice = torch.device(\"cuda:0\")\nImageFile.LOAD_TRUNCATED_IMAGES = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F  #(uncomment if needed,but you likely already have it)\n\n#Mish - \"Mish: A Self Regularized Non-Monotonic Neural Activation Function\"\n#https://arxiv.org/abs/1908.08681v1\n#implemented for PyTorch / FastAI by lessw2020 \n#github: https://github.com/lessw2020/mish\n\nclass Mish(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, x):\n        #inlining this saves 1 second per epoch (V100 GPU) vs having a temp x and then returning x(!)\n        return x *( torch.tanh(F.softplus(x)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_relu_to_mish(model):\n    for child_name, child in model.named_children():\n        if isinstance(child, nn.ReLU):\n            setattr(model, child_name, Mish())\n        else:\n            convert_relu_to_mish(child)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_sigmoid_to_mish(model):\n    for child_name, child in model.named_children():\n        if isinstance(child, nn.Sigmoid):\n            setattr(model, child_name, Mish())\n        else:\n            convert_relu_to_mish(child)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AdaptiveConcatPool2d(nn.Module):\n    \"Layer that concats `AdaptiveAvgPool2d` and `AdaptiveMaxPool2d`.\"\n    def __init__(self, sz):\n      super().__init__()\n      \"Output will be 2*sz or 2 if sz is None\"\n      self.output_size = sz or 1\n      self.ap = nn.AdaptiveAvgPool2d(self.output_size)\n      self.mp = nn.AdaptiveMaxPool2d(self.output_size)\n\n    def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Flatten(nn.Module):\n    \"Flatten `x` to a single dimension, often used at the end of a model. `full` for rank-1 tensor\"\n    def __init__(self, full:bool=False): \n      super().__init__()\n      self.full = full\n    def forward(self, x): return x.view(-1) if self.full else x.view(x.size(0), -1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def bn_drop_lin( n_in:int, n_out:int, actn:nn.Module,bn:bool=True, p:float=0.):\n    \"Sequence of batchnorm (if `bn`), dropout (with `p`) and linear (`n_in`,`n_out`) layers followed by `actn`.\"\n    layers = [nn.BatchNorm1d(n_in)] if bn else []\n    if p != 0: layers.append(nn.Dropout(p))\n    layers.append(nn.Linear(n_in, n_out))\n    if actn is not None: layers.append(actn)\n    return layers","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"class RetinopathyDatasetTrain(Dataset):\n\n    def __init__(self, csv_file , transform):\n\n        self.data = pd.read_csv(csv_file)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join('../input/aptos2019-blindness-detection/train_images', self.data.loc[idx, 'id_code'] + '.png')\n        image = Image.open(img_name)\n        image = image.resize((224, 224), resample=Image.BILINEAR)\n        image = np.asarray( image, dtype=np.uint8 )\n        if self.transform:\n            result = self.transform(image=image)\n            image = result['image']\n        label = torch.tensor(self.data.loc[idx, 'diagnosis'])\n        return {'image': transforms.ToTensor()(image),\n                'labels': label\n                }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Head(nn.Module):\n    def __init__(self, nc, n, ps=0.5):\n        super(Head,self).__init__()\n        layers = [AdaptiveConcatPool2d(1), Mish(), Flatten()] + \\\n            bn_drop_lin(nc*2, 512, Mish(),True, ps) + \\\n            bn_drop_lin(512, n, None, True, ps)\n        self.fc = nn.Sequential(*layers)\n        self._init_weight()\n        \n    def _init_weight(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                torch.nn.init.kaiming_normal_(m.weight)\n            elif isinstance(m, nn.BatchNorm2d):\n                m.weight.data.fill_(1.0)\n                m.bias.data.zero_()\n        \n    def forward(self, x):\n        return self.fc(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"arch = pretrainedmodels.__dict__['se_resnext50_32x4d']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DRModel(nn.Module):\n    def __init__(self, arch = arch , pre=True):\n        super(DRModel,self).__init__()\n        m = arch(pretrained='imagenet') if pre else arch(pretrained=None)\n        conv = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False)\n        #w = (m.layer0.conv1.weight.sum(1)).unsqueeze(1)\n        #conv.weight = nn.Parameter(w)\n        self.layer0 = nn.Sequential(conv, m.layer0.bn1, m.layer0.relu1, m.layer0.pool)\n        self.layer1 = m.layer1\n        self.layer2 = m.layer2\n        self.layer3 = m.layer3\n        self.layer4 = nn.Sequential(m.layer4[0], m.layer4[1], m.layer4[2])\n\n        \n        nc = self.layer4[-1].se_module.fc2.out_channels       # changes as per architecture\n        self.head = Head(nc,1)\n#         self.head1 = Head(nc,n[0])\n#         self.base_model = m\n#         self.head = Head(1000,1)\n#         convert_sigmoid_to_mish(self.base_model)\n#         convert_relu_to_mish(self.base_model)\n        \n        \n    def forward(self, x):    \n        x = self.layer0(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        \n        x = self.head(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = DRModel(pre=True)\nmodel=model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = torchvision.models.resnet101(pretrained=False)\n# model.load_state_dict(torch.load(\"../input/pytorch-pretrained-models/resnet101-5d3b4d8f.pth\"))\n# num_features = model.fc.in_features\n# model.fc = nn.Linear(2048, 1)\n\n# model = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transforms = Compose([ \n    ShiftScaleRotate(p=1,rotate_limit=180, shift_limit=0.2,scale_limit=0.2),\n    Normalize(mean=(0.0692),std=(0.2051))])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = RetinopathyDatasetTrain(csv_file='../input/aptos2019-blindness-detection/train.csv',transform = train_transforms)\ndata_loader = torch.utils.data.DataLoader(train_dataset, batch_size=64, shuffle=True, num_workers=4)\n\n\n\noptimizer = optim.Adam(model.parameters(), lr=2e-3)\nscheduler = lr_scheduler.StepLR(optimizer, step_size=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"since = time.time()\ncriterion = nn.SmoothL1Loss()\nnum_epochs = 25\nfor epoch in  range(num_epochs):\n    print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n    print('-' * 10)\n    scheduler.step()\n    model.train()\n    running_loss = 0.0\n    tk0 = tqdm(data_loader, total=int(len(data_loader)))\n    counter = 0\n    for bi, d in enumerate(tk0):\n        inputs = d[\"image\"]\n        labels = d[\"labels\"].view(-1, 1)\n        inputs = inputs.to(device, dtype=torch.float)\n        labels = labels.to(device, dtype=torch.float)\n        optimizer.zero_grad()\n        with torch.set_grad_enabled(True):\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n        running_loss += loss.item()\n        counter += 1\n        tk0.set_postfix(loss=(running_loss / (counter * data_loader.batch_size)))\n    epoch_loss = running_loss / len(data_loader)\n    print('Training Loss: {:.4f}'.format(epoch_loss))\n\ntime_elapsed = time.time() - since\nprint('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\ntorch.save(model.state_dict(), \"model.pth\")\n\nPkl_Filename = \"Pickle_SeresModel.pkl\"  \n\nwith open(Pkl_Filename, 'wb') as file:  \n    pickle.dump(model, file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}