{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%%capture\n%%bash\npip install efficientnet_pytorch torchtoolbox","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom tqdm.notebook import tqdm\nimport multiprocessing as mp\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim.lr_scheduler import LambdaLR\nfrom torch.utils.data import Dataset, DataLoader\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom efficientnet_pytorch import EfficientNet\n\nimport warnings\nwarnings.simplefilter('ignore')\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"base = \"/kaggle/input/data-saving/im_resized/\"\ntrain_df = pd.read_csv(base+\"train.csv\")\nvalid_df = pd.read_csv(base+\"val.csv\")\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p = 0.5\ntrain_tfms = A.Compose([\n    A.Cutout(p=p),\n    A.RandomRotate90(p=p),\n    A.Flip(p=p),\n    A.OneOf([\n        A.RandomBrightnessContrast(brightness_limit=0.2,\n                                   contrast_limit=0.2,\n                                   ),\n        A.HueSaturationValue(\n            hue_shift_limit=20,\n            sat_shift_limit=50,\n            val_shift_limit=50)\n    ], p=p),\n    A.OneOf([\n        A.IAAAdditiveGaussianNoise(),\n        A.GaussNoise(),\n    ], p=p),\n    A.OneOf([\n        A.MotionBlur(p=0.2),\n        A.MedianBlur(blur_limit=3, p=0.1),\n        A.Blur(blur_limit=3, p=0.1),\n    ], p=p),\n    A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=p),\n    A.OneOf([\n        A.OpticalDistortion(p=0.3),\n        A.GridDistortion(p=0.1),\n        A.IAAPiecewiseAffine(p=0.3),\n    ], p=p), \n    A.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],\n    ),\n    ToTensorV2()\n])\n    \ntest_tfms = A.Compose([\n    A.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225],\n    ),\n    ToTensorV2()\n])\n\nclass Data(Dataset):\n    def __init__(self, df, size, base, transform=None, is_test=False):\n        self.labels = df[\"target\"].values.astype(np.float32)[:,None]\n        self.size = size\n        self.base = base\n        self.current = None\n        self.transform = transform\n        self.is_test = is_test\n        \n    def __len__(self):\n        return len(self.labels)\n    \n    def __getitem__(self, i):\n        batch = i // self.size\n        if self.current != self.base + str(batch) + \".npy\":\n            self.current = self.base + str(batch) + \".npy\"\n            self.current_batch = np.load(self.current) #.transpose((0,3,1,2))\n        i = i % self.size\n        image = self.current_batch[i]\n        if self.transform:\n            image = self.transform(image=image)['image']\n            \n        if self.is_test:\n            return image\n        return image, self.labels[i]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BS = 128\ntrain_ds = Data(train_df, 128, base+\"train_batch_\", train_tfms)\nvalid_ds = Data(valid_df, 128, base+\"valid_batch_\", test_tfms)\ntrain_dl = DataLoader(train_ds, BS, drop_last=True, num_workers=mp.cpu_count())\nvalid_dl = DataLoader(valid_ds, BS, num_workers=mp.cpu_count())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyEfficientNet(nn.Module):\n    def __init__(self, base=\"efficientnet-b0\"):\n        super().__init__()\n\n        # EfficientNet\n        self.base = EfficientNet.from_pretrained(base)\n        # freeze _fc\n        for p in self.base._fc.parameters(): p.requires_grad=False\n        \n        # Replace last layer\n        self.fc = nn.Sequential(nn.Linear(self.base._fc.in_features, 512), \n                                         nn.ReLU(),  \n                                         nn.Dropout(0.25),\n                                         nn.Linear(512, 128), \n                                         nn.ReLU(),  \n                                         nn.Dropout(0.25), \n                                         nn.Linear(128,1))\n    \n    def forward(self, x):\n        pool = F.adaptive_avg_pool2d(self.base.extract_features(x), 1)\n        pool = pool.view(x.shape[0], -1)\n        return self.fc(pool)\n\nmodel = MyEfficientNet()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"a1 = 1 / train_df[\"target\"].mean()\na2 = 1 / (1 - train_df[\"target\"].mean())\n\nclass WeightedFocalLoss(nn.Module):\n    \"Non weighted version of Focal Loss\"\n    def __init__(self, a1, a2, gamma=2):\n        super().__init__()\n        self.alpha = torch.tensor([a1, a2]).cuda()\n        self.gamma = gamma\n\n    def forward(self, inputs, targets):\n        BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction='none')\n        targets = targets.type(torch.long)\n        at = self.alpha.gather(0, targets.data.view(-1))\n        pt = torch.exp(-BCE_loss)\n        F_loss = at*(1-pt)**self.gamma * BCE_loss\n        return F_loss.mean()\n    \nloss_fn = WeightedFocalLoss(a1, a2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Optimizer","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"blocks = []\n\nfor n,p in model.base.named_parameters():\n    if p.requires_grad:\n        if n.startswith(\"_blocks.\"):\n            n = \".\".join(n.split(\".\", maxsplit=2)[:2])\n        else:\n            n = n.split(\".\", maxsplit=1)[0]\n        if n not in blocks:\n            blocks.append(n)\n\nblocks = [\"base.\"+block for block in blocks]\nblocks += [\"fc\"]\nblocks = [block+\".\" for block in blocks]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_range = [1e-9, 1e-5]\nmul = (lr_range[1] / lr_range[0]) ** (1/(len(blocks)-1))\nlrs = [lr_range[0]*mul**i for i in range(len(blocks))]\n\nparam_list = []\nfor lr, block in zip(lrs, blocks):\n    param_list.extend([{'params':p ,'lr':lr} for n,p in model.named_parameters() if n.startswith(block)])\noptimizer = torch.optim.Adam(param_list)\n\nlr_sched = lambda batch: 1.1**batch\nscheduler = LambdaLR(optimizer, lr_lambda=[lr_sched]*len(param_list))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Save initial weights before LR finding","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"weights = []\n\nfor param in model.parameters():\n    weights.append(param.clone())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lrs = []\nlosses = []\nmin_loss = 1e9\n\nmodel.train()\nmodel = model.to(device)\nfor x, y in tqdm(train_dl):\n    x, y = x.to(device), y.to(device)\n    optimizer.zero_grad()\n    \n    lrs.append(optimizer.param_groups[-1]['lr'])\n    y_pred = model(x)\n    loss = loss_fn(y_pred, y)\n    loss.backward()\n    optimizer.step()\n    scheduler.step()\n    losses.append(loss.detach().cpu().numpy())\n    \n    print(f\"\\rLoss: {loss:.4f}, lr {lrs[-1]:.6f}\", end=\"\")\n    if loss < min_loss:\n        min_loss = loss\n    if loss > 20 * min_loss:\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(lrs, losses)\nplt.xscale('log')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Put original weights back into model.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"for w, p in zip(weights, model.parameters()):\n    p.data = w","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}