{"cells":[{"metadata":{},"cell_type":"markdown","source":"## About this notebook\n\n* No ensembling is used\n* Single efficientnet-b2 model is trained(achieved **0.921 score on public LB**)\n* Anyone with a kaggle account can reproduce the results\n* Trained only on the gpu provided by the kaggle platform\n* No cross validation used\n* Minimum data augmentation is applied\n\nThe inference notebook can be found [here](https://www.kaggle.com/bipinkrishnan/ranzcr-clip-inference-notebook)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from PIL import Image\nimport sys\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\nimport torch\nfrom torch import optim\nfrom torchvision.transforms import transforms\nfrom torch import nn\nfrom torch.utils.data import DataLoader, Dataset\n\nsys.path.append('../input/efficient-net-deps-1/')\nfrom efficientnet_pytorch.model import EfficientNet\nfrom tqdm.notebook import tqdm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"root_path = '../input/ranzcr-clip-catheter-line-classification/'\ntrain_imgs = root_path + 'train/'\ntest_imgs = root_path+ 'test/'\n\ndf = pd.read_csv(root_path+'train.csv')\nsubmission = pd.read_csv(root_path+'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv, val_csv = train_test_split(df, test_size=0.2, random_state=42)\nlen(train_csv), len(val_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LoadData(Dataset):\n    def __init__(self, df, transform, test=False):\n        super().__init__()\n        self.df = df\n        self.transform = transform\n        self.test = test\n        \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        if not self.test:\n            img = Image.open(train_imgs+row[0]+'.jpg').convert('RGB')\n            labels = torch.from_numpy(row[1:-1].astype(np.float32).values)\n        \n            return self.transform(img), labels \n        else:\n            img = Image.open(test_imgs+row[0]+'.jpg').convert('RGB')\n            return self.transform(img)\n        \n    def __len__(self): return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Trainer(nn.Module):\n    def __init__(self, device, model_name, scheduler):\n        super().__init__()\n        self.model_name = model_name\n        self.device = device\n        self.model = self.build_model().to(self.device)\n        self.criterion = nn.BCEWithLogitsLoss()\n        self.scheduler = scheduler\n        \n        if scheduler=='reducelronplat':\n            self.opt = optim.Adam(self.model.parameters())\n            self.scheduler = optim.lr_scheduler.ReduceLROnPlateau(self.opt, \n                                                                  mode='max', \n                                                                  patience=1, \n                                                                  verbose=True)\n        elif scheduler=='cosineannealing':\n            self.opt = optim.Adam(self.model.parameters(), lr=1e-4)\n            self.scheduler = optim.lr_scheduler.CosineAnnealingLR(self.opt, \n                                                                  T_max=3,\n                                                                  verbose=False)\n            \n        \n    def build_model(self):\n        #building the model for transfer learning\n        if self.model_name=='effnet-b5':\n            effnet = EfficientNet.from_pretrained('efficientnet-b5', num_classes=11)\n            return effnet\n        elif self.model_name=='effnet-b2':\n            effnet = EfficientNet.from_pretrained('efficientnet-b2', num_classes=11)\n            return effnet\n        \n    def train_loop(self, data, label):\n        data, label = data.to(self.device), label.to(self.device)\n        self.opt.zero_grad()\n        train_out = self.model(data)\n        train_loss = self.criterion(train_out, label)\n        train_loss.backward()\n        self.opt.step()\n        \n        return train_loss, train_out\n    \n    def val_loop(self, val_data, val_label):\n        val_data, val_label = val_data.to(self.device), val_label.to(self.device)\n        val_out = self.model(val_data)\n        val_loss = self.criterion(val_out, val_label)\n        \n        return val_loss, val_out\n            \n    def freeze_fit(self, epoch, train_dl):\n        print(\"-------------Starting freezed fit-------------\")\n        if self.model_name=='effnet-b5':\n            self.model = self.unfreeze_linear(self.freeze_all(self.model), 2048, 11)\n        elif self.model_name=='effnet-b2':\n            self.model = self.unfreeze_linear(self.freeze_all(self.model), 1408, 11)\n        self.model.train()\n        for i in range(epoch):\n            for data, label in tqdm(train_dl, total=len(train_dl), leave=False):\n                loss, out = self.train_loop(data, label)\n            print(f\"Epoch: {i+1}/{epoch}  train_loss: {loss}\")            \n        \n    def fit(self, epochs, train_dl, val_dl):\n        #unfreeze the model and\n        #training for specified number of epochs\n        print(\"-------------Starting unfreezed fit-------------\")\n        self.model = self.unfreeze_all(self.model)\n        current_score = 0.0\n        for epoch in range(epochs):\n            val_preds, val_labels = [], []\n            self.model.train()\n            for i, (data, label) in enumerate(tqdm(train_dl, total=len(train_dl), leave=False), 1):\n                train_loss, train_out = self.train_loop(data, label)\n            \n            self.model.eval()\n            with torch.no_grad():\n                for j, (val_data, val_label) in enumerate(tqdm(val_dl, total=len(val_dl), leave=False), 1):\n                    val_loss, val_out = self.val_loop(val_data, val_label)\n                    val_preds.append(val_out.cpu())\n                    val_labels.append(val_label.cpu())\n            \n            val_preds, val_labels = np.concatenate(val_preds), np.concatenate(val_labels)\n            avg_score, scores = self.get_score(val_preds, val_labels)\n            if self.scheduler=='reducelronplat':\n                self.scheduler.step(avg_score)\n            elif self.scheduler=='cosineannealing':\n                self.scheduler.step()\n            \n            if avg_score > current_score:\n                torch.save(self.model.state_dict(), \"model.pt\")\n                current_score = avg_score\n\n            print(f\"{epoch+1}/{epochs}  train_loss: {train_loss}  val_loss: {val_loss}  score: {avg_score}\")\n    \n    def get_score(self, preds, labels):\n        #Calculates ROC AUC score\n        scores = []\n        for i in range(labels.shape[1]):\n            score = roc_auc_score(labels[:, i], preds[:, i])\n            scores.append(score)\n        avg_score = np.mean(scores)\n        return avg_score, scores\n            \n    def load_model(self, path):\n        return self.model.load_state_dict(torch.load(path))\n          \n    def freeze_all(self, model):\n        #freezes all layers of the model\n        for params in model.parameters():\n            params.requires_grad = False\n        return model\n    \n    def unfreeze_linear(self, model, in_features, out_features):\n        #unfreezes the specified layers of the model\n        for params in model.parameters():\n            if params.shape==torch.Size([out_features, in_features]) or params.shape==torch.Size([out_features]):\n                params.requires_grad = True\n        return model\n    \n    def unfreeze_all(self, model):\n        #unfreezes all layers of the model\n        for params in model.parameters():\n            params.requires_grad = True\n        return model        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 32\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nIMG_SIZE = 256\n\ntransform = transforms.Compose([transforms.Resize((IMG_SIZE, IMG_SIZE)),\n                                transforms.ToTensor(),\n                                transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                     std=[0.229, 0.224, 0.225])])\n\ntrain_ds = LoadData(train_csv, transform, test=False)\nval_ds = LoadData(val_csv, transform, test=False)\ntest_ds = LoadData(submission, transform, test=True)\n\ntrain_dl = DataLoader(train_ds, BATCH_SIZE, shuffle=True)\nval_dl = DataLoader(val_ds, BATCH_SIZE, shuffle=False)\ntest_dl = DataLoader(test_ds, BATCH_SIZE, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer = Trainer(device=DEVICE, model_name='effnet-b2', scheduler='reducelronplat')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"trainer.freeze_fit(1, train_dl)\ntrainer.fit(5, train_dl, val_dl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"model_path = './model.pt'\ntrainer.opt.param_groups[0]['lr'] = 1e-4\ntrainer.load_model(model_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"trainer.fit(5, train_dl, val_dl)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}