{"cells":[{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"!pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport torch\nfrom torch import nn\nfrom torch import optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import transforms\nfrom efficientnet_pytorch import EfficientNet\nimport os\nfrom PIL import Image\nimport numpy as np\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import accuracy_score, roc_auc_score\n\nfrom tqdm.notebook import tqdm\nimport warnings\nwarnings.simplefilter('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\ntrain_path = '../input/siim-isic-melanoma-classification/jpeg/train/'\ntest_path = '../input/siim-isic-melanoma-classification/jpeg/test/'\n\ndata = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\ndf_test = pd.read_csv('../input/siim-isic-melanoma-classification/test.csv')\nmalignant = data.iloc[data[data['target'] == 1].index]\nbenign = data.iloc[(data[data['target'] == 0].index)[:586]]\ndf = pd.concat([benign, malignant], axis=0)\ndf['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_meta(df):\n    create_meta.dic = {}\n    \n    for i, j in enumerate(df['anatom_site_general_challenge'].unique()):\n      create_meta.dic.update({j:i})\n    \n    sex = np.array(df['sex'].map({'female': 0, 'male': 1}).fillna(-1)).reshape(-1, 1)\n    age = np.array(df['age_approx'].fillna(-1)).reshape(-1, 1)\n    part = np.array(pd.get_dummies(df['anatom_site_general_challenge'].map(create_meta.dic))).reshape(-1, 7)\n    \n    return np.concatenate((sex, age, part), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LoadDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, img_path: str, train: bool=True, transform=None, meta_features=None):\n        super().__init__()\n        self.df = df\n        self.img_path = img_path\n        self.train = train\n        self.transform = transform\n        self.meta = meta_features\n        \n    def __getitem__(self, idx):\n        img_file = os.path.join(self.img_path, self.df['image_name'][idx] + '.jpg')\n        img = Image.open(img_file)\n        \n        if self.transform:\n            img = self.transform(img)\n        \n        if self.train:\n            return (img, self.meta[idx]), self.df.iloc[idx]['target']\n        else:\n            return (img, self.meta[idx])\n        \n    def __len__(self):\n        return len(self.df)\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, arch, n_meta: int):\n        super().__init__()\n        self.arch = arch\n        self.n_meta = n_meta\n        \n        self.arch._fc = nn.Linear(self.arch._fc.in_features, 500)\n        \n        self.meta = nn.Sequential(nn.Linear(self.n_meta, 500),\n                                  nn.BatchNorm1d(500),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2),\n                                  nn.Linear(500, 250),\n                                  nn.BatchNorm1d(250),\n                                  nn.ReLU(),\n                                  nn.Dropout(p=0.2))\n        \n        self.output = nn.Linear(500+250, 1)\n        \n    def forward(self, inputs):\n        img, meta = inputs\n        \n        cnn_out = self.arch(img)\n        meta_out = self.meta(meta)\n        \n        features = torch.cat((cnn_out, meta_out), dim=1)\n        out = self.output(features)\n        \n        return out\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transform = transforms.Compose([\n                                        transforms.RandomResizedCrop(size=256, scale=(0.8, 1.0)),\n                                        transforms.RandomHorizontalFlip(),\n                                        transforms.RandomVerticalFlip(),\n                                        transforms.ToTensor(),\n                                        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\ntest_transform = transforms.Compose([\n                                        transforms.Resize((256, 256)),\n                                        transforms.ToTensor(),\n                                        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n])\n\ntrain_df = create_meta(df)\ntest_df = create_meta(df_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"gkf = GroupKFold(5)\ncnn = EfficientNet.from_pretrained('efficientnet-b1')\ncnn = cnn.to(device)\nfor params in cnn.parameters():\n    params.requires_grad = False\n\nmodel = Model(cnn, train_df.shape[1])\nmodel = model.to(device)\n\nfor params in model.parameters():\n    if params.requires_grad == True:\n        print(params.shape)\n\nnext(model.parameters()).is_cuda","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt = optim.Adam(model.parameters(), 0.001)\ncriterion = nn.BCEWithLogitsLoss()\n\nbs = 4\nepochs = 12\nes_patience = 3\ncorrect = 0.0\nbest_roc = 0\n\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(\n                                                    optimizer=opt, \n                                                    mode='max', \n                                                    patience=1, \n                                                    verbose=True, \n                                                    factor=0.2\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold, (train_idx, val_idx) in enumerate(gkf.split(df, df['target'], df['patient_id']), 1):\n    print(\"Fold\", fold)\n    \n    cnn = EfficientNet.from_pretrained('efficientnet-b1')\n    cnn = cnn.to(device)\n    for params in cnn.parameters():\n        params.requires_grad = False\n\n    model = Model(cnn, train_df.shape[1])\n    model = model.to(device)\n\n    trainset = LoadDataset(df.iloc[train_idx].reset_index(), train_path, True, train_transform, train_df)\n    valset = LoadDataset(df.iloc[val_idx].reset_index(), train_path, True, train_transform, train_df)\n    testset = LoadDataset(df_test, test_path, False, test_transform, test_df)\n    \n    trainloader = DataLoader(trainset, bs, shuffle=True, num_workers=2)\n    valloader = DataLoader(valset, bs, shuffle=False, num_workers=2)   \n    testloader = DataLoader(testset, bs, shuffle=False, num_workers=2)\n    \n    for epoch in range(epochs):\n        model.train()\n        \n        for (img, meta_feature), label in tqdm(trainloader, total=len(trainloader)):\n            img = img.to(device)\n            meta = meta_feature.type(torch.float32).to(device)\n            label = label.type(torch.float32).to(device)\n            \n            opt.zero_grad()\n            out = model((img, meta))\n            loss = criterion(out, label.unsqueeze(1))\n            loss.backward()\n            \n            opt.step()\n            \n            pred = torch.sigmoid(out)\n            correct += (pred.cpu()==label.cpu().unsqueeze(1)).sum().item()  \n            \n        train_acc = correct / len(trainloader)\n        \n        #validaton step\n        model.eval()\n        with torch.no_grad():\n            val_pred = torch.zeros((len(val_idx), 1), dtype=torch.float32, device=device)\n            \n            for idx, ((img_val, meta_feature_val), label_val) in enumerate(valloader):\n                img_val = img_val.to(device)\n                meta_val = meta_feature_val.type(torch.float32).to(device)\n                label_val = label_val.type(torch.float32).to(device)\n                \n                out_val = torch.sigmoid(model((img_val, meta_val)))\n                loss_val = criterion(out_val, label_val.unsqueeze(1))\n                val_pred[idx*valloader.batch_size: idx*valloader.batch_size+valloader.batch_size] = out_val\n                \n      \n            roc_val = roc_auc_score(df.iloc[val_idx]['target'], val_pred.cpu())\n                \n                \n            print(\"Epoch: {}/{}  train_loss: {:0.4f}  val_loss: {:0.4f}  roc: {:0.4f}\".format(\n                epoch+1, epochs, loss.item(), loss_val.item(), roc_val))\n               \n            scheduler.step(roc_val)\n            \n            if roc_val >= best_roc:\n                save_path = f'model_{fold}.pth'\n                best_roc = roc_val\n                patience = es_patience\n                torch.save(model, save_path)\n#             else:\n#                 patience -= 1\n                \n#         if patience == 0:\n#             print(\"Early stopping\\tBest roc score: {:0.3f}\".format(best_roc))\n#             break\n        \n        model = torch.load(save_path)\n        model.eval()   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}