{"cells":[{"metadata":{"_uuid":"5e87694dcaf86e48e7423752ee27ffa3a7cc4525"},"cell_type":"markdown","source":"## Preparing libraries"},{"metadata":{"trusted":true,"_uuid":"b4c89006c3126aaf0e9ccbe873b92985d1b9c96e"},"cell_type":"code","source":"! pip install albumentations\n! pip install pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# libraries\nimport numpy as np\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold,StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport torch.backends.cudnn as cudnn\nimport time \nimport tqdm\nimport random\nfrom PIL import Image\ntrain_on_gpu = True\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\n\nimport cv2\n\nimport albumentations\nfrom albumentations import torch as AT\nimport pretrainedmodels\n\nimport scipy.special\n\ncudnn.benchmark = True","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"SEED = 323\nbase_dir = '../input/'\ndef seed_everything(seed=SEED):\n    random.seed(seed)\n    os.environ['PYHTONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5f92a7b85b373990c8aff56efd61fc09193ba337"},"cell_type":"markdown","source":"## Preparing data & Simple EDA"},{"metadata":{"trusted":true,"_uuid":"167f1b76dfabcd323c938693f595e36d65aa8353"},"cell_type":"code","source":"labels = pd.read_csv(base_dir+'train_labels.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3a540c680ad23ec07be65ccae8040e52e15e201"},"cell_type":"code","source":"print(len(os.listdir(base_dir+\"train\")))\nprint(len(os.listdir(base_dir+\"test\")))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2bb5d7682d9b376f483488b36f50026cd736a90"},"cell_type":"code","source":"fig = plt.figure(figsize=(25, 4))\n# display 20 images\ntrain_imgs = os.listdir(base_dir+\"train\")\nfor idx, img in enumerate(np.random.choice(train_imgs, 20)):\n    ax = fig.add_subplot(2, 20//2, idx+1, xticks=[], yticks=[])\n    im = Image.open(base_dir+\"train/\" + img)\n    plt.imshow(im)\n    lab = labels.loc[labels['id'] == img.split('.')[0], 'label'].values[0]\n    ax.set_title('Label: %s'%lab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b11f22eaf24ad933e08392b66de7466038c58405"},"cell_type":"code","source":"labels.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d20610fd043960ddbee0f2089bf03515c3fba1fd"},"cell_type":"code","source":"tr, val = train_test_split(labels.label, stratify=labels.label, test_size=0.15, random_state=SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eaf7d715bd01e6ccbb6abc2ce343b28a62d806c8"},"cell_type":"code","source":"img_class_dict = {k:v for k, v in zip(labels.id, labels.label)}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f2c6e4883abfb0869f277e512f343c54c70779d6"},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true,"_uuid":"8d56ecdd72319e2c0156930e73bcd1af21d8e7d2"},"cell_type":"code","source":"class CancerDataset(Dataset):\n    def __init__(self, datafolder, datatype='train', transform = transforms.Compose([transforms.CenterCrop(64),transforms.ToTensor()]), labels_dict={}):\n        self.datafolder = datafolder\n        self.datatype = datatype\n        self.image_files_list = [s for s in os.listdir(datafolder)]\n        self.transform = transform\n        self.labels_dict = labels_dict\n        if self.datatype == 'train':\n            self.labels = [labels_dict[i.split('.')[0]] for i in self.image_files_list]\n        else:\n            self.labels = [0 for _ in range(len(self.image_files_list))]\n\n    def __len__(self):\n        return len(self.image_files_list)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join(self.datafolder, self.image_files_list[idx])\n        img = cv2.imread(img_name)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        image = self.transform(image=img)\n        image = image['image']\n\n        img_name_short = self.image_files_list[idx].split('.')[0]\n\n        if self.datatype == 'train':\n            label = self.labels_dict[img_name_short]\n        else:\n            label = 0\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74a6d1c81be396c2993e668f0375c16a095cd793"},"cell_type":"code","source":"data_transforms = albumentations.Compose([\n    #albumentations.CenterCrop(64, 64),\n    albumentations.RandomRotate90(p=0.5),\n    albumentations.Transpose(p=0.5),\n    albumentations.Flip(p=0.5),\n    albumentations.OneOf([\n        albumentations.CLAHE(clip_limit=2), albumentations.IAASharpen(), albumentations.IAAEmboss(), \n        albumentations.RandomBrightness(), albumentations.RandomContrast(),\n        albumentations.JpegCompression(), albumentations.Blur(), albumentations.GaussNoise()], p=0.5), \n    albumentations.HueSaturationValue(p=0.5), \n    albumentations.ShiftScaleRotate(shift_limit=0.15, scale_limit=0.15, rotate_limit=45, p=0.5),\n    albumentations.Normalize(),\n    AT.ToTensor()\n    ])\n\ndata_transforms_test = albumentations.Compose([\n    #albumentations.CenterCrop(64, 64),\n    albumentations.Normalize(),\n    AT.ToTensor()\n    ])\n\ndata_transforms_tta0 = albumentations.Compose([\n    #albumentations.CenterCrop(64, 64),\n    albumentations.RandomRotate90(p=0.5),\n    albumentations.Transpose(p=0.5),\n    albumentations.Flip(p=0.5),\n    albumentations.Normalize(),\n    AT.ToTensor()\n    ])\n\ndata_transforms_tta1 = albumentations.Compose([\n    #albumentations.CenterCrop(64, 64),\n    albumentations.RandomRotate90(p=1),\n    albumentations.Normalize(),\n    AT.ToTensor()\n    ])\n\ndata_transforms_tta2 = albumentations.Compose([\n    #albumentations.CenterCrop(64, 64),\n    albumentations.Transpose(p=1),\n    albumentations.Normalize(),\n    AT.ToTensor()\n    ])\n\ndata_transforms_tta3 = albumentations.Compose([\n    #albumentations.CenterCrop(64, 64),\n    albumentations.Flip(p=1),\n    albumentations.Normalize(),\n    AT.ToTensor()\n    ])\n\ndataset = CancerDataset(datafolder=base_dir+'train/', datatype='train', transform=data_transforms, labels_dict=img_class_dict)\nval_set = CancerDataset(datafolder=base_dir+'train/', datatype='train', transform=data_transforms_test, labels_dict=img_class_dict)\ntest_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_test)\ntrain_sampler = SubsetRandomSampler(list(tr.index)) \nvalid_sampler = SubsetRandomSampler(list(val.index))\nbatch_size = 256\nnum_workers = 0\n# prepare data loaders (combine dataset and sampler)\ntrain_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers)\nvalid_loader = torch.utils.data.DataLoader(val_set, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers)\ntest_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b8d3de18c8a43415de2c93494c436423d67ebc0"},"cell_type":"code","source":"model_conv = pretrainedmodels.se_resnext50_32x4d()\nmodel_conv.avg_pool = nn.AdaptiveAvgPool2d((1, 1))\nmodel_conv.last_linear = nn.Linear(in_features=2048, out_features=1, bias=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8277f8cbcecd0829f9440ee936758c3b11edb7b3"},"cell_type":"code","source":"# class FocalLoss(nn.Module):\n#     def __init__(self, alpha=0.42, gamma=2, logits=False, reduce=True):\n#         super(FocalLoss, self).__init__()\n#         self.alpha = alpha\n#         self.gamma = gamma\n#         self.logits = logits\n#         self.reduce = reduce\n\n#     def forward(self, inputs, targets):\n#         if self.logits:\n#             BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduce=False)\n#         else:\n#             BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)\n#         pt = torch.exp(-BCE_loss)\n#         F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n\n#         if self.reduce:\n#             return torch.mean(F_loss)\n#         else:\n#             return F_loss","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b137050e2ac7ad1229e7e8ace3bc3a3175340859"},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true,"_uuid":"5d6cefbb5bca7ee5c01c77ddf1e5d6199837582c"},"cell_type":"code","source":"model_conv.cuda()\ncriterion = nn.BCEWithLogitsLoss()\n#criterion = FocalLoss()\n\noptimizer = optim.Adam(model_conv.parameters(), lr=0.00005)\nscheduler = StepLR(optimizer, 1, gamma=0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d384f23b71a56031ae141e5fa4e8e216c862749c","scrolled":true},"cell_type":"code","source":"valid_loss_min = np.Inf\npatience = 5\n# current number of tests, where validation loss didn't increase\np = 0\n# whether training should be stopped\nstop = False\n\n# number of epochs to train the model\nn_epochs = 6\nfor epoch in range(1, n_epochs+1):\n    print(time.ctime(), 'Epoch:', epoch)\n    \n    if stop:\n        print(\"Training stop.\")\n        break\n\n    train_loss = []\n    scheduler.step(epoch)\n    train_auc = []\n\n    for tr_batch_i, (data, target) in enumerate(train_loader):\n        model_conv.train()\n        #time_s = time.time()\n\n        data, target = data.cuda(), target.cuda()\n\n        optimizer.zero_grad()\n        output = model_conv(data)\n        loss = criterion(output[:,0], target.float())\n        train_loss.append(loss.item())\n        \n        a = target.data.cpu().numpy()\n        b = output[:,0].detach().cpu().numpy()\n        train_auc.append(roc_auc_score(a, b))\n\n        loss.backward()\n        optimizer.step()\n        \n        #time_e = time.time()\n        #delta_t = time_e - time_s\n        #print(\"training.... (time cost:%.3f s)\"% delta_t)\n        \n        if (tr_batch_i+1)%100 == 0:    \n            model_conv.eval()\n            val_loss = []\n            val_auc = []\n            for val_batch_i, (data, target) in enumerate(valid_loader):\n                data, target = data.cuda(), target.cuda()\n                output = model_conv(data)\n\n                loss = criterion(output[:,0], target.float())\n\n                val_loss.append(loss.item()) \n                a = target.data.cpu().numpy()\n                b = output[:,0].detach().cpu().numpy()\n                val_auc.append(roc_auc_score(a, b))\n\n            # print(f'Epoch {epoch}, train loss: {np.mean(train_loss):.4f}, valid loss: {np.mean(val_loss):.4f}, train auc: {np.mean(train_auc):.4f}, valid auc: {np.mean(val_auc):.4f}')\n            print('Epoch %d, batches:%d, train loss: %.4f, valid loss: %.4f.'%(epoch, tr_batch_i, np.mean(train_loss), np.mean(val_loss)) +\n                  '  train auc: %.4f, valid auc: %.4f'%(np.mean(train_auc),np.mean(val_auc)))\n            train_loss = []\n            train_auc = []\n            valid_loss = np.mean(val_loss)\n            if valid_loss <= valid_loss_min:\n                print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n                valid_loss_min,\n                valid_loss))\n                #torch.save(model_conv.state_dict(), 'save/model_valloss%d.pt'%(valid_loss*10000))\n                torch.save(model_conv.state_dict(), 'model.pt')\n                valid_loss_min = valid_loss\n                p = 0\n            else:\n                p += 1\n            if p > patience:\n                print('Early stop training')\n                stop = True\n                break          ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a88a78608a97536127d769975d4a2c32398ae130"},"cell_type":"code","source":"torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"964ed06f462a6c6e6503f5f4259f705e650066a4"},"cell_type":"markdown","source":"## Inference"},{"metadata":{"trusted":true,"_uuid":"71258d16737b84ad8ef4b6f69960224f429d73e5"},"cell_type":"code","source":"model_conv.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d427380ffd08093b5bc2a751cba7ca08c75332d6"},"cell_type":"code","source":"saved_dict = torch.load('model.pt')\nmodel_conv.load_state_dict(saved_dict)\npreds = []\nfor batch_i, (data, target) in enumerate(test_loader):\n    data, target = data.cuda(), target.cuda()\n    output = model_conv(data).detach()\n\n    pr = output[:,0].cpu().numpy()\n    for i in pr:\n        preds.append(i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc454091ca3eb27a0a352d2332128a3f64121867"},"cell_type":"code","source":"test_preds = pd.DataFrame({'imgs': test_set.image_files_list, 'preds': preds})\ntest_preds['imgs'] = test_preds['imgs'].apply(lambda x: x.split('.')[0])\nsub = pd.read_csv('../input/sample_submission.csv')\nsub = pd.merge(sub, test_preds, left_on='id', right_on='imgs')\nsub = sub[['id', 'preds']]\nsub.columns = ['id', 'label']\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24f7b9511eb4d2b5072697d3056f330c3b5e8141"},"cell_type":"code","source":"sub.to_csv('sub.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5013176b39c0142ff6dfd2a3f013ea2ba2f2560c"},"cell_type":"markdown","source":"## TTA inference"},{"metadata":{"trusted":true,"_uuid":"591fa5e73bf4fedf11a15e50c5fcc788cbd4a0b8"},"cell_type":"code","source":"NUM_TTA = 8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9dacb064b3fbbf82a9ef49eb0a5eb1b33a74d06f"},"cell_type":"code","source":"sigmoid = lambda x: scipy.special.expit(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"01a2b59c1fe53196e7b8d1f390eae59d3e982b3f"},"cell_type":"code","source":"for num_tta in range(NUM_TTA):\n    if num_tta==0:\n        test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_test)\n        test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)\n    elif num_tta==1:\n        test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_tta1)\n        test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)\n    elif num_tta==2:\n        test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_tta2)\n        test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)\n    elif num_tta==3:\n        test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_tta3)\n        test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)\n    else:\n        test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_tta0)\n        test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)\n    \n    preds = []\n    for batch_i, (data, target) in enumerate(test_loader):\n        data, target = data.cuda(), target.cuda()\n        output = model_conv(data).detach()\n        pr = output[:,0].cpu().numpy()\n        for i in pr:\n            preds.append(sigmoid(i)/NUM_TTA)\n    if num_tta==0:\n        test_preds = pd.DataFrame({'imgs': test_set.image_files_list, 'preds': preds})\n        test_preds['imgs'] = test_preds['imgs'].apply(lambda x: x.split('.')[0])\n    else:\n        test_preds['preds']+=np.array(preds)\n    \nsub = pd.read_csv('../input/sample_submission.csv')\nsub = pd.merge(sub, test_preds, left_on='id', right_on='imgs')\nsub = sub[['id', 'preds']]\nsub.columns = ['id', 'label']\nsub.to_csv('sub_tta.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"01c137e8c7c8e45a4bc966fa9d815669b7c2d0be"},"cell_type":"code","source":"print(\"Finished\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b97f38d61ab89a31eb87e7ba3282714431d41af"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}