{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev\n!pip install wtfml\n!pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import warnings\nimport torch_xla\nimport torch_xla.debug.metrics as met\nimport torch_xla.distributed.data_parallel as dp\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.utils.utils as xu\nimport torch_xla.core.xla_model as xm \nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla.test.test_utils as test_utils\nimport warnings\nimport gc\n\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\nimport torchvision\n\nimport cv2\n\nimport numpy as np \nimport pandas as pd\nimport os\n\nfrom torch.utils.data import DataLoader,TensorDataset,Dataset\nimport matplotlib.pyplot as plt\nimport albumentations\nfrom sklearn import model_selection\nfrom sklearn.metrics import roc_auc_score\nfrom efficientnet_pytorch import EfficientNet\n\nfrom wtfml.utils import EarlyStopping","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/melanoma-merged-external-data-512x512-jpeg/folds_13062020.csv')\nprint(data.columns)\n\nbenign_data = data[(data['source']== 'ISIC20') & (data['target']==0)]\nmalign_data = data[data['target']==1]\ndata = benign_data.append(malign_data)\n\ndata.drop('source',axis=1,inplace=True)\ndata.to_csv('train_data.csv',index=False)\n\ndel data, benign_data, malign_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append('../input/autoaug')\nfrom auto_augment import AutoAugment, Cutout\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = '/kaggle/input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/512x512-dataset-melanoma/'\n# test_path = '/kaggle/input/melanoma-merged-external-data-512x512-jpeg/512x512-test/512x512-test/'\ndf= pd.read_csv('./train_data.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['fold'] = -1\nY = df['target'].values\nkf = model_selection.StratifiedKFold(n_splits=3,shuffle=True)\nidx = kf.get_n_splits(X=df,y=Y)\nprint(idx)\nfor fold,(x,y) in enumerate(kf.split(X=df,y=Y)):\n    df.loc[y,'fold'] = fold\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.columns =['image_name', 'patient_id', 'target', 'sex', 'age_approx',\n       'anatom_site_general_challenge', 'stratify_group', 'kfold']\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self,path,name,target,aug):\n        super(CustomDataset,self).__init__()\n        self.path = path\n        self.name = name\n        self.target = target\n        self.aug = aug\n        \n        \n    def __len__(self):\n        return len(self.name)\n    \n    def __getitem__(self,index):\n        \n        im_name = self.name[index]\n        y = self.target[index]\n        img_path = os.path.join(self.path,im_name + '.jpg')\n        img = cv2.resize(cv2.imread(img_path),dsize=(384,384))\n        image = self.aug(image=img)\n        l = image['image']\n        image = np.transpose(l, (2, 0, 1)).astype(np.float32)\n        \n        return torch.tensor(image,dtype=torch.float),torch.tensor(y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Data_Loader():\n    def __init__(self,path,name,target,aug):\n        self.path = path\n        self.name = name\n        self.target = target\n        self.aug = aug\n        self.dataset = CustomDataset(self.path,self.name,self.target,self.aug)\n        \n    def get(self,batch_size,shuffle,num_workers):\n        \n        sampler = torch.utils.data.distributed.DistributedSampler(self.dataset,\n                                                                  num_replicas = xm.xrt_world_size(),\n                                                                  rank = xm.get_ordinal(),\n                                                                  shuffle = shuffle)\n        dataloader = torch.utils.data.DataLoader(self.dataset,\n                                                 batch_size=batch_size,\n                                                 shuffle=False,\n                                                 sampler=sampler,\n                                                 num_workers=num_workers)\n        return dataloader\n       ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EffNet(nn.Module):\n    def __init__(self,model='b0'):\n        super(EffNet,self).__init__()\n        \n        model_name = 'efficientnet' + model\n        self.feature = EfficientNet.from_pretrained(\"efficientnet-b1\")\n        self.drop = nn.Dropout(0.25)\n        self.l0 = nn.Linear(1280,1) #b4-1792 # b3 - 1536, b2 - 1408, b0 = 1280, b1 = 1280\n        self.l1 = nn.Sigmoid()\n        \n        \n        \n    def forward(self,img):\n        batch_size = img.shape[0]\n        \n        x = self.feature.extract_features(img)\n        #print(x.shape)\n        \n        x = nn.functional.adaptive_avg_pool2d(x,1).reshape(batch_size,-1)\n        #print(x.shape)\n        \n        x = self.drop(x)\n        #print(x.shape)\n        out = self.l0(x)\n        #print(out.shape)\n        out = self.l1(out)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model,fold):\n    \n    batch_t = 128\n    batch_v = 128\n    best_score = 0\n    device = xm.xla_device() \n    \n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225)\n    train_transpose = albumentations.Compose([\n                albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True),\n                albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),\n                albumentations.Flip(p=0.5)\n            ])\n    valid_transpose = albumentations.Compose(\n        [ albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True) \n        ])\n    \n    image_path = '../input/melanoma-merged-external-data-512x512-jpeg/512x512-dataset-melanoma/512x512-dataset-melanoma/' #'/kaggle/input/siic-isic-224x224-images/train/'\n    train_df = df[df.kfold!=fold].reset_index(drop=True)  #kfold to fold\n    valid_df = df[df.kfold==fold].reset_index(drop=True)\n    train_im = train_df.image_name.values.tolist()\n    train_y = train_df.target.values\n    valid_im = valid_df.image_name.values.tolist()\n    valid_y = valid_df.target.values\n    train_dataset = Data_Loader(image_path,train_im,train_y,\n                                train_transpose).get(batch_size=batch_t,shuffle=True,num_workers=4)\n    valid_dataset = Data_Loader(image_path,valid_im,valid_y,\n                               valid_transpose).get(batch_size=batch_v,shuffle=False,num_workers=4)\n    \n    \n    \n    \n    \n    #model = Resnext50_32x4d()\n    #model = EffNet()\n    model = model.to(device)\n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n    schedular = torch.optim.lr_scheduler.ReduceLROnPlateau(    #to update the learning rate if model auc score does not increase\n        optimizer,                                             #for 3 succesive epochs\n        patience=3,           \n        threshold=0.001,\n        mode=\"max\"\n    )\n\n    es = EarlyStopping(patience=5, mode=\"max\",tpu=True)  #early stopping function to stop training if auc score does not increase over 5 epochs\n    criterion = nn.BCEWithLogitsLoss()\n    #criterion = FocalLoss()\n    epochs = 10\n    best_score = 0\n    \n    \n    \n    for epoch in range(epochs):\n            #train mode for training the model and updating the losses\n            model.train()\n            batch = 0\n            #para_loader = pl.ParallelLoader(train_dataset,[device])\n            #train_loader = para_loader.per_device_loader(device)\n        \n            #for _,(train_data,label) in enumerate(train_loader):\n            for train_data,label in train_dataset:\n                train_data = train_data.to(device)\n                label = torch.tensor(label,dtype = torch.float32)\n                label = label.to(device)\n                \n                optimizer.zero_grad()\n                out = model(train_data)\n                loss = criterion(out,label.unsqueeze(1).type_as(out))\n                #loss = criterion(out,label)\n                batch +=1\n                del train_data,label\n                gc.collect()\n                if batch%100==0 : print(\"EPOCH {}  Loss {}  batch  {}\".format(epoch,loss.item(),batch))\n                \n                loss.backward()\n                xm.optimizer_step(optimizer,barrier=True)\n            #evaluate mode to evaluate the model on cv and update learning rate based on auc score\n            #del para_loader,train_loader\n            gc.collect()\n            model.eval()\n            preds = []\n            batch = 0\n            #para_loader = pl.ParallelLoader(valid_dataset,[device])\n            #valid_loader = para_loader.per_device_loader(device)\n            \n            #for _,(valid_data,valid_label) in enumerate(valid_dataset):\n            for valid_data,valid_label in valid_dataset:\n                valid_data = valid_data.to(device)\n                valid_label = torch.tensor(valid_label,dtype = torch.float32)\n                valid_label = valid_label.to(device)\n                batch +=1\n                \n                \n                with torch.no_grad():\n                    out = model(valid_data)\n                    #loss = criterion(out,valid_label)\n                    loss = criterion(out,valid_label.unsqueeze(1).type_as(out))\n                    preds.append(out.cpu())\n                    if batch%50==0 : xm.master_print('Valid Loss {}  batch  {}'.format(loss.item(),batch))\n                del valid_data,valid_label\n                gc.collect()\n            #del para_loader,valid_loader\n            gc.collect()\n            pred=np.vstack((preds)).ravel()\n            #print('pred',pred)\n            auc_score = roc_auc_score(valid_y.astype(np.float32),pred)\n            print(\"EPOCH {}  AUC Score {}\".format(epoch,auc_score))\n            schedular.step(auc_score)\n            es(auc_score, model, model_path=f\"model_fold_{fold}.bin\")\n            if es.early_stop:\n                print(\"Early stopping\")\n                break\n            gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = EffNet()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train(model,0)\ntrain(model,1)\ntrain(model,2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Prediction","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(fold):\n    test_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')   #/kaggle/input/siim-isic-melanoma-classification/test.csv\n    im_path = '/kaggle/input/melanoma-merged-external-data-512x512-jpeg/512x512-test/512x512-test/' #'/kaggle/input/siic-isic-224x224-images/test/'\n    batch_t = 32\n    #model_path = '../working/model_fold_'+str(fold)+'.bin'\n    model_path = '../working/model_fold_'+str(fold)+'.bin' \n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225)\n    test_transpose = albumentations.Compose(\n        [ albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True) \n        ])\n    \n    test_im = test_df.image_name.values.tolist()\n    test_y = np.ones(len(test_im))\n    test_dataset = Data_Loader(im_path,test_im,test_y,test_transpose).get(batch_t,shuffle=False,num_workers=4)\n    device = xm.xla_device()\n    \n    \n    model = EffNet()\n    model.load_state_dict(torch.load(model_path))\n    model.to(device)\n    model.eval()\n    preds = []\n    batch = 0\n    #for i in range(5):\n    for test_data,test_label in test_dataset:\n                test_data = test_data.to(device)\n                batch +=1\n                \n                with torch.no_grad():\n                    out = model(test_data)\n                    preds.append(out.cpu())\n                    if batch%50==0 : print('Batch  {}'.format(batch))\n\n    pred=np.vstack((preds)).ravel()\n    return pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_1 = predict(0)\npredict_2 = predict(1)\npredict_3 = predict(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\nsub['target'] = predict_1\nsub.to_csv('b1_signmoid_1_fold.csv',index=False)\n# sub = pd.read_csv('../input/siim-isic-melanoma-classification/sample_submission.csv')\nsub['target'] = predict_2\nsub.to_csv('b1_signmoid_2_fold.csv',index=False)\n\nsub['target'] = predict_3\nsub.to_csv('b1_signmoid_3_fold.csv',index=False)\n\nsub['target'] = (predict_1 + predict_2 + predict_3)/3\nsub.to_csv('b1_signmoid_3folds_avg.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}