{"cells":[{"metadata":{"_kg_hide-output":true,"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev\n!pip install wtfml\n!pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nimport torch_xla\nimport torch_xla.debug.metrics as met\nimport torch_xla.distributed.data_parallel as dp\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.utils.utils as xu\nimport torch_xla.core.xla_model as xm \nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla.test.test_utils as test_utils\nimport warnings\nimport gc\n\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\nimport torchvision\n\nimport cv2\n\nimport numpy as np \nimport pandas as pd\nimport os\n\nfrom torch.utils.data import DataLoader,TensorDataset,Dataset\nimport matplotlib.pyplot as plt\nimport albumentations\nfrom sklearn import model_selection\nfrom sklearn.metrics import roc_auc_score\nfrom efficientnet_pytorch import EfficientNet\n\nfrom wtfml.utils import EarlyStopping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/jpeg-melanoma-384x384/train.csv')  #/kaggle/input/siim-isic-melanoma-classification/train.csv\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = train_df.sample(frac=1).reset_index(drop=True)\ndf['kfold'] = -1\ny = train_df.target.values\nkf = model_selection.StratifiedKFold(n_splits=5,shuffle=True)\nidx = kf.get_n_splits(X=df,y=y)\nprint(idx)\nfor fold,(x,y) in enumerate(kf.split(X=df,y=y)):\n    df.loc[y,'kfold'] = fold\n    \ndf.to_csv('train_fold_tpu.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('train_fold_tpu.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self,path,name,target,aug):\n        super(CustomDataset,self).__init__()\n        self.path = path\n        self.name = name\n        self.target = target\n        self.aug = aug\n        \n        \n    def __len__(self):\n        return len(self.name)\n    \n    def __getitem__(self,index):\n        \n        im_name = self.name[index]\n        y = self.target[index]\n        img_path = os.path.join(self.path,im_name + '.jpg')\n        img = cv2.resize(cv2.imread(img_path),dsize=(384,384))\n        image = self.aug(image=img)\n        l = image['image']\n        image = np.transpose(l, (2, 0, 1)).astype(np.float32)\n        \n        return torch.tensor(image,dtype=torch.float),torch.tensor(y)\n        \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Data_Loader():\n    def __init__(self,path,name,target,aug):\n        self.path = path\n        self.name = name\n        self.target = target\n        self.aug = aug\n        self.dataset = CustomDataset(self.path,self.name,self.target,self.aug)\n        \n    def get(self,batch_size,shuffle,num_workers):\n        \n        sampler = torch.utils.data.distributed.DistributedSampler(self.dataset,\n                                                                  num_replicas = xm.xrt_world_size(),\n                                                                  rank = xm.get_ordinal(),\n                                                                  shuffle = shuffle)\n        dataloader = torch.utils.data.DataLoader(self.dataset,\n                                                 batch_size=batch_size,\n                                                 shuffle=False,\n                                                 sampler=sampler,\n                                                 num_workers=num_workers)\n        return dataloader\n        \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EffNet(nn.Module):\n    def __init__(self,model='b4'):\n        super(EffNet,self).__init__()\n        \n        model_name = 'efficientnet' + model\n        self.feature = EfficientNet.from_pretrained(\"efficientnet-b4\")\n        self.drop = nn.Dropout(0.3)\n        self.l0 = nn.Linear(1792,1) # b3 - 1536 b2 - 1408\n        \n        \n    def forward(self,img):\n        batch_size = img.shape[0]\n        \n        x = self.feature.extract_features(img)\n        #print(x.shape)\n        \n        x = nn.functional.adaptive_avg_pool2d(x,1).reshape(batch_size,-1)\n        #print(x.shape)\n        \n        x = self.drop(x)\n        #print(x.shape)\n        out = self.l0(x)\n        #print(out.shape)\n        \n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self,alpha=1,gamma=2):\n        super(FocalLoss,self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        \n    def forward(self,preds,truth):\n        criterion = nn.BCEWithLogitsLoss()\n        logits = criterion(preds,truth.unsqueeze(-1).type_as(preds))\n        pt = torch.exp(-logits)\n        focal_loss = self.alpha*(1-pt)**self.gamma*logits\n        \n        return torch.mean(focal_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model,fold):\n    \n    batch_t = 128\n    batch_v = 128\n    best_score = 0\n    device = xm.xla_device() \n    \n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225)\n    train_transpose = albumentations.Compose([\n                albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True),\n                albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),\n                albumentations.Flip(p=0.5)               \n            ])\n    valid_transpose = albumentations.Compose(\n        [ albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True) \n        ])\n    \n    image_path = '/kaggle/input/jpeg-melanoma-384x384/train/' #'/kaggle/input/siic-isic-224x224-images/train/'\n    train_df = df[df.kfold!=fold].reset_index(drop=True)  #kfold to fold\n    valid_df = df[df.kfold==fold].reset_index(drop=True)\n    train_im = train_df.image_name.values.tolist()\n    train_y = train_df.target.values\n    valid_im = valid_df.image_name.values.tolist()\n    valid_y = valid_df.target.values\n    train_dataset = Data_Loader(image_path,train_im,train_y,\n                                train_transpose).get(batch_size=batch_t,shuffle=True,num_workers=4)\n    valid_dataset = Data_Loader(image_path,valid_im,valid_y,\n                               valid_transpose).get(batch_size=batch_v,shuffle=False,num_workers=4)\n    \n\n    model = model.to(device)\n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)\n    schedular = torch.optim.lr_scheduler.ReduceLROnPlateau(    #to update the learning rate if model auc score does not increase\n        optimizer,                                             #for 3 succesive epochs\n        patience=100,           \n        threshold=0.001,\n        mode=\"max\"\n    )\n\n    es = EarlyStopping(patience=100, mode=\"max\",tpu=True)  #early stopping function to stop training if auc score does not increase over 5 epochs\n    criterion = nn.BCEWithLogitsLoss()   \n    epochs = 250\n    best_score = 0\n    \n    \n    \n    for epoch in range(epochs):\n            #train mode for training the model and updating the losses\n            model.train()\n            batch = 0\n            #para_loader = pl.ParallelLoader(train_dataset,[device])\n            #train_loader = para_loader.per_device_loader(device)\n        \n            #for _,(train_data,label) in enumerate(train_loader):\n            for train_data,label in train_dataset:\n                train_data = train_data.to(device)\n                label = torch.tensor(label,dtype = torch.float32)\n                label = label.to(device)\n                \n                optimizer.zero_grad()\n                out = model(train_data)\n                loss = criterion(out,label.unsqueeze(1).type_as(out))\n                #loss = criterion(out,label)\n                batch +=1\n                del train_data,label\n                gc.collect()\n                if batch%100==0 : print(\"EPOCH {}  Loss {}  batch  {}\".format(epoch,loss.item(),batch))\n                \n                loss.backward()\n                xm.optimizer_step(optimizer,barrier=True)\n            #evaluate mode to evaluate the model on cv and update learning rate based on auc score\n            #del para_loader,train_loader\n            gc.collect()\n            model.eval()\n            preds = []\n            batch = 0\n            #para_loader = pl.ParallelLoader(valid_dataset,[device])\n            #valid_loader = para_loader.per_device_loader(device)\n            \n            #for _,(valid_data,valid_label) in enumerate(valid_dataset):\n            for valid_data,valid_label in valid_dataset:\n                valid_data = valid_data.to(device)\n                valid_label = torch.tensor(valid_label,dtype = torch.float32)\n                valid_label = valid_label.to(device)\n                batch +=1\n                \n                \n                with torch.no_grad():\n                    out = model(valid_data)\n                    #loss = criterion(out,valid_label)\n                    loss = criterion(out,valid_label.unsqueeze(1).type_as(out))\n                    preds.append(out.cpu())\n                    if batch%50==0 : xm.master_print('Valid Loss {}  batch  {}'.format(loss.item(),batch))\n                del valid_data,valid_label\n                gc.collect()\n            #del para_loader,valid_loader\n            gc.collect()\n            pred=np.vstack((preds)).ravel()\n            #print('pred',pred)\n            auc_score = roc_auc_score(valid_y.astype(np.float32),pred)\n            print(\"EPOCH {}  AUC Score {}\".format(epoch,auc_score))\n            schedular.step(auc_score)\n            es(auc_score, model, model_path=f\"model_fold_{fold}.bin\")\n            if es.early_stop:\n                print(\"Early stopping\")\n                break\n            gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = EffNet()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def _mp_fn(rank, flags):\n    torch.set_default_tensor_type('torch.FloatTensor')\n    a = train(model,0)\n\nFLAGS={}\nxmp.spawn(_mp_fn, args=(FLAGS,), nprocs=2, start_method='fork')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train(model,0)\ntrain(model,1)\ntrain(model,2)\ntrain(model,3)\ntrain(model,4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def image_aug(path,image_name,valid=False):\n    mean = (0.485, 0.456, 0.406)\n    std = (0.229, 0.224, 0.225)\n    train_transpose = albumentations.Compose([\n            albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True),\n            albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),\n            albumentations.Flip(p=0.5)\n            #albumentations.CenterCrop(150,150,always_apply=True)\n        ])\n    valid_transpose = albumentations.Compose(\n    [ albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True) \n    ])\n    \n    if valid==True :\n        \n            im_path = os.path.join(path,image_name + '.jpg')\n            #img = cv2.imread(im_path)\n            img = cv2.resize(cv2.imread(im_path),dsize=(384,384))\n            aug = valid_transpose(image=img)\n            l = aug['image']\n            #print(\"Validation set image augmented\")\n               \n    else:\n        \n            im_path = os.path.join(path,image_name + '.jpg')\n            img = cv2.resize(cv2.imread(im_path),dsize=(384,384))\n            aug = train_transpose(image=img)\n            l =aug['image']\n            #print(\"Train set image augmented\")\n            \n    image = np.transpose(l, (2, 0, 1)).astype(np.float32)\n    return torch.tensor(image, dtype=torch.float)\n\n\n\nclass Data_Loader(Dataset):\n    def __init__(self,image_path,im_name,target,valid=False):\n        self.name = im_name\n        self.target = target\n        self.path = image_path\n        self.valid = valid\n        \n    def __len__(self):\n        return (len(self.name))\n    \n    def __getitem__(self,index):\n        \n        if self.valid==False:\n            im = self.name[index]\n            self.train_y = self.target[index]\n            im_tensor = image_aug(self.path,im)\n            \n            return im_tensor,self.train_y\n        \n        else:\n            im = self.name[index]\n            self.valid_y = self.target[index]\n            im_tensor = image_aug(self.path,im,valid=True)\n            \n            return im_tensor,self.valid_y\n        \n        \n        \n        \n        \ndef train(fold):\n    \n    batch_t = 16\n    batch_v = 16\n    best_score = 0\n    device = 'cuda'\n    image_path = '/kaggle/input/jpeg-melanoma-384x384/train/' #'/kaggle/input/siic-isic-224x224-images/train/'\n    train_df = df[df.kfold!=fold].reset_index(drop=True)  #kfold to fold\n    valid_df = df[df.kfold==fold].reset_index(drop=True)\n    train_im = train_df.image_name.values.tolist()\n    train_y = train_df.target.values\n    valid_im = valid_df.image_name.values.tolist()\n    valid_y = valid_df.target.values\n    train_dataset = Data_Loader(image_path,train_im,train_y)\n    train_dataset = DataLoader(train_dataset,batch_t,shuffle=False,num_workers=4)\n    valid_dataset = Data_Loader(image_path,valid_im,valid_y,valid=True)\n    valid_dataset = DataLoader(valid_dataset,batch_v,shuffle=False,num_workers=4)\n    \n      \n    model = EffNet()\n    model = model.cuda()\n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)\n    schedular = torch.optim.lr_scheduler.ReduceLROnPlateau(    #to update the learning rate if model auc score does not increase\n        optimizer,                                             #for 3 succesive epochs\n        patience=100,           \n        threshold=0.001,\n        mode=\"max\"\n    )\n\n    es = EarlyStopping(patience=100, mode=\"max\")  #early stopping function to stop training if auc score does not increase over 5 epochs\n    criterion = nn.BCEWithLogitsLoss()\n    #criterion = FocalLoss()\n    epochs = 250\n    best_score = 0\n    \n    \n    \n    for epoch in range(epochs):\n            #train mode for training the model and updating the losses\n            model.train()\n            batch = 0\n        \n            for train_data,label in train_dataset:\n                train_data = train_data.to(device)\n                label = torch.tensor(label,dtype = torch.float32)\n                label = label.to(device)\n                \n                optimizer.zero_grad()\n                out = model(train_data)\n                loss = criterion(out,label.unsqueeze(1).type_as(out))\n                #loss = criterion(out,label)\n                batch +=1\n                if batch%200==0 : print(\"EPOCH {}  Loss {}  batch  {}\".format(epoch,loss.item(),batch))\n                \n                loss.backward()\n                optimizer.step()\n            #evaluate mode to evaluate the model on cv and update learning rate based on auc score\n            model.eval()\n            preds = []\n            batch = 0\n            for valid_data,valid_label in valid_dataset:\n                valid_data = valid_data.to(device)\n                valid_label = torch.tensor(valid_label,dtype = torch.float32)\n                valid_label = valid_label.to(device)\n                batch +=1\n                \n                \n                with torch.no_grad():\n                    out = model(valid_data)\n                    #loss = criterion(out,valid_label)\n                    loss = criterion(out,valid_label.unsqueeze(1).type_as(out))\n                    preds.append(out.cpu())\n                    if batch%50==0 : print('Valid Loss {}  batch  {}'.format(loss.item(),batch))\n\n            pred=np.vstack((preds)).ravel()\n            #print('pred',pred)\n            auc_score = roc_auc_score(valid_y.astype(np.float32),pred)\n            print(\"EPOCH {}  AUC Score {}\".format(epoch,auc_score))\n            schedular.step(auc_score)\n            es(auc_score, model, model_path=f\"model_fold_{fold}.bin\")\n            if es.early_stop:\n                print(\"Early stopping\")\n                break        \n            #if auc_score>best_score:\n                        #best_score = auc_score \n                        #torch.save(model,'best_model.pth')\n                        #print(\"Validation Score Improved ======>>>>>> Saving Model\")\n            del train_data,valid_data,label,valid_label\n            gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train(0)\ntrain(1)\ntrain(2)\ntrain(3)\ntrain(4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Prediction**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(fold):\n    test_df = pd.read_csv('/kaggle/input/jpeg-melanoma-384x384/test.csv')   #/kaggle/input/siim-isic-melanoma-classification/test.csv\n    im_path = '/kaggle/input/jpeg-melanoma-384x384/test/' #'/kaggle/input/siic-isic-224x224-images/test/'\n    batch_t = 16    \n    model_path = '/kaggle/input/tpu-model/model_fold_'+str(fold)+'.bin' \n    \n    \n    test_im = test_df.image_name.values.tolist()\n    test_y = np.ones(len(test_im))\n    test_dataset = Data_Loader(im_path,test_im,test_y,valid=False)\n    test_dataset = DataLoader(test_dataset,batch_t,shuffle=False,num_workers=4)\n    device = 'cuda'\n    \n    \n    model = EffNet()\n    model.load_state_dict(torch.load(model_path))\n    model.to(device)\n    model.eval()\n    preds = []\n    batch = 0\n    #for i in range(5):\n    for test_data,test_label in test_dataset:\n                test_data = test_data.to(device)\n                batch +=1\n                \n                with torch.no_grad():\n                    out = model(test_data)\n                    preds.append(out.cpu())\n                    if batch%50==0 : print('Batch  {}'.format(batch))\n\n    pred=np.vstack((preds)).ravel()\n    return pred\n        ","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"predict_1 = predict(0)\npredict_2 = predict(1)\npredict_3 = predict(2)\npredict_4 = predict(3)\npredict_5 = predict(4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction = (predict_1+predict_2+predict_3+predict_4+predict_5)/5\nsubmission = pd.read_csv(\"../input/jpeg-melanoma-384x384/sample_submission.csv\")\nsubmission.loc[:,'target'] = prediction\nsubmission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"s1 = pd.read_csv('../working/submission.csv')  # ../input/new-submit/submit_bce.csv\ns2 = pd.read_csv('../input/new-submit/submit_fl.csv')\ns3 = pd.read_csv('../input/new-submit/submit_bce.csv')\ntarget_res = s3.target.values\ntarget_eff = s2.target.values\ntarget_eff_1 = s1.target.values\nresult = (target_res + target_eff + target_eff_1)/3\n#result_1 = (target_eff + target_eff_1)/2\nsubmission = pd.read_csv(\"../input/jpeg-melanoma-384x384/sample_submission.csv\")\nsubmission.loc[:,'target'] = result\nsubmission.to_csv('submit_1.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../working/submission.csv')\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"reduced_loss = xm.mesh_reduce(\"loss_reduce\", loss, reduce_fn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}