{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Installing XLA","execution_count":null},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!export XLA_USE_BF16=1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Importing Libraries","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport time\nimport os\nimport torch\nimport torch.nn as nn\nimport torch.utils.data as D\nfrom torch.optim.lr_scheduler import ExponentialLR, ReduceLROnPlateau\n\nfrom torchvision import models as M, transforms as T\n\nfrom tqdm import tqdm\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom glob import glob\n\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp\n\nimport gc\nimport matplotlib.pyplot as plt\nimport pickle\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom joblib import Parallel, delayed\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nprint(\"Required libraries installed /-\\-/-\\...\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Seeding","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_torch(seed_value):\n    #random.seed(seed_value) # Python\n    np.random.seed(seed_value) # cpu vars\n    torch.manual_seed(seed_value) # cpu  vars    \n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(seed_value)\n        torch.cuda.manual_seed_all(seed_value) # gpu vars\n    if torch.backends.cudnn.is_available:\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\nseed_torch(42)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Reading CSV's","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE_PATH = \"../input/jpeg-melanoma-256x256\"\ndf_train = pd.read_csv(BASE_PATH + \"/train.csv\")\ndf_test = pd.read_csv(BASE_PATH + \"/test.csv\")\ndf_sub = pd.read_csv(BASE_PATH + \"/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp = plt.imread(\"../input/siic-isic-224x224-images/train/ISIC_4232172.png\")\nplt.xticks([])\nplt.yticks([])\nplt.imshow(temp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"groups_by_patient = df_train.patient_id.copy().to_list()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training Images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"BINGO_PATH = \"/kaggle/input/siic-isic-224x224-images\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Defining Architecture","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"classes = 2\n\nclass Net(nn.Module):\n    def __init__(self, arch):\n        super(Net, self).__init__()\n        self.arch = arch\n        self.arch._avg_pool = nn.modules.pooling.AdaptiveAvgPool2d(output_size = 32)\n        self.arch._fc = nn.Linear(in_features = 1280, out_features = 2, bias = True)\n        \n    def forward(self, x):\n        x = self.arch(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Preparing Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImagesDS(D.Dataset):\n    def __init__(self, df, dir, mode = \"train\", transforms = None):\n        self.records = df.to_records(index = False)\n        self.mode = mode\n        self.dir = dir\n        self.len = df.shape[0]\n        self.transforms = transforms\n        \n    @staticmethod\n    def _load_train_img_as_tensor(filename):\n        with Image.open(filename) as img:\n            return T.Compose([\n                T.RandomResizedCrop(size = 224, scale = (0.7, 1.0)), \n                                  T.RandomHorizontalFlip(), \n                                  T.RandomVerticalFlip(), \n                                  T.ColorJitter(brightness = 32. / 255., saturation = 0.5),\n                                  T.ToTensor(), \n                                  T.Normalize(mean = [0.485, 0.456, 0.406], std = [0.229, 0.224, 0.225])])(img)\n    \n    @staticmethod\n    def _load_test_img_as_tensor(filename):\n        with Image.open(filename) as img:\n            return T.Compose([T.ToTensor(), T.Normalize(mean = [0.485, 0.456, 0.406], std = [0.229, 0.224, 0.225])])(img)\n    \n    def _get_image_path(self, index):\n        image_id = self.records[index].image_name\n        return \"/\".join([self.dir, self.mode, f\"{image_id}.png\"])\n    \n    \n    def __getitem__(self, index):\n        path = self._get_image_path(index)\n        \n        if self.transforms == \"train\":\n            img = self._load_train_img_as_tensor(path)\n        else:\n            img = self._load_test_img_as_tensor(path)\n\n        if self.mode == \"train\":\n            return img, self.records[index].target\n        else:\n            return img\n        \n    def __len__(self):\n        return self.len","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter:\n    \"\"\"\n    Computes and stores the average and current value\n    \"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_fn(values):\n    return sum(values)/len(values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gkf = GroupKFold(n_splits = 3)\ncv = []","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fitter","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = EfficientNet.from_pretrained(\"efficientnet-b0\")\nnet = Net(arch = model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Fitter:\n    def __init__(self, batch_size, epoch):\n        self.device = xm.xla_device()\n        xm.master_print(f'Fitter prepared. Device is {self.device}')\n        xm.master_print(f\"Device: {xm.get_ordinal()}, Num_Replicas: {xm.xrt_world_size()}\")\n        self.best_auc = 0\n        self.n_epochs = epoch\n        self.bs = batch_size\n        \n    def train_model(self, model, epoch, loader, device):\n        model.train()\n        losses = AverageMeter()\n        avg_loss = 0\n        for i, data in enumerate(loader.per_device_loader(device)):\n            # Get the inputs\n            inputs, labels = data\n            inputs, labels = inputs.to(device), labels.to(device)\n            # Forward + backward + optimize\n            self.optimizer.zero_grad()\n        \n            outputs = model(inputs)\n            loss = self.criterion(outputs, labels)\n            loss.backward()\n            xm.optimizer_step(self.optimizer, barrier = True)\n            reduced_loss = xm.mesh_reduce(\"loss_reduce\", loss, reduce_fn)\n            losses.update(reduced_loss.item(), inputs.size(0))\n        return avg_loss\n\n    def test_model(self, model, val_loader, device):\n        model.eval()\n    \n        losses = AverageMeter()\n        avg_val_loss = 0.\n        valid_preds, valid_targets = [], []\n    \n        with torch.no_grad():\n#         .per_device_loader(device)\n            for i, data in enumerate(val_loader.per_device_loader(device)):\n                inputs, labels = data\n                inputs, labels = inputs.to(device), labels.to(device)\n                outputs = model(inputs)\n                loss = self.criterion(outputs, labels)\n            \n#                 avg_val_loss += loss.item()/len(val_loader)\n                reduced_loss = xm.mesh_reduce(\"loss_reduce\", loss, reduce_fn)\n                losses.update(reduced_loss.item(), inputs.size(0))\n                valid_preds.append(torch.softmax(outputs, 1)[:, 1].detach().cpu().numpy())\n                valid_targets.append(labels.detach().cpu().numpy())\n            \n            valid_preds = np.concatenate(valid_preds)\n            valid_targets = np.concatenate(valid_targets)\n            val_auc = roc_auc_score(valid_targets, valid_preds)\n            val_acc = accuracy_score(valid_targets, np.round(valid_preds))\n    \n        return avg_val_loss, val_auc, val_acc\n\n    def start(self):\n        for fold, (train_idx, val_idx) in enumerate(gkf.split(X = np.zeros(len(df_train)), y = df_train[\"target\"], groups = groups_by_patient), 1):\n            xm.master_print(\"*\"*40, \"Fold \", fold, \"*\"*40)\n            xm.master_print(\"xla:\", xm.get_ordinal())\n            self.best_auc = 0\n            ds = ImagesDS(df_train.iloc[train_idx], BINGO_PATH, mode = \"train\", transforms = \"train\")\n            ds_val = ImagesDS(df_train.iloc[val_idx], BINGO_PATH, mode = \"train\", transforms = \"test\")\n            \n            train_sampler = D.distributed.DistributedSampler(ds, num_replicas = xm.xrt_world_size(), rank = xm.get_ordinal(), shuffle = True)\n            loader = D.DataLoader(ds, batch_size = self.bs, sampler = train_sampler, num_workers = 0)\n            \n            val_sampler = D.distributed.DistributedSampler(ds_val, num_replicas = xm.xrt_world_size(), rank = xm.get_ordinal(), shuffle = True)\n \n            val_loader = D.DataLoader(ds_val, batch_size = self.bs, sampler = val_sampler, num_workers = 0)\n    \n#             model = EfficientNet.from_pretrained(\"efficientnet-b0\")\n#             net = Net(arch = model)\n            net.to(self.device)\n            \n            self.criterion = nn.CrossEntropyLoss()\n            self.optimizer = torch.optim.Adam(net.parameters(), lr = 0.0001)\n            self.scheduler = ReduceLROnPlateau(self.optimizer, mode = \"max\", patience = 3, verbose = True, factor = 0.2)\n\n            for epoch in range(self.n_epochs):\n                para_loader = pl.ParallelLoader(loader, [self.device])\n                avg_loss = self.train_model(net, epoch, para_loader, self.device)\n            \n                para_loader = pl.ParallelLoader(val_loader, [self.device])\n                avg_val_loss, val_auc, val_acc = self.test_model(net, para_loader, self.device)\n\n                if val_auc > self.best_auc:\n                    self.best_auc = val_auc\n                    xm.save(net.state_dict(), str(fold) + 'weight.pt')\n                xm.master_print('current_val_auc: ', val_auc, '| best_val_auc: ', self.best_auc, \"| Average loss: \", avg_loss, \"| Average val loss: \", avg_val_loss, \"| Validation accuracy: \", val_acc)\n        \n                self.scheduler.step(val_auc)\n\n            cv.append(self.best_auc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _mp_fn(rank, flags):\n    torch.set_default_tensor_type('torch.FloatTensor')\n    fitter = Fitter(64, 5)\n    if rank==0:\n        time.sleep(1)\n    fitter.start()\nFLAGS={}\nxmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(cv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = xm.xla_device()\nmodel = EfficientNet.from_pretrained(\"efficientnet-b0\")\nmodel1 = Net(arch = model)\nmodel1.to(device)\nmodel1.load_state_dict(torch.load(\"./1weight.pt\"))\n\nmodel2 = Net(arch = model)\nmodel2.to(device)\nmodel2.load_state_dict(torch.load(\"./2weight.pt\"))\n\nmodel3 = Net(arch = model)\nmodel3.to(device)\nmodel3.load_state_dict(torch.load(\"./3weight.pt\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"model1.eval()\nmodel2.eval()\nmodel3.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds_test = ImagesDS(df_test, BINGO_PATH, mode = \"test\")\ntest_loader = D.DataLoader(ds_test, batch_size = batch_size, shuffle = False, num_workers = 4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tta = 4\ntest_pred = np.zeros((len(df_test),))\n\nwith torch.no_grad():\n    for i, data in enumerate(tqdm(test_loader, position = 0, leave = True)):\n        images = data\n        images = images.to(device)\n        \n        pred = (model1(images) + model2(images) + model3(images)) \\\n             + (model1(images) + model2(images) + model3(images)) \\\n            + (model1(images) + model2(images) + model3(images)) \\\n            + (model1(images) + model2(images) + model3(images))\n        \n        pred = torch.softmax(pred,1).cpu().detach().numpy()[:,1]\n    \n        test_pred[i*batch_size: (i+1)*batch_size] = pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.kdeplot(pd.Series(test_pred.reshape(-1, )))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub.target = test_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub.to_csv(\"submission.csv\", index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}