{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"!pip install pretrainedmodels\n!pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Objective\nIn this notebook i am going to use 8TPU cores for 8Folds training\n\nReference : https://www.kaggle.com/abhishek/super-duper-fast-pytorch-tpu-kernel\n\nPlease upvote if you like it. It motivates me. Thank you ☺️ .","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn import model_selection\n\nimport os\nimport random\n\nfrom sklearn import metrics\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\n\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp\n\n\nimport albumentations\nfrom PIL import Image, ImageFile\n\n\nimport pretrainedmodels\nfrom efficientnet_pytorch import EfficientNet\n\nfrom joblib import Parallel, delayed\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom tqdm.autonotebook import tqdm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create_folds","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLDS = 8\n\ntrain_df = pd.read_csv(\"../input/siim-isic-melanoma-classification/train.csv\")\ntrain_df[\"kfold\"] = -1\ntrain_df = train_df.sample(frac=1).reset_index(drop=True)\ny = train_df.target.values\nkfold = model_selection.StratifiedKFold(n_splits=FOLDS)\n\nfor f, (tra_, val_) in enumerate(kfold.split(X=train_df, y=y)):\n    train_df.loc[val_, \"kfold\"] = f\n\n\ntrain_df.kfold.value_counts()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class SIIMDataset:\n    def __init__(self, args, df, mode=\"train\", fold=0):\n\n        self.mode = mode\n\n        mean = (0.485, 0.456, 0.406)\n        std = (0.229, 0.224, 0.225)\n\n        if self.mode == \"train\":\n            df = df[~df.kfold.isin([fold])].dropna()\n            self.image_names = df.image_name.values\n            self.targets = df.target.values\n\n            self.aug = albumentations.Compose(\n                [\n                    albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True),\n                    albumentations.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=15),\n                    albumentations.Flip(p=0.5)\n                ]\n            )\n\n        if self.mode == \"valid\":\n            df = df[df.kfold.isin([fold])].dropna()\n            self.image_names = df.image_name.values\n            self.targets = df.target.values\n\n            self.aug = albumentations.Compose(\n                [\n                    albumentations.Normalize(mean, std, max_pixel_value=255.0, always_apply=True)\n                ]\n            )\n\n    def __len__(self):\n        return len(self.image_names)\n        \n    def __getitem__(self,item):\n        \n        image_path = f\"../input/siic-isic-224x224-images/train/{self.image_names[item]}.png\"\n        image = Image.open(image_path)\n        target = self.targets[item]\n\n        image = np.array(image)\n        augmented = self.aug(image=image)\n        image = augmented[\"image\"]\n\n        image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n\n        return {\n            \"image\" : torch.tensor(image, dtype=torch.float),\n            \"target\": torch.tensor(target, dtype=torch.long)\n        }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResNet50(nn.Module):\n    def __init__(self, pretrained=\"imagenet\"):\n        super(ResNet50, self).__init__()\n\n        self.base_model = pretrainedmodels.__dict__[\n            \"resnet50\"\n        ](pretrained=pretrained)\n\n        self.l0 = nn.Linear(2048, 1)\n\n    def forward(self, image):\n        bs, _, _, _ = image.shape\n\n        x = self.base_model.features(image)\n        x = F.adaptive_avg_pool2d(x, 1).reshape(bs, -1)\n\n        out = self.l0(x)\n\n        return out\n\nclass EfficientNet3(nn.Module):\n    def __init__(self, pretrained=\"imagenet\"):\n        super(EfficientNet3, self).__init__()\n\n        self.base_model = EfficientNet.from_pretrained(\"efficientnet-b0\")\n\n        self.l0 = nn.Linear(1536, 1)\n\n    def forward(self, image):\n        bs, _, _, _ = image.shape\n\n        x = self.base_model.extract_features(image)\n        x = F.adaptive_avg_pool2d(x, 1).reshape(bs, -1)\n\n        out = self.l0(x)\n\n        return out","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utils","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"\nclass EarlyStopping:\n    def __init__(self, patience=7, mode=\"max\", delta=0.0001):\n        self.patience = patience\n        self.counter = 0\n        self.mode = mode\n        self.best_score = None\n        self.early_stop = False\n        self.delta = delta\n        if self.mode == \"min\":\n            self.val_score = np.Inf\n        else:\n            self.val_score = -np.Inf\n\n    def __call__(self, epoch_score, model, model_path):\n        if self.mode == \"min\":\n            score = -1.0 * epoch_score\n        else:\n            score = np.copy(epoch_score)\n\n        if self.best_score is None:\n            self.best_score = score\n            #self.save_checkpoint(epoch_score, model, model_path)\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            #print(\n            #    \"EarlyStopping counter: {} out of {}\".format(\n            #        self.counter, self.patience\n            #    )\n            #)\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            #self.save_checkpoint(epoch_score, model, model_path)\n            self.counter = 0\n\n    def save_checkpoint(self, epoch_score, model, model_path):\n        if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:\n            print(\n                \"Validation score improved ({} --> {}). Saving model!\".format(\n                    self.val_score, epoch_score\n                )\n            )\n            torch.save(model.state_dict(), model_path)\n        self.val_score = epoch_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train & Valid loaders","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_list(tensor):\n    return tensor.detach().cpu().tolist()\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current values\"\"\"\n    def __init__(self):\n        self.reset()\n    \n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\ndef reduce_fn(vals):\n    return sum(vals) / len(vals)\n\ndef loss_fn(preds, labels):\n    return nn.BCEWithLogitsLoss()(preds, labels.view(-1, 1).type_as(preds))\n\n\ndef train(args, train_loader, model, device, optimizer, epoch):\n    total_loss = AverageMeter()\n\n    model.train()\n\n    t = tqdm(train_loader, disable=not xm.is_master_ordinal())\n    for step, d in enumerate(t):\n        \n        image = d[\"image\"].to(device)\n        target = d[\"target\"].to(device)\n\n        model.zero_grad()\n\n        logits = model(\n            image\n        )\n\n        loss = loss_fn(logits, target)\n        n_position1 = target.shape[0]\n        total_loss.update(loss.item())\n        \n        loss.backward()\n        xm.optimizer_step(optimizer, barrier=True)\n        print_loss = total_loss.avg\n        \n        t.set_description(f\"Train E:{epoch+1} - Loss:{print_loss:0.4f}\")\n    \n    return total_loss.avg\n\n\ndef valid(args, valid_loader, model, device, epoch):\n    total_loss = AverageMeter()\n    final_predictions = []\n    final_targets = []\n\n    model.eval()\n\n    with torch.no_grad():\n        t = tqdm(valid_loader, disable=not xm.is_master_ordinal())\n        for step, d in enumerate(t):\n            \n            image = d[\"image\"].to(device)\n            target = d[\"target\"].to(device)\n\n            model.zero_grad()\n\n            logits = model(\n                image\n            )\n\n            loss = loss_fn(logits, target)\n            n_position1 = target.shape[0]\n            total_loss.update(loss.item(), n_position1)\n\n            print_loss = total_loss.avg\n            t.set_description(f\"Train E:{epoch+1} - Loss:{print_loss:0.4f}\")\n\n            predictions = to_list(logits)\n            targets = to_list(target)\n\n            final_predictions.append(predictions)\n            final_targets.append(targets)\n\n    final_predictions = np.concatenate(final_predictions)\n    final_targets = np.concatenate(final_targets)\n\n    auc = metrics.roc_auc_score(final_targets, final_predictions)\n    \n    return total_loss.avg, auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"    \ndef run(fold_index):\n    \n    args.fold_index = fold_index\n    \n    MX = ResNet50(pretrained=None)\n    \n\n    device = xm.xla_device(fold_index+1)\n    model = MX.to(device)\n    \n    args.save_path = os.path.join(args.output_dir, args.exp_name)\n\n    if not os.path.exists(args.save_path):\n        os.makedirs(args.save_path)\n\n    # DataLoaders\n    train_dataset = SIIMDataset(\n        args=args, \n        df=train_df, \n        mode=\"train\", \n        fold=args.fold_index\n    )\n\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=args.batch_size,\n        drop_last=False,\n        shuffle=True\n    )\n\n    valid_dataset =  SIIMDataset(\n        args=args, \n        df=train_df, \n        mode=\"valid\", \n        fold=args.fold_index\n    )\n    \n    valid_loader = DataLoader(\n        valid_dataset,\n        batch_size=args.batch_size,\n        drop_last=False,\n        shuffle=True\n    )\n\n\n    optimizer = torch.optim.Adam(\n        model.parameters(),\n        lr=args.learning_rate\n    )\n\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n        optimizer,\n        patience=3,\n        threshold=0.001,\n        mode=\"max\"\n    )\n\n    best_auc = 0\n    early_stopping = EarlyStopping(patience=3, mode=\"max\")\n\n    for epoch in range(args.epochs):\n        \n        train_loss = train(\n            args, \n            train_loader,\n            model,\n            device,\n            optimizer,\n            epoch\n        )\n\n        valid_loss, valid_auc = valid(\n            args, \n            valid_loader,\n            model,\n            device,\n            epoch,\n        )\n\n\n        auc = valid_auc\n        val_loss = valid_loss\n\n        scheduler.step(val_loss)\n\n        print(f\"Fold {fold_index} ** Epoch {epoch+1} **==>** AUC = {auc}\")\n        print(f\"Fold {fold_index} ** Epoch {epoch+1} **==>** valid_loss = {val_loss}\")\n\n        if auc > best_auc:\n            xm.save(model.state_dict(), os.path.join(args.save_path, f\"fold_{fold_index}.bin\"))\n            best_auc = auc\n\n        early_stopping(auc, model, \"none\")\n\n        if early_stopping.early_stop:\n            print(\"Early stopping\")\n            break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class args:\n    \n    learning_rate = 0.00002\n    epochs = 5\n    batch_size = 64\n    output_dir = \"resnet50\"\n    exp_name = \"base_model\"\n    seed = 42","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Parallel(n_jobs=8, backend=\"threading\")(delayed(run)(i) for i in range(8))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Please upvote if you like it. It motivates me. Thank you ☺️ .","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}