{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py > /dev/null\n!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev > /dev/null\n\n!pip install -q --upgrade efficientnet-pytorch\n!pip install -q --upgrade pytorchcv","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"NOTE : There could exist implementation bug,if you find any please let me know in the comment box so this solo worker can learn, thank you :)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"* # Import libraries and utility scripts","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold\nimport cv2\nfrom skimage import io\nimport torch\nimport torch.nn.functional as F\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nfrom tqdm import tqdm\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nimport sklearn\nimport warnings\nfrom pytorchcv.model_provider import get_model as ptcv_get_model\nfrom efficientnet_pytorch import EfficientNet\nfrom sklearn import metrics\n\nimport warnings  \nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import torch_xla\nimport torch_xla.debug.metrics as met\nimport torch_xla.distributed.data_parallel as dp\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.utils.utils as xu\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla.test.test_utils as test_utils\n\n\nfrom sklearn.model_selection import train_test_split\n\nfrom sklearn import metrics\nfrom transformers import AdamW\nfrom transformers import get_linear_schedule_with_warmup\n\nimport time\nimport torchvision\nimport torch.nn as nn\nfrom tqdm import tqdm_notebook as tqdm\n\nfrom PIL import Image, ImageFile\nfrom torch.utils.data import Dataset\nimport torch\nimport torch.optim as optim\nfrom torchvision import transforms\nfrom torch.optim import lr_scheduler\n\n\nimport sys\n\nimport gc\nimport os\nimport random\n\nimport skimage.io\nimport cv2\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\n\nimport sklearn.metrics\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom functools import partial\n\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision.models as models\n\nfrom albumentations import Compose, Normalize, HorizontalFlip, VerticalFlip\nfrom albumentations.pytorch import ToTensorV2\n\n\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom collections import defaultdict, Counter\n\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \nseed_everything(seed=42)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"BASE_PATH = \"/kaggle/input/alaska2-image-steganalysis\"\nDATA_ROOT_PATH = '../input/alaska2-image-steganalysis'\nfold = 0\n\ndataset_train = pd.read_csv('../input/alaska2split/fold_{}_train.csv'.format(fold))# .iloc[:1000, :]\ndataset_val = pd.read_csv('../input/alaska2split/fold_{}_valid.csv'.format(fold))# .iloc[:1000, :]\nsub = pd.read_csv('/kaggle/input/alaska2-image-steganalysis/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def onehot(size, target):\n    vec = torch.zeros(size, dtype=torch.float32)\n    vec[target] = 1.\n    return vec\n\n# https://www.kaggle.com/shonenkov/train-inference-gpu-baseline\nclass ALASKA2Dataset(Dataset):\n\n    def __init__(self, kinds, image_names, labels, transforms=None):\n        super().__init__()\n        self.kinds = kinds\n        self.image_names = image_names\n        self.labels = labels\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n        kind, image_name, label = self.kinds[index], self.image_names[index], self.labels[index]\n        image = cv2.imread(f'{DATA_ROOT_PATH}/{kind}/{image_name}', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        if self.transforms:\n            sample = {'image': image}\n            sample = self.transforms(**sample)\n            image = sample['image']\n\n        target = onehot(4, label)\n        return image, target\n\n    def __len__(self) -> int:\n        return self.image_names.shape[0]\n\n    def get_labels(self):\n        return list(self.labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transform = A.Compose([A.HorizontalFlip(p=0.5),\n                             A.VerticalFlip(p=0.5),\n                             A.Transpose(p=0.5),\n                             A.Resize(height=512, width=512, p=1.0),\n                             ToTensorV2(p=1.0),\n                             ], p=1.0)\n\nval_transform = A.Compose([A.Resize(height=512, width=512, p=1.0),\n                           ToTensorV2(p=1.0),\n                           ], p=1.0)\n\ntrain_dataset = ALASKA2Dataset(\n    kinds=dataset_train.kind.values,\n    image_names=dataset_train.image_name.values,\n    labels=dataset_train.label.values,\n    transforms=train_transform,\n)\n\nvalid_dataset = ALASKA2Dataset(\n    kinds=dataset_val.kind.values,\n    image_names=dataset_val.image_name.values,\n    labels=dataset_val.label.values,\n    transforms=val_transform,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(train_dataset))\nprint(len(valid_dataset))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Identity(nn.Module):\n    def __init__(self):\n        super(Identity, self).__init__()\n\n    def forward(self, x):\n        return x\n\n\nclass Mish(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n    def forward(self, x):\n        return x * torch.tanh(F.softplus(x))\n\n\nclass Alaska2Model(nn.Module):\n\n    def __init__(self, model_name='efficientnet_b3', num_classes=4):\n        super().__init__()\n        if model_name.lower() == 'resnet34':\n            self.backbone = ptcv_get_model(\"resnet34\", pretrained=True)\n\n            self.backbone.features.final_pool = nn.AdaptiveAvgPool2d(1)\n            self.backbone.output = nn.Linear(512, num_classes)\n            # self.backbone.output = nn.Sequential(nn.Linear(512, 128),\n            #                                      swish(),\n            #                                      nn.Dropout(p=0.5),\n            #                                      nn.Linear(128, num_classes))\n        elif model_name.lower() == 'efficientnet_b7':\n            self.backbone = EfficientNet.from_pretrained('efficientnet-b7')\n            # self.backbone._fc = nn.Linear(2560, num_classes)\n\n            self.backbone._fc = nn.Sequential(nn.Linear(2560, 256),\n                                              Mish(),\n                                              nn.Dropout(p=0.5),\n                                              nn.Linear(256, num_classes))\n        elif model_name.lower() == 'efficientnet_b2':\n            self.backbone = EfficientNet.from_pretrained('efficientnet-b2')\n            # self.backbone._fc = nn.Linear(2560, num_classes)\n\n            self.backbone._fc = nn.Sequential(nn.Linear(1408, 256),\n                                              Mish(),\n                                              nn.Dropout(p=0.5),\n                                              nn.Linear(256, num_classes))\n        elif model_name.lower() == 'efficientnet_b3':\n            self.backbone = EfficientNet.from_pretrained('efficientnet-b3')\n            # self.backbone._fc = nn.Linear(2560, num_classes)\n\n            self.backbone._fc = nn.Sequential(nn.Linear(1536, 256),\n                                              Mish(),\n                                              nn.Dropout(p=0.5),\n                                              nn.Linear(256, num_classes))\n        elif model_name.lower() == 'se_resnext101':\n            self.backbone = ptcv_get_model(\"seresnext101_32x4d\", pretrained=True)\n\n            self.backbone.features.final_pool = nn.AdaptiveAvgPool2d(1)\n            self.backbone.output = nn.Sequential(nn.Linear(2048, 256),\n                                                 Mish(),\n                                                 nn.Dropout(p=0.5),\n                                                 nn.Linear(256, num_classes))\n        elif model_name.lower() == 'inceptionresnetv2':\n            self.backbone = ptcv_get_model(\"inceptionresnetv2\", pretrained=True)\n\n            self.backbone.features.final_pool = nn.AdaptiveAvgPool2d(1)\n            self.backbone.output = nn.Sequential(nn.Linear(1536, 128),\n                                                 Mish(),\n                                                 nn.Dropout(p=0.5),\n                                                 nn.Linear(128, num_classes))\n        elif model_name.lower() == 'pnasnet5large':\n            self.backbone = ptcv_get_model(\"pnasnet5large\", pretrained=True)\n            self.backbone.features.final_pool = nn.AdaptiveAvgPool2d(1)\n            self.backbone.output = nn.Sequential(nn.Linear(4320, 512),\n                                                 Mish(),\n                                                 nn.Dropout(p=0.5),\n                                                 nn.Linear(512, num_classes))\n        else:\n            raise NotImplementedError\n\n    def forward(self, x):\n\n        x = self.backbone(x)\n\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Alaska2Model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_epochs = 2\nNUM_EPOCH = num_epochs\nfrom torch.optim.lr_scheduler import OneCycleLR\n\nBATCH_SIZE = 12\n#model = torchvision.models.resnext50_32x4d(pretrained=True)\n#model.load_state_dict(torch.load(\"../input/pytorch-pretrained-models/resnet101-5d3b4d8f.pth\"))\n# model = EfficientNet.from_name('efficientnet-b3')\n\n#model.avg_pool = nn.AdaptiveAvgPool2d(1)\n# num_ftrs = model._fc.in_features\n# model._fc = nn.Linear(num_ftrs, 4)\n#model.load_state_dict(torch.load(\"../input/pytorch-transfer-learning-baseline/model.bin\"))\n#device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef alaska_weighted_auc(y_true, y_valid):\n    \"\"\"\n    https://www.kaggle.com/anokas/weighted-auc-metric-updated\n    \"\"\"\n    tpr_thresholds = [0.0, 0.4, 1.0]\n    weights = [2, 1]\n\n    fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)\n\n    # size of subsets\n    areas = np.array(tpr_thresholds[1:]) - np.array(tpr_thresholds[:-1])\n\n    # The total area is normalized by the sum of weights such that the final weighted AUC is between 0 and 1.\n    normalization = np.dot(areas, weights)\n\n    competition_metric = 0\n    for idx, weight in enumerate(weights):\n        y_min = tpr_thresholds[idx]\n        y_max = tpr_thresholds[idx + 1]\n        mask = (y_min < tpr) & (tpr < y_max)\n\n        if mask.sum() == 0:\n            continue\n\n        x_padding = np.linspace(fpr[mask][-1], 1, 100)\n\n        x = np.concatenate([fpr[mask], x_padding])\n        y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])\n        y = y - y_min  # normalize such that curve starts at y=0\n        score = metrics.auc(x, y)\n        submetric = score * weight\n        best_subscore = (y_max - y_min) * weight\n        competition_metric += submetric\n\n    return competition_metric / normalization\n\n\nclass RocAucMeter(object):\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.y_true = np.array([0, 1])\n        self.y_pred = np.array([0.5, 0.5])\n        self.score = 0\n\n    def update(self, y_true, y_pred):\n        y_true = y_true.cpu().numpy().argmax(axis=1).clip(min=0, max=1).astype(int)\n        y_pred = 1 - nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:, 0]\n        self.y_true = np.hstack((self.y_true, y_true))\n        self.y_pred = np.hstack((self.y_pred, y_pred))\n        self.score = alaska_weighted_auc(self.y_true, self.y_pred)\n\n    @property\n    def avg(self):\n        return self.score\n\n\nclass LabelSmoothing(nn.Module):\n    def __init__(self, smoothing=0.0):\n        super(LabelSmoothing, self).__init__()\n        self.confidence = 1.0 - smoothing\n        self.smoothing = smoothing\n\n    def forward(self, x, target):\n        if self.training:\n            x = x.float()\n            target = target.float()\n            logprobs = torch.nn.functional.log_softmax(x, dim=-1)\n\n            nll_loss = -logprobs * target\n            nll_loss = nll_loss.sum(-1)\n\n            smooth_loss = -logprobs.mean(dim=-1)\n\n            loss = self.confidence * nll_loss + self.smoothing * smooth_loss\n\n            return loss.mean()\n        else:\n            return torch.nn.functional.cross_entropy(x, target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/dhananjay3/pytorch-xla-for-tpu-with-multiprocessing\n#https://www.kaggle.com/abhishek/very-simple-pytorch-training-0-59/data\n\ndef train_model():\n    global train_dataset, valid_dataset\n        \n    train_sampler = torch.utils.data.distributed.DistributedSampler(\n        train_dataset,\n        num_replicas=xm.xrt_world_size(),\n        rank=xm.get_ordinal(),\n        shuffle=True)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=BATCH_SIZE,\n        sampler=train_sampler,\n        num_workers=0,\n        drop_last=True) \n        \n    valid_sampler = torch.utils.data.distributed.DistributedSampler(\n        valid_dataset,\n        num_replicas=xm.xrt_world_size(),\n        rank=xm.get_ordinal(),\n        )\n        \n    valid_loader = torch.utils.data.DataLoader(\n        valid_dataset,\n        batch_size=BATCH_SIZE ,\n        sampler=valid_sampler,\n        shuffle=False,\n        num_workers=0,\n        drop_last=True)\n    \n    #xm.master_print(f\"Train for {len(train_loader)} steps per epoch\")\n#     LOGGER.debug(f\"Train for {len(train_loader)} steps per epoch\")\n    # Scale learning rate to num cores\n    lr  = 0.001 * xm.xrt_world_size()\n\n    # Get loss function, optimizer, and model\n    device = xm.xla_device()\n\n    #model = model()\n    '''\n    for param in model.base_model.parameters(): # freeze some layers\n        param.requires_grad = False'''\n    \n    \n    global model\n    \n    model = model.to(device)\n\n    criterion = LabelSmoothing().to(device) #  BCEWithLogitsLoss\n    #criterion = torch.nn.CrossEntropyLoss()\n    optimizer = optim.AdamW(model.parameters(), lr=lr)\n#     scheduler = lr_scheduler.StepLR(optimizer, step_size=10)\n    \n    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n    scheduler_params = dict(\n        mode='min',\n        factor=0.5,\n        patience=1,\n        verbose=False,\n        threshold=0.0001,\n        threshold_mode='abs',\n        cooldown=0,\n        min_lr=1e-8,\n        eps=1e-08\n    )\n    \n    scheduler = SchedulerClass(optimizer, **scheduler_params)\n\n    \n    def train_loop_fn(loader):\n        tracker = xm.RateTracker()\n        model.train()\n        \n        summary_loss = AverageMeter()\n        final_scores = RocAucMeter()\n\n        xm.master_print(f\"Training Start:{(time.ctime())}\")\n        for step, (inputs, labels) in enumerate(loader):\n            inputs = inputs.to(device, dtype=torch.float)\n            labels = labels.to(device, dtype=torch.float)\n            \n            batch_size = inputs.shape[0]\n            \n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            xm.optimizer_step(optimizer)\n            \n            final_scores.update(labels, outputs)\n            summary_loss.update(loss.detach().item(), batch_size)\n            if step % 200 == 0:\n                xm.master_print(f\"Train Step {step}/{len(loader)} - Loss:{summary_loss.avg:0.4f} - Score:{final_scores.avg:0.4f}\")\n        scheduler.step(final_scores.avg)\n        \n#         xm.master_print(f\"Train Step {step}/{len(loader)} - Loss:{summary_loss.avg:0.4f} - Score:{final_scores.avg:0.4f}\")\n    \n    def test_loop_fn(loader):\n        summary_loss = AverageMeter()\n        final_scores = RocAucMeter()\n        for step, (inputs, labels) in enumerate(loader):\n            inputs = inputs.to(device, dtype=torch.float)\n            labels = labels.to(device, dtype=torch.float)\n            batch_size = inputs.shape[0]\n            outputs = model(inputs)\n            \n            loss = criterion(outputs, labels)\n                    \n            final_scores.update(labels, outputs)\n            summary_loss.update(loss.detach().item(), batch_size)\n            if step % 200 == 0:\n                xm.master_print(f\"Val Step {step}/{len(loader)} - Loss:{summary_loss.avg:0.4f} - Score:{final_scores.avg:0.4f}\")        \n        \n        #auc_score = alaska_weighted_auc(labels.cpu().numpy(), output.cpu().numpy())\n        #LOGGER.debug(\"auc_score according to competition metric = {} \".format(auc_score))\n        #print('[xla:{}] Accuracy={:.4f}%'.format(xm.get_ordinal(), accuracy), flush=True)\n        model.train()\n        return final_scores.avg\n    \n    # Train - valid  loop\n    for epoch in range(1, num_epochs + 1):\n        start = time.time()\n        para_loader = pl.ParallelLoader(train_loader, [device])\n        train_loop_fn(para_loader.per_device_loader(device))\n        \n        para_loader = pl.ParallelLoader(valid_loader, [device])\n        result = test_loop_fn(para_loader.per_device_loader(device))       \n        \n        xm.master_print(\"Finished training epoch {}  Val-Acc {:.4f} in {:.4f} sec\".format(epoch, result,  time.time() - start))   \n        if(epoch>0):\n            xm.save(model.state_dict(), f\"./epoch{epoch}valauc{result}.bin\")\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":false,"_kg_hide-input":false},"cell_type":"code","source":"# Start training processes\n\ndef _mp_fn(rank, flags):\n    torch.set_default_tensor_type('torch.FloatTensor')\n    res = train_model()\n\nFLAGS={}\nxmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xm.get_xla_supported_devices()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}