{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Alaska2 Baseline PyTorch\n\n* This notebook is based on the great work of [Alex Shonenkov](https://www.kaggle.com/shonenkov/train-inference-gpu-baseline)\n* I have classified the dataset into 12 classes, 3 quality factors for each folder","metadata":{}},{"cell_type":"markdown","source":"# Dependencies","metadata":{}},{"cell_type":"code","source":"# Changes:\n# Seed = 2\n# Fold = 2\n# Sampler downsize\n# smoothing = 0.01\n#      net._fc = nn.Sequential(nn.Linear(1408, 1024),\n#                                  nn.ReLU(),\n#                                  nn.Dropout(0.2 , inplace = True),\n#                                  nn.Linear(1024,512),\n#                                  nn.ReLU(),\n#                                  nn.Dropout(0.2 , inplace = True),\n#                                  nn.Linear(512,256),\n#                                  nn.ReLU(),\n#                                  nn.Dropout(0.2 , inplace = True),\n#                                  nn.Linear(256,2),\n#                                  nn.LogSoftmax(dim=1)).to('cuda') \n\n\n#model link:\n#    https://drive.google.com/file/d/14Z9F-TABfD7-ibuVX1C9DrrPSDrKIo9Y/view?usp=sharing\n        ","metadata":{"execution":{"iopub.status.busy":"2022-03-09T06:59:51.707577Z","iopub.execute_input":"2022-03-09T06:59:51.707988Z","iopub.status.idle":"2022-03-09T06:59:51.71686Z","shell.execute_reply.started":"2022-03-09T06:59:51.707948Z","shell.execute_reply":"2022-03-09T06:59:51.715994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DCT = False\nresetLR = False\nTransferLearning = False\nUpdateLayerInFitter = False\nUpdateLayer2InFitter = False \nUpdateLayerInModel = False\nUpdateLayer2InModel = True\nLoad_model = True\nLoadedFileName = 'shaykhon-19/model-shaykhon-v19.bin'\n#LoadedFileName = '12afteralex/modelafteralex12.bin'\nn_epochs = 1\nSavedFile = 'model-shaykhon-v20-new.bin'\nQualityNum = 0 # 0 for all qualities, 90,95,75 for certain quality","metadata":{"execution":{"iopub.status.busy":"2022-03-09T06:59:51.726117Z","iopub.execute_input":"2022-03-09T06:59:51.726717Z","iopub.status.idle":"2022-03-09T06:59:51.736997Z","shell.execute_reply.started":"2022-03-09T06:59:51.726675Z","shell.execute_reply":"2022-03-09T06:59:51.735971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q efficientnet_pytorch > /dev/null","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-09T06:59:51.748583Z","iopub.execute_input":"2022-03-09T06:59:51.749196Z","iopub.status.idle":"2022-03-09T07:00:00.776039Z","shell.execute_reply.started":"2022-03-09T06:59:51.749157Z","shell.execute_reply":"2022-03-09T07:00:00.775213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nimport sklearn\nimport pdb\n\n\nSEED = 2\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:00.778296Z","iopub.execute_input":"2022-03-09T07:00:00.778673Z","iopub.status.idle":"2022-03-09T07:00:03.788008Z","shell.execute_reply.started":"2022-03-09T07:00:00.778617Z","shell.execute_reply":"2022-03-09T07:00:03.787235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GroupKFold splitting\n\nThanks to [Remi Cogranne](https://www.kaggle.com/remicogranne/jpeg-explanations?scriptVersionId=33893706) for his explanation of JPEG and this part of the quantization table corresponding to each quality factor","metadata":{}},{"cell_type":"code","source":"df_fn =  pd.read_csv('../input/quality-ds/quality_ds.csv') \nif QualityNum != 0:\n    df_fn = df_fn[df_fn['Quality']==QualityNum]\ndf_fn","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:03.789435Z","iopub.execute_input":"2022-03-09T07:00:03.789884Z","iopub.status.idle":"2022-03-09T07:00:03.884559Z","shell.execute_reply.started":"2022-03-09T07:00:03.789843Z","shell.execute_reply":"2022-03-09T07:00:03.883665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n%%time\n#count =0\ndataset = []\n\nfor label, kind in enumerate(['Cover', 'JMiPOD', 'JUNIWARD', 'UERD']):\n    if label >1 :\n        l = 1\n    else:\n        l = label\n    for i,row in df_fn.iterrows():\n      #  l = 0\n      #  if row['Quality'] == 95:\n      #      if label >= 1:\n      #          l = 2\n      #  if (row['Quality'] == 90) | (row['Quality'] == 75):\n      #      if label >= 1:\n      #          l = 1\n        dataset.append({\n            'kind': kind,\n            'image_name': row['ImageName'], #path.split('/')[-1],\n            'label': l\n        })\n     \n\nrandom.shuffle(dataset)\ndataset = pd.DataFrame(dataset)\n\ngkf = GroupKFold(n_splits=5)\n\ndataset.loc[:, 'fold'] = 0\nfor fold_number, (train_index, val_index) in enumerate(gkf.split(X=dataset.index, y=dataset['label'], groups=dataset['image_name'])):    \n    dataset.loc[dataset.iloc[val_index].index, 'fold'] = fold_number ","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:03.885917Z","iopub.execute_input":"2022-03-09T07:00:03.886271Z","iopub.status.idle":"2022-03-09T07:00:31.162904Z","shell.execute_reply.started":"2022-03-09T07:00:03.886233Z","shell.execute_reply":"2022-03-09T07:00:31.161969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Simple Augs: Flips","metadata":{}},{"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose([#transforms.Compose([\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.5),\n            A.Resize(height=512, width=512, p=1.0),\n            #transforms.ToTensor(),\n            ToTensorV2(p=1.0),\n    ], p=1.0)\n\ndef get_valid_transforms():\n    return A.Compose([\n            A.Resize(height=512, width=512, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.0)","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:31.166397Z","iopub.execute_input":"2022-03-09T07:00:31.166692Z","iopub.status.idle":"2022-03-09T07:00:31.173864Z","shell.execute_reply.started":"2022-03-09T07:00:31.166658Z","shell.execute_reply":"2022-03-09T07:00:31.172412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"if DCT:\n    ! git clone https://github.com/dwgoon/jpegio\n    !pip install jpegio/.\n    import jpegio as jio\n","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:31.176445Z","iopub.execute_input":"2022-03-09T07:00:31.177012Z","iopub.status.idle":"2022-03-09T07:00:31.188004Z","shell.execute_reply.started":"2022-03-09T07:00:31.176967Z","shell.execute_reply":"2022-03-09T07:00:31.187182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_ROOT_PATH = '../input/alaska2-image-steganalysis'\n\n# Define 2D DCT\ndef dct2(a):\n    # Return the Discrete Cosine Transform of arbitrary type sequence x.\n    return fftpack.dct(fftpack.dct( a, axis=0, norm='ortho' ), axis=1, norm='ortho')\n\ndef onehot(size, target):\n    vec = torch.zeros(size, dtype=torch.float32)\n    vec[target] = 1.\n    return vec\n\nclass DatasetRetriever(Dataset):\n\n    def __init__(self, kinds, image_names, labels, transforms=None):\n        super().__init__()\n        self.kinds = kinds\n        self.image_names = image_names\n        self.labels = labels\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n        kind, image_name, label = self.kinds[index], self.image_names[index], self.labels[index]\n\n        image = cv2.imread(f'{DATA_ROOT_PATH}/{kind}/{image_name}', cv2.IMREAD_COLOR)   \n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n   \n            #DCT\n            ##image = np.zeros([512,512,3])\n            ##image_jpeg = jio.read(f'{DATA_ROOT_PATH}/{kind}/{image_name}')\n            #image = JPEGdecompressYCbCr(image_jpeg)\n            ##image[:,:,0] = image_jpeg.coef_arrays[0]  \n            ##image[:,:,1] = image_jpeg.coef_arrays[1]  \n            ##image[:,:,2] = image_jpeg.coef_arrays[2]\n            ##image /= 255.0\n            ###image = mpimg.imread(f'{DATA_ROOT_PATH}/{kind}/{image_name}')\n            ###imsize = (512, 512, 3)\n            ###dct = np.zeros([512,512,3])\n            ###for i in r_[:imsize[0]:8]:\n            ###    for j in r_[:imsize[1]:8]:\n            ###        dct[i:(i+8),j:(j+8)] = dct2( image[i:(i+8),j:(j+8)] )\n            \n            ###thresh = 0.002\n            ###image = dct * (abs(dct) > (thresh*np.max(dct)))\n        \n        if self.transforms:\n            sample = {'image': image}\n            sample = self.transforms(**sample)\n            image = sample['image']\n           \n        target = onehot(2, label)\n        return image, target\n\n    def __len__(self) -> int:\n        return self.image_names.shape[0]\n\n    def get_labels(self):\n        return list(self.labels)","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:31.191337Z","iopub.execute_input":"2022-03-09T07:00:31.191661Z","iopub.status.idle":"2022-03-09T07:00:31.206378Z","shell.execute_reply.started":"2022-03-09T07:00:31.191591Z","shell.execute_reply":"2022-03-09T07:00:31.205417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold_number = 2 #2 #0\n\ntrain_dataset = DatasetRetriever(\n    kinds=dataset[dataset['fold'] != fold_number].kind.values,\n    image_names=dataset[dataset['fold'] != fold_number].image_name.values,\n    labels=dataset[dataset['fold'] != fold_number].label.values,\n    transforms=get_train_transforms(),\n)\n\nvalidation_dataset = DatasetRetriever(\n    kinds=dataset[dataset['fold'] == fold_number].kind.values,\n    image_names=dataset[dataset['fold'] == fold_number].image_name.values,\n    labels=dataset[dataset['fold'] == fold_number].label.values,\n    transforms=get_valid_transforms(),\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:31.208044Z","iopub.execute_input":"2022-03-09T07:00:31.208746Z","iopub.status.idle":"2022-03-09T07:00:31.255997Z","shell.execute_reply.started":"2022-03-09T07:00:31.208705Z","shell.execute_reply":"2022-03-09T07:00:31.255115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#image, target = train_dataset[79000]\nimage, target = train_dataset[0]\nnumpy_image = image.permute(1,2,0).cpu().numpy()\n\nfig, ax = plt.subplots(1, 1, figsize=(16, 8))\n    \nax.set_axis_off()\nax.imshow(numpy_image);","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:31.258456Z","iopub.execute_input":"2022-03-09T07:00:31.258873Z","iopub.status.idle":"2022-03-09T07:00:31.540912Z","shell.execute_reply.started":"2022-03-09T07:00:31.258833Z","shell.execute_reply":"2022-03-09T07:00:31.540022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Metrics","metadata":{}},{"cell_type":"code","source":"from sklearn import metrics\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n        \n        \ndef alaska_weighted_auc(y_true, y_valid):\n    \"\"\"\n    https://www.kaggle.com/anokas/weighted-auc-metric-updated\n    \"\"\"\n    tpr_thresholds = [0.0, 0.4, 1.0]\n    weights = [2, 1]\n\n    fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)\n\n    # size of subsets\n    areas = np.array(tpr_thresholds[1:]) - np.array(tpr_thresholds[:-1])\n\n    # The total area is normalized by the sum of weights such that the final weighted AUC is between 0 and 1.\n    normalization = np.dot(areas, weights)\n\n    competition_metric = 0\n    for idx, weight in enumerate(weights):\n        y_min = tpr_thresholds[idx]\n        y_max = tpr_thresholds[idx + 1]\n        mask = (y_min < tpr) & (tpr < y_max)\n        if len(fpr[mask])==0:\n            pdb.set_trace()\n\n        x_padding = np.linspace(fpr[mask][-1], 1, 100)\n\n        x = np.concatenate([fpr[mask], x_padding])\n        y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])\n        y = y - y_min  # normalize such that curve starts at y=0\n        score = metrics.auc(x, y)\n        submetric = score * weight\n        best_subscore = (y_max - y_min) * weight\n        competition_metric += submetric\n\n    return competition_metric / normalization\n        \nclass RocAucMeter(object):\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.y_true = np.array([0,1])\n        self.y_pred = np.array([0.5,0.5])\n        self.score = 0\n\n    def update(self, y_true, y_pred):\n        y_true = y_true.cpu().numpy().argmax(axis=1).clip(min=0, max=1).astype(int)\n        y_pred = 1 - nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:,0]\n        self.y_true = np.hstack((self.y_true, y_true))\n        self.y_pred = np.hstack((self.y_pred, y_pred))\n        self.score = alaska_weighted_auc(self.y_true, self.y_pred)\n    \n    @property\n    def avg(self):\n        return self.score","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:31.541994Z","iopub.execute_input":"2022-03-09T07:00:31.54231Z","iopub.status.idle":"2022-03-09T07:00:31.568296Z","shell.execute_reply.started":"2022-03-09T07:00:31.542278Z","shell.execute_reply":"2022-03-09T07:00:31.567485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Label Smoothing","metadata":{}},{"cell_type":"markdown","source":"# Fitter","metadata":{}},{"cell_type":"code","source":"class LabelSmoothing(nn.Module):\n    def __init__(self, smoothing = 0.01): #0.0158#0.01):\n        super(LabelSmoothing, self).__init__()\n        self.confidence = 1.0 - smoothing\n        self.smoothing = smoothing\n\n    def forward(self, x, target):\n        \n        if self.training:\n            x = x.float()\n            target = target.float()\n            logprobs = torch.nn.functional.log_softmax(x, dim = -1)\n\n            nll_loss = -logprobs * target\n            nll_loss = nll_loss.sum(-1)\n    \n            smooth_loss = -logprobs.mean(dim=-1)\n\n            loss = self.confidence * nll_loss + self.smoothing * smooth_loss\n\n            return loss.mean()\n        else:\n            return torch.nn.functional.cross_entropy(x, target)","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:31.569739Z","iopub.execute_input":"2022-03-09T07:00:31.570477Z","iopub.status.idle":"2022-03-09T07:00:31.581809Z","shell.execute_reply.started":"2022-03-09T07:00:31.570392Z","shell.execute_reply":"2022-03-09T07:00:31.580911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\n\nwarnings.filterwarnings(\"ignore\")\n\nclass Fitter:\n    \n    def __init__(self, model, device, config):\n        self.config = config\n        self.epoch = 0\n        \n        self.base_dir = './'\n        self.log_path = f'{self.base_dir}/log.txt'\n        self.best_summary_loss = 10**5\n\n        self.model = model\n        self.device = device\n\n        param_optimizer = list(self.model.named_parameters())\n        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n        optimizer_grouped_parameters = [\n            {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n            {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n        ] \n\n        self.optimizer = torch.optim.AdamW(self.model.parameters(), lr=config.lr)\n        self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)\n        self.criterion = LabelSmoothing().to(self.device)\n        self.log(f'Fitter prepared. Device is {self.device}')\n\n    def fit(self, train_loader, validation_loader):\n        for e in range(n_epochs):\n            if self.config.verbose:\n                lr = self.optimizer.param_groups[0]['lr']\n                timestamp = datetime.utcnow().isoformat()\n                self.log(f'\\n{timestamp}\\nLR: {lr}')\n\n            t = time.time()\n            summary_loss, final_scores = self.train_one_epoch(train_loader)\n\n            self.log(f'[RESULT]: Train. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')\n            self.save(f'{self.base_dir}/last-checkpoint.bin')\n\n            t = time.time()\n            summary_loss, final_scores = self.validation(validation_loader)\n\n            self.log(f'[RESULT]: Val. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')\n            if summary_loss.avg < self.best_summary_loss:\n                self.best_summary_loss = summary_loss.avg\n                self.model.eval()\n                self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch).zfill(3)}epoch.bin')\n                for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin'))[:-3]:\n                    os.remove(path)\n\n            if self.config.validation_scheduler:\n                self.scheduler.step(metrics=summary_loss.avg)\n\n            self.epoch += 1\n\n    def validation(self, val_loader):\n        self.model.eval()\n        summary_loss = AverageMeter()\n        final_scores = RocAucMeter()\n        t = time.time()\n        for step, (images, targets) in enumerate(val_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Val Step {step}/{len(val_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            with torch.no_grad():\n                targets = targets.to(self.device).float()\n                batch_size = images.shape[0]\n                images = images.to(self.device).float()\n                outputs = self.model(images)\n                loss = self.criterion(outputs, targets)\n                final_scores.update(targets, outputs)\n                summary_loss.update(loss.detach().item(), batch_size)\n\n        return summary_loss, final_scores\n\n    def train_one_epoch(self, train_loader):\n        self.model.train()\n        summary_loss = AverageMeter()\n        final_scores = RocAucMeter()\n        t = time.time()\n        for step, (images, targets) in enumerate(train_loader):\n            if self.config.verbose:\n                if step % self.config.verbose_step == 0:\n                    print(\n                        f'Train Step {step}/{len(train_loader)}, ' + \\\n                        f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \\\n                        f'time: {(time.time() - t):.5f}', end='\\r'\n                    )\n            \n            targets = targets.to(self.device).float()\n            images = images.to(self.device).float()\n            batch_size = images.shape[0]\n\n            self.optimizer.zero_grad()\n            outputs = self.model(images)\n            loss = self.criterion(outputs, targets)\n            loss.backward()\n            \n            final_scores.update(targets, outputs)\n            summary_loss.update(loss.detach().item(), batch_size)\n\n            self.optimizer.step()\n\n            if self.config.step_scheduler:\n                self.scheduler.step()\n\n        return summary_loss, final_scores\n    \n    def save(self, path):\n        self.model.eval()\n        torch.save({\n            'model_state_dict': self.model.state_dict(),\n            'optimizer_state_dict': self.optimizer.state_dict(),\n            'scheduler_state_dict': self.scheduler.state_dict(),\n            'best_summary_loss': self.best_summary_loss,\n            'epoch': self.epoch,\n        }, path)\n\n    def load(self, path):\n        checkpoint = torch.load(path)\n        #pdb.set_trace()\n        self.model.load_state_dict(checkpoint['model_state_dict'])\n        self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n        self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])\n        self.best_summary_loss = checkpoint['best_summary_loss']\n        self.epoch = checkpoint['epoch'] + 1\n        \n    def log(self, message):\n        if self.config.verbose:\n            print(message)\n        with open(self.log_path, 'a+') as logger:\n            logger.write(f'{message}\\n')","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:31.58339Z","iopub.execute_input":"2022-03-09T07:00:31.583976Z","iopub.status.idle":"2022-03-09T07:00:31.621104Z","shell.execute_reply.started":"2022-03-09T07:00:31.583936Z","shell.execute_reply":"2022-03-09T07:00:31.620362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EfficientNet","metadata":{}},{"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\n\ndef get_net():\n    net = EfficientNet.from_pretrained('efficientnet-b2')\n    if TransferLearning:\n        for param in net.parameters():\n            param.requires_grad = False\n    if UpdateLayerInModel:\n        net._fc = nn.Sequential(nn.Linear(1408, 256),\n                                 nn.ReLU(),\n                                 nn.Dropout(0.2),\n                                 nn.Linear(256, 2),\n                                 nn.LogSoftmax(dim=1)).to('cuda')\n    elif UpdateLayer2InModel:\n        net._fc = nn.Sequential(nn.Linear(1408, 1024),\n                                 nn.ReLU(),\n                                 nn.Dropout(0.2 , inplace = True),\n                                 nn.Linear(1024,512),\n                                 nn.ReLU(),\n                                 nn.Dropout(0.2 , inplace = True),\n                                 nn.Linear(512,256),\n                                 nn.ReLU(),\n                                 nn.Dropout(0.2 , inplace = True),\n                                 nn.Linear(256,2),\n                                 nn.LogSoftmax(dim=1)).to('cuda')        \n    else:\n        net._fc = nn.Linear(in_features=1408, out_features=4, bias=True)\n     \n    return net\n\nnet = get_net().cuda()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-03-09T07:00:31.622535Z","iopub.execute_input":"2022-03-09T07:00:31.623093Z","iopub.status.idle":"2022-03-09T07:00:38.385467Z","shell.execute_reply.started":"2022-03-09T07:00:31.623055Z","shell.execute_reply":"2022-03-09T07:00:38.383758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:38.391306Z","iopub.execute_input":"2022-03-09T07:00:38.392058Z","iopub.status.idle":"2022-03-09T07:00:38.404847Z","shell.execute_reply.started":"2022-03-09T07:00:38.392016Z","shell.execute_reply":"2022-03-09T07:00:38.404017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class TrainGlobalConfig:\n    num_workers = 4\n    batch_size = 16 \n    #n_epochs = 1 #5#25\n    lr = 0.0001\n    #0.001#0.0009#0.002#0.0002#0.001\n\n    # -------------------\n    verbose = True\n    verbose_step = 1\n    # -------------------\n\n    # --------------------\n    step_scheduler = False  # do scheduler.step after optimizer.step\n    validation_scheduler = True  # do scheduler.step after validation stage loss\n\n\n    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n    scheduler_params = dict(\n        mode='min',\n        factor=0.5,#0.1,#0.5\n        patience=1,#5,#1\n        verbose=False, \n        threshold=0.0001, #0.0001\n        threshold_mode='abs',\n        cooldown=0, \n        min_lr=1e-8,#0.0001#1e-4,#1e-8\n        eps=1e-12,#1e-04#1e-8\n    )\n    # --------------------","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:38.406771Z","iopub.execute_input":"2022-03-09T07:00:38.407225Z","iopub.status.idle":"2022-03-09T07:00:38.415932Z","shell.execute_reply.started":"2022-03-09T07:00:38.40716Z","shell.execute_reply":"2022-03-09T07:00:38.415109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class Balance \"on fly\" from [@CatalystTeam](https://github.com/catalyst-team/catalyst)","metadata":{}},{"cell_type":"code","source":"from catalyst.data.sampler import BalanceClassSampler\n\ndef run_training():\n    device = torch.device('cuda')\n\n    train_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        sampler=BalanceClassSampler(labels=train_dataset.get_labels(), mode=\"downsampling\"),\n        batch_size=TrainGlobalConfig.batch_size,\n        pin_memory=False,\n        drop_last=True,\n        num_workers=TrainGlobalConfig.num_workers,\n    )\n    val_loader = torch.utils.data.DataLoader(\n        validation_dataset, \n        batch_size=TrainGlobalConfig.batch_size,\n        num_workers=TrainGlobalConfig.num_workers,\n        shuffle=False,\n        sampler=SequentialSampler(validation_dataset),\n        pin_memory=False,\n    )\n\n    fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)\n    if Load_model:\n        fitter.load('../input/' + LoadedFileName)\n    #fitter.load('../input/new-layer-1/modelafteralex14.bin')\n    if UpdateLayerInFitter:\n        fitter.model._fc = nn.Sequential(nn.Linear(1408, 256),\n                                 nn.ReLU(),\n                                 nn.Dropout(0.2),\n                                 nn.Linear(256, 2),\n                                 nn.LogSoftmax(dim=1)).to('cuda')\n        fitter.optimizer = torch.optim.AdamW(fitter.model.parameters(), lr=TrainGlobalConfig.lr)\n    elif UpdateLayer2InFitter:\n        fitter.model._fc = nn.Sequential(nn.Linear(1408, 1024),\n                                 nn.ReLU(),\n                                 nn.Dropout(0.2),\n                                 nn.Linear(1024,512),\n                                 nn.ReLU(),\n                                 nn.Dropout(0.2),\n                                 nn.Linear(512,256),\n                                 nn.ReLU(),\n                                 nn.Dropout(0.2),\n                                 nn.Linear(256,2),\n                                 nn.LogSoftmax(dim=1)).to('cuda')        \n        fitter.optimizer = torch.optim.AdamW(fitter.model.parameters(), lr=TrainGlobalConfig.lr)\n    if resetLR:\n        fitter.optimizer.param_groups[0]['lr']=0.001\n    fitter.fit(train_loader, val_loader)\n    fitter.save(SavedFile)","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:38.417342Z","iopub.execute_input":"2022-03-09T07:00:38.418001Z","iopub.status.idle":"2022-03-09T07:00:43.130683Z","shell.execute_reply.started":"2022-03-09T07:00:38.417961Z","shell.execute_reply":"2022-03-09T07:00:43.129852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training\n\nI have used 1xV100 for training model, in kaggle kernel it works also. You can make fork and check it, but I would like to share with you my logs","metadata":{}},{"cell_type":"code","source":"run_training()","metadata":{"execution":{"iopub.status.busy":"2022-03-09T07:00:43.132146Z","iopub.execute_input":"2022-03-09T07:00:43.132472Z","iopub.status.idle":"2022-03-09T08:23:50.649064Z","shell.execute_reply.started":"2022-03-09T07:00:43.132437Z","shell.execute_reply":"2022-03-09T08:23:50.648053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"branchName = SavedFile\n\n!git config --global user.name \"RTCTeam\"\n!git config --global user.email \"tareksherif.courses@gmail.com\"\n\n!git checkout -b origin\n!git init\n!git checkout master\n!git add .\n!git commit -m  {branchName}\n!git remote add origin https://RTCTeam:RTC_Team_2020@github.com/RTCTeam/ALASKA2_shaykhon.git\n!git push -u  --force origin master \n \n# delete branch if exist \n!git branch -d {branchName}  --force\n!git push origin --delete {branchName}  --force\n\n# create branch\n!git checkout -b  {branchName}\n!git commit -m  rm{branchName}\n!git push origin {branchName}","metadata":{"execution":{"iopub.status.busy":"2022-03-09T08:23:50.651086Z","iopub.execute_input":"2022-03-09T08:23:50.65147Z","iopub.status.idle":"2022-03-09T08:24:15.606414Z","shell.execute_reply.started":"2022-03-09T08:23:50.651427Z","shell.execute_reply":"2022-03-09T08:24:15.605508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"#checkpoint = torch.load('../input/afteralex1epoch/last-checkpoint.bin')\n#net.load_state_dict(checkpoint['model_state_dict']);\nnet.eval();","metadata":{"execution":{"iopub.status.busy":"2022-03-09T08:24:15.608856Z","iopub.execute_input":"2022-03-09T08:24:15.609243Z","iopub.status.idle":"2022-03-09T08:24:15.616112Z","shell.execute_reply.started":"2022-03-09T08:24:15.6092Z","shell.execute_reply":"2022-03-09T08:24:15.615047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DatasetSubmissionRetriever(Dataset):\n\n    def __init__(self, image_names, transforms=None):\n        super().__init__()\n        self.image_names = image_names\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n        image_name = self.image_names[index]\n        #jpeg = jio.read(f'{DATA_ROOT_PATH}/Test/{image_name}')\n        #image = JPEGdecompressYCbCr(jpeg)\n        image = cv2.imread(f'{DATA_ROOT_PATH}/Test/{image_name}', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        \n        if self.transforms:\n            sample = {'image': image}\n            sample = self.transforms(**sample)\n            image = sample['image']\n\n        return image_name, image\n\n    def __len__(self) -> int:\n        return self.image_names.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-03-09T08:24:15.617908Z","iopub.execute_input":"2022-03-09T08:24:15.618583Z","iopub.status.idle":"2022-03-09T08:24:15.629752Z","shell.execute_reply.started":"2022-03-09T08:24:15.618544Z","shell.execute_reply":"2022-03-09T08:24:15.628905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = DatasetSubmissionRetriever(\n    image_names=np.array([path.split('/')[-1] for path in glob('../input/alaska2-image-steganalysis/Test/*.jpg')]),\n    transforms=get_valid_transforms(),\n)\n\n\ndata_loader = DataLoader(\n    dataset,\n    batch_size=8,\n    shuffle=False,\n    num_workers=2,\n    drop_last=False,\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-09T08:24:15.631543Z","iopub.execute_input":"2022-03-09T08:24:15.631976Z","iopub.status.idle":"2022-03-09T08:24:16.246388Z","shell.execute_reply.started":"2022-03-09T08:24:15.631903Z","shell.execute_reply":"2022-03-09T08:24:16.245652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nresult = {'Id': [], 'Label': []}\nfor step, (image_names, images) in enumerate(data_loader):\n    print(step, end='\\r')\n    \n    y_pred = net(images.cuda())\n    #cover95 = nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:,0]\n    #cover90 = nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:,1]\n    #cover75 = nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:,2]\n    y_pred = 1 - nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:,0]\n    #1- (cover95 + cover90 + cover75)\n    \n    result['Id'].extend(image_names)\n    result['Label'].extend(y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-03-09T08:24:16.24983Z","iopub.execute_input":"2022-03-09T08:24:16.250096Z","iopub.status.idle":"2022-03-09T08:25:26.961637Z","shell.execute_reply.started":"2022-03-09T08:24:16.250069Z","shell.execute_reply":"2022-03-09T08:25:26.959807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(result)\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-09T08:25:26.964045Z","iopub.execute_input":"2022-03-09T08:25:26.96446Z","iopub.status.idle":"2022-03-09T08:25:27.213531Z","shell.execute_reply.started":"2022-03-09T08:25:26.964412Z","shell.execute_reply":"2022-03-09T08:25:27.212493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#FileLink(r'submission95_2.csv')","metadata":{"execution":{"iopub.status.busy":"2022-03-09T08:25:27.215002Z","iopub.execute_input":"2022-03-09T08:25:27.215406Z","iopub.status.idle":"2022-03-09T08:25:27.219745Z","shell.execute_reply.started":"2022-03-09T08:25:27.215363Z","shell.execute_reply":"2022-03-09T08:25:27.218783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}