{"cells":[{"metadata":{},"cell_type":"markdown","source":"This kernel is prepared by the help of tutorial of Abhishek Thakur with some modification.\nUsing this kernel you can train EfficientNet+ gridMask model and save it weights, I have commented out some code as i dont have sufficient GPU time left to train the model."},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install git+https://github.com/qubvel/segmentation_models\n!pip install iterative-stratification\n!pip install pretrainedmodels\n!pip install --upgrade efficientnet-pytorch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys\nimport pretrainedmodels\n\nimport glob\nimport torch\nimport albumentations\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport numpy as np # linear algebra\nfrom tqdm import tqdm\nfrom PIL import Image\nimport joblib\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nfrom efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DIR_INPUT='../input/bengaliai-cv19/'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Follwing code can be used for creating train data with respective folds. "},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# df = pd.read_csv(DIR_INPUT+\"train.csv\")\n# print(df.head())\n# df.loc[:,'kfold']=-1\n# df = df.sample(frac=1).reset_index(drop=True)\n# X = df.image_id.values\n# y = df[['grapheme_root','vowel_diacritic','consonant_diacritic']].values\n# mskf = MultilabelStratifiedKFold(n_splits=5)\n# for fold, (trn_,val_) in enumerate(mskf.split(X,y)):\n#   print(\"TRAIN: \",trn_,\"VAL: \",val_)\n#   df.loc[val_,\"kfold\"] = fold\n# print(df.kfold.value_counts())\n# df.to_csv(DIR_INPUT+\"/train_folds.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Follwing code can be used for creating Image pickles. "},{"metadata":{"trusted":true},"cell_type":"code","source":"# files = glob.glob(\"/content/train_*.parquet\")\n# for f in files:\n#   df = pd.read_parquet(f)\n#   image_ids = df.image_id.values\n#   df = df.drop(\"image_id\", axis=1)\n#   image_array = df.values\n#   for j, image_id in tqdm(enumerate(image_ids), total=len(image_ids)):\n#       joblib.dump(image_array[j, :], f\"/content/image_pickles/{image_id}.pkl\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Grid mask code"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\n\nimport torch\n\nfrom PIL import Image\nfrom albumentations.core.transforms_interface import ImageOnlyTransform\nfrom albumentations.augmentations import functional as F1\nfrom PIL import Image, ImageOps, ImageEnhance\n\nfrom albumentations.core.transforms_interface import DualTransform\n\nclass GridMask(DualTransform):\n    \"\"\"GridMask augmentation for image classification and object detection.\n\n    Args:\n        num_grid (int): number of grid in a row or column.\n        fill_value (int, float, lisf of int, list of float): value for dropped pixels.\n        rotate ((int, int) or int): range from which a random angle is picked. If rotate is a single int\n            an angle is picked from (-rotate, rotate). Default: (-90, 90)\n        mode (int):\n            0 - cropout a quarter of the square of each grid (left top)\n            1 - reserve a quarter of the square of each grid (left top)\n            2 - cropout 2 quarter of the square of each grid (left top & right bottom)\n\n    Targets:\n        image, mask\n\n    Image types:\n        uint8, float32\n\n    Reference:\n    |  https://arxiv.org/abs/2001.04086\n    |  https://github.com/akuxcw/GridMask\n    \"\"\"\n\n    def __init__(self, num_grid=3, fill_value=0, rotate=0, mode=0, always_apply=False, p=0.5):\n        super(GridMask, self).__init__(always_apply, p)\n        if isinstance(num_grid, int):\n            num_grid = (num_grid, num_grid)\n        if isinstance(rotate, int):\n            rotate = (-rotate, rotate)\n        self.num_grid = num_grid\n        self.fill_value = fill_value\n        self.rotate = rotate\n        self.mode = mode\n        self.masks = None\n        self.rand_h_max = []\n        self.rand_w_max = []\n\n    def init_masks(self, height, width):\n        if self.masks is None:\n            self.masks = []\n            n_masks = self.num_grid[1] - self.num_grid[0] + 1\n            for n, n_g in enumerate(range(self.num_grid[0], self.num_grid[1] + 1, 1)):\n                grid_h = height / n_g\n                grid_w = width / n_g\n                this_mask = np.ones((int((n_g + 1) * grid_h), int((n_g + 1) * grid_w))).astype(np.uint8)\n                for i in range(n_g + 1):\n                    for j in range(n_g + 1):\n                        this_mask[\n                             int(i * grid_h) : int(i * grid_h + grid_h / 2),\n                             int(j * grid_w) : int(j * grid_w + grid_w / 2)\n                        ] = self.fill_value\n                        if self.mode == 2:\n                            this_mask[\n                                 int(i * grid_h + grid_h / 2) : int(i * grid_h + grid_h),\n                                 int(j * grid_w + grid_w / 2) : int(j * grid_w + grid_w)\n                            ] = self.fill_value\n                \n                if self.mode == 1:\n                    this_mask = 1 - this_mask\n\n                self.masks.append(this_mask)\n                self.rand_h_max.append(grid_h)\n                self.rand_w_max.append(grid_w)\n\n    def apply(self, image, mask, rand_h, rand_w, angle, **params):\n        h, w = image.shape[:2]\n        mask = F1.rotate(mask, angle) if self.rotate[1] > 0 else mask\n        mask = mask[:,:,np.newaxis] if image.ndim == 3 else mask\n        image *= mask[rand_h:rand_h+h, rand_w:rand_w+w].astype(image.dtype)\n        return image\n\n    def get_params_dependent_on_targets(self, params):\n        img = params['image']\n        height, width = img.shape[:2]\n        self.init_masks(height, width)\n\n        mid = np.random.randint(len(self.masks))\n        mask = self.masks[mid]\n        rand_h = np.random.randint(self.rand_h_max[mid])\n        rand_w = np.random.randint(self.rand_w_max[mid])\n        angle = np.random.randint(self.rotate[0], self.rotate[1]) if self.rotate[1] > 0 else 0\n\n        return {'mask': mask, 'rand_h': rand_h, 'rand_w': rand_w, 'angle': angle}\n\n    @property\n    def targets_as_params(self):\n        return ['image']\n\n    def get_transform_init_args_names(self):\n        return ('num_grid', 'fill_value', 'rotate', 'mode')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image\n\nclass BengaliDatasetTrain:\n    def __init__(self, folds, img_height, img_width, mean, std):\n        df = pd.read_csv(\"../input/train-data/train_folds.csv\")\n        df = df[[\"image_id\", \"grapheme_root\", \"vowel_diacritic\", \"consonant_diacritic\", \"kfold\"]]\n\n        df = df[df.kfold.isin(folds)].reset_index(drop=True)\n        \n        self.image_ids = df.image_id.values\n        self.grapheme_root = df.grapheme_root.values\n        self.vowel_diacritic = df.vowel_diacritic.values\n        self.consonant_diacritic = df.consonant_diacritic.values\n\n        if len(folds) == 1:\n            self.aug = albumentations.Compose([\n                albumentations.Resize(img_height, img_width, always_apply=True),\n                albumentations.Normalize(mean, std, always_apply=True)\n            ])\n        else:\n             self.aug = albumentations.Compose([\n                albumentations.Resize(img_height, img_width, always_apply=True),\n                # albumentations.ShiftScaleRotate(shift_limit=0.0625,\n                #                                scale_limit=0.1, \n                #                                rotate_limit=5,\n                #                                p=0.9),\n                albumentations.Normalize(mean, std, always_apply=True),\n                albumentations.OneOf([\n                    GridMask(num_grid=3, mode=0, rotate=15),\n                    GridMask(num_grid=3, mode=2, rotate=15),\n                ], p=0.75)\n            ])\n\n\n\n    def __len__(self):\n        return len(self.image_ids)\n    \n    def __getitem__(self, item):\n        image = joblib.load(f\"/content/image_pickles/{self.image_ids[item]}.pkl\")\n        image = image.reshape(137, 236).astype(float)\n        image = Image.fromarray(image).convert(\"RGB\")\n        image = self.aug(image=np.array(image))[\"image\"]\n        image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n\n        return {\n            \"image\": torch.tensor(image, dtype=torch.float),\n            \"grapheme_root\": torch.tensor(self.grapheme_root[item], dtype=torch.long),\n            \"vowel_diacritic\": torch.tensor(self.vowel_diacritic[item], dtype=torch.long),\n            \"consonant_diacritic\": torch.tensor(self.consonant_diacritic[item], dtype=torch.long)\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EfficientNetB1(nn.Module):\n    def __init__(self, pretrained):\n        super(EfficientNetB1, self).__init__()\n\n        if pretrained is True:\n            self.model = EfficientNet.from_pretrained(\"efficientnet-b1\")\n        else:\n            self.model = EfficientNet.from_name('efficientnet-b1')\n        \n        self.l0 = nn.Linear(1280, 168)\n        self.l1 = nn.Linear(1280, 11)\n        self.l2 = nn.Linear(1280, 7)\n\n    def forward(self, x):\n        bs, _, _, _ = x.shape\n        x = self.model.extract_features(x)\n        x = F.adaptive_avg_pool2d(x, 1).reshape(bs, -1)\n        l0 = self.l0(x)\n        l1 = self.l1(x)\n        l2 = self.l2(x)\n\n        return l0, l1, l2\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nMODEL_DISPATCHER = {\n    \"efficientNetb1\": EfficientNetB1\n}  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport torch\n\nclass EarlyStopping:\n    \"\"\"Early stops the training if validation loss doesn't improve after a given patience.\"\"\"\n    def __init__(self, patience=7, verbose=False, delta=0):\n        \"\"\"\n        Args:\n            patience (int): How long to wait after last time validation loss improved.\n                            Default: 7\n            verbose (bool): If True, prints a message for each validation loss improvement. \n                            Default: False\n            delta (float): Minimum change in the monitored quantity to qualify as an improvement.\n                            Default: 0\n        \"\"\"\n        self.patience = patience\n        self.verbose = verbose\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.val_loss_min = np.Inf\n        self.delta = delta\n\n    def __call__(self, val_loss, model):\n\n        score = -val_loss\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            print(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n            self.counter = 0\n\n    def save_checkpoint(self, val_loss, model):\n        '''Saves model when validation loss decrease.'''\n        if self.verbose:\n            print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n        torch.save(model.state_dict(), 'checkpoint.pt')\n        self.val_loss_min = val_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport os\nimport ast\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport sklearn.metrics\nfrom tqdm import tqdm\n\nDEVICE = \"cuda\"\nIMG_HEIGHT=224\nIMG_WIDTH=224\nEPOCHS=20\nTRAIN_BATCH_SIZE=64\nTEST_BATCH_SIZE=64\nMODEL_MEAN=[0.485, 0.456, 0.406]\nMODEL_STD=[0.229, 0.224, 0.225]\nBASE_MODEL=\"efficientNetb1\"\nTRAINING_FOLDS_CSV=\"../input/train-data/train_folds.csv\"\n\nTRAINING_FOLDS=[0,1,2,3]\nVALIDATION_FOLDS=[4]\n\n\n\ndef macro_recall(pred_y, y, n_grapheme=168, n_vowel=11, n_consonant=7):\n    \n    pred_y = torch.split(pred_y, [n_grapheme, n_vowel, n_consonant], dim=1)\n    pred_labels = [torch.argmax(py, dim=1).cpu().numpy() for py in pred_y]\n\n    y = y.cpu().numpy()\n\n    recall_grapheme = sklearn.metrics.recall_score(pred_labels[0], y[:, 0], average='macro')\n    recall_vowel = sklearn.metrics.recall_score(pred_labels[1], y[:, 1], average='macro')\n    recall_consonant = sklearn.metrics.recall_score(pred_labels[2], y[:, 2], average='macro')\n    scores = [recall_grapheme, recall_vowel, recall_consonant]\n    final_score = np.average(scores, weights=[2, 1, 1])\n    print(f'recall: grapheme {recall_grapheme}, vowel {recall_vowel}, consonant {recall_consonant}, 'f'total {final_score}, y {y.shape}')\n    \n    return final_score\n\n\ndef loss_fn(outputs, targets):\n    o1, o2, o3 = outputs\n    t1, t2, t3 = targets\n    l1 = nn.CrossEntropyLoss()(o1, t1)\n    l2 = nn.CrossEntropyLoss()(o2, t2)\n    l3 = nn.CrossEntropyLoss()(o3, t3)\n    return (l1 + l2 + l3) / 3\n\n\n\ndef train(dataset, data_loader, model, optimizer):\n    model.train()\n    final_loss = 0\n    counter = 0\n    final_outputs = []\n    final_targets = []\n\n    for bi, d in tqdm(enumerate(data_loader), total=int(len(dataset)/data_loader.batch_size)):\n        counter = counter + 1\n        image = d[\"image\"]\n        grapheme_root = d[\"grapheme_root\"]\n        vowel_diacritic = d[\"vowel_diacritic\"]\n        consonant_diacritic = d[\"consonant_diacritic\"]\n\n        image = image.to(DEVICE, dtype=torch.float)\n        grapheme_root = grapheme_root.to(DEVICE, dtype=torch.long)\n        vowel_diacritic = vowel_diacritic.to(DEVICE, dtype=torch.long)\n        consonant_diacritic = consonant_diacritic.to(DEVICE, dtype=torch.long)\n        \n        print(image.shape)\n\n        optimizer.zero_grad()\n        outputs = model(image)\n        targets = (grapheme_root, vowel_diacritic, consonant_diacritic)\n        loss = loss_fn(outputs, targets)\n\n        loss.backward()\n        optimizer.step()\n\n        final_loss += loss\n\n        o1, o2, o3 = outputs\n        t1, t2, t3 = targets\n        final_outputs.append(torch.cat((o1,o2,o3), dim=1))\n        final_targets.append(torch.stack((t1,t2,t3), dim=1))\n\n        #if bi % 10 == 0:\n        #    break\n    final_outputs = torch.cat(final_outputs)\n    final_targets = torch.cat(final_targets)\n\n    print(\"=================Train=================\")\n    macro_recall_score = macro_recall(final_outputs, final_targets)\n    \n    return final_loss/counter , macro_recall_score\n\n\n\ndef evaluate(dataset, data_loader, model):\n    with torch.no_grad():\n        model.eval()\n        final_loss = 0\n        counter = 0\n        final_outputs = []\n        final_targets = []\n        for bi, d in tqdm(enumerate(data_loader), total=int(len(dataset)/data_loader.batch_size)):\n            counter = counter + 1\n            image = d[\"image\"]\n            grapheme_root = d[\"grapheme_root\"]\n            vowel_diacritic = d[\"vowel_diacritic\"]\n            consonant_diacritic = d[\"consonant_diacritic\"]\n\n            image = image.to(DEVICE, dtype=torch.float)\n            grapheme_root = grapheme_root.to(DEVICE, dtype=torch.long)\n            vowel_diacritic = vowel_diacritic.to(DEVICE, dtype=torch.long)\n            consonant_diacritic = consonant_diacritic.to(DEVICE, dtype=torch.long)\n\n            outputs = model(image)\n            targets = (grapheme_root, vowel_diacritic, consonant_diacritic)\n            loss = loss_fn(outputs, targets)\n            final_loss += loss\n\n            o1, o2, o3 = outputs\n            t1, t2, t3 = targets\n            #print(t1.shape)\n            final_outputs.append(torch.cat((o1,o2,o3), dim=1))\n            final_targets.append(torch.stack((t1,t2,t3), dim=1))\n        \n        final_outputs = torch.cat(final_outputs)\n        final_targets = torch.cat(final_targets)\n\n        print(\"=================Train=================\")\n        macro_recall_score = macro_recall(final_outputs, final_targets)\n\n    return final_loss/counter , macro_recall_score\n\n\n\ndef main():\n    model = MODEL_DISPATCHER[BASE_MODEL](pretrained=True)\n    model.to(DEVICE)\n\n    train_dataset = BengaliDatasetTrain(\n        folds=TRAINING_FOLDS,\n        img_height = IMG_HEIGHT,\n        img_width = IMG_WIDTH,\n        mean = MODEL_MEAN,\n        std = MODEL_STD\n    )\n\n    train_loader = torch.utils.data.DataLoader(\n        dataset=train_dataset,\n        batch_size= TRAIN_BATCH_SIZE,\n        shuffle=True,\n        num_workers=4\n    )\n\n    valid_dataset = BengaliDatasetTrain(\n        folds=VALIDATION_FOLDS,\n        img_height = IMG_HEIGHT,\n        img_width = IMG_WIDTH,\n        mean = MODEL_MEAN,\n        std = MODEL_STD\n    )\n\n    valid_loader = torch.utils.data.DataLoader(\n        dataset=valid_dataset,\n        batch_size= TEST_BATCH_SIZE,\n        shuffle=True,\n        num_workers=4\n    )\n\n    optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, \n                                                            mode=\"min\", \n                                                            patience=5, \n                                                            factor=0.3,verbose=True)\n\n    early_stopping = EarlyStopping(patience=5, verbose=True)\n\n    #if torch.cuda.device_count() > 1:\n    #    model = nn.DataParallel(model)\n\n    best_score = -1\n\n    print(\"FOLD : \", VALIDATION_FOLDS[0] )\n    \n    for epoch in range(1, EPOCHS+1):\n\n        train_loss, train_score = train(train_dataset,train_loader, model, optimizer)\n        val_loss, val_score = evaluate(valid_dataset, valid_loader, model)\n\n        scheduler.step(val_loss)\n\n        \n\n        if val_score > best_score:\n            best_score = val_score\n            torch.save(model.state_dict(), f\"{BASE_MODEL}_fold{VALIDATION_FOLDS[0]}.pth\")\n\n        epoch_len = len(str(EPOCHS))\n        print_msg = (f'[{epoch:>{epoch_len}}/{EPOCHS:>{epoch_len}}] ' +\n                     f'train_loss: {train_loss:.5f} ' +\n                     f'train_score: {train_score:.5f} ' +\n                     f'valid_loss: {val_loss:.5f} ' +\n                     f'valid_score: {val_score:.5f}'\n                    )\n        \n        print(print_msg)\n\n        early_stopping(val_score, model)\n        if early_stopping.early_stop:\n            print(\"Early stopping\")\n            break\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* you can comment out all below codes to train the model."},{"metadata":{"trusted":true},"cell_type":"code","source":"# TRAINING_FOLDS=[0,1,2,3]\n# VALIDATION_FOLDS=[4]\n# main()\n\n\n# TRAINING_FOLDS=[0,1,2,4]\n# VALIDATION_FOLDS=[3]\n# main()\n\n# TRAINING_FOLDS=[0,1,3,4]\n# VALIDATION_FOLDS=[2]\n# main()\n\n\n# TRAINING_FOLDS=[0,2,3,4]\n# VALIDATION_FOLDS=[1]\n# main()\n\n# TRAINING_FOLDS=[1,2,3,4]\n# VALIDATION_FOLDS=[0]\n# main()\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}