{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pickle\nimport gc\nimport os\nimport cv2 \n\nfrom cv2 import resize\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nfrom pathlib import Path\n\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom typing import Union\nfrom typing import List","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DEFAULT_H, DEFAULT_W = 137, 236\n\nSIZE = 128\n\nLABEL_PATH = Path('train.csv')\n\nDATADIR = Path('/kaggle/input/bengaliai-cv19')\n\nTEST_FORM = 'test_image_data_ID.parquet'\n\nWEIGHTS_FILE = '/kaggle/input/mobilenet-v2-30-epochs/model_2020-03-03T12_16_37.627015.pt'\n\n\ntest = pd.read_csv(DATADIR/'test.csv')\ntrain = pd.read_csv(DATADIR/'train.csv')\ntrain_labels = train[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].values\nsubmission_df = pd.read_csv(DATADIR/'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data loading functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# loading PARQUET format files \ndef load_images(train_test, indices=['0', '1', '2', '3']):\n    \"\"\"\n    Utility function to Load the images from both the location and return them\n    :param train_test:\n    :return:\n    \"\"\"\n\n    path_form = {\n        'test': TEST_FORM\n    }[train_test]\n\n    imgs_list = []\n\n    # sequentially load all four files.\n    for id in indices:\n\n        # Form the path of the files.\n        path = DATADIR / path_form.replace('ID', id)\n        print('Loading', path)\n        df = pd.read_parquet(path)\n        imgs = df.iloc[:, 1:].to_numpy()\n        imgs_list.append(imgs)\n    del imgs\n    gc.collect()\n    imgs_list = np.concatenate(imgs_list)\n    imgs_list = imgs_list.reshape(-1, DEFAULT_H, DEFAULT_W)\n\n    return imgs_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data(train_test, indices=['0', '1', '2', '3']):\n    \"\"\"\n     A combined function to load both trian and label?\n    :return:\n    \"\"\"\n    # Load all images into a variable.\n    imgs = load_images(train_test, indices=indices)\n    \n    if train_test == 'train':\n        labels = load_labels()\n        all_data = list(zip(imgs, labels))\n    else:\n        all_data = imgs\n\n    return all_data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset and Transforms"},{"metadata":{"trusted":true},"cell_type":"code","source":"# use a dictionary as config settings\ndataset_cfg = {'aug_cfg': {\n                    'resize_shape': (128, 128),\n                    'crop': True,\n                    'to_rgb': True,\n                    'normalize_mean': [0.485, 0.456, 0.406],\n                    'normalize_std': [0.229, 0.224, 0.225]\n                          }\n              }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def content_crop(img, pad_to_square: bool, white_background: bool):\n    \"\"\"\n    https://www.kaggle.com/iafoss/image-preprocessing-128x128\n\n    :param img: grapheme image matrix\n    :param pad_to_square:  whether pad to square (preserving aspect ratio)\n    :param white_background: whether the image\n    :return: cropped image matrix\n    \"\"\"\n    # remove the surrounding 5 pixels\n    img = img[5:-5, 5:-5]\n    if white_background:\n        y_list, x_list = np.where(img < 235)\n    else:\n        y_list, x_list = np.where(img > 80)\n\n    # get xy min max\n    xmin, xmax = np.min(x_list), np.max(x_list)\n    ymin, ymax = np.min(y_list), np.max(y_list)\n\n    xmin = xmin - 13 if (xmin > 13) else 0\n    ymin = ymin - 10 if (ymin > 10) else 0\n    xmax = xmax + 13 if (xmax < 223) else 236\n    ymax = ymax + 10 if (ymax < 127) else 137\n    img = img[ymin:ymax, xmin:xmax]\n\n    # remove lo intensity pixels as noise\n    if white_background:\n        img[img > 235] = 255\n    else:\n        img[img < 28] = 0\n\n    if pad_to_square:\n        lx, ly = xmax - xmin, ymax - ymin\n        l = max(lx, ly) + 16\n        # make sure that the aspect ratio is kept in rescaling\n        if white_background:\n            constant_pad = 255\n        else:\n            constant_pad = 0\n        img = np.pad(img, [((l - ly) // 2,), ((l - lx) // 2,)], mode='constant', constant_values=constant_pad)\n\n    return img\n\nclass Preprocessor(object):\n    \n    def __init__(self, dataset_cfg):\n        aug_cfg = dataset_cfg['aug_cfg']\n        self.resize_shape = aug_cfg['resize_shape']\n        self.crop = aug_cfg['crop']\n        self.to_rgb = aug_cfg['to_rgb']\n        self.normalize_mean = aug_cfg['normalize_mean']\n        self.normalize_std = aug_cfg['normalize_std']\n                                     \n    def __call__(self, img, normalize=True):\n            \n        if self.crop:\n            img = content_crop(img, pad_to_square=True, white_background=True)\n        \n        img = resize(img, self.resize_shape)\n        \n        if self.to_rgb: \n            img = np.repeat(np.expand_dims(img, axis=-1), 3, axis=-1)\n        \n        if not normalize:\n            return img\n        \n        # normalize to 0-1\n        img = img / 255.\n        \n        if self.normalize_mean is not None:\n            img = (img - self.normalize_mean) / self.normalize_std\n       \n        img = torch.tensor(img)\n        img = img.permute([2, 0, 1])\n\n        return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# return an image and the name of the image\nclass BengaliDataset(Dataset):\n    \"\"\"\n    Torch data set object for the bengali data\n    \"\"\"\n\n    def __init__(self, data_list, data_cfg, fname, indices=None):\n        \"\"\"\n        :param data_list: list of raw data consists of (image, labels)\n        :param data_cfg:  data config node\n        \"\"\"\n        self.data_list = data_list\n        self.data_size = len(data_list)\n\n        if indices is None:\n            indices = np.arange(self.data_size)\n        self.indices = indices\n        self.preprocessor = Preprocessor(data_cfg)\n        \n        # get image names\n        if fname:\n            self.df = pd.read_parquet(DATADIR / fname)\n        self.fname = fname\n\n    def __len__(self) -> int:\n        return len(self.indices)\n\n    def __getitem__(self, idx: int) -> (np.ndarray, np.ndarray):\n        idx = self.indices[idx]      \n        img = self.data_list[idx]\n        img = self.preprocessor(img)\n        name = self.df.iloc[idx, 0]\n        return img, name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Use collator if batch size > 1\n# class BengaliDataBatchCollator(object):\n#     \"\"\"\n#     Custom collator\n#     \"\"\"\n\n#     def __init__(self):\n#         pass\n    \n#     def __call__(self, batch: List) -> (torch.Tensor, torch.Tensor):\n#         \"\"\"\n#         :param batch:\n#         :return:\n#         \"\"\"\n\n#         inputs = np.array([x[0] for x in batch])\n#         inputs = torch.tensor(inputs)\n#         inputs = inputs.permute([0, 3, 1, 2])\n#         names = [x[1] for x in batch]\n#         return inputs, names","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"[](http://)## Model - MobileNet V2"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nCopy entirely torchvision.models.mobilenet_v2 simply to force myself to read through mobilenet V2\n\"\"\"\n\nimport torch\nfrom torch import nn\n#from yacs.config import CfgNode\n#from .build import BACKBONE_REGISTRY\n\n\ndef _make_divisible(v, divisor, min_value=None):\n    \"\"\"\n    This function is taken from the original tf repo.\n    It ensures that all layers have a channel number that is divisible by 8\n    It can be seen here:\n    https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py\n    :param v:\n    :param divisor:\n    :param min_value:\n    :return:\n    \"\"\"\n    if min_value is None:\n        min_value = divisor\n    new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)\n    # Make sure that round down does not go down by more than 10%.\n    if new_v < 0.9 * v:\n        new_v += divisor\n    return new_v\n\n\nclass ConvBNReLU(nn.Sequential):\n    def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1):\n        padding = (kernel_size - 1) // 2\n        super(ConvBNReLU, self).__init__(\n            nn.Conv2d(in_planes, out_planes, kernel_size, stride, padding, groups=groups, bias=False),\n            nn.BatchNorm2d(out_planes),\n            nn.ReLU6(inplace=True)\n        )\n\n\nclass InvertedResidual(nn.Module):\n    def __init__(self, inp, oup, stride, expand_ratio):\n        super(InvertedResidual, self).__init__()\n        self.stride = stride\n        assert stride in [1, 2]\n\n        hidden_dim = int(round(inp * expand_ratio))\n        self.use_res_connect = self.stride == 1 and inp == oup\n\n        layers = []\n        if expand_ratio != 1:\n            # pw\n            layers.append(ConvBNReLU(inp, hidden_dim, kernel_size=1))\n        layers.extend([\n            # dw\n            ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim),\n            # pw-linear\n            nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),\n            nn.BatchNorm2d(oup),\n        ])\n        self.conv = nn.Sequential(*layers)\n\n    def forward(self, x):\n        if self.use_res_connect:\n            return x + self.conv(x)\n        else:\n            return self.conv(x)\n\n\nclass MobileNetV2(nn.Module):\n    def __init__(self,\n                 width_mult=1.0,\n                 inverted_residual_setting=None,\n                 round_nearest=8,\n                 block=None):\n        \"\"\"\n        MobileNet V2 main class\n\n        \"\"\"\n        super(MobileNetV2, self).__init__()\n\n        if block is None:\n            block = InvertedResidual\n        input_channel = 32\n        last_channel = 1280\n\n        if inverted_residual_setting is None:\n            inverted_residual_setting = [\n                # t, c, n, s\n                [1, 16, 1, 1],\n                [6, 24, 2, 2],\n                [6, 32, 3, 2],\n                [6, 64, 4, 2],\n                [6, 96, 3, 1],\n                [6, 160, 3, 2],\n                [6, 320, 1, 1],\n            ]\n\n        # only check the first element, assuming user knows t,c,n,s are required\n        if len(inverted_residual_setting) == 0 or len(inverted_residual_setting[0]) != 4:\n            raise ValueError(\"inverted_residual_setting should be non-empty \"\n                             \"or a 4-element list, got {}\".format(inverted_residual_setting))\n\n        # building first layer\n        input_channel = _make_divisible(input_channel * width_mult, round_nearest)\n        self.last_channel = _make_divisible(last_channel * max(1.0, width_mult), round_nearest)\n        features = [ConvBNReLU(3, input_channel, stride=2)]\n        # building inverted residual blocks\n        for t, c, n, s in inverted_residual_setting:\n            output_channel = _make_divisible(c * width_mult, round_nearest)\n            for i in range(n):\n                stride = s if i == 0 else 1\n                features.append(block(input_channel, output_channel, stride, expand_ratio=t))\n                input_channel = output_channel\n        # building last several layers\n        features.append(ConvBNReLU(input_channel, self.last_channel, kernel_size=1))\n        # make it nn.Sequential\n        self.features = nn.Sequential(*features)\n\n        # weight initialization\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out')\n                if m.bias is not None:\n                    nn.init.zeros_(m.bias)\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.ones_(m.weight)\n                nn.init.zeros_(m.bias)\n            elif isinstance(m, nn.Linear):\n                nn.init.normal_(m.weight, 0, 0.01)\n                nn.init.zeros_(m.bias)\n\n    def _forward_impl(self, x):\n        # This exists since TorchScript doesn't support inheritance, so the superclass method\n        # (this one) needs to have a name other than `forward` that can be accessed in a subclass\n        x = self.features(x)\n        x = x.mean([2, 3])\n        return x\n\n    def forward(self, x):\n        return self._forward_impl(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_mobilenet_backbone(backbone_cfg, **kwargs):\n    \"\"\"\n    :param backbone_cfg: backbone config node\n    :param kwargs:\n    :return: backbone module\n    \"\"\"\n    model = MobileNetV2(**kwargs)\n    if backbone_cfg.get('pretrained_path'):\n        pretrained_path = backbone_cfg['pretrained_path']\n        state_dict = torch.load(pretrained_path, map_location='cpu')\n        model.load_state_dict(state_dict, strict=False)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# head and backbone config\nmodel_cfg = {\n            'head_cfg': {\n                'head_name': 'simple_head',\n                'activation': 'leaky_relu',\n                'output_dims': [168, 11, 7],\n                'input_dims': 1280,   # densenet121\n                'hidden_dims': [512, 256],\n                'bn': True,\n                'dropout': -1\n                        },\n            'backbone_cfg': {\n#                 'pretrained_path': '/kaggle/input/julien-4-epochs-densenet121-bengali/model.pt'\n                            }\n             }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch import nn\nimport torch.nn.functional as F\nfrom typing import Union\n\nACTIVATION_FN = {\n    'relu': F.relu,\n    'relu6': F.relu6,\n    'elu': F.elu,\n    'leaky_relu': F.leaky_relu,\n    None: None\n}\n\nclass LinearLayer(nn.Module):\n\n    def __init__(self, input_dim, output_dim, activation, bn, dropout_rate = -1):\n        super(LinearLayer, self).__init__()\n        self.input_dim = input_dim\n        self.output_dim = output_dim\n        self.linear = nn.Linear(input_dim, output_dim)\n        self.activation_fn = ACTIVATION_FN[activation]\n        if bn:\n            self.bn = nn.BatchNorm1d(self.output_dim)\n        else:\n            self.bn = None\n        if dropout_rate > 0:\n            self.dropout = nn.Dropout(p=dropout_rate)\n        else:\n            self.dropout = None\n\n    def forward(self, x):\n        # LINEAR -> BN -> ACTIVATION -> DROPOUT\n        x = self.linear(x)\n        if self.bn is not None:\n            x = self.bn(x)\n        if self.activation_fn is not None:\n            x = self.activation_fn(x, inplace=True)\n        if self.dropout is not None:\n            x = self.dropout(x)\n        return x\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch import nn\n\ndef build_head(head_cfg):\n    return SimplePredictionHead(head_cfg)\n\n\nclass SimplePredictionHead(nn.Module):\n\n    def __init__(self, head_cfg):\n        super(SimplePredictionHead, self).__init__()\n        self.fc_layers = []\n        input_dim = head_cfg['input_dims']\n        # first hidden layers\n        for hidden_dim in head_cfg['hidden_dims']:\n            self.fc_layers.append(\n                LinearLayer(input_dim, hidden_dim, bn=head_cfg['bn'], activation=head_cfg['activation'],\n                            dropout_rate=head_cfg['dropout'])\n            )\n            input_dim = hidden_dim\n\n        output_dims = head_cfg['output_dims']\n\n        # prediction layer\n        self.fc_layers.append(\n            LinearLayer(input_dim, sum(output_dims), bn=False, activation=None, dropout_rate=-1)\n        )\n\n        self.fc_layers = nn.Sequential(*self.fc_layers)\n        for m in self.modules():\n            if isinstance(m, nn.BatchNorm1d):\n                nn.init.ones_(m.weight)\n                nn.init.zeros_(m.bias)\n            elif isinstance(m, nn.Linear):\n                nn.init.normal_(m.weight, 0, 0.01)\n                nn.init.zeros_(m.bias)\n\n    def forward(self, x):\n\n        return self.fc_layers(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BaselineModel(nn.Module):\n\n    def __init__(self, model_cfg):\n        super(BaselineModel, self).__init__()\n        self.backbone = build_mobilenet_backbone(model_cfg['backbone_cfg'])\n        self.head = build_head(model_cfg['head_cfg'])\n        self.heads_dims = model_cfg['head_cfg']['output_dims']\n\n    def forward(self, x):\n        x = self.backbone(x)\n        x = self.head(x)\n        grapheme_logits, vowel_logits, consonant_logits = torch.split(x, self.heads_dims, dim=1)\n        return grapheme_logits, vowel_logits, consonant_logits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ndevice = \"cpu\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = BaselineModel(model_cfg)\nstate_dict = torch.load(WEIGHTS_FILE, map_location='cpu')\nmodel.load_state_dict(state_dict['model_state'])\nmodel.to(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_eval():    \n    model.eval()\n    test_data = ['test_image_data_0.parquet','test_image_data_1.parquet','test_image_data_2.parquet','test_image_data_3.parquet']\n    row_id,target = [],[]\n    batch_size=1\n    for idx, fname in enumerate(test_data):\n        test_images = get_data('test', indices=[str(idx)])\n        test_dataset = BengaliDataset(test_images, dataset_cfg, fname=fname)\n        \n        # test_collator = BengaliDataBatchCollator() ---> don't need batch collator for batch size of 1\n        \n        test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False,  \n                             num_workers=4)\n\n        with torch.no_grad():\n            for inputs, name in test_loader:\n                inputs = inputs.to(device)\n                name = str(name).strip(\"'(),'\")\n                grapheme_logits, vowel_logits, consonant_logits = model(inputs.float())\n\n                grapheme_logits  = grapheme_logits.argmax(-1)\n                vowel_logits     = vowel_logits.argmax(-1)\n                consonant_logits = consonant_logits.argmax(-1)\n                \n                # use a for loop if batch_size > 1\n                row_id += [f'{name}_grapheme_root',f'{name}_vowel_diacritic',\n                               f'{name}_consonant_diacritic']\n                target += [grapheme_logits.item(), vowel_logits.item(), \n                           consonant_logits.item()]\n            del test_images, test_dataset, test_loader\n            gc.collect()\n\n    return pd.DataFrame({'row_id': row_id, 'target': target})\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = test_eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}