{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CycleGAN Classifier Training","metadata":{}},{"cell_type":"markdown","source":"## Environment","metadata":{}},{"cell_type":"code","source":"!cp -r ../input/efficientnetpytorch/ ./efficientnetpytorch\n!pip install ./efficientnetpytorch/\n!rm -r ./efficientnetpytorch/","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('1')\nimport pandas as pd\nimport numpy as np\nimport torch\nfrom torch import nn\nimport torchvision\nfrom torchvision import transforms\nimport albumentations as A\nfrom efficientnet_pytorch import EfficientNet\nimport gc\nimport cv2\nfrom tqdm import tqdm\nimport sklearn.metrics\nimport json\nprint('1')","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MEAN = [0.5, 0.5, 0.5]\nSTD = [0.5, 0.5, 0.5]\nIMG_HEIGHT = 224\nIMG_WIDTH = 224\nBATCH_SIZE = 32\nEPOCH = 40\nTQDM_DISABLE = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"## Load Dataset","metadata":{}},{"cell_type":"code","source":"def load_images(paths):\n    all_images = []\n    for path in paths:\n        image_df = pd.read_parquet(path)\n        images = image_df.iloc[:, 1:].values.reshape(-1, 137, 236).astype(np.uint8)\n        del image_df\n        gc.collect()\n        all_images.append(images)\n    all_images = np.concatenate(all_images)\n    return all_images","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_data = pd.read_csv('../input/bengaliai-cv19/train.csv')\n# multi_diacritics_train_data = pd.read_csv('../input/bengaliai-cv19/train_multi_diacritics.csv')\n# train_data = train_data.set_index('image_id')\n# multi_diacritics_train_data = multi_diacritics_train_data.set_index('image_id')\n# train_data.update(multi_diacritics_train_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_images = load_images([\n#     '../input/bengaliai-cv19/train_image_data_0.parquet',\n#     '../input/bengaliai-cv19/train_image_data_1.parquet',\n#     '../input/bengaliai-cv19/train_image_data_2.parquet',\n#     '../input/bengaliai-cv19/train_image_data_3.parquet',\n# ])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"font_data = pd.read_csv('../input/bengaliai-cv19-font/font.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"font_images = load_images([\n    '../input/bengaliai-cv19-font/font_image_data_0.parquet',\n    '../input/bengaliai-cv19-font/font_image_data_1.parquet',\n    '../input/bengaliai-cv19-font/font_image_data_2.parquet',\n    '../input/bengaliai-cv19-font/font_image_data_3.parquet',\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"## Create  Datset","metadata":{}},{"cell_type":"code","source":"class GraphemeDataset(torch.utils.data.Dataset):\n    \n    def __init__(self, data, images, transform=None, num_grapheme_root=168, num_vowel_diacritic=11, num_consonant_diacritic=8):\n        self.data = data\n        self.grapheme_root_list = np.array(data['grapheme_root'].tolist(), dtype=np.int64)\n        self.vowel_diacritic_list = np.array(data['vowel_diacritic'].tolist(), dtype=np.int64)\n        self.consonant_diacritic_list = np.array(data['consonant_diacritic'].tolist(), dtype=np.int64)\n        self.num_grapheme_root = num_grapheme_root\n        self.num_vowel_diacritic = num_vowel_diacritic\n        self.num_consonant_diacritic = num_consonant_diacritic\n        self.images = images\n        self.transform = transform\n            \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        grapheme_root = self.grapheme_root_list[idx]\n        vowel_diacritic = self.vowel_diacritic_list[idx]\n        consonant_diacritic = self.consonant_diacritic_list[idx]\n        label = (grapheme_root*self.num_vowel_diacritic+vowel_diacritic)*self.num_consonant_diacritic+consonant_diacritic\n        np_image = self.images[idx].copy()\n        out_image = self.transform(np_image)\n        return out_image, label\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Albumentations:\n    def __init__(self, augmentations):\n        self.augmentations = A.Compose(augmentations)\n    \n    def __call__(self, image):\n        image = self.augmentations(image=image)['image']\n        return image\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocess = [\n    A.CenterCrop(height=137, width=IMG_WIDTH),\n    A.Resize(height=IMG_HEIGHT, width=IMG_WIDTH, always_apply=True),\n]\n\naugmentations = [\n    A.PadIfNeeded(min_height=256, min_width=256, border_mode=cv2.BORDER_CONSTANT, value=[255, 255, 255], always_apply=True),\n    A.imgaug.transforms.IAAAffine(shear=20, mode='constant', cval=255, always_apply=True),\n    A.ShiftScaleRotate(rotate_limit=20, border_mode=cv2.BORDER_CONSTANT, value=[255, 255, 255], mask_value=[255, 255, 255], always_apply=True),\n    A.RandomCrop(height=IMG_HEIGHT, width=IMG_WIDTH, always_apply=True),\n    A.Cutout(num_holes=1, max_h_size=112, max_w_size=112, fill_value=128, always_apply=True),\n]\n\ntrain_transform = transforms.Compose([\n    np.uint8,\n    transforms.Lambda(lambda x: np.array([x, x, x]).transpose((1, 2, 0)) ),\n    np.uint8,\n    Albumentations(preprocess + augmentations),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=MEAN, std=STD),\n#     transforms.ToPILImage(),\n])\nvalid_transform = transforms.Compose([\n    np.uint8,\n    transforms.Lambda(lambda x: np.array([x, x, x]).transpose((1, 2, 0)) ),\n    np.uint8,\n    Albumentations(preprocess),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=MEAN, std=STD),\n#     transforms.ToPILImage(),\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"font_dataset = GraphemeDataset(font_data, font_images, train_transform)\nvalid_dataset = GraphemeDataset(font_data, font_images, valid_transform)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"## Create Model","metadata":{}},{"cell_type":"code","source":"class BengalModel(nn.Module):\n    def __init__(self, backbone, hidden_size=2560, class_num=168*11*7):\n        super(BengalModel, self).__init__()\n        self.backbone = backbone\n        self._avg_pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(hidden_size, class_num)\n        self.ln = nn.LayerNorm(hidden_size)\n\n        \n    def forward(self, inputs):\n        bs = inputs.shape[0]\n        feature = self.backbone.extract_features(inputs)\n        feature_vector = self._avg_pooling(feature)\n        feature_vector = feature_vector.view(bs, -1)\n        feature_vector = self.ln(feature_vector)\n\n        out = self.fc(feature_vector)\n        return out   \n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"backbone = EfficientNet.from_name('efficientnet-b0')\nclassifier = BengalModel(backbone, hidden_size=1280, class_num=168*11*8).to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"## Create Data Loader","metadata":{}},{"cell_type":"code","source":"font_sampler = torch.utils.data.RandomSampler(font_dataset, True, int(len(font_dataset))*(EPOCH))\nvalid_sampler = torch.utils.data.RandomSampler(valid_dataset, True, int(len(valid_dataset))*(EPOCH))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"font_loader = torch.utils.data.DataLoader(\n    font_dataset, \n    batch_size=BATCH_SIZE, \n    shuffle=False, \n    num_workers=1, \n    pin_memory=True, \n    drop_last=True, \n    sampler=font_sampler)\nvalid_loader = torch.utils.data.DataLoader(\n    valid_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=False,\n    num_workers=1,\n    pin_memory=True,\n    drop_last=True,\n    sampler=valid_sampler)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"font_loader_iter = iter(font_loader)\nvalid_loader_iter = iter(valid_loader)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"def train_step(model, train_iter, criterion, optimizer, scheduler, device):\n    image, label = next(train_iter)\n    image = image.to(device)\n    label = label.to(device)\n    optimizer.zero_grad()\n    out = model(image)\n    loss = criterion(out, label)\n    loss.backward()\n    optimizer.step()\n    scheduler.step()\n    return loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = torch.optim.AdamW(classifier.parameters())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier_loss = nn.CrossEntropyLoss()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_step_per_epoch = len(font_loader)//EPOCH\nnum_valid_step_per_epoch = len(valid_loader)//EPOCH\ntrain_steps = num_step_per_epoch*EPOCH\nWARM_UP_STEP = train_steps*0.5\n\ndef warmup_linear_decay(step):\n    if step < WARM_UP_STEP:\n        return 1.0\n    else:\n        return (train_steps-step)/(train_steps-WARM_UP_STEP)\nscheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, warmup_linear_decay)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log = []\nbest_score = 0.\n\n\nfor epoch in range(EPOCH):\n    classifier.train()\n    metric = {}\n    losses = []\n    for i in tqdm(range(num_step_per_epoch), disable=TQDM_DISABLE):\n        loss = train_step(classifier,\n                  font_loader_iter,\n                  classifier_loss,\n                  optimizer,\n                  scheduler,\n                  device)        \n        losses.append(loss.item())\n    metric['train/loss'] = sum(losses)/len(losses)\n    classifier.eval()\n    preds = []\n    labels = []\n    for i in tqdm(range(num_valid_step_per_epoch), disable=TQDM_DISABLE):\n        image, label = next(valid_loader_iter)\n        image = image.to(device)\n        with torch.no_grad():\n            out = classifier(image)\n            pred = out.argmax(dim=1).cpu().numpy()\n        \n        preds.append(pred)\n        labels.append(label.numpy())\n    \n    preds = np.concatenate(preds)\n    labels = np.concatenate(labels)\n    accuracy = sklearn.metrics.accuracy_score(y_pred=preds, y_true=labels)\n    metric['valid/accuracy'] = accuracy\n    metric['epoch'] = epoch\n    \n    log.append(metric)\n    \n    if accuracy > best_score:\n        best_score = accuracy\n        torch.save(classifier.state_dict(), 'best.pth')\n    torch.save(classifier.state_dict(), 'model.pth')\n    with open('log.json', 'w') as fout:\n        json.dump(log , fout, indent=4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CycleGAN Training","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nfrom torch import nn\nfrom torch.nn import init\nimport torchvision\nfrom torchvision import transforms\nimport albumentations as A\nfrom efficientnet_pytorch import EfficientNet\nimport gc\nimport cv2\nfrom tqdm import tqdm\nimport sklearn.metrics\nimport json\nimport functools\nimport itertools\nimport random","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MEAN = [0.5, 0.5, 0.5]\nSTD = [0.5, 0.5, 0.5]\nIMG_HEIGHT = 224\nIMG_WIDTH = 224\nBATCH_SIZE = 8\nEPOCH = 1\nTQDM_DISABLE = True","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_images(paths):\n    all_images = []\n    for path in paths:\n        image_df = pd.read_parquet(path)\n        images = image_df.iloc[:, 1:].values.reshape(-1, 137, 236).astype(np.uint8)\n        del image_df\n        gc.collect()\n        all_images.append(images)\n    all_images = np.concatenate(all_images)\n    return all_images","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"font_data = pd.read_csv('../input/bengaliai-cv19-font/font.csv')\nfont_images = load_images([\n    '../input/bengaliai-cv19-font/font_image_data_0.parquet',\n    '../input/bengaliai-cv19-font/font_image_data_1.parquet',\n    '../input/bengaliai-cv19-font/font_image_data_2.parquet',\n    '../input/bengaliai-cv19-font/font_image_data_3.parquet',\n])\nnp.save('font_images.npy', font_images)\ndel font_images\ngc.collect()\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ntrain_data = pd.read_csv('../input/bengaliai-cv19/train.csv')\nmulti_diacritics_train_data = pd.read_csv('../input/bengaliai-cv19/train_multi_diacritics.csv')\ntrain_data = train_data.set_index('image_id')\nmulti_diacritics_train_data = multi_diacritics_train_data.set_index('image_id')\ntrain_data.update(multi_diacritics_train_data)\ntrain_images = load_images([\n    '../input/bengaliai-cv19/train_image_data_0.parquet',\n    '../input/bengaliai-cv19/train_image_data_1.parquet',\n    '../input/bengaliai-cv19/train_image_data_2.parquet',\n    '../input/bengaliai-cv19/train_image_data_3.parquet',\n])\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nfont_images = np.load('font_images.npy')\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm ./font_images.npy","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GraphemeDataset(torch.utils.data.Dataset):\n    \n    def __init__(self, data, images, transform=None, num_grapheme_root=168, num_vowel_diacritic=11, num_consonant_diacritic=8):\n        self.data = data\n        self.grapheme_root_list = np.array(data['grapheme_root'].tolist(), dtype=np.int64)\n        self.vowel_diacritic_list = np.array(data['vowel_diacritic'].tolist(), dtype=np.int64)\n        self.consonant_diacritic_list = np.array(data['consonant_diacritic'].tolist(), dtype=np.int64)\n        self.num_grapheme_root = num_grapheme_root\n        self.num_vowel_diacritic = num_vowel_diacritic\n        self.num_consonant_diacritic = num_consonant_diacritic\n        self.images = images\n        self.transform = transform\n            \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        grapheme_root = self.grapheme_root_list[idx]\n        vowel_diacritic = self.vowel_diacritic_list[idx]\n        consonant_diacritic = self.consonant_diacritic_list[idx]\n        label = (grapheme_root*self.num_vowel_diacritic+vowel_diacritic)*self.num_consonant_diacritic+consonant_diacritic\n        np_image = self.images[idx].copy()\n        out_image = self.transform(np_image)\n        return out_image, label\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Albumentations:\n    def __init__(self, augmentations):\n        self.augmentations = A.Compose(augmentations)\n    \n    def __call__(self, image):\n        image = self.augmentations(image=image)['image']\n        return image","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocess = [\n    A.CenterCrop(height=137, width=IMG_WIDTH),\n    A.Resize(height=IMG_HEIGHT, width=IMG_WIDTH, always_apply=True),\n]\n\naugmentations = [\n    A.PadIfNeeded(min_height=256, min_width=256, border_mode=cv2.BORDER_CONSTANT, value=[255, 255, 255], always_apply=True),\n    A.imgaug.transforms.IAAAffine(shear=5, mode='constant', cval=255, always_apply=True),\n    A.ShiftScaleRotate(rotate_limit=5, border_mode=cv2.BORDER_CONSTANT, value=[255, 255, 255], mask_value=[255, 255, 255], always_apply=True),\n    A.RandomCrop(height=IMG_HEIGHT, width=IMG_WIDTH, always_apply=True),\n]\n\n\ntrain_transform = transforms.Compose([\n    np.uint8,\n    transforms.Lambda(lambda x: np.array([x, x, x]).transpose((1, 2, 0)) ),\n    np.uint8,\n    Albumentations(preprocess + augmentations),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=MEAN, std=STD),\n#     transforms.ToPILImage(),\n])\nvalid_transform = transforms.Compose([\n    np.uint8,\n    transforms.Lambda(lambda x: np.array([x, x, x]).transpose((1, 2, 0)) ),\n    np.uint8,\n    Albumentations(preprocess),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=MEAN, std=STD),\n#     transforms.ToPILImage(),\n])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nhand_dataset = GraphemeDataset(train_data, train_images, valid_transform)\nfont_dataset = GraphemeDataset(font_data, font_images, train_transform)\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CycleGAN Model","metadata":{}},{"cell_type":"code","source":"class ResnetGenerator(nn.Module):\n    \"\"\"Resnet-based generator that consists of Resnet blocks between a few downsampling/upsampling operations.\n    We adapt Torch code and idea from Justin Johnson's neural style transfer project(https://github.com/jcjohnson/fast-neural-style)\n    \"\"\"\n\n    def __init__(self, input_nc, output_nc, ngf=64, norm_layer=nn.BatchNorm2d, use_dropout=False, n_blocks=6, padding_type='reflect'):\n        \"\"\"Construct a Resnet-based generator\n        Parameters:\n            input_nc (int)      -- the number of channels in input images\n            output_nc (int)     -- the number of channels in output images\n            ngf (int)           -- the number of filters in the last conv layer\n            norm_layer          -- normalization layer\n            use_dropout (bool)  -- if use dropout layers\n            n_blocks (int)      -- the number of ResNet blocks\n            padding_type (str)  -- the name of padding layer in conv layers: reflect | replicate | zero\n        \"\"\"\n        assert(n_blocks >= 0)\n        super(ResnetGenerator, self).__init__()\n        if type(norm_layer) == functools.partial:\n            use_bias = norm_layer.func == nn.InstanceNorm2d\n        else:\n            use_bias = norm_layer == nn.InstanceNorm2d\n\n        model = [nn.ReflectionPad2d(3),\n                 nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0, bias=use_bias),\n                 norm_layer(ngf),\n                 nn.ReLU(True)]\n\n        n_downsampling = 2\n        for i in range(n_downsampling):  # add downsampling layers\n            mult = 2 ** i\n            model += [nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=1, bias=use_bias),\n                      norm_layer(ngf * mult * 2),\n                      nn.ReLU(True)]\n\n        mult = 2 ** n_downsampling\n        for i in range(n_blocks):       # add ResNet blocks\n\n            model += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)]\n\n        for i in range(n_downsampling):  # add upsampling layers\n            mult = 2 ** (n_downsampling - i)\n            model += [nn.ConvTranspose2d(ngf * mult, int(ngf * mult / 2),\n                                         kernel_size=3, stride=2,\n                                         padding=1, output_padding=1,\n                                         bias=use_bias),\n                      norm_layer(int(ngf * mult / 2)),\n                      nn.ReLU(True)]\n        model += [nn.ReflectionPad2d(3)]\n        model += [nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)]\n        model += [nn.Tanh()]\n\n        self.model = nn.Sequential(*model)\n\n    def forward(self, input):\n        \"\"\"Standard forward\"\"\"\n        return self.model(input)\n\n\nclass ResnetBlock(nn.Module):\n    \"\"\"Define a Resnet block\"\"\"\n\n    def __init__(self, dim, padding_type, norm_layer, use_dropout, use_bias):\n        \"\"\"Initialize the Resnet block\n        A resnet block is a conv block with skip connections\n        We construct a conv block with build_conv_block function,\n        and implement skip connections in <forward> function.\n        Original Resnet paper: https://arxiv.org/pdf/1512.03385.pdf\n        \"\"\"\n        super(ResnetBlock, self).__init__()\n        self.conv_block = self.build_conv_block(dim, padding_type, norm_layer, use_dropout, use_bias)\n\n    def build_conv_block(self, dim, padding_type, norm_layer, use_dropout, use_bias):\n        \"\"\"Construct a convolutional block.\n        Parameters:\n            dim (int)           -- the number of channels in the conv layer.\n            padding_type (str)  -- the name of padding layer: reflect | replicate | zero\n            norm_layer          -- normalization layer\n            use_dropout (bool)  -- if use dropout layers.\n            use_bias (bool)     -- if the conv layer uses bias or not\n        Returns a conv block (with a conv layer, a normalization layer, and a non-linearity layer (ReLU))\n        \"\"\"\n        conv_block = []\n        p = 0\n        if padding_type == 'reflect':\n            conv_block += [nn.ReflectionPad2d(1)]\n        elif padding_type == 'replicate':\n            conv_block += [nn.ReplicationPad2d(1)]\n        elif padding_type == 'zero':\n            p = 1\n        else:\n            raise NotImplementedError('padding [%s] is not implemented' % padding_type)\n\n        conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), norm_layer(dim), nn.ReLU(True)]\n        if use_dropout:\n            conv_block += [nn.Dropout(0.5)]\n\n        p = 0\n        if padding_type == 'reflect':\n            conv_block += [nn.ReflectionPad2d(1)]\n        elif padding_type == 'replicate':\n            conv_block += [nn.ReplicationPad2d(1)]\n        elif padding_type == 'zero':\n            p = 1\n        else:\n            raise NotImplementedError('padding [%s] is not implemented' % padding_type)\n        conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), norm_layer(dim)]\n\n        return nn.Sequential(*conv_block)\n\n    def forward(self, x):\n        \"\"\"Forward function (with skip connections)\"\"\"\n        out = x + self.conv_block(x)  # add skip connections\n        return out","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class NLayerDiscriminator(nn.Module):\n    \"\"\"Defines a PatchGAN discriminator\"\"\"\n\n    def __init__(self, input_nc, ndf=64, n_layers=3, norm_layer=nn.BatchNorm2d):\n        \"\"\"Construct a PatchGAN discriminator\n        Parameters:\n            input_nc (int)  -- the number of channels in input images\n            ndf (int)       -- the number of filters in the last conv layer\n            n_layers (int)  -- the number of conv layers in the discriminator\n            norm_layer      -- normalization layer\n        \"\"\"\n        super(NLayerDiscriminator, self).__init__()\n        if type(norm_layer) == functools.partial:  # no need to use bias as BatchNorm2d has affine parameters\n            use_bias = norm_layer.func == nn.InstanceNorm2d\n        else:\n            use_bias = norm_layer == nn.InstanceNorm2d\n\n        kw = 4\n        padw = 1\n        sequence = [nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)]\n        nf_mult = 1\n        nf_mult_prev = 1\n        for n in range(1, n_layers):  # gradually increase the number of filters\n            nf_mult_prev = nf_mult\n            nf_mult = min(2 ** n, 8)\n            sequence += [\n                nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=2, padding=padw, bias=use_bias),\n                norm_layer(ndf * nf_mult),\n                nn.LeakyReLU(0.2, True)\n            ]\n\n        nf_mult_prev = nf_mult\n        nf_mult = min(2 ** n_layers, 8)\n        sequence += [\n            nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=1, padding=padw, bias=use_bias),\n            norm_layer(ndf * nf_mult),\n            nn.LeakyReLU(0.2, True)\n        ]\n\n        sequence += [nn.Conv2d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)]  # output 1 channel prediction map\n        self.model = nn.Sequential(*sequence)\n\n    def forward(self, input):\n        \"\"\"Standard forward.\"\"\"\n        return self.model(input)\n\ndef init_weight(net, init_gain):\n    def init_func(m):\n        classname = m.__class__.__name__\n        if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1):\n            init.normal_(m.weight.data, 0.0, init_gain)\n            if hasattr(m, 'bias'):\n                init.constant_(m.bias.data, 0.0)\n        elif classname.find('BatchNorm2d') != -1:\n            init.normal_(m.weight.data, 1.0, init_gain)\n            init.constant_(m.bias.data, 0.0)\n    net.apply(init_func)\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImagePool():\n    \"\"\"This class implements an image buffer that stores previously generated images.\n    This buffer enables us to update discriminators using a history of generated images\n    rather than the ones produced by the latest generators.\n    \"\"\"\n\n    def __init__(self, pool_size):\n        \"\"\"Initialize the ImagePool class\n        Parameters:\n            pool_size (int) -- the size of image buffer, if pool_size=0, no buffer will be created\n        \"\"\"\n        self.pool_size = pool_size\n        if self.pool_size > 0:  # create an empty pool\n            self.num_imgs = 0\n            self.images = []\n\n    def query(self, images):\n        \"\"\"Return an image from the pool.\n        Parameters:\n            images: the latest generated images from the generator\n        Returns images from the buffer.\n        By 50/100, the buffer will return input images.\n        By 50/100, the buffer will return images previously stored in the buffer,\n        and insert the current images to the buffer.\n        \"\"\"\n        if self.pool_size == 0:  # if the buffer size is 0, do nothing\n            return images\n        return_images = []\n        for image in images:\n            image = torch.unsqueeze(image.data, 0)\n            if self.num_imgs < self.pool_size:   # if the buffer is not full; keep inserting current images to the buffer\n                self.num_imgs = self.num_imgs + 1\n                self.images.append(image)\n                return_images.append(image)\n            else:\n                p = random.uniform(0, 1)\n                if p > 0.5:  # by 50% chance, the buffer will return a previously stored image, and insert the current image into the buffer\n                    random_id = random.randint(0, self.pool_size - 1)  # randint is inclusive\n                    tmp = self.images[random_id].clone()\n                    self.images[random_id] = image\n                    return_images.append(tmp)\n                else:       # by another 50% chance, the buffer will return the current image\n                    return_images.append(image)\n        return_images = torch.cat(return_images, 0)   # collect all the images and return\n        return return_images","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" class GANLoss(nn.Module):\n    \"\"\"Define different GAN objectives.\n    The GANLoss class abstracts away the need to create the target label tensor\n    that has the same size as the input.\n    \"\"\"\n\n    def __init__(self, gan_mode, target_real_label=1.0, target_fake_label=0.0):\n        \"\"\" Initialize the GANLoss class.\n        Parameters:\n            gan_mode (str) - - the type of GAN objective. It currently supports vanilla, lsgan, and wgangp.\n            target_real_label (bool) - - label for a real image\n            target_fake_label (bool) - - label of a fake image\n        Note: Do not use sigmoid as the last layer of Discriminator.\n        LSGAN needs no sigmoid. vanilla GANs will handle it with BCEWithLogitsLoss.\n        \"\"\"\n        super(GANLoss, self).__init__()\n        self.register_buffer('real_label', torch.tensor(target_real_label))\n        self.register_buffer('fake_label', torch.tensor(target_fake_label))\n        self.gan_mode = gan_mode\n        if gan_mode == 'lsgan':\n            self.loss = nn.MSELoss()\n        elif gan_mode == 'vanilla':\n            self.loss = nn.BCEWithLogitsLoss()\n        elif gan_mode in ['wgangp']:\n            self.loss = None\n        else:\n            raise NotImplementedError('gan mode %s not implemented' % gan_mode)\n\n    def get_target_tensor(self, prediction, target_is_real):\n        \"\"\"Create label tensors with the same size as the input.\n        Parameters:\n            prediction (tensor) - - tpyically the prediction from a discriminator\n            target_is_real (bool) - - if the ground truth label is for real images or fake images\n        Returns:\n            A label tensor filled with ground truth label, and with the size of the input\n        \"\"\"\n\n        if target_is_real:\n            target_tensor = self.real_label\n        else:\n            target_tensor = self.fake_label\n        return target_tensor.expand_as(prediction)\n\n    def __call__(self, prediction, target_is_real):\n        \"\"\"Calculate loss given Discriminator's output and grount truth labels.\n        Parameters:\n            prediction (tensor) - - tpyically the prediction output from a discriminator\n            target_is_real (bool) - - if the ground truth label is for real images or fake images\n        Returns:\n            the calculated loss.\n        \"\"\"\n        if self.gan_mode in ['lsgan', 'vanilla']:\n            target_tensor = self.get_target_tensor(prediction, target_is_real)\n            loss = self.loss(prediction, target_tensor)\n        elif self.gan_mode == 'wgangp':\n            if target_is_real:\n                loss = -prediction.mean()\n            else:\n                loss = prediction.mean()\n        return loss","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BengalModel(nn.Module):\n    def __init__(self, backbone, hidden_size=2560, class_num=168*11*7):\n        super(BengalModel, self).__init__()\n        self.backbone = backbone\n        self._avg_pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(hidden_size, class_num)\n        self.ln = nn.LayerNorm(hidden_size)\n\n        \n    def forward(self, inputs):\n        bs = inputs.shape[0]\n        feature = self.backbone.extract_features(inputs)\n        feature_vector = self._avg_pooling(feature)\n        feature_vector = feature_vector.view(bs, -1)\n        feature_vector = self.ln(feature_vector)\n\n        out = self.fc(feature_vector)\n        return out   ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"norm_layer = functools.partial(nn.InstanceNorm2d, affine=False, track_running_stats=False)\ngenerator_a = ResnetGenerator(input_nc=3, output_nc=3, ngf=64, norm_layer=norm_layer, use_dropout=False, n_blocks=9)\ngenerator_b = ResnetGenerator(input_nc=3, output_nc=3, ngf=64, norm_layer=norm_layer, use_dropout=False, n_blocks=9)\n\ndiscriminator_a = NLayerDiscriminator(input_nc=3, ndf=64, n_layers=3, norm_layer=norm_layer)\ndiscriminator_b = NLayerDiscriminator(input_nc=3, ndf=64, n_layers=3, norm_layer=norm_layer)\nbackbone = EfficientNet.from_name('efficientnet-b0')\nclassifier = BengalModel(backbone, hidden_size=1280, class_num=168*11*8)\ninit_weight(generator_a, 0.02)\ninit_weight(generator_b, 0.02)\ninit_weight(discriminator_a, 0.02)\ninit_weight(discriminator_b, 0.02)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"discriminator_loss = GANLoss('lsgan', target_real_label=1.0, target_fake_label=0.0)\nclassifier_loss = nn.CrossEntropyLoss()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier.load_state_dict(torch.load('../input/cyclegan-classifier-results/best.pth'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CycleGan(nn.Module):\n    \n    def __init__(self, \n                 generator_a, generator_b, discriminator_a, discriminator_b, classifier, \n                 discriminator_loss, classifier_loss, \n                 lambda_a, lambda_b, lambda_cls,\n                 device):\n        super(CycleGan, self).__init__()\n        self.generator_a = generator_a\n        self.generator_b = generator_b\n        self.discriminator_a = discriminator_a\n        self.discriminator_b = discriminator_b\n        self.classifier = classifier.eval()\n        CycleGan.set_requires_grad(self.classifier, requires_grad=False)\n        self.discriminator_loss = discriminator_loss\n        self.classifier_loss = classifier_loss\n        self.reconstruct_loss = nn.L1Loss()\n        self.device = device\n        \n        self.image_pool_a = ImagePool(50)\n        self.image_pool_b = ImagePool(50)\n        \n        self.lambda_a = lambda_a\n        self.lambda_b = lambda_b\n        self.lambda_cls = lambda_cls\n        \n        self.real_images_a = None\n        self.real_images_b = None\n        self.labels_a = None\n        self.labels_b = None\n        self.fake_images_a = None\n        self.fake_images_b = None\n        self.rec_images_a = None\n        self.rec_images_b = None\n        self.generator_a = torch.nn.DataParallel(self.generator_a)\n        self.generator_b = torch.nn.DataParallel(self.generator_b)\n        self.discriminator_a = torch.nn.DataParallel(self.discriminator_a)\n        self.discriminator_b = torch.nn.DataParallel(self.discriminator_b)\n        self.to(device)\n        \n    def forward(self):\n        self.fake_images_a = self.generator_a(self.real_images_b)\n        self.fake_images_b = self.generator_b(self.real_images_a)\n        self.rec_images_a = self.generator_a(self.fake_images_b)\n        self.rec_images_b = self.generator_b(self.fake_images_a)\n    \n        \n    @staticmethod\n    def set_requires_grad(nets, requires_grad=False):\n        \"\"\"Set requies_grad=Fasle for all the networks to avoid unnecessary computations\n        Parameters:\n            nets (network list)   -- a list of networks\n            requires_grad (bool)  -- whether the networks require gradients or not\n        \"\"\"\n        if not isinstance(nets, list):\n            nets = [nets]\n        for net in nets:\n            if net is not None:\n                for param in net.parameters():\n                    param.requires_grad = requires_grad\n                    \n                    \n    def generator_step(self):\n        CycleGan.set_requires_grad([self.discriminator_a, self.discriminator_b], False)\n        \n        loss_a = self.discriminator_loss(self.discriminator_a(self.fake_images_a), True)\n        loss_b = self.discriminator_loss(self.discriminator_b(self.fake_images_b), True)\n        cycle_a = self.reconstruct_loss(self.rec_images_a, self.real_images_a)*self.lambda_a\n        cycle_b = self.reconstruct_loss(self.rec_images_b, self.real_images_b)*self.lambda_b\n        cls_loss = self.classifier_loss(self.classifier(self.fake_images_b), self.labels_a)*self.lambda_cls\n        \n        loss = loss_a + loss_b + cycle_a + cycle_b + cls_loss\n        loss.backward()\n        CycleGan.set_requires_grad([self.discriminator_a, self.discriminator_b], True)\n        return loss, loss_a, loss_b, cycle_a, cycle_b, cls_loss\n        \n    def discriminator_step(self):\n        pred_real_a = self.discriminator_a(self.real_images_a)\n        loss_real_a = self.discriminator_loss(pred_real_a, True)\n        fake_images_a = self.image_pool_a.query(self.fake_images_a).detach()\n        pred_fake_a = self.discriminator_a(fake_images_a)\n        loss_fake_a = self.discriminator_loss(pred_fake_a, False)\n        \n        pred_real_b = self.discriminator_b(self.real_images_b)\n        loss_real_b = self.discriminator_loss(pred_real_b, True)\n        fake_images_b = self.image_pool_b.query(self.fake_images_b).detach()\n        pred_fake_b = self.discriminator_b(fake_images_b)\n        loss_fake_b = self.discriminator_loss(pred_fake_b, False)\n        \n        loss = (loss_real_a + loss_fake_a)/2 + (loss_real_b + loss_fake_b)/2\n        loss.backward()\n        return loss, loss_real_a, loss_fake_a, (loss_real_a + loss_fake_a)/2, loss_real_b, loss_fake_b, (loss_real_b+loss_fake_b)/2\n    \n    def set_input(self, images_a, images_b, labels_a, labels_b):\n        self.real_images_a = images_a.to(self.device)\n        self.real_images_b = images_b.to(self.device)\n        self.labels_a = labels_a\n        self.labels_b = labels_b","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CycleGan(generator_a=generator_a,\n                generator_b=generator_b,\n                discriminator_a=discriminator_a,\n                discriminator_b=discriminator_b,\n                classifier=classifier,\n                discriminator_loss=discriminator_loss,\n                classifier_loss=classifier_loss,\n                lambda_a=10.0,\n                lambda_b=10.0,\n                lambda_cls=1.0,\n                device=device\n                )\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hand_sampler = torch.utils.data.RandomSampler(hand_dataset, True, int(max(len(hand_dataset), len(font_dataset)))*(EPOCH))\nfont_sampler = torch.utils.data.RandomSampler(font_dataset, True, int(max(len(hand_dataset), len(font_dataset)))*(EPOCH))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hand_loader = torch.utils.data.DataLoader(\n    hand_dataset, \n    batch_size=BATCH_SIZE, \n    shuffle=False, \n    num_workers=1, \n    pin_memory=True, \n    drop_last=True, \n    sampler=hand_sampler)\nfont_loader = torch.utils.data.DataLoader(\n    font_dataset, \n    batch_size=BATCH_SIZE, \n    shuffle=False, \n    num_workers=1, \n    pin_memory=True, \n    drop_last=True, \n    sampler=font_sampler)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nhand_loader_iter = iter(hand_loader)\nfont_loader_iter = iter(font_loader)\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_step(model, a_iter, b_iter, generator_optimizer, discriminator_optimizer, generator_scheduler, discriminator_scheduler, device):\n    a_image, a_label = next(a_iter)\n    b_image, b_label = next(b_iter)\n    a_image = a_image.to(device)\n    b_image = b_image.to(device)\n    a_label = a_label.to(device)\n    b_label = b_label.to(device)\n    model.set_input(a_image, b_image, a_label, b_label)\n    model.forward()\n    generator_optimizer.zero_grad()\n    generator_loss, generator_loss_a, generator_loss_b, cycle_a, cycle_b, cls_loss = model.generator_step()\n    generator_optimizer.step()\n    discriminator_optimizer.zero_grad()\n    discriminator_loss, loss_real_a, loss_fake_a, discriminator_loss_a, loss_real_b, loss_fake_b, discriminator_loss_b = model.discriminator_step()\n    discriminator_optimizer.step()\n    generator_scheduler.step()\n    discriminator_scheduler.step()\n    return generator_loss, generator_loss_a, generator_loss_b, cycle_a, cycle_b, cls_loss, discriminator_loss, loss_real_a, loss_fake_a, discriminator_loss_a, loss_real_b, loss_fake_b, discriminator_loss_b","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"generator_optimizer = torch.optim.Adam(itertools.chain(generator_a.parameters(), generator_b.parameters()), lr=0.0002, betas=(0.5, 0.999))\ndiscriminator_optimizer = torch.optim.Adam(itertools.chain(discriminator_a.parameters(), discriminator_b.parameters()), lr=0.0002, betas=(0.5, 0.999))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_step_per_epoch = len(hand_loader)//EPOCH\ntrain_steps = num_step_per_epoch*EPOCH\nWARM_UP_STEP = train_steps*0.5\n\ndef warmup_linear_decay(step):\n    if step < WARM_UP_STEP:\n        return 1.0\n    else:\n        return (train_steps-step)/(train_steps-WARM_UP_STEP)\ngenerator_scheduler = torch.optim.lr_scheduler.LambdaLR(generator_optimizer, warmup_linear_decay)\ndiscriminator_scheduler = torch.optim.lr_scheduler.LambdaLR(discriminator_optimizer, warmup_linear_decay)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LossAverager:\n    def __init__(self, prefix):\n        self.prefix = prefix\n        self.generator_loss = []\n        self.generator_loss_a = []\n        self.generator_loss_b = []\n        self.cycle_a = []\n        self.cycle_b = []\n        self.cls_loss = []\n        self.discriminator_loss = []\n        self.loss_real_a = []\n        self.loss_fake_a = []\n        self.discriminator_loss_a = []\n        self.loss_real_b = []\n        self.loss_fake_b = []\n        self.discriminator_loss_b = []\n    \n    def append(self, generator_loss, generator_loss_a, generator_loss_b, cycle_a, cycle_b, cls_loss, discriminator_loss, loss_real_a, loss_fake_a, discriminator_loss_a, loss_real_b, loss_fake_b, discriminator_loss_b):\n        self.generator_loss.append(generator_loss.item())\n        self.generator_loss_a.append(generator_loss_a.item())\n        self.generator_loss_b.append(generator_loss_b.item())\n        self.cycle_a.append(cycle_a.item())\n        self.cycle_b.append(cycle_b.item())\n        self.cls_loss.append(cls_loss.item())\n        self.discriminator_loss.append(discriminator_loss.item())\n        self.loss_real_a.append(loss_real_a.item())\n        self.loss_fake_a.append(loss_fake_a.item())\n        self.discriminator_loss_a.append(discriminator_loss_a.item())\n        self.loss_real_b.append(loss_real_b.item())\n        self.loss_fake_b.append(loss_fake_b.item())\n        self.discriminator_loss_b.append(discriminator_loss_b.item())\n\n    def average(self):\n        metric = {}\n        for key, value in self.__dict__.items():\n            if isinstance(value, list):\n                metric[self.prefix+'/'+key] = sum(value)/len(value)\n        return metric","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log = []\n\n\nfor epoch in range(EPOCH):\n    model.train()\n    model.classifier.eval()\n    loss_averager = LossAverager('train')\n    for i in tqdm(range(num_step_per_epoch)):\n        losses = train_step(model, hand_loader_iter, font_loader_iter, generator_optimizer, discriminator_optimizer, generator_scheduler, discriminator_scheduler, device)\n        loss_averager.append(*losses)\n    metric = loss_averager.average()\n    metric['epoch'] = epoch\n    model.eval()\n    log.append(metric)\n    torch.save(generator_b.state_dict(), 'generator.pth')\n    with open('log.json', 'w') as fout:\n        json.dump(log , fout, indent=4)\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CycleGAN Submision ","metadata":{}},{"cell_type":"code","source":"%config Completer.use_jedi = False\n\nimport os\nimport json\nimport functools\n\nimport torch\nimport torchvision\nfrom torch import nn\nfrom torchvision import transforms\nfrom efficientnet_pytorch import EfficientNet\nimport numpy as np\nfrom tqdm import tqdm\nimport pandas as pd\nfrom PIL import Image\nimport albumentations as A\nimport sklearn.metrics","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mode = 'test'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nNUM_GRAPHEME_ROOT = 168\nNUM_VOWEL_DIACRITIC = 11\nNUM_CONSONANT_DIACRITIC = 8\nclass_map = pd.read_csv('../input/bengaliai-cv19/class_map.csv')\ngrapheme_root = class_map[class_map['component_type'] == 'grapheme_root']\nvowel_diacritic = class_map[class_map['component_type'] == 'vowel_diacritic']\nconsonant_diacritic = class_map[class_map['component_type'] == 'consonant_diacritic']\ngrapheme_root_list = grapheme_root['component'].tolist()\nvowel_diacritic_list = vowel_diacritic['component'].tolist()\nconsonant_diacritic_list = consonant_diacritic['component'].tolist()\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BengalModel(nn.Module):\n    def __init__(self, backbone, hidden_size=2560, class_num=168*11*7):\n        super(BengalModel, self).__init__()\n        self.backbone = backbone\n        self._avg_pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(hidden_size, class_num)\n        self.ln = nn.LayerNorm(hidden_size)\n\n        \n    def forward(self, inputs):\n        bs = inputs.shape[0]\n        feature = self.backbone.extract_features(inputs)\n        feature_vector = self._avg_pooling(feature)\n        feature_vector = feature_vector.view(bs, -1)\n        feature_vector = self.ln(feature_vector)\n\n        out = self.fc(feature_vector)\n        return out   \n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Albumentations:\n    def __init__(self, augmentations):\n        self.augmentations = A.Compose(augmentations)\n    \n    def __call__(self, image):\n        image = self.augmentations(image=image)['image']\n        return image","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestGraphemeDataset(torch.utils.data.Dataset):\n    \n    def __init__(self, images, transform=None):\n        self.images = images\n        self.transform = transform\n            \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self, idx):\n        np_image = self.images[idx].copy()\n        out_image = self.transform(np_image)\n        return out_image, idx","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def label_to_grapheme(grapheme_root, vowel_diacritic, consonant_diacritic):\n    if consonant_diacritic == 0:\n        if vowel_diacritic == 0:\n            return grapheme_root_list[grapheme_root]\n        else:\n            return grapheme_root_list[grapheme_root] + vowel_diacritic_list[vowel_diacritic]\n    elif consonant_diacritic == 1:\n        if vowel_diacritic == 0:\n            return grapheme_root_list[grapheme_root] + consonant_diacritic_list[consonant_diacritic]\n        else:\n            return grapheme_root_list[grapheme_root] + vowel_diacritic_list[vowel_diacritic] + consonant_diacritic_list[consonant_diacritic]\n    elif consonant_diacritic == 2:\n        if vowel_diacritic == 0:\n            return consonant_diacritic_list[consonant_diacritic] + grapheme_root_list[grapheme_root]\n        else:\n            return consonant_diacritic_list[consonant_diacritic] + grapheme_root_list[grapheme_root] + vowel_diacritic_list[vowel_diacritic]\n    elif consonant_diacritic == 3:\n        if vowel_diacritic == 0:\n            return consonant_diacritic_list[consonant_diacritic][:2] + grapheme_root_list[grapheme_root] + consonant_diacritic_list[consonant_diacritic][1:]\n        else:\n            return consonant_diacritic_list[consonant_diacritic][:2] + grapheme_root_list[grapheme_root] + consonant_diacritic_list[consonant_diacritic][1:] + vowel_diacritic_list[vowel_diacritic]\n    elif consonant_diacritic == 4:\n        if vowel_diacritic == 0:\n            return grapheme_root_list[grapheme_root] + consonant_diacritic_list[consonant_diacritic]\n        else:\n            if grapheme_root == 123 and vowel_diacritic == 1:\n                return grapheme_root_list[grapheme_root] + '\\u200d' + consonant_diacritic_list[consonant_diacritic] + vowel_diacritic_list[vowel_diacritic]\n            return grapheme_root_list[grapheme_root]  + consonant_diacritic_list[consonant_diacritic] + vowel_diacritic_list[vowel_diacritic]\n    elif consonant_diacritic == 5:\n        if vowel_diacritic == 0:\n            return grapheme_root_list[grapheme_root] + consonant_diacritic_list[consonant_diacritic]\n        else:\n            return grapheme_root_list[grapheme_root] + consonant_diacritic_list[consonant_diacritic] + vowel_diacritic_list[vowel_diacritic]\n    elif consonant_diacritic == 6:\n        if vowel_diacritic == 0:\n            return grapheme_root_list[grapheme_root] + consonant_diacritic_list[consonant_diacritic]\n        else:\n            return grapheme_root_list[grapheme_root] + consonant_diacritic_list[consonant_diacritic] + vowel_diacritic_list[vowel_diacritic]\n    elif consonant_diacritic == 7:\n        if vowel_diacritic == 0:\n            return consonant_diacritic_list[2] + grapheme_root_list[grapheme_root] + consonant_diacritic_list[2][::-1]\n        else:\n            return consonant_diacritic_list[2] + grapheme_root_list[grapheme_root] + consonant_diacritic_list[2][::-1] + vowel_diacritic_list[vowel_diacritic]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResnetGenerator(nn.Module):\n    \"\"\"Resnet-based generator that consists of Resnet blocks between a few downsampling/upsampling operations.\n    We adapt Torch code and idea from Justin Johnson's neural style transfer project(https://github.com/jcjohnson/fast-neural-style)\n    \"\"\"\n\n    def __init__(self, input_nc, output_nc, ngf=64, norm_layer=nn.BatchNorm2d, use_dropout=False, n_blocks=6, padding_type='reflect'):\n        \"\"\"Construct a Resnet-based generator\n        Parameters:\n            input_nc (int)      -- the number of channels in input images\n            output_nc (int)     -- the number of channels in output images\n            ngf (int)           -- the number of filters in the last conv layer\n            norm_layer          -- normalization layer\n            use_dropout (bool)  -- if use dropout layers\n            n_blocks (int)      -- the number of ResNet blocks\n            padding_type (str)  -- the name of padding layer in conv layers: reflect | replicate | zero\n        \"\"\"\n        assert(n_blocks >= 0)\n        super(ResnetGenerator, self).__init__()\n        if type(norm_layer) == functools.partial:\n            use_bias = norm_layer.func == nn.InstanceNorm2d\n        else:\n            use_bias = norm_layer == nn.InstanceNorm2d\n\n        model = [nn.ReflectionPad2d(3),\n                 nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0, bias=use_bias),\n                 norm_layer(ngf),\n                 nn.ReLU(True)]\n\n        n_downsampling = 2\n        for i in range(n_downsampling):  # add downsampling layers\n            mult = 2 ** i\n            model += [nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=1, bias=use_bias),\n                      norm_layer(ngf * mult * 2),\n                      nn.ReLU(True)]\n\n        mult = 2 ** n_downsampling\n        for i in range(n_blocks):       # add ResNet blocks\n\n            model += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)]\n\n        for i in range(n_downsampling):  # add upsampling layers\n            mult = 2 ** (n_downsampling - i)\n            model += [nn.ConvTranspose2d(ngf * mult, int(ngf * mult / 2),\n                                         kernel_size=3, stride=2,\n                                         padding=1, output_padding=1,\n                                         bias=use_bias),\n                      norm_layer(int(ngf * mult / 2)),\n                      nn.ReLU(True)]\n        model += [nn.ReflectionPad2d(3)]\n        model += [nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)]\n        model += [nn.Tanh()]\n\n        self.model = nn.Sequential(*model)\n\n    def forward(self, input):\n        \"\"\"Standard forward\"\"\"\n        return self.model(input)\n\n\nclass ResnetBlock(nn.Module):\n    \"\"\"Define a Resnet block\"\"\"\n\n    def __init__(self, dim, padding_type, norm_layer, use_dropout, use_bias):\n        \"\"\"Initialize the Resnet block\n        A resnet block is a conv block with skip connections\n        We construct a conv block with build_conv_block function,\n        and implement skip connections in <forward> function.\n        Original Resnet paper: https://arxiv.org/pdf/1512.03385.pdf\n        \"\"\"\n        super(ResnetBlock, self).__init__()\n        self.conv_block = self.build_conv_block(dim, padding_type, norm_layer, use_dropout, use_bias)\n\n    def build_conv_block(self, dim, padding_type, norm_layer, use_dropout, use_bias):\n        \"\"\"Construct a convolutional block.\n        Parameters:\n            dim (int)           -- the number of channels in the conv layer.\n            padding_type (str)  -- the name of padding layer: reflect | replicate | zero\n            norm_layer          -- normalization layer\n            use_dropout (bool)  -- if use dropout layers.\n            use_bias (bool)     -- if the conv layer uses bias or not\n        Returns a conv block (with a conv layer, a normalization layer, and a non-linearity layer (ReLU))\n        \"\"\"\n        conv_block = []\n        p = 0\n        if padding_type == 'reflect':\n            conv_block += [nn.ReflectionPad2d(1)]\n        elif padding_type == 'replicate':\n            conv_block += [nn.ReplicationPad2d(1)]\n        elif padding_type == 'zero':\n            p = 1\n        else:\n            raise NotImplementedError('padding [%s] is not implemented' % padding_type)\n\n        conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), norm_layer(dim), nn.ReLU(True)]\n        if use_dropout:\n            conv_block += [nn.Dropout(0.5)]\n\n        p = 0\n        if padding_type == 'reflect':\n            conv_block += [nn.ReflectionPad2d(1)]\n        elif padding_type == 'replicate':\n            conv_block += [nn.ReplicationPad2d(1)]\n        elif padding_type == 'zero':\n            p = 1\n        else:\n            raise NotImplementedError('padding [%s] is not implemented' % padding_type)\n        conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), norm_layer(dim)]\n\n        return nn.Sequential(*conv_block)\n\n    def forward(self, x):\n        \"\"\"Forward function (with skip connections)\"\"\"\n        out = x + self.conv_block(x)  # add skip connections\n        return out\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EnsembleModel(nn.Module):\n    def __init__(self, models):\n        super(EnsembleModel, self).__init__()\n        self.model1 = models[0]\n        self.model2 = models[1]\n        \n    def forward(self, images):\n\n        return self.model1(images), self.model2(images)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_merc(classifier_load_path1, generator_load_path1, classifier_load_path2, generator_load_path2, images):\n    MEAN = [0.5, 0.5, 0.5]\n    STD = [0.5, 0.5, 0.5]\n    IMG_HEIGHT = 224\n    IMG_WIDTH = 224\n    BATCH_SIZE = 32\n    norm_layer = functools.partial(nn.InstanceNorm2d, affine=False, track_running_stats=False)\n    generator_b1 = ResnetGenerator(input_nc=3, output_nc=3, ngf=64, norm_layer=norm_layer, use_dropout=False, n_blocks=9)\n    backbone1 = EfficientNet.from_name('efficientnet-b0')\n    classifier1 = BengalModel(backbone1, hidden_size=1280, class_num=NUM_GRAPHEME_ROOT*NUM_VOWEL_DIACRITIC*NUM_CONSONANT_DIACRITIC)\n\n    classifier1.load_state_dict(torch.load(classifier_load_path1))\n    generator_b1.load_state_dict(torch.load(generator_load_path1))\n    model1 = nn.Sequential(generator_b1, classifier1)\n    \n    generator_b2 = ResnetGenerator(input_nc=3, output_nc=3, ngf=64, norm_layer=norm_layer, use_dropout=False, n_blocks=9)\n    backbone2 = EfficientNet.from_name('efficientnet-b0')\n    classifier2 = BengalModel(backbone2, hidden_size=1280, class_num=NUM_GRAPHEME_ROOT*NUM_VOWEL_DIACRITIC*NUM_CONSONANT_DIACRITIC)\n\n    classifier2.load_state_dict(torch.load(classifier_load_path2))\n    generator_b2.load_state_dict(torch.load(generator_load_path2))\n    model2 = nn.Sequential(generator_b2, classifier2)\n    model = EnsembleModel([model1, model2])\n    model.to(device)\n    model.eval()\n    \n    grapheme_root_map = np.zeros((NUM_GRAPHEME_ROOT*NUM_VOWEL_DIACRITIC*NUM_CONSONANT_DIACRITIC, ), dtype=np.int64)\n    vowel_diacritic_map = np.zeros((NUM_GRAPHEME_ROOT*NUM_VOWEL_DIACRITIC*NUM_CONSONANT_DIACRITIC, ), dtype=np.int64)\n    consonant_diacritic_map = np.zeros((NUM_GRAPHEME_ROOT*NUM_VOWEL_DIACRITIC*NUM_CONSONANT_DIACRITIC, ), dtype=np.int64)\n    for grapheme_root in range(168):\n        for vowel_diacritic in range(11):\n            for consonant_diacritic in range(8):\n                i = (grapheme_root*NUM_VOWEL_DIACRITIC + vowel_diacritic)*NUM_CONSONANT_DIACRITIC + consonant_diacritic\n                grapheme_root_map[i] = grapheme_root\n                vowel_diacritic_map[i] = vowel_diacritic\n                consonant_diacritic_map[i] = consonant_diacritic\n    \n    preprocess = [\n        A.CenterCrop(height=137, width=IMG_WIDTH),\n        A.Resize(height=IMG_HEIGHT, width=IMG_WIDTH, always_apply=True),\n    ]\n\n    transform = transforms.Compose([\n        np.uint8,\n        transforms.Lambda(lambda x: np.array([x, x, x]).transpose((1, 2, 0)) ),\n        np.uint8,\n        Albumentations(preprocess),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=MEAN, std=STD),\n    #     transforms.ToPILImage(),\n    ])\n    \n    dataset = TestGraphemeDataset(images, transform)\n    loader = torch.utils.data.DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, pin_memory=True, drop_last=False)\n    def out2pred(out):\n        out1, out2 = out\n        softmax = nn.Softmax(dim=1)\n        out1 = softmax(out1)\n        out2 = softmax(out2)\n        out = out1+out2\n        \n        box_out1 = out1.reshape((-1, NUM_GRAPHEME_ROOT, NUM_VOWEL_DIACRITIC, NUM_CONSONANT_DIACRITIC))\n        grapheme_root_out = box_out1.sum(dim=(2, 3))\n        vowel_diacritic_out = box_out1.sum(dim=(1, 3))\n        consonant_diacritic_out = box_out1.sum(dim=(1, 2))\n        box_out2 = out2.reshape((-1, NUM_GRAPHEME_ROOT, NUM_VOWEL_DIACRITIC, NUM_CONSONANT_DIACRITIC))\n        grapheme_root_out += box_out2.sum(dim=(2, 3))\n        vowel_diacritic_out += box_out2.sum(dim=(1, 3))\n        consonant_diacritic_out += box_out2.sum(dim=(1, 2))\n        grapheme_root_preds = grapheme_root_out.argmax(dim=1).cpu().numpy()\n        vowel_diacritic_preds = vowel_diacritic_out.argmax(dim=1).cpu().numpy()\n        consonant_diacritic_preds = consonant_diacritic_out.argmax(dim=1).cpu().numpy()\n        preds = (grapheme_root_preds*NUM_VOWEL_DIACRITIC+vowel_diacritic_preds)*NUM_CONSONANT_DIACRITIC+consonant_diacritic_preds\n#         confidences, preds = out.max(dim=1)\n\n#         confidences = confidences.cpu().numpy()\n#         preds = preds.cpu().numpy()\n#         grapheme_root_preds = grapheme_root_map[preds]\n#         vowel_diacritic_preds = vowel_diacritic_map[preds]\n#         consonant_diacritic_preds = consonant_diacritic_map[preds]\n        ret = []\n        for p, g, v, co in zip(preds, grapheme_root_preds, vowel_diacritic_preds, consonant_diacritic_preds):\n            x = {\n                'pred': p,\n#                 'confidence': c,\n                'grapheme_root': g,\n                'vowel_diacritic': v,\n                'consonant_diacritic': co,\n                'grapheme': label_to_grapheme(g, v, co),\n            }\n            ret.append(x)\n        return ret\n        \n    \n    return model, loader, out2pred","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluator(model, loader, out2pred):\n    ret = []\n    model.eval()\n    softmax = nn.Softmax(dim=1)\n    for (images, idx) in tqdm(loader):\n        images = images.to(device)\n        with torch.no_grad():\n            out = model(images)\n            ret += out2pred(out)\n    return ret","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_images(path):\n    image_df = pd.read_parquet(path)\n    images = image_df.iloc[:, 1:].values.reshape(-1, 137, 236)\n    return images\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nmerc_result = []\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = load_images('../input/bengaliai-cv19/{}_image_data_0.parquet'.format(mode))\nmodel, loader, out2pred = create_merc('../input/cyclegan-classifier-results/best.pth', '../input/cyclegan-training-results/generator.pth', '../input/cyclegan-classifier-results/best.pth', '../input/cyclegan-training-results/generator.pth', images)\nmerc_result += evaluator(model, loader, out2pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = load_images('../input/bengaliai-cv19/{}_image_data_1.parquet'.format(mode))\nmodel, loader, out2pred = create_merc('../input/cyclegan-classifier-results/best.pth', '../input/cyclegan-training-results/generator.pth', '../input/cyclegan-classifier-results/best.pth', '../input/cyclegan-training-results/generator.pth', images)\nmerc_result += evaluator(model, loader, out2pred)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nimages = load_images('../input/bengaliai-cv19/{}_image_data_2.parquet'.format(mode))\nmodel, loader, out2pred = create_merc('../input/cyclegan-classifier-results/best.pth', '../input/cyclegan-training-results/generator.pth', '../input/cyclegan-classifier-results/best.pth', '../input/cyclegan-training-results/generator.pth', images)\nmerc_result += evaluator(model, loader, out2pred)\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nimages = load_images('../input/bengaliai-cv19/{}_image_data_3.parquet'.format(mode))\nmodel, loader, out2pred = create_merc('../input/cyclegan-classifier-results/best.pth', '../input/cyclegan-training-results/generator.pth', '../input/cyclegan-classifier-results/best.pth', '../input/cyclegan-training-results/generator.pth', images)\nmerc_result += evaluator(model, loader, out2pred)\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nmerc_result_df = pd.DataFrame(merc_result)\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_inference(merc_result_df):\n    ret = []\n    for merc_row in merc_result_df.itertuples():\n        inference = {}\n        inference['grapheme_root'] = merc_row.grapheme_root\n        inference['vowel_diacritic'] = merc_row.vowel_diacritic\n        inference['consonant_diacritic'] = merc_row.consonant_diacritic\n        inference['grapheme'] = merc_row.grapheme\n        ret.append(inference)\n        \n    return pd.DataFrame(ret)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference = create_inference(merc_result_df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def submit(inference):\n    row_id_list = []\n    target_list = []\n    for i, row in inference.iterrows():\n        row_id_list.append('Test_{}_grapheme_root'.format(i))\n        target_list.append(row.grapheme_root)\n        row_id_list.append('Test_{}_vowel_diacritic'.format(i))\n        target_list.append(row.vowel_diacritic)\n        row_id_list.append('Test_{}_consonant_diacritic'.format(i))\n        if row.consonant_diacritic == 7:\n            target_list.append(2)\n        else:\n            target_list.append(row.consonant_diacritic)\n    raw_submission = {\n        'row_id': row_id_list,\n        'target': target_list\n    }\n    submission = pd.DataFrame(raw_submission)\n    display(submission)\n    submission.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nsubmit(inference)\n\n","metadata":{},"execution_count":null,"outputs":[]}]}