{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CycleGAN Classifier Training","metadata":{}},{"cell_type":"markdown","source":"## Environment","metadata":{}},{"cell_type":"code","source":"!cp -r ../input/efficientnetpytorch/ ./efficientnetpytorch\n!pip install ./efficientnetpytorch/\n!rm -r ./efficientnetpytorch/","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-19T07:56:09.964851Z","iopub.execute_input":"2023-07-19T07:56:09.965253Z","iopub.status.idle":"2023-07-19T07:56:43.219292Z","shell.execute_reply.started":"2023-07-19T07:56:09.965198Z","shell.execute_reply":"2023-07-19T07:56:43.218183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nfrom torch import nn\nimport torchvision\nfrom torchvision import transforms\nimport albumentations as A\nfrom efficientnet_pytorch import EfficientNet\nimport gc\nimport cv2\nfrom tqdm import tqdm\nimport sklearn.metrics\nimport json","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2023-07-19T07:56:43.221391Z","iopub.execute_input":"2023-07-19T07:56:43.221812Z","iopub.status.idle":"2023-07-19T07:56:46.205574Z","shell.execute_reply.started":"2023-07-19T07:56:43.221754Z","shell.execute_reply":"2023-07-19T07:56:46.204596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MEAN = [0.5, 0.5, 0.5]\nSTD = [0.5, 0.5, 0.5]\nIMG_HEIGHT = 224\nIMG_WIDTH = 224\nBATCH_SIZE = 32\nEPOCH = 40\nTQDM_DISABLE = True","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:56:46.207038Z","iopub.execute_input":"2023-07-19T07:56:46.207438Z","iopub.status.idle":"2023-07-19T07:56:46.214552Z","shell.execute_reply.started":"2023-07-19T07:56:46.207375Z","shell.execute_reply":"2023-07-19T07:56:46.213435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\")","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:56:46.216368Z","iopub.execute_input":"2023-07-19T07:56:46.217167Z","iopub.status.idle":"2023-07-19T07:56:46.222759Z","shell.execute_reply.started":"2023-07-19T07:56:46.217115Z","shell.execute_reply":"2023-07-19T07:56:46.221822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"## Load Dataset","metadata":{}},{"cell_type":"code","source":"def load_images(paths):\n    all_images = []\n    for path in paths:\n        image_df = pd.read_parquet(path)\n        images = image_df.iloc[:, 1:].values.reshape(-1, 137, 236).astype(np.uint8)\n        del image_df\n        gc.collect()\n        all_images.append(images)\n    all_images = np.concatenate(all_images)\n    return all_images","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:56:46.226853Z","iopub.execute_input":"2023-07-19T07:56:46.227223Z","iopub.status.idle":"2023-07-19T07:56:46.235186Z","shell.execute_reply.started":"2023-07-19T07:56:46.227151Z","shell.execute_reply":"2023-07-19T07:56:46.234039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_data = pd.read_csv('../input/bengaliai-cv19/train.csv')\n# multi_diacritics_train_data = pd.read_csv('../input/bengaliai-cv19/train_multi_diacritics.csv')\n# train_data = train_data.set_index('image_id')\n# multi_diacritics_train_data = multi_diacritics_train_data.set_index('image_id')\n# train_data.update(multi_diacritics_train_data)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:56:46.238228Z","iopub.execute_input":"2023-07-19T07:56:46.238982Z","iopub.status.idle":"2023-07-19T07:56:46.243424Z","shell.execute_reply.started":"2023-07-19T07:56:46.238766Z","shell.execute_reply":"2023-07-19T07:56:46.242422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_images = load_images([\n#     '../input/bengaliai-cv19/train_image_data_0.parquet',\n#     '../input/bengaliai-cv19/train_image_data_1.parquet',\n#     '../input/bengaliai-cv19/train_image_data_2.parquet',\n#     '../input/bengaliai-cv19/train_image_data_3.parquet',\n# ])","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:56:46.245774Z","iopub.execute_input":"2023-07-19T07:56:46.246479Z","iopub.status.idle":"2023-07-19T07:56:46.251501Z","shell.execute_reply.started":"2023-07-19T07:56:46.246197Z","shell.execute_reply":"2023-07-19T07:56:46.250140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"font_data = pd.read_csv('../input/bengaliai-cv19-font/font.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:56:46.253253Z","iopub.execute_input":"2023-07-19T07:56:46.253839Z","iopub.status.idle":"2023-07-19T07:56:46.378592Z","shell.execute_reply.started":"2023-07-19T07:56:46.253729Z","shell.execute_reply":"2023-07-19T07:56:46.377746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"font_images = load_images([\n    '../input/bengaliai-cv19-font/font_image_data_0.parquet',\n    '../input/bengaliai-cv19-font/font_image_data_1.parquet',\n    '../input/bengaliai-cv19-font/font_image_data_2.parquet',\n    '../input/bengaliai-cv19-font/font_image_data_3.parquet',\n])","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:56:46.380205Z","iopub.execute_input":"2023-07-19T07:56:46.380616Z","iopub.status.idle":"2023-07-19T07:57:40.338687Z","shell.execute_reply.started":"2023-07-19T07:56:46.380566Z","shell.execute_reply":"2023-07-19T07:57:40.337562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"## Create  Datset","metadata":{}},{"cell_type":"code","source":"class GraphemeDataset(torch.utils.data.Dataset):\n    \n    def __init__(self, data, images, transform=None, num_grapheme_root=168, num_vowel_diacritic=11, num_consonant_diacritic=8):\n        self.data = data\n        self.grapheme_root_list = np.array(data['grapheme_root'].tolist(), dtype=np.int64)\n        self.vowel_diacritic_list = np.array(data['vowel_diacritic'].tolist(), dtype=np.int64)\n        self.consonant_diacritic_list = np.array(data['consonant_diacritic'].tolist(), dtype=np.int64)\n        self.num_grapheme_root = num_grapheme_root\n        self.num_vowel_diacritic = num_vowel_diacritic\n        self.num_consonant_diacritic = num_consonant_diacritic\n        self.images = images\n        self.transform = transform\n            \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        grapheme_root = self.grapheme_root_list[idx]\n        vowel_diacritic = self.vowel_diacritic_list[idx]\n        consonant_diacritic = self.consonant_diacritic_list[idx]\n        label = (grapheme_root*self.num_vowel_diacritic+vowel_diacritic)*self.num_consonant_diacritic+consonant_diacritic\n        np_image = self.images[idx].copy()\n        out_image = self.transform(np_image)\n        return out_image, label\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:57:40.340325Z","iopub.execute_input":"2023-07-19T07:57:40.340796Z","iopub.status.idle":"2023-07-19T07:57:40.354247Z","shell.execute_reply.started":"2023-07-19T07:57:40.340747Z","shell.execute_reply":"2023-07-19T07:57:40.352944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Albumentations:\n    def __init__(self, augmentations):\n        self.augmentations = A.Compose(augmentations)\n    \n    def __call__(self, image):\n        image = self.augmentations(image=image)['image']\n        return image\n        ","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:57:40.356126Z","iopub.execute_input":"2023-07-19T07:57:40.356952Z","iopub.status.idle":"2023-07-19T07:57:40.366657Z","shell.execute_reply.started":"2023-07-19T07:57:40.356898Z","shell.execute_reply":"2023-07-19T07:57:40.365787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocess = [\n    A.CenterCrop(height=137, width=IMG_WIDTH),\n    A.Resize(height=IMG_HEIGHT, width=IMG_WIDTH, always_apply=True),\n]\n\naugmentations = [\n    A.PadIfNeeded(min_height=256, min_width=256, border_mode=cv2.BORDER_CONSTANT, value=[255, 255, 255], always_apply=True),\n    A.imgaug.transforms.IAAAffine(shear=20, mode='constant', cval=255, always_apply=True),\n    A.ShiftScaleRotate(rotate_limit=20, border_mode=cv2.BORDER_CONSTANT, value=[255, 255, 255], mask_value=[255, 255, 255], always_apply=True),\n    A.RandomCrop(height=IMG_HEIGHT, width=IMG_WIDTH, always_apply=True),\n    A.Cutout(num_holes=1, max_h_size=112, max_w_size=112, fill_value=128, always_apply=True),\n]\n\ntrain_transform = transforms.Compose([\n    np.uint8,\n    transforms.Lambda(lambda x: np.array([x, x, x]).transpose((1, 2, 0)) ),\n    np.uint8,\n    Albumentations(preprocess + augmentations),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=MEAN, std=STD),\n#     transforms.ToPILImage(),\n])\nvalid_transform = transforms.Compose([\n    np.uint8,\n    transforms.Lambda(lambda x: np.array([x, x, x]).transpose((1, 2, 0)) ),\n    np.uint8,\n    Albumentations(preprocess),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=MEAN, std=STD),\n#     transforms.ToPILImage(),\n])","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:57:40.368341Z","iopub.execute_input":"2023-07-19T07:57:40.368953Z","iopub.status.idle":"2023-07-19T07:57:40.384993Z","shell.execute_reply.started":"2023-07-19T07:57:40.368726Z","shell.execute_reply":"2023-07-19T07:57:40.383670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"font_dataset = GraphemeDataset(font_data, font_images, train_transform)\nvalid_dataset = GraphemeDataset(font_data, font_images, valid_transform)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:57:40.386695Z","iopub.execute_input":"2023-07-19T07:57:40.387326Z","iopub.status.idle":"2023-07-19T07:57:40.499698Z","shell.execute_reply.started":"2023-07-19T07:57:40.387067Z","shell.execute_reply":"2023-07-19T07:57:40.497853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"## Create Model","metadata":{}},{"cell_type":"code","source":"class BengalModel(nn.Module):\n    def __init__(self, backbone, hidden_size=2560, class_num=168*11*7):\n        super(BengalModel, self).__init__()\n        self.backbone = backbone\n        self._avg_pooling = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(hidden_size, class_num)\n        self.ln = nn.LayerNorm(hidden_size)\n\n        \n    def forward(self, inputs):\n        bs = inputs.shape[0]\n        feature = self.backbone.extract_features(inputs)\n        feature_vector = self._avg_pooling(feature)\n        feature_vector = feature_vector.view(bs, -1)\n        feature_vector = self.ln(feature_vector)\n\n        out = self.fc(feature_vector)\n        return out   \n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:57:40.506101Z","iopub.execute_input":"2023-07-19T07:57:40.506679Z","iopub.status.idle":"2023-07-19T07:57:40.532509Z","shell.execute_reply.started":"2023-07-19T07:57:40.506495Z","shell.execute_reply":"2023-07-19T07:57:40.528956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"backbone = EfficientNet.from_name('efficientnet-b0')\nclassifier = BengalModel(backbone, hidden_size=1280, class_num=168*11*8).to(device)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:57:40.534206Z","iopub.execute_input":"2023-07-19T07:57:40.535281Z","iopub.status.idle":"2023-07-19T07:57:43.908778Z","shell.execute_reply.started":"2023-07-19T07:57:40.534598Z","shell.execute_reply":"2023-07-19T07:57:43.907833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"## Create Data Loader","metadata":{}},{"cell_type":"code","source":"font_sampler = torch.utils.data.RandomSampler(font_dataset, True, int(len(font_dataset))*(EPOCH))\nvalid_sampler = torch.utils.data.RandomSampler(valid_dataset, True, int(len(valid_dataset))*(EPOCH))","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:57:43.910230Z","iopub.execute_input":"2023-07-19T07:57:43.910647Z","iopub.status.idle":"2023-07-19T07:57:43.917124Z","shell.execute_reply.started":"2023-07-19T07:57:43.910594Z","shell.execute_reply":"2023-07-19T07:57:43.915923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"font_loader = torch.utils.data.DataLoader(\n    font_dataset, \n    batch_size=BATCH_SIZE, \n    shuffle=False, \n    num_workers=1, \n    pin_memory=True, \n    drop_last=True, \n    sampler=font_sampler)\nvalid_loader = torch.utils.data.DataLoader(\n    valid_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=False,\n    num_workers=1,\n    pin_memory=True,\n    drop_last=True,\n    sampler=valid_sampler)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:57:43.918748Z","iopub.execute_input":"2023-07-19T07:57:43.919409Z","iopub.status.idle":"2023-07-19T07:57:43.930767Z","shell.execute_reply.started":"2023-07-19T07:57:43.919347Z","shell.execute_reply":"2023-07-19T07:57:43.929990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"font_loader_iter = iter(font_loader)\nvalid_loader_iter = iter(valid_loader)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:57:43.933554Z","iopub.execute_input":"2023-07-19T07:57:43.934286Z","iopub.status.idle":"2023-07-19T07:57:44.433878Z","shell.execute_reply.started":"2023-07-19T07:57:43.934234Z","shell.execute_reply":"2023-07-19T07:57:44.432136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"def train_step(model, train_iter, criterion, optimizer, scheduler, device):\n    image, label = next(train_iter)\n    image = image.to(device)\n    label = label.to(device)\n    optimizer.zero_grad()\n    out = model(image)\n    loss = criterion(out, label)\n    loss.backward()\n    optimizer.step()\n    scheduler.step()\n    return loss","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:57:44.440426Z","iopub.execute_input":"2023-07-19T07:57:44.443504Z","iopub.status.idle":"2023-07-19T07:57:44.530505Z","shell.execute_reply.started":"2023-07-19T07:57:44.443394Z","shell.execute_reply":"2023-07-19T07:57:44.529027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = torch.optim.AdamW(classifier.parameters())","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:57:44.537444Z","iopub.execute_input":"2023-07-19T07:57:44.546658Z","iopub.status.idle":"2023-07-19T07:57:44.559228Z","shell.execute_reply.started":"2023-07-19T07:57:44.546596Z","shell.execute_reply":"2023-07-19T07:57:44.558274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier_loss = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:57:44.564627Z","iopub.execute_input":"2023-07-19T07:57:44.574158Z","iopub.status.idle":"2023-07-19T07:57:44.581714Z","shell.execute_reply.started":"2023-07-19T07:57:44.574095Z","shell.execute_reply":"2023-07-19T07:57:44.580624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_step_per_epoch = len(font_loader)//EPOCH\nnum_valid_step_per_epoch = len(valid_loader)//EPOCH\ntrain_steps = num_step_per_epoch*EPOCH\nWARM_UP_STEP = train_steps*0.5\n\ndef warmup_linear_decay(step):\n    if step < WARM_UP_STEP:\n        return 1.0\n    else:\n        return (train_steps-step)/(train_steps-WARM_UP_STEP)\nscheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, warmup_linear_decay)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:57:44.588053Z","iopub.execute_input":"2023-07-19T07:57:44.590691Z","iopub.status.idle":"2023-07-19T07:57:44.612038Z","shell.execute_reply.started":"2023-07-19T07:57:44.590635Z","shell.execute_reply":"2023-07-19T07:57:44.610874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log = []\nbest_score = 0.\n\n\nfor epoch in range(EPOCH):\n    classifier.train()\n    metric = {}\n    losses = []\n    for i in tqdm(range(num_step_per_epoch), disable=TQDM_DISABLE):\n        loss = train_step(classifier,\n                  font_loader_iter,\n                  classifier_loss,\n                  optimizer,\n                  scheduler,\n                  device)        \n        losses.append(loss.item())\n    metric['train/loss'] = sum(losses)/len(losses)\n    classifier.eval()\n    preds = []\n    labels = []\n    for i in tqdm(range(num_valid_step_per_epoch), disable=TQDM_DISABLE):\n        image, label = next(valid_loader_iter)\n        image = image.to(device)\n        with torch.no_grad():\n            out = classifier(image)\n            pred = out.argmax(dim=1).cpu().numpy()\n        \n        preds.append(pred)\n        labels.append(label.numpy())\n    \n    preds = np.concatenate(preds)\n    labels = np.concatenate(labels)\n    accuracy = sklearn.metrics.accuracy_score(y_pred=preds, y_true=labels)\n    metric['valid/accuracy'] = accuracy\n    metric['epoch'] = epoch\n    \n    log.append(metric)\n    \n    if accuracy > best_score:\n        best_score = accuracy\n        torch.save(classifier.state_dict(), 'best.pth')\n    torch.save(classifier.state_dict(), 'model.pth')\n    with open('log.json', 'w') as fout:\n        json.dump(log , fout, indent=4)","metadata":{"execution":{"iopub.status.busy":"2023-07-19T07:57:44.618061Z","iopub.execute_input":"2023-07-19T07:57:44.621590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}