{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this notebook, I will introduce a baseline using catalyst. I perfer the `console` version rather than this kind of `notebook` version. \nHowever, I am going to try my best to make it to be more clear. \nIn our local experiment, I got 0.953 CV (one fold) and 0.954 LB (one fold) with following settings:\n* Resnet34 + 3 heads (refer the code bellow) \n* Optimizer AdamW \n* Loss: CrossEntropyLoss for each head\n* Strategies: \n    * Freeze the backbone:\n        * lr = 0.001\n        * num_epochs = 3\n    * Unfreeze the backbone:\n        * lr = 0.0001\n        * num_epochs = 15\n        * Scheduler: OneCycleLRWithWarmup:\n            * num_steps: 15\n            * warmup_steps: 5\n            * lr_range: [0.0005, 0.00001]\n            * momentum_range: [0.85, 0.95]\n            \n            \nIn this notebook, the settings are different from my exp above because of computing resource and time limitation. You are welcome to experiment in your local environment"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install cnn-finetune","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from typing import Callable, List, Tuple\n\nimport os\nimport torch\nimport catalyst\n\nfrom catalyst.dl import utils\n\nprint(f\"torch: {torch.__version__}, catalyst: {catalyst.__version__}\")\n\n# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"  # \"\" - CPU, \"0\" - 1 GPU, \"0,1\" - MultiGPU\n\nSEED = 2411\nutils.set_global_seed(SEED)\nutils.prepare_cudnn(deterministic=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\nfrom cnn_finetune import make_model\n\n\nclass BegaliaiModel(nn.Module):\n    def __init__(self, model_name, pretrained, num_classes):\n        super(BegaliaiModel, self).__init__()\n        self.model = make_model(\n            model_name=model_name,\n            pretrained=pretrained,\n            num_classes=1000,\n        )\n\n        in_features = self.model._classifier.in_features\n\n        self.head_grapheme_root = nn.Linear(in_features, num_classes[0])\n        self.head_vowel_diacritic = nn.Linear(in_features, num_classes[1])\n        self.head_consonant_diacritic = nn.Linear(in_features, num_classes[2])\n\n    def freeze(self):\n        for param in self.model._features.parameters():\n            param.requires_grad = False\n\n    def unfreeze(self):\n        for param in self.model._features.parameters():\n            param.requires_grad = True\n\n    def forward(self, x):\n        features = self.model._features(x)\n        features = F.adaptive_avg_pool2d(features, 1)\n        features = features.view(features.size(0), -1)\n\n        logit_grapheme_root = self.head_grapheme_root(features)\n        logit_vowel_diacritic = self.head_vowel_diacritic(features)\n        logit_consonant_diacritic = self.head_consonant_diacritic(features)\n\n        return logit_grapheme_root, logit_vowel_diacritic, logit_consonant_diacritic","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport os\nimport cv2\nimport pandas as pd\nfrom torch.utils.data import Dataset, DataLoader\n\n\ndef load_image(path):\n    image = cv2.imread(path, 0)\n    image = np.stack((image, image, image), axis=-1)\n    return image\n\n\nclass BengaliaiDataset(Dataset):\n\n    def __init__(self, df, root, transform):\n        self.image_ids = df['image_id'].values\n        self.grapheme_roots = df['grapheme_root'].values\n        self.vowel_diacritics = df['vowel_diacritic'].values\n        self.consonant_diacritics = df['consonant_diacritic'].values\n\n        self.root = root\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def __getitem__(self, idx):\n        image_id = self.image_ids[idx]\n        grapheme_root = self.grapheme_roots[idx]\n        vowel_diacritic = self.vowel_diacritics[idx]\n        consonant_diacritic = self.consonant_diacritics[idx]\n\n        image_id = os.path.join(self.root, image_id + '.png')\n        image = load_image(image_id)\n        if self.transform:\n            image = self.transform(image=image)['image']\n            image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n\n        return {\n            'images': image,\n            'grapheme_roots': grapheme_root,\n            'vowel_diacritics': vowel_diacritic,\n            'consonant_diacritics': consonant_diacritic\n        }\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentations"},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import Compose, Resize, Rotate, HorizontalFlip, Normalize\n\ndef train_aug(image_size):\n    return Compose([\n        Resize(*image_size),\n        Rotate(10),\n        HorizontalFlip(),\n        Normalize()\n    ], p=1)\n\n\ndef valid_aug(image_size):\n    return Compose([\n        Resize(*image_size),\n        Normalize()\n    ], p=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Callbacks"},{"metadata":{},"cell_type":"markdown","source":"This is a callback for `hierarchical macro-averaged recall`"},{"metadata":{"trusted":true},"cell_type":"code","source":"from typing import Any, List, Optional, Union  # isort:skip\n# import logging\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom catalyst.dl.core import Callback, CallbackOrder, RunnerState\nfrom sklearn.metrics import recall_score\nimport numpy as np\n\nclass HMacroAveragedRecall(Callback):\n    def __init__(\n        self,\n        input_grapheme_root_key: str = \"grapheme_roots\",\n        input_consonant_diacritic_key: str = \"consonant_diacritics\",\n        input_vowel_diacritic_key: str = \"vowel_diacritics\",\n\n        output_grapheme_root_key: str = \"logit_grapheme_root\",\n        output_consonant_diacritic_key: str = \"logit_consonant_diacritic\",\n        output_vowel_diacritic_key: str = \"logit_vowel_diacritic\",\n\n        prefix: str = \"hmar\",\n    ):\n        self.input_grapheme_root_key = input_grapheme_root_key\n        self.input_consonant_diacritic_key = input_consonant_diacritic_key\n        self.input_vowel_diacritic_key = input_vowel_diacritic_key\n\n        self.output_grapheme_root_key = output_grapheme_root_key\n        self.output_consonant_diacritic_key = output_consonant_diacritic_key\n        self.output_vowel_diacritic_key = output_vowel_diacritic_key\n        self.prefix = prefix\n\n        super().__init__(CallbackOrder.Metric)\n\n    def on_batch_end(self, state: RunnerState):\n        input_grapheme_root = state.input[self.input_grapheme_root_key].detach().cpu().numpy()\n        input_consonant_diacritic = state.input[self.input_consonant_diacritic_key].detach().cpu().numpy()\n        input_vowel_diacritic = state.input[self.input_vowel_diacritic_key].detach().cpu().numpy()\n\n        output_grapheme_root = state.output[self.output_grapheme_root_key]\n        output_grapheme_root = F.softmax(output_grapheme_root, 1)\n        _, output_grapheme_root = torch.max(output_grapheme_root, 1)\n        output_grapheme_root = output_grapheme_root.detach().cpu().numpy()\n\n        output_consonant_diacritic = state.output[self.output_consonant_diacritic_key]\n        output_consonant_diacritic = F.softmax(output_consonant_diacritic, 1)\n        _, output_consonant_diacritic = torch.max(output_consonant_diacritic, 1)\n        output_consonant_diacritic = output_consonant_diacritic.detach().cpu().numpy()\n\n        output_vowel_diacritic = state.output[self.output_vowel_diacritic_key]\n        output_vowel_diacritic = F.softmax(output_vowel_diacritic, 1)\n        _, output_vowel_diacritic = torch.max(output_vowel_diacritic, 1)\n        output_vowel_diacritic = output_vowel_diacritic.detach().cpu().numpy()\n\n\n        scores = []\n        scores.append(recall_score(input_grapheme_root, output_grapheme_root, average='macro'))\n        scores.append(recall_score(input_consonant_diacritic, output_consonant_diacritic, average='macro'))\n        scores.append(recall_score(input_vowel_diacritic, output_vowel_diacritic, average='macro'))\n\n        final_score = np.average(scores, weights=[2, 1, 1])\n        state.metrics.add_batch_value(name=self.prefix, value=final_score)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loaders"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import collections\n\ndata_root = \"../input/bengaliai/256_train/256/\"\ndf = pd.read_csv(\"../input/bengaliai-cv19/train.csv\")\ntrain_df, valid_df = train_test_split(df, test_size=0.2, shuffle=True, random_state=2411)\n\n# image size\nimage_size = [224, 224]\n\n# transforms \ntrain_transform = train_aug(image_size)\nvalid_transform = valid_aug(image_size)\n\ntrain_dataset = BengaliaiDataset(\n    df=train_df, \n    root=data_root, \n    transform=train_transform\n)\ntrain_loader = DataLoader(\n    dataset=train_dataset,\n    batch_size=32,\n    num_workers=4,\n    shuffle=True,\n    drop_last=False\n)\n\n\nvalid_dataset = BengaliaiDataset(\n    df=valid_df, \n    root=data_root, \n    transform=valid_transform\n)\nvalid_loader = DataLoader(\n    dataset=valid_dataset,\n    batch_size=32,\n    num_workers=4,\n    shuffle=False,\n    drop_last=False\n)\n\nloaders = collections.OrderedDict()\nloaders[\"train\"] = train_loader\nloaders[\"valid\"] = valid_loader","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Experiments"},{"metadata":{},"cell_type":"markdown","source":"## Criterions"},{"metadata":{},"cell_type":"markdown","source":"You can train each head with different loss functions, just define it as a dictionary"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch import nn\n\n# we have multiple criterions\ncriterion = {\n    \"ce\": nn.CrossEntropyLoss(),\n    # Define your awesome losses in here. Ex: Focal, lovasz, etc\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = BegaliaiModel(\n    model_name='resnet34',\n    num_classes=[168, 11, 7],\n    pretrained=True\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Freeze the network"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.freeze()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch import optim\nfrom catalyst.contrib.optimizers import RAdam, Lookahead\n\nlearning_rate = 0.001\noptimizer = optim.AdamW(\n    model.parameters(), \n    lr=learning_rate\n)\nscheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[10], gamma=0.3) # Hack","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from catalyst.dl.runner import SupervisedRunner\nnum_epochs = 3\nlogdir = \"./logs/bengaliai/\"\n\ndevice = utils.get_device()\nprint(f\"device: {device}\")\n\n\nfp16_params = None\n\n\n# by default SupervisedRunner uses \"features\" and \"targets\",\n# in our case we get \"image\" and \"mask\" keys in dataset __getitem__\nrunner = SupervisedRunner(\n    device=device,\n    input_key=\"images\",\n    output_key=(\"logit_grapheme_root\", \"logit_vowel_diacritic\", \"logit_consonant_diacritic\"),\n    input_target_key=(\"grapheme_roots\", \"vowel_diacritics\", \"consonant_diacritics\"),\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from catalyst.dl.callbacks import DiceCallback, IouCallback, \\\n  CriterionCallback, CriterionAggregatorCallback\n\nrunner.train(\n    model=model,\n    criterion=criterion,\n    optimizer=optimizer,\n    scheduler=scheduler,\n    \n    # our dataloaders\n    loaders=loaders,\n    \n    callbacks=[\n        # Each criterion is calculated separately.\n        # Criterion for the grapheme root head. Select `criterion_key` to determine which loss you want to use for this head\n        # It is similar to another heads.\n        CriterionCallback(\n            input_key=\"grapheme_roots\",\n            output_key=\"logit_grapheme_root\",\n            criterion_key='ce',\n            prefix='loss_gr',\n        ),\n        CriterionCallback(\n            input_key=\"vowel_diacritics\",\n            output_key=\"logit_vowel_diacritic\",\n            criterion_key='ce',\n            prefix='loss_wd',\n        ),\n        CriterionCallback(\n            input_key=\"consonant_diacritics\",\n            output_key=\"logit_consonant_diacritic\",\n            criterion_key='ce',\n            prefix='loss_cd',\n        ),\n\n        # And only then we aggregate everything into one loss.\n        # Actually you can compute weighted loss, but the catalyst version should be 19.12.1.\n        CriterionAggregatorCallback(\n            prefix=\"loss\",\n            loss_aggregate_fn=\"sum\", # It can be \"sum\", \"weighted_sum\" or \"mean\" in 19.12.1 version\n            loss_keys=['loss_gr', 'loss_wd', 'loss_cd']\n            # because we want weighted sum, we need to add scale for each loss\n#             loss_keys={\"loss_gr\": 1.0, \"loss_wd\": 1.0, \"loss_cd\": 1.0},\n        ),\n        \n        # metrics\n        HMacroAveragedRecall(),\n    ],\n    # path to save logs\n    logdir=logdir,\n    \n    num_epochs=num_epochs,\n    \n    # save our best checkpoint by IoU metric\n    main_metric=\"hmar\",\n    # IoU needs to be maximized.\n    minimize_metric=False,\n    \n    # for FP16. It uses the variable from the very first cell\n    fp16=None,\n    \n    # for external monitoring tools, like Alchemy\n    monitoring_params=None,\n    \n    # prints train logs\n    verbose=True,\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Unfreeze"},{"metadata":{},"cell_type":"markdown","source":"Now unfreeze the model and train with different settings. It is upto you !.\nUncomment this cell to continue"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.unfreeze()\nlogdir = \"./logs/bengaliai/\"\n\nlearning_rate = 0.0001\nnum_epochs = 5\noptimizer = optim.AdamW(\n    model.parameters(), \n    lr=learning_rate\n)\nscheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[3], gamma=0.1) # Hack\n\nrunner = SupervisedRunner(\n    device=device,\n    input_key=\"images\",\n    output_key=(\"logit_grapheme_root\", \"logit_vowel_diacritic\", \"logit_consonant_diacritic\"),\n    input_target_key=(\"grapheme_roots\", \"vowel_diacritics\", \"consonant_diacritics\"),\n)\n\nrunner.train(\n    model=model,\n    criterion=criterion,\n    optimizer=optimizer,\n    scheduler=scheduler,\n    \n    # our dataloaders\n    loaders=loaders,\n    \n    callbacks=[\n        # Each criterion is calculated separately.\n        # Criterion for the grapheme root head. Select `criterion_key` to determine which loss you want to use for this head\n        # It is similar to another heads.\n        CriterionCallback(\n            input_key=\"grapheme_roots\",\n            output_key=\"logit_grapheme_root\",\n            criterion_key='ce',\n            prefix='loss_gr',\n        ),\n        CriterionCallback(\n            input_key=\"vowel_diacritics\",\n            output_key=\"logit_vowel_diacritic\",\n            criterion_key='ce',\n            prefix='loss_wd',\n        ),\n        CriterionCallback(\n            input_key=\"consonant_diacritics\",\n            output_key=\"logit_consonant_diacritic\",\n            criterion_key='ce',\n            prefix='loss_cd',\n        ),\n\n        # And only then we aggregate everything into one loss.\n        # Actually you can compute weighted loss, but the catalyst version should be 19.12.1.\n        CriterionAggregatorCallback(\n            prefix=\"loss\",\n            loss_aggregate_fn=\"sum\", # It can be \"sum\", \"weighted_sum\" or \"mean\" in 19.12.1 version\n            loss_keys=['loss_gr', 'loss_wd', 'loss_cd']\n            # because we want weighted sum, we need to add scale for each loss\n#             loss_keys={\"loss_gr\": 1.0, \"loss_wd\": 1.0, \"loss_cd\": 1.0},\n        ),\n        \n        # metrics\n        HMacroAveragedRecall(),\n    ],\n    # path to save logs\n    logdir=logdir,\n    \n    num_epochs=num_epochs,\n    \n    # save our best checkpoint by IoU metric\n    main_metric=\"hmar\",\n    # IoU needs to be maximized.\n    minimize_metric=False,\n    \n    # for FP16. It uses the variable from the very first cell\n    fp16=None,\n    \n    # for external monitoring tools, like Alchemy\n    monitoring_params=None,\n    \n    # prints train logs\n    verbose=True,\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Take your weights and do inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}