{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Simple pytorch training\nYou can use this simple notebook as your starter code for training.\n\n\n#### References\n- [Image Dataset](http://www.kaggle.com/dataset/a318f9ccd11aea9ede828487914dbbcb76776b72aeb4ef85b51709cfbbe004d3)\n- [Pretrained weights](https://www.kaggle.com/pytorch/resnet18)\n- [Inference kernel](https://www.kaggle.com/pestipeti/simple-pytorch-inference)\n- [EDA Kernel](https://www.kaggle.com/pestipeti/bengali-quick-eda)"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nimport torch.optim as optim\nimport torch.nn.functional as F\nimport os\nimport cv2\nimport torchvision\nimport sklearn.metrics\n\nfrom tqdm import tqdm\nfrom torch.utils.data import Dataset\nfrom albumentations import Compose, ShiftScaleRotate, Resize\nfrom albumentations.pytorch import ToTensorV2\n\nINPUT_PATH = '/kaggle/input/bengaliai-cv19'\nINPUT_PATH_TRAIN_IMAGES = '/kaggle/input/bengaliai/256_train/256'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ======================\n# Params\nBATCH_SIZE = 32\nN_WORKERS = 4\nN_EPOCHS = 4\n\n# Disable training in kaggle\nTRAIN_ENABLED = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset\nFor training I use the 256x256 image dataset, you can find it [here](http://www.kaggle.com/dataset/a318f9ccd11aea9ede828487914dbbcb76776b72aeb4ef85b51709cfbbe004d3)."},{"metadata":{"trusted":true},"cell_type":"code","source":"class BengaliImageDataset(Dataset):\n\n    def __init__(self, csv_file, path, labels, transform=None):\n\n        self.data = pd.read_csv(csv_file)\n        self.data_dummie_labels = pd.get_dummies(\n            self.data[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']],\n            columns=['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']\n        )\n        self.path = path\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n\n        image_name = os.path.join(self.path, self.data.loc[idx, 'image_id'] + '.png')\n        img = cv2.imread(image_name)\n\n        if self.transform:\n            transformed = self.transform(image=img)\n            img = transformed['image']\n\n        if self.labels:\n            return {\n                'image': img,\n                'l_graph': torch.tensor(self.data_dummie_labels.iloc[idx, 0:168]),\n                'l_vowel': torch.tensor(self.data_dummie_labels.iloc[idx, 168:179]),\n                'l_conso': torch.tensor(self.data_dummie_labels.iloc[idx, 179:186]),\n            }\n        else:\n            return {'image': img}\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model\nSimple resnet for now."},{"metadata":{"trusted":true},"cell_type":"code","source":"class BengaliModel(torch.nn.Module):\n\n    def __init__(self):\n        super().__init__()\n        self.backbone = torchvision.models.resnet18(pretrained=False)\n        self.backbone.load_state_dict(torch.load(\"/kaggle/input/resnet18/resnet18.pth\"))\n\n        in_features = self.backbone.fc.in_features\n\n        self.fc_graph = torch.nn.Linear(in_features, 168)\n        self.fc_vowel = torch.nn.Linear(in_features, 11)\n        self.fc_conso = torch.nn.Linear(in_features, 7)\n\n    def forward(self, x):\n\n        x = self.backbone.conv1(x)\n        x = self.backbone.bn1(x)\n        x = self.backbone.relu(x)\n        x = self.backbone.maxpool(x)\n\n        x = self.backbone.layer1(x)\n        x = self.backbone.layer2(x)\n        x = self.backbone.layer3(x)\n        x = self.backbone.layer4(x)\n\n        x = self.backbone.avgpool(x)\n        x = torch.flatten(x, 1)\n\n        fc_graph = self.fc_graph(x)\n        fc_vowel = self.fc_vowel(x)\n        fc_conso = self.fc_conso(x)\n\n        return fc_graph, fc_vowel, fc_conso\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transform_train = Compose([\n    ToTensorV2()\n])\n\ntrain_dataset = BengaliImageDataset(\n    csv_file=INPUT_PATH + '/train.csv',\n    path=INPUT_PATH_TRAIN_IMAGES,\n    transform=transform_train, labels=True\n)\n\ndata_loader_train = torch.utils.data.DataLoader(\n    train_dataset,\n    batch_size=BATCH_SIZE,\n    num_workers=N_WORKERS,\n    shuffle=True\n)\n\ndevice = torch.device(\"cuda:0\")\nmodel = BengaliModel()\nmodel = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = torch.nn.BCEWithLogitsLoss()\nplist = [{'params': model.parameters(), 'lr': 2e-5}]\noptimizer = optim.Adam(plist, lr=2e-5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"# TRAIN_ENABLED is just for faster committing.\n# Feel free to remove it.\nif TRAIN_ENABLED:\n    for epoch in range(N_EPOCHS):\n\n        print('Epoch {}/{}'.format(epoch, N_EPOCHS - 1))\n        print('-' * 10)\n\n        model.train()\n        tr_loss = 0\n\n        tk0 = tqdm(data_loader_train, desc=\"Iteration\")\n\n        for step, batch in enumerate(tk0):\n            inputs = batch[\"image\"]\n            l_graph = batch[\"l_graph\"]\n            l_vowel = batch[\"l_vowel\"]\n            l_conso = batch[\"l_conso\"]\n\n            inputs = inputs.to(device, dtype=torch.float)\n            l_graph = l_graph.to(device, dtype=torch.float)\n            l_vowel = l_vowel.to(device, dtype=torch.float)\n            l_conso = l_conso.to(device, dtype=torch.float)\n\n            out_graph, out_vowel, out_conso = model(inputs)\n\n            loss_graph = criterion(out_graph, l_graph)\n            loss_vowel = criterion(out_vowel, l_vowel)\n            loss_conso = criterion(out_conso, l_conso)\n\n            loss = loss_graph + loss_vowel + loss_conso\n            loss.backward()\n\n            tr_loss += loss.item()\n\n            optimizer.step()\n            optimizer.zero_grad()\n\n        epoch_loss = tr_loss / len(data_loader_train)\n        print('Training Loss: {:.4f}'.format(epoch_loss))\n\ntorch.save(model.state_dict(), './baseline_weights.pth')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---------------\n**Thanks for reading. If you find this notebook useful, please vote**"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}