{"cells":[{"metadata":{},"cell_type":"markdown","source":"**Simple PyTorch DenseNet161 Training + Infernece with no Augmentation and 10 epoch only Private LB 0.9334**\nThings i did:\n* Used pretrained model by cadene with imagenet weight\n* no augmentation , simple training with 224*224*1 image\n* used batch size 32, larger batch caused gpu to run out of memory\n* my score is based on only 10 epochs, each takes 50 minutes"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null # no output","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport os\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torchvision import transforms,models\nfrom tqdm import tqdm_notebook as tqdm\nimport math\nimport torch.nn.functional as F\nfrom torch.nn import init\nimport gc\nimport cv2\nimport pretrainedmodels\nimport torchvision","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"HEIGHT = 137\nWIDTH = 236\nTRAIN = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_image(dataType = 'train', indices = [0,1,2,3]):\n    assert dataType in ['train', 'test']\n    HEIGHT = 137\n    WIDTH = 236\n    images = []\n    for i in indices:\n        image_df = pd.read_parquet(f'../input/bengaliai-cv19/{dataType}_image_data_{i}.parquet') \n        images.append(image_df.iloc[:,1:].values.reshape(-1,HEIGHT,WIDTH))\n        del image_df\n        gc.collect()\n    \n    \n    images = np.concatenate(images, axis = 0)\n    return images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if TRAIN:\n    train = pd.read_csv('/kaggle/input/bengaliai-cv19/train.csv')\n    train_labels = train[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].values\n    train_images = prepare_image()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def bbox(img):\n    rows = np.any(img, axis=1)\n    cols = np.any(img, axis=0)\n    rmin, rmax = np.where(rows)[0][[0, -1]]\n    cmin, cmax = np.where(cols)[0][[0, -1]]\n    return rmin, rmax, cmin, cmax","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_resize(img0, size=128, pad=16):\n    #crop a box around pixels large than the threshold \n    #some images contain line at the sides\n    ymin,ymax,xmin,xmax = bbox(img0[5:-5,5:-5] > 80)\n    #cropping may cut too much, so we need to add it back\n    xmin = xmin - 13 if (xmin > 13) else 0\n    ymin = ymin - 10 if (ymin > 10) else 0\n    xmax = xmax + 13 if (xmax < WIDTH - 13) else WIDTH\n    ymax = ymax + 10 if (ymax < HEIGHT - 10) else HEIGHT\n    img = img0[ymin:ymax,xmin:xmax]\n    #remove lo intensity pixels as noise\n    img[img < 28] = 0\n    lx, ly = xmax-xmin,ymax-ymin\n    l = max(lx,ly) + pad\n    #make sure that the aspect ratio is kept in rescaling\n    img = np.pad(img, [((l-ly)//2,), ((l-lx)//2,)], mode='constant')\n    return cv2.resize(img,(size,size))\n    #return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BengaliAIDataset(Dataset):\n    def __init__(self, images, labels=None, indices=None):\n        super(BengaliAIDataset, self).__init__()\n        self.images = images\n        self.labels = labels\n        if indices is None:\n            indices = np.arange(len(images))\n        self.indices = indices\n        self.train = labels is not None\n\n    def __len__(self):\n        \"\"\"return length of this dataset\"\"\"\n        return len(self.indices)\n\n    def __getitem__(self, i):\n        \"\"\"Return i-th data\"\"\"\n        i = self.indices[i]\n        x = self.images[i]\n        # Opposite white and black: background will be white and\n        # for future Affine transformation\n        x = (255 - x).astype(np.float32) \n        x = (x*(255.0/x.max())).astype(np.float32)\n        x = crop_resize(x,224)\n        x = np.stack((x,)*3,axis=-1)\n        x = x * [0.229, 0.224, 0.225]\n        x = np.rollaxis(x, 2, 0)\n        x = x.astype(np.float32)/255.0\n        if self.train:\n            y = self.labels[i]\n            return x, y\n        else:\n            return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def MyModel():\n    model = pretrainedmodels.__dict__['densenet161'](pretrained=None)\n    model.last_linear = nn.Linear(model.last_linear.in_features, 186)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device('cuda:0')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = MyModel().to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if TRAIN:\n    optimizer = torch.optim.Adam(model.parameters(), lr=4e-4)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=2, factor=0.3)\n    criterion = nn.CrossEntropyLoss()\n    batch_size=32\n    num_images = len(train_images)\n    train_data_size = int(num_images*0.9)\n    test_data_size = num_images - train_data_size\n    perm = np.random.RandomState(111).permutation(num_images)\n    train_dataset = BengaliAIDataset(   train_images, train_labels, indices=perm[:train_data_size]   )\n    valid_dataset = BengaliAIDataset(   train_images, train_labels, indices=perm[train_data_size:train_data_size+test_data_size]   )\n    train_loader = torch.utils.data.DataLoader(train_dataset,batch_size=batch_size,shuffle=True)\n    valid_loader = torch.utils.data.DataLoader(valid_dataset,batch_size=batch_size,shuffle=True)\n    epochs = 10\n    losses=[]\n    accs=[]\n    train_root_accs = []\n    val_root_accs = []\n    train_vowel_accs = []\n    val_vowel_accs = []\n    train_con_accs = []\n    val_con_accs = []\n    train_accs = []\n    val_accs = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not TRAIN:\n    load_model_path = '/kaggle/input/dense161/seresnet50 (3).pth'\n    checkpoint = torch.load(load_model_path)\n    model.load_state_dict(checkpoint['model_state_dict'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if TRAIN:\n    for epoch in range(epochs):\n\n        gc.collect()\n        print('epochs {}/{} '.format(epoch+1,epochs))\n        running_loss = 0.0\n        root_acc = 0.0\n        vowel_acc = 0.0\n        con_acc = 0.0\n        model.train()\n        for idx, (inputs,labels) in tqdm(enumerate(train_loader),total=len(train_loader)):\n\n            inputs = inputs.to(device)\n            labels = labels.to(device)\n            output = model(inputs.float())\n            outputs = torch.split(output, [168, 11, 7], dim=1)\n            loss1 = criterion(outputs[0],labels[:,0])\n            loss2 = criterion(outputs[1],labels[:,1])\n            loss3 = criterion(outputs[2],labels[:,2])    \n            running_loss += (2*loss1 + loss2 + loss3)\n            root_acc += (outputs[0].argmax(1)==labels[:,0]).float().mean()\n            vowel_acc += (outputs[1].argmax(1)==labels[:,1]).float().mean()\n            con_acc += (outputs[2].argmax(1)==labels[:,2]).float().mean()\n            (2*loss1 + loss2 + loss3).backward()\n            optimizer.step()\n            del inputs\n            del labels\n\n        losses.append(running_loss/len(train_loader))\n        train_root_acc = root_acc/(len(train_loader))\n        train_vowel_acc = vowel_acc/(len(train_loader))\n        train_con_acc = con_acc/(len(train_loader))\n        train_root_accs.append(train_root_acc)\n        train_vowel_accs.append(train_vowel_acc)\n        train_con_accs.append(train_con_acc)\n        act_train = (2*train_root_acc+train_vowel_acc+train_con_acc)/4.0\n        accs.append(act_train)\n\n        print('acc : {:.4f}'.format(act_train))\n        print('root_acc : {:.4f}'.format(train_root_acc))\n        print('vowel_acc : {:.4f}'.format(train_vowel_acc))\n        print('con acc : {:.4f}'.format(train_con_acc))\n        print('loss : {:.4f}'.format(running_loss/len(train_loader)))\n\n\n\n\n        val_loss = 0.0\n        root_acc = 0.0\n        vowel_acc = 0.0\n        con_acc = 0.0\n        model.eval()\n        with torch.no_grad():\n            for idx, (inputs,labels) in tqdm(enumerate(valid_loader),total=len(valid_loader)):\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n                #print(inputs.shape)\n                #print(labels.shape)\n                output = model(inputs.float())\n                outputs = torch.split(output, [168, 11, 7], dim=1)\n                loss11 = criterion(outputs[0],labels[:,0])\n                loss21 = criterion(outputs[1],labels[:,1])\n                loss31 = criterion(outputs[2],labels[:,2])\n                val_loss += (2*loss11 + loss21 + loss31)\n                root_acc += (outputs[0].argmax(1)==labels[:,0]).float().mean()\n                vowel_acc += (outputs[1].argmax(1)==labels[:,1]).float().mean()\n                con_acc += (outputs[2].argmax(1)==labels[:,2]).float().mean()\n                del inputs\n                del labels\n                gc.collect()\n\n        val_root_acc = root_acc/(len(valid_loader))\n        val_vowel_acc = vowel_acc/(len(valid_loader))\n        val_con_acc = con_acc/(len(valid_loader))\n        val_root_accs.append(val_root_acc)\n        val_vowel_accs.append(val_vowel_acc)\n        val_con_accs.append(val_con_acc)\n        act_val = (2*val_root_acc+val_vowel_acc+val_con_acc)/4.0\n        val_loss = val_loss/3.0\n        scheduler.step(val_loss)\n\n        print('val_acc : {:.4f}'.format(act_val))\n        print('root_acc : {:.4f}'.format(val_root_acc))\n        print('vowel_acc : {:.4f}'.format(val_vowel_acc))\n        print('con acc : {:.4f}'.format(val_con_acc))\n        print('loss : {:.4f}'.format(val_loss/len(valid_loader)))\n\n        file_name = str(act_train)+'_'+str(act_val)+'_.tar'\n        torch.save({\n            'model_state_dict':model.state_dict(),\n            'optimizer_state_dict':optimizer.state_dict(),\n\n        },'weight.pth')\n        print()\n        print()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_type = 'test'\ntest_preds_list = []\nfor i in range(4):\n    gc.collect()\n    indices = [i]\n    test_images = prepare_image(data_type,indices=indices)\n    n_dataset = len(test_images)\n    print(f'i={i}, n_dataset={n_dataset}')\n    # test_data_size = 200 if debug else int(n_dataset * 0.9)\n    test_dataset = BengaliAIDataset(test_images )\n    print('test_dataset', len(test_dataset))\n    test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=16, shuffle=False)\n    for  idx, inputs in enumerate(test_loader):\n        inputs = inputs.to(device)\n        output = model(inputs.float())\n        outputs = torch.split(output, [168, 11, 7], dim=1)\n        outputs = outputs[0].argmax(1), outputs[1].argmax(1), outputs[2].argmax(1)\n        test_preds_list.append(outputs)\n    del test_images\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p0 = np.concatenate([test_preds[0].cpu().detach().numpy() for test_preds in test_preds_list], axis=0)\np1 = np.concatenate([test_preds[1].cpu().detach().numpy() for test_preds in test_preds_list], axis=0)\np2 = np.concatenate([test_preds[2].cpu().detach().numpy() for test_preds in test_preds_list], axis=0)\nprint('p0', p0.shape, 'p1', p1.shape, 'p2', p2.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"row_id = []\ntarget = []\nfor i in range(len(p0)):\n    row_id += [f'Test_{i}_consonant_diacritic', f'Test_{i}_grapheme_root',\n               f'Test_{i}_vowel_diacritic']\n    target += [p2[i], p0[i], p1[i]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.DataFrame({'row_id': row_id, 'target': target})\nsubmission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}