{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom  torch.cuda.amp import autocast, GradScaler\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\nimport timm #from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom\nfrom sklearn.metrics import log_loss","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 324,\n    'model_arch': 'SE-Net',#'tf_efficientnet_b4_ns', #'seresnext50_32x4d',\n    'img_size': 28,\n    'epochs': 10,\n    'train_bs': 16,\n    'valid_bs': 32,\n    'T_0': 10,\n    'lr': 1e-2,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_workers': 2,\n    'accum_iter': 4, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'tta': 15,\n    'used_epochs': [8],\n    'weights': [1,1,1,1,1]\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/Kannada-MNIST/sample_submission.csv')\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\ndef show_batch_imgs(examples, df, transforms=None):\n    imgs = df.sample(frac=1)[:examples]\n    plt.figure(figsize=(8,8))\n    for i in range(examples):\n        img = imgs.iloc[i][1:].values.astype(np.uint8).reshape((28,28))\n        if transforms:\n            img = transforms(image=img)['image']\n            img = img.numpy().reshape((28,28))\n        # print(img.shape)\n        plt.subplot(1, examples, i%examples+1)\n        plt.axis('off')\n        plt.imshow(img)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class KMnistDataset(Dataset):\n    def __init__(self, data, transforms=None, do_fmix=None, fmix_params=None, do_cutmix=None, cutmix_params={'alpha':1}):\n        self.data = data\n        self.transforms = transforms\n        self.do_fmix = do_fmix\n        self.fmix_params = fmix_params\n        self.do_cutmix = do_cutmix\n        self.cutmix_params = cutmix_params\n  \n    def __len__(self):\n        return len(self.data)\n  \n    def __getitem__(self, index):\n        # get images\n        img = self.data.iloc[index, 1:].values.astype(np.float32).reshape((28,28))\n\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n\n        # if self.do_cutmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n\n        # if self.do_fmix and np.random.uniform(0., 1., size=1)[0] > 0.5:\n\n        return img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Train\\Validation Image Augmentations","metadata":{}},{"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            # RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            # Transpose(p=0.5), # not good and make confuse\n            # HorizontalFlip(p=0.5), # make confuse\n            # VerticalFlip(p=0.5), # should be avioded\n            ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=30, p=0.5), # should control the rotate limit\n            # RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5), # not help in grayscale\n            CoarseDropout(p=0.1),\n            Cutout(p=0.1),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            # CenterCrop(CFG['img_size'], CFG['img_size'], p=1.), # may lose parts of the character\n            # Resize(CFG['img_size'], CFG['img_size']),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\ndef get_inference_transforms():\n    return Compose([\n            # CenterCrop(CFG['img_size'], CFG['img_size'], p=1.), # may lose parts of the character\n            # Resize(CFG['img_size'], CFG['img_size']),\n            ToTensorV2(p=1.0),\n        ], p=1.)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class Sq_Ex_Block(nn.Module):\n    def __init__(self, in_ch, r):\n        super(Sq_Ex_Block, self).__init__()\n        self.se = nn.Sequential(\n            GlobalAvgPool(),\n            nn.Linear(in_ch, in_ch//r),\n            nn.ReLU(inplace=True),\n            nn.Linear(in_ch//r, in_ch),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        se_weight = self.se(x).unsqueeze(-1).unsqueeze(-1)\n        return x.mul(se_weight)\n\nclass GlobalAvgPool(nn.Module):\n    def __init__(self):\n        super(GlobalAvgPool, self).__init__()\n    def forward(self, x):\n        return x.view(*(x.shape[:-2]),-1).mean(-1)\n\nclass SE_Net(nn.Module):\n    def __init__(self,in_channels):\n        super(SE_Net,self).__init__()\n        #torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride=1, padding=0, \n        #                dilation=1, groups=1, bias=True, padding_mode='zeros')\n        self.c1 = nn.Conv2d(in_channels=in_channels, out_channels=64,kernel_size=3,stride=1,padding=0)\n        self.bn1 = nn.BatchNorm2d(num_features=64,eps=1e-3,momentum=0.01)\n        self.c2 = nn.Conv2d(64,64,3,1,0)\n        self.bn2 = nn.BatchNorm2d(64,1e-3,0.01)\n        self.c3 = nn.Conv2d(64,64,5,1,2)\n        self.bn3 = nn.BatchNorm2d(64,1e-3,0.01)\n        self.m1 = nn.MaxPool2d(2)\n        self.d1 = nn.Dropout(0.4)\n        \n        self.c4 = nn.Conv2d(64,128,3,1,0)\n        self.bn4 = nn.BatchNorm2d(128,1e-3,0.01)\n        self.c5 = nn.Conv2d(128,128,3,1,0)\n        self.bn5 = nn.BatchNorm2d(128,1e-3,0.01)\n        self.c6 = nn.Conv2d(128,128,5,1,2)\n        self.bn6 = nn.BatchNorm2d(128,1e-3,0.01)\n        self.m2 = nn.MaxPool2d(2)\n        self.d2 = nn.Dropout(0.4)\n        \n        self.c7 = nn.Conv2d(128,256,3,1,0)\n        self.bn7 = nn.BatchNorm2d(256,1e-3,0.01)\n        self.se3 = Sq_Ex_Block(in_ch=256,r=8)\n        self.m3 = nn.MaxPool2d(2)\n        self.d3 = nn.Dropout(0.4)\n\n        self.fc1 = nn.Linear(256*1*1,256)\n        self.bn8 = nn.BatchNorm1d(256,1e-3,0.01)\n        \n        self.out = nn.Linear(256,10)\n        \n        self.init_linear_weights()\n        \n    def forward(self,x):\n        x = self.bn1(F.leaky_relu(self.c1(x),0.1))\n        x = self.bn2(F.leaky_relu(self.c2(x),0.1))\n        x = self.bn3(F.leaky_relu(self.c3(x),0.1))\n        x = self.d1(self.m1(x))\n        \n        x = self.bn4(F.leaky_relu(self.c4(x),0.1))\n        x = self.bn5(F.leaky_relu(self.c5(x),0.1))\n        x = self.bn6(F.leaky_relu(self.c6(x),0.1))\n        x = self.d2(self.m2(x))\n        \n        x = self.bn7(F.leaky_relu(self.c7(x),0.1))\n        x = self.se3(x)\n        x = self.d3(self.m3(x))\n        \n        x = x.view(-1, 256*1*1) #reshape\n        x = self.bn8(F.leaky_relu(self.fc1(x),0.1))\n        return self.out(x)\n    \n    def init_linear_weights(self):\n        nn.init.kaiming_normal_(self.fc1.weight, mode='fan_in')  #default mode: fan_in\n        nn.init.kaiming_normal_(self.out.weight, mode='fan_in')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main Loop","metadata":{}},{"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)   #output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(CFG['seed'])\n\ntest_data = pd.read_csv(\"../input/Kannada-MNIST/test.csv\")\n\ntest_ds = KMnistDataset(test_data, transforms=get_inference_transforms())\n\ntst_preds = []\n\ndevice = torch.device(CFG['device'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_paths = ['../input/eesm5720/SE-Net_fold_0_9', \n               '../input/eesm5720/SE-Net_fold_1_9',\n               '../input/eesm5720/SE-Net_fold_2_9',\n               '../input/eesm5720/SE-Net_fold_3_9',\n               '../input/eesm5720/SE-Net_fold_4_9',]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for o, model_path in enumerate(model_paths):\n    model = SE_Net(in_channels=1).to(device)\n\n    tst_loader = torch.utils.data.DataLoader(\n        test_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=False,\n    )\n    \n    print(\"using the model from {}\".format(model_path))\n    \n    model.load_state_dict(torch.load(model_path))\n\n    with torch.no_grad():\n        for _ in range(CFG['tta']):\n            tst_preds += [1/CFG['tta']*inference_one_epoch(model, tst_loader, device)]\n\ntst_preds = np.mean(tst_preds, axis=0) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data['label'] = np.argmax(tst_preds, axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = pd.DataFrame(test_data, columns=['id','label'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For the training model, please refer to [EESM 5720 SE-Net baseline for Kannada MNIST](https://www.kaggle.com/dongjai04/se-net-baseline-eesm5720-train)\n\nThank you for the [notebook](https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-inference-tta) from [khyeh0719](https://www.kaggle.com/khyeh0719) and the [notebook](https://www.kaggle.com/ccchang801023/se-net-my-top1-baseline-model-with-pytorch) from [ccchang801023](https://www.kaggle.com/ccchang801023).","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}