{"cells":[{"metadata":{"_cell_guid":"44e0e8dd-3f69-4b09-bbf9-4b61ab211837","_uuid":"6ea60225-2ed8-4464-94ea-426eede4f725","execution":{"iopub.execute_input":"2021-01-26T12:29:21.030518Z","iopub.status.busy":"2021-01-26T12:29:21.029695Z","iopub.status.idle":"2021-01-26T12:29:21.032123Z","shell.execute_reply":"2021-01-26T12:29:21.032765Z"},"papermill":{"duration":0.027472,"end_time":"2021-01-26T12:29:21.032888","exception":false,"start_time":"2021-01-26T12:29:21.005416","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"package_path = '../input/pytorch-image-models/pytorch-image-models-master' #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\nimport sys; sys.path.append(package_path)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5ec73de8-81f5-4331-8a73-4c6ab37e5923","_uuid":"3607d2c4-66c6-4e40-8432-9640aefbc6ed","execution":{"iopub.execute_input":"2021-01-26T12:29:21.084172Z","iopub.status.busy":"2021-01-26T12:29:21.083476Z","iopub.status.idle":"2021-01-26T12:29:24.49087Z","shell.execute_reply":"2021-01-26T12:29:24.489729Z"},"papermill":{"duration":3.43735,"end_time":"2021-01-26T12:29:24.491006","exception":false,"start_time":"2021-01-26T12:29:21.053656","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom  torch.cuda.amp import autocast, GradScaler\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\nimport timm #from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom\nfrom sklearn.metrics import log_loss","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"2dfd253a-3417-4db4-90e3-0e81f9416617","_uuid":"7c0aa19c-8715-4c3c-8959-d8c1f5d06700","execution":{"iopub.execute_input":"2021-01-26T12:29:24.539908Z","iopub.status.busy":"2021-01-26T12:29:24.537983Z","iopub.status.idle":"2021-01-26T12:29:24.540644Z","shell.execute_reply":"2021-01-26T12:29:24.541109Z"},"papermill":{"duration":0.029912,"end_time":"2021-01-26T12:29:24.541226","exception":false,"start_time":"2021-01-26T12:29:24.511314","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 10,\n    'seed': 719,\n    'model_arch': 'tf_efficientnet_b3_ns',\n    'img_size': 512,\n    'epochs': 25,\n    'train_bs': 32,\n    'valid_bs': 32,\n    'lr': 1e-4,\n    'num_workers': 4,\n    'accum_iter': 1, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'tta': 3,\n    'used_epochs': [6,7,8,9],\n    'weights': [1,1,1,1]\n}","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"61aa057d-14f4-4b45-b934-79c00bd77c95","_uuid":"87a6b7b3-e2cd-420a-9f2e-bd1feb6ad934","execution":{"iopub.execute_input":"2021-01-26T12:29:24.588367Z","iopub.status.busy":"2021-01-26T12:29:24.587752Z","iopub.status.idle":"2021-01-26T12:29:24.632883Z","shell.execute_reply":"2021-01-26T12:29:24.631873Z"},"papermill":{"duration":0.071336,"end_time":"2021-01-26T12:29:24.632995","exception":false,"start_time":"2021-01-26T12:29:24.561659","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b70744ae-b98c-40d8-9d0c-a0bc86a3ff66","_uuid":"465e1674-75f5-4d79-adbf-2814556a90f4","execution":{"iopub.execute_input":"2021-01-26T12:29:24.679182Z","iopub.status.busy":"2021-01-26T12:29:24.678607Z","iopub.status.idle":"2021-01-26T12:29:24.687525Z","shell.execute_reply":"2021-01-26T12:29:24.68797Z"},"papermill":{"duration":0.033966,"end_time":"2021-01-26T12:29:24.688093","exception":false,"start_time":"2021-01-26T12:29:24.654127","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6646e1d7-882a-43ce-8f54-3fd43d320630","_uuid":"c7dcb74a-ade9-4223-aae8-14c431012f77","papermill":{"duration":0.021499,"end_time":"2021-01-26T12:29:24.730639","exception":false,"start_time":"2021-01-26T12:29:24.70914","status":"completed"},"tags":[]},"cell_type":"markdown","source":"> We could do stratified validation split in each fold to make each fold's train and validation set looks like the whole train set in target distributions."},{"metadata":{"_cell_guid":"ba5f5e45-051b-4294-994a-cb7736452696","_uuid":"a799e950-6ccc-42f2-8729-352289259641","execution":{"iopub.execute_input":"2021-01-26T12:29:24.778127Z","iopub.status.busy":"2021-01-26T12:29:24.77754Z","iopub.status.idle":"2021-01-26T12:29:24.790583Z","shell.execute_reply":"2021-01-26T12:29:24.791152Z"},"papermill":{"duration":0.039575,"end_time":"2021-01-26T12:29:24.791264","exception":false,"start_time":"2021-01-26T12:29:24.751689","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"132b91c5-dde6-49c1-b2a0-6035ac923dc7","_uuid":"be34b35c-a1f0-4af4-9a2f-03552381b87e","papermill":{"duration":0.021832,"end_time":"2021-01-26T12:29:24.835688","exception":false,"start_time":"2021-01-26T12:29:24.813856","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Helper Functions"},{"metadata":{"_cell_guid":"46bd2623-1b6a-48c1-8788-87e28ce52802","_uuid":"ede84a90-2726-4515-9215-e357861b7b19","execution":{"iopub.execute_input":"2021-01-26T12:29:24.887393Z","iopub.status.busy":"2021-01-26T12:29:24.886712Z","iopub.status.idle":"2021-01-26T12:29:25.173193Z","shell.execute_reply":"2021-01-26T12:29:25.173683Z"},"papermill":{"duration":0.316153,"end_time":"2021-01-26T12:29:25.173815","exception":false,"start_time":"2021-01-26T12:29:24.857662","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nplt.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"625f4195-6334-481d-9421-1ae91e0619b0","_uuid":"171be0f4-8cd6-4a0c-a5df-dddaa5a3b13c","papermill":{"duration":0.026225,"end_time":"2021-01-26T12:29:25.226664","exception":false,"start_time":"2021-01-26T12:29:25.200439","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"_cell_guid":"d0971419-2664-4fa7-b07b-56313828a953","_uuid":"4ece1b4d-4050-4053-9ea6-b857d2fa420f","execution":{"iopub.execute_input":"2021-01-26T12:29:25.290431Z","iopub.status.busy":"2021-01-26T12:29:25.289526Z","iopub.status.idle":"2021-01-26T12:29:25.297779Z","shell.execute_reply":"2021-01-26T12:29:25.298595Z"},"papermill":{"duration":0.045939,"end_time":"2021-01-26T12:29:25.298769","exception":false,"start_time":"2021-01-26T12:29:25.25283","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(\n        self, df, data_root, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        \n        img  = get_img(path)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"44c3d0b5-10ef-4204-9b65-9a0dbc6c86d1","_uuid":"01ec5b7a-aa4f-45a2-8e00-d33e07f905db","papermill":{"duration":0.041391,"end_time":"2021-01-26T12:29:25.381657","exception":false,"start_time":"2021-01-26T12:29:25.340266","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Define Train\\Validation Image Augmentations"},{"metadata":{"_cell_guid":"881eddac-48cd-4e94-9137-edf7b39a1f08","_uuid":"de12a378-4de0-4fa1-8e68-82461b2b3920","execution":{"iopub.execute_input":"2021-01-26T12:29:25.478531Z","iopub.status.busy":"2021-01-26T12:29:25.477541Z","iopub.status.idle":"2021-01-26T12:29:26.285846Z","shell.execute_reply":"2021-01-26T12:29:26.287046Z"},"papermill":{"duration":0.867591,"end_time":"2021-01-26T12:29:26.287221","exception":false,"start_time":"2021-01-26T12:29:25.41963","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\ndef get_inference_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ee14ecdd-ef35-40ce-960a-45c85b1c885b","_uuid":"d4bd4ce4-6636-4e2f-ae48-7768f3e4cbc2","papermill":{"duration":0.03982,"end_time":"2021-01-26T12:29:26.366674","exception":false,"start_time":"2021-01-26T12:29:26.326854","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Model"},{"metadata":{"_cell_guid":"6ddbd2ec-d85d-45bd-ab12-5397f3bd0e6d","_uuid":"7f2b7c09-92f9-49b5-b400-7af3fe89bd60","execution":{"iopub.execute_input":"2021-01-26T12:29:26.469426Z","iopub.status.busy":"2021-01-26T12:29:26.468357Z","iopub.status.idle":"2021-01-26T12:29:26.473304Z","shell.execute_reply":"2021-01-26T12:29:26.475879Z"},"papermill":{"duration":0.069649,"end_time":"2021-01-26T12:29:26.476078","exception":false,"start_time":"2021-01-26T12:29:26.406429","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c7fd7378-0478-4f47-9d3c-a547bb7c4f01","_uuid":"ce0a875d-d570-4408-8dd0-cfeac755a89b","papermill":{"duration":0.026729,"end_time":"2021-01-26T12:29:26.536786","exception":false,"start_time":"2021-01-26T12:29:26.510057","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Main Loop"},{"metadata":{"_cell_guid":"51830ef0-50d2-4d14-8123-e2717d18b59f","_uuid":"e40787da-b43f-4e15-bf16-a968bc661c25","execution":{"iopub.execute_input":"2021-01-26T12:29:26.601691Z","iopub.status.busy":"2021-01-26T12:29:26.599862Z","iopub.status.idle":"2021-01-26T12:29:26.602442Z","shell.execute_reply":"2021-01-26T12:29:26.602922Z"},"papermill":{"duration":0.038266,"end_time":"2021-01-26T12:29:26.603036","exception":false,"start_time":"2021-01-26T12:29:26.56477","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)   #output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c610e286-a881-49dd-8ac6-fa6ff86fd1e9","_uuid":"c006ba82-6d0e-40d5-b5e8-290aa36414f3","execution":{"iopub.execute_input":"2021-01-26T12:29:26.682989Z","iopub.status.busy":"2021-01-26T12:29:26.682119Z","iopub.status.idle":"2021-01-26T12:29:40.301202Z","shell.execute_reply":"2021-01-26T12:29:40.302034Z"},"papermill":{"duration":13.669203,"end_time":"2021-01-26T12:29:40.302238","exception":false,"start_time":"2021-01-26T12:29:26.633035","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        # we'll train fold 0 first\n        if fold > 0:\n            break \n\n        print('Inference fold {} started'.format(fold))\n\n        valid_ = train.loc[val_idx,:].reset_index(drop=True)\n        valid_ds = CassavaDataset(valid_, '../input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms(), output_label=False)\n        \n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms(), output_label=False)\n        \n        val_loader = torch.utils.data.DataLoader(\n            valid_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n        \n        tst_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n\n        device = torch.device(CFG['device'])\n        model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique()).to(device)\n        \n        val_preds = []\n        tst_preds = []\n        \n        #for epoch in range(CFG['epochs']-3):\n        for i, epoch in enumerate(CFG['used_epochs']):    \n            model.load_state_dict(torch.load('../input/cassava-eff-results-1/eff_b3_without_snap_32_5_10/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))\n            \n            with torch.no_grad():\n                for _ in range(CFG['tta']):\n                    #val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]\n                    tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]\n\n        #val_preds = np.mean(val_preds, axis=0) \n        tst_preds = np.sum(tst_preds, axis=0) \n        \n        #print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))\n        #print('fold {} validation accuracy = {:.5f}'.format(fold, (valid_.label.values==np.argmax(val_preds, axis=1)).mean()))\n        \n        del model\n        torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"44fe5f4d-9518-4a2b-804f-39c3243c4555","_uuid":"609df7b0-d3a5-490b-82ca-44c1af93def8","execution":{"iopub.execute_input":"2021-01-26T12:29:40.389639Z","iopub.status.busy":"2021-01-26T12:29:40.388779Z","iopub.status.idle":"2021-01-26T12:29:40.392666Z","shell.execute_reply":"2021-01-26T12:29:40.392153Z"},"papermill":{"duration":0.048579,"end_time":"2021-01-26T12:29:40.39277","exception":false,"start_time":"2021-01-26T12:29:40.344191","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"'''test['label'] = np.argmax(tst_preds, axis=1)\ntest.head()'''","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-26T12:29:40.479424Z","iopub.status.busy":"2021-01-26T12:29:40.478619Z","iopub.status.idle":"2021-01-26T12:29:40.481695Z","shell.execute_reply":"2021-01-26T12:29:40.481117Z"},"papermill":{"duration":0.047104,"end_time":"2021-01-26T12:29:40.481797","exception":false,"start_time":"2021-01-26T12:29:40.434693","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"eff_preds = tst_preds","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"243a7d06-5afe-4c66-be9c-603ccf5ec3c0","_uuid":"2c5bff1e-8ea8-4d78-8876-64c38be180f4","execution":{"iopub.execute_input":"2021-01-26T12:29:40.568025Z","iopub.status.busy":"2021-01-26T12:29:40.567272Z","iopub.status.idle":"2021-01-26T12:29:40.56971Z","shell.execute_reply":"2021-01-26T12:29:40.570281Z"},"papermill":{"duration":0.047406,"end_time":"2021-01-26T12:29:40.570409","exception":false,"start_time":"2021-01-26T12:29:40.523003","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"package_path = '../input/pytorch-image-models/pytorch-image-models-master' #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\nimport sys; sys.path.append(package_path)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7353e1f6-dd73-48db-a43b-7af43594a037","_uuid":"9bb7df35-ee67-4933-a0e5-1da42a6c8feb","execution":{"iopub.execute_input":"2021-01-26T12:29:40.668732Z","iopub.status.busy":"2021-01-26T12:29:40.666644Z","iopub.status.idle":"2021-01-26T12:29:40.669498Z","shell.execute_reply":"2021-01-26T12:29:40.670044Z"},"papermill":{"duration":0.059083,"end_time":"2021-01-26T12:29:40.670182","exception":false,"start_time":"2021-01-26T12:29:40.611099","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 719,\n    'model_arch': 'vit_base_patch16_384',\n    'img_size': 384,\n    'epochs': 10,\n    'train_bs': 32,\n    'valid_bs': 32,\n    'lr': 1e-4,\n    'num_workers': 4,\n    'accum_iter': 1, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'tta': 3,\n    'used_epochs': [6,7,8,9],\n    'weights': [1,1,1,1]\n}","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"54fe56fb-985e-431b-9d01-a6282daad0a2","_uuid":"d4628a00-6904-4d5f-8cfb-ea7c052064b5","execution":{"iopub.execute_input":"2021-01-26T12:29:40.770131Z","iopub.status.busy":"2021-01-26T12:29:40.769542Z","iopub.status.idle":"2021-01-26T12:29:40.775878Z","shell.execute_reply":"2021-01-26T12:29:40.775127Z"},"papermill":{"duration":0.060272,"end_time":"2021-01-26T12:29:40.775985","exception":false,"start_time":"2021-01-26T12:29:40.715713","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"bcb2d7fb-75f1-4fa2-bb17-d5599b575cfe","_uuid":"e82212ef-e558-474f-ba89-fcfdd107e4f3","execution":{"iopub.execute_input":"2021-01-26T12:29:40.868033Z","iopub.status.busy":"2021-01-26T12:29:40.867168Z","iopub.status.idle":"2021-01-26T12:29:40.871372Z","shell.execute_reply":"2021-01-26T12:29:40.870854Z"},"papermill":{"duration":0.052768,"end_time":"2021-01-26T12:29:40.871473","exception":false,"start_time":"2021-01-26T12:29:40.818705","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        MODEL_PATH = \"../input/vit-base-models-pretrained-pytorch/jx_vit_base_p16_384-83fb41ba.pth\"\n        self.model = timm.create_model(\"vit_base_patch16_384\", pretrained=False)\n        \n        self.model.load_state_dict(torch.load(MODEL_PATH))\n\n        self.model.head = nn.Linear(self.model.head.in_features, 5)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"72bfa458-c33a-4031-b184-b48e3bbef24b","_uuid":"6e7f3af3-ae45-4794-8114-abc860bc4672","execution":{"iopub.execute_input":"2021-01-26T12:29:40.962747Z","iopub.status.busy":"2021-01-26T12:29:40.962094Z","iopub.status.idle":"2021-01-26T12:29:40.966007Z","shell.execute_reply":"2021-01-26T12:29:40.965447Z"},"papermill":{"duration":0.053087,"end_time":"2021-01-26T12:29:40.966105","exception":false,"start_time":"2021-01-26T12:29:40.913018","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)   #output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8026c939-1ad8-40ff-8316-1dec66a2b739","_uuid":"2ac914e6-8bcf-4998-9fec-e9d1a1613601","execution":{"iopub.execute_input":"2021-01-26T12:29:41.069611Z","iopub.status.busy":"2021-01-26T12:29:41.068723Z","iopub.status.idle":"2021-01-26T12:30:05.676078Z","shell.execute_reply":"2021-01-26T12:30:05.675309Z"},"papermill":{"duration":24.66718,"end_time":"2021-01-26T12:30:05.676226","exception":false,"start_time":"2021-01-26T12:29:41.009046","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        # we'll train fold 0 first\n        if fold ==0 or fold==1:\n            print('Inference fold {} started'.format(fold))\n\n            valid_ = train.loc[val_idx,:].reset_index(drop=True)\n            valid_ds = CassavaDataset(valid_, '../input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms(), output_label=False)\n\n            test = pd.DataFrame()\n            test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n            test_ds = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms(), output_label=False)\n\n            val_loader = torch.utils.data.DataLoader(\n                valid_ds, \n                batch_size=CFG['valid_bs'],\n                num_workers=CFG['num_workers'],\n                shuffle=False,\n                pin_memory=False,\n            )\n\n            tst_loader = torch.utils.data.DataLoader(\n                test_ds, \n                batch_size=CFG['valid_bs'],\n                num_workers=CFG['num_workers'],\n                shuffle=False,\n                pin_memory=False,\n            )\n\n            device = torch.device(CFG['device'])\n            model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique()).to(device)\n\n            val_preds = []\n            tst_preds = []\n\n            #for epoch in range(CFG['epochs']-3):\n            for i, epoch in enumerate(CFG['used_epochs']):    \n                model.load_state_dict(torch.load('../input/cassava-eff-results-1/vision_trans_64_4_10/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))\n\n                with torch.no_grad():\n                    for _ in range(CFG['tta']):\n                        #val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]\n                        tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]\n\n            #val_preds = np.mean(val_preds, axis=0) \n            tst_preds = np.sum(tst_preds, axis=0) \n\n            #print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))\n            #print('fold {} validation accuracy = {:.5f}'.format(fold, (valid_.label.values==np.argmax(val_preds, axis=1)).mean()))\n\n            del model\n            torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vit_preds=tst_preds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# resnext"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Directory settings\n# ====================================================\nimport os\n\nOUTPUT_DIR = './'\nMODEL_DIR = '../input/cassava-eff-results-1/resnext_snap_inc_32_5_10/'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\nTRAIN_PATH = '../input/cassava-leaf-disease-classification/train_images'\nTEST_PATH = '../input/cassava-leaf-disease-classification/test_images'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# CFG\n# ====================================================\nclass CFG:\n    debug=False\n    num_workers=4\n    model_name='resnext50_32x4d'\n    size=512\n    batch_size=16\n    seed=42\n    target_size=5\n    target_col='label'\n    n_fold=5\n    trn_fold=[0, 1, 2, 3, 4]\n    train=False\n    inference=True\n    tta=3\n    used_epochs= [8,9,10,11,12]\n    weights= [1,1,1,1,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Library\n# ====================================================\nimport sys\nsys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n\nimport os\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nfrom albumentations import *\n# (\n#     Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip, \n#     RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout, \n#     IAAAdditiveGaussianNoise, Transpose\n#     )\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport timm\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n\ndef init_logger(log_file=OUTPUT_DIR+'inference.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\n#LOGGER = init_logger()\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntest = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # ====================================================\n# # Dataset\n# # ====================================================\n# class TestDataset(Dataset):\n#     def __init__(self, df, transform=None):\n#         self.df = df\n#         self.file_names = df['image_id'].values\n#         self.transform = transform\n        \n#     def __len__(self):\n#         return len(self.df)\n\n#     def __getitem__(self, idx):\n#         file_name = self.file_names[idx]\n#         file_path = f'{TEST_PATH}/{file_name}'\n#         image = cv2.imread(file_path)\n#         print(file_path)\n#         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n#         if self.transform:\n#             augmented = self.transform(image=image)\n#             image = augmented['image']\n#         return image\nclass CassavaDataset(Dataset):\n    def __init__(\n        self, df, data_root, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        \n        img  = get_img(path)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Transforms\n# ====================================================\ndef get_transforms(*, data):\n    \n    if data == 'train':\n        return Compose([\n            #Resize(CFG.size, CFG.size),\n            RandomResizedCrop(CFG.size, CFG.size),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\n\n    elif data == 'valid':\n        return Compose([\n            Resize(CFG.size, CFG.size),\n            Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225],\n            ),\n            ToTensorV2(),\n        ])\ndef get_inference_transforms():\n    return Compose([\n            RandomResizedCrop(CFG.size, CFG.size),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomResNext(nn.Module):\n    def __init__(self, model_name='resnext50_32x4d', pretrained=False):\n        super().__init__()\n        backbone = timm.create_model(model_name, pretrained=pretrained)\n        n_features = backbone.fc.in_features\n        self.model = nn.Sequential(*backbone.children())[:-2]\n        self.classifier = nn.Linear(n_features, CFG.target_size)\n        self.pool = nn.AdaptiveAvgPool2d((1, 1))\n        \n    def forward_features(self, x):\n        x = self.model(x)\n        return x\n\n    def forward(self, x):\n        feats = self.forward_features(x)\n        x = self.pool(feats).view(x.size(0), -1)\n        x = self.classifier(x)\n        return x, feats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef inference(model, states, test_loader, device):\n    model.to(device)\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state['model'])\n            model.eval()\n            with torch.no_grad():\n                #y_preds = model(images)\n                y_preds,_ = model(images) #snapmix\n            avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs\ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        #image_preds = model(imgs)\n        image_preds, _ = model(imgs)   #for snapmix inference\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG.seed)\n    \n    folds = StratifiedKFold(n_splits=CFG.n_fold).split(np.arange(train.shape[0]), train.label.values)\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        # we'll train fold 0 first\n        if fold > 0:\n            break \n\n        print('Inference fold {} started'.format(fold))\n\n        valid_ = train.loc[val_idx,:].reset_index(drop=True)\n        valid_ds = CassavaDataset(valid_, '../input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms(), output_label=False)\n        \n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms(), output_label=False)\n        \n        \n        val_loader = torch.utils.data.DataLoader(\n            valid_ds, \n            batch_size=CFG.batch_size,\n            num_workers=CFG.num_workers,\n            shuffle=False,\n            pin_memory=False,\n        )\n        \n        tst_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size=CFG.batch_size,\n            num_workers=CFG.num_workers,\n            shuffle=False,\n            pin_memory=False,\n        )\n\n        device = torch.device('cuda:0')\n        model = CustomResNext(CFG.model_name, pretrained=False).to(device)\n        \n        val_preds = []\n        resv1tta = []\n        \n        #for epoch in range(CFG['epochs']-3):\n        for i, epoch in enumerate(CFG.used_epochs):    \n            model.load_state_dict(torch.load(MODEL_DIR+f'{CFG.model_name}_fold{i}_best.pth')['model'])\n            \n            with torch.no_grad():\n                for _ in range(CFG.tta):\n                    #val_preds += [CFG.weights[i]/sum(CFG.weights)/CFG.tta*inference_one_epoch(model, val_loader, device)]\n                    resv1tta += [CFG.weights[i]/sum(CFG.weights)/CFG.tta*inference_one_epoch(model, tst_loader, device)]\n\n        #val_preds = np.mean(val_preds, axis=0) \n        resv1tta = np.sum(resv1tta, axis=0) \n        \n        #print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))\n        #print('fold {} validation accuracy = {:.5f}'.format(fold, (valid_.label.values==np.argmax(val_preds, axis=1)).mean()))\n        \n        del model\n        torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\nclass TestDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TEST_PATH}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        return image\n    \n# ====================================================\n# Helper functions\n# ====================================================\ndef inference(model, states, test_loader, device):\n    model.to(device)\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state['model'])\n            model.eval()\n            with torch.no_grad():\n                #y_preds = model(images)\n                y_preds, _ = model(images) #for snapmix\n            avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs\n\n# ====================================================\n# inference\n# ====================================================\nmodel = CustomResNext(CFG.model_name, pretrained=False)\nstates = [torch.load(MODEL_DIR+f'{CFG.model_name}_fold{fold}_best.pth') for fold in range(5)]\ntest_dataset = TestDataset(test, transform=get_transforms(data='valid'))\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, \n                         num_workers=CFG.num_workers, pin_memory=True)\nresv1notta = inference(model, states, test_loader, device)\n# submission\n# test['label'] = predictions.argmax(1)\n# test[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)\n# test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL_DIR = '../input/cassava-resnext50-results-1/resnext50_master_512_32_5_10/'\n# ====================================================\n# CFG\n# ====================================================\nclass CFG:\n    debug=False\n    num_workers=4\n    model_name='resnext50_32x4d'\n    size=512\n    batch_size=16\n    seed=42\n    target_size=5\n    target_col='label'\n    n_fold=5\n    trn_fold=[0, 1, 2, 3, 4]\n    train=False\n    inference=True\n    tta=3\n    #used_epochs= [0,1,2,3,4]\n    used_folds = [0,2,3,4]\n    weights= [1,1,1,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ====================================================\n# inference\n# ====================================================\nmodel = CustomResNext(CFG.model_name, pretrained=False)\nstates = [torch.load(MODEL_DIR+f'{CFG.model_name}_fold{fold}_best.pth') for fold in CFG.used_folds]\ntest_dataset = TestDataset(test, transform=get_transforms(data='valid'))\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, \n                         num_workers=CFG.num_workers, pin_memory=True)\nresv2notta = inference(model, states, test_loader, device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res_preds = 0.5*resv1tta + 0.25*resv1notta + 0.25*resv2notta\nres_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 10,\n    'seed': 719,\n    'model_arch': 'tf_efficientnet_b3_ns',\n    'img_size': 512,\n    'epochs': 25,\n    'train_bs': 32,\n    'valid_bs': 32,\n    'lr': 1e-4,\n    'num_workers': 4,\n    'accum_iter': 1, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'tta': 3,\n    'used_epochs': [6,7,8,9],\n    'weights': [1,1,1,1]\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        backbone = timm.create_model(CFG['model_arch'], pretrained=pretrained)\n        n_features = backbone.classifier.in_features  #backbone.classifier.in_features\n        self.model = nn.Sequential(*backbone.children())[:-2]\n        self.classifier = nn.Linear(n_features, n_class)\n        self.pool = nn.AdaptiveAvgPool2d((1, 1))\n\n    def forward_features(self, x):\n        x = self.model(x)\n        return x\n\n    def forward(self, x):\n        feats = self.forward_features(x)\n        x = self.pool(feats).view(x.size(0), -1)\n        x = self.classifier(x)\n        return x, feats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\ndef get_inference_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds,_ = model(imgs)   #output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        # we'll train fold 0 first\n        if fold > 0:\n            break \n\n        print('Inference fold {} started'.format(fold))\n\n        valid_ = train.loc[val_idx,:].reset_index(drop=True)\n        valid_ds = CassavaDataset(valid_, '../input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms(), output_label=False)\n        \n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms(), output_label=False)\n        \n        val_loader = torch.utils.data.DataLoader(\n            valid_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n        \n        tst_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n\n        device = torch.device(CFG['device'])\n        model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique()).to(device)\n        \n        val_preds = []\n        tst_preds = []\n        \n        #for epoch in range(CFG['epochs']-3):\n        for i, epoch in enumerate(CFG['used_epochs']):    \n            model.load_state_dict(torch.load('../input/cassava-eff-results-2/eff_b3_smapmix_512_32_5_10/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))\n            \n            with torch.no_grad():\n                for _ in range(CFG['tta']):\n                    #val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]\n                    tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]\n\n        #val_preds = np.mean(val_preds, axis=0) \n        tst_preds = np.sum(tst_preds, axis=0) \n        \n        #print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))\n        #print('fold {} validation accuracy = {:.5f}'.format(fold, (valid_.label.values==np.argmax(val_preds, axis=1)).mean()))\n        \n        del model\n        torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"snapmix_preds=tst_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 719,\n    'model_arch': 'resnest50d',\n    'img_size': 512,\n    'epochs': 10,\n    'train_bs': 32,\n    'valid_bs': 32,\n    'lr': 1e-4,\n    'num_workers': 4,\n    'accum_iter': 1, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'tta': 5,\n    'used_epochs': [6,7,8,9],\n    'weights': [1,1,1,1]\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()\n\n# %% [code]\ntrain.label.value_counts()\n\n# %% [markdown]\n# > We could do stratified validation split in each fold to make each fold's train and validation set looks like the whole train set in target distributions.\n\n# %% [code]\nsubmission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(\n        self, df, data_root, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        \n        img  = get_img(path)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\ndef get_inference_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\n# %% [markdown]\n# # Model\n\n# %% [code]\nclass CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        backbone = timm.create_model('resnest50d', pretrained=pretrained)\n        n_features = backbone.fc.in_features  #backbone.classifier.in_features\n        self.model = nn.Sequential(*backbone.children())[:-2]\n        self.classifier = nn.Linear(n_features, n_class)\n        self.pool = nn.AdaptiveAvgPool2d((1, 1))\n\n    def forward_features(self, x):\n        x = self.model(x)\n        return x\n\n    def forward(self, x):\n        feats = self.forward_features(x)\n        x = self.pool(feats).view(x.size(0), -1)\n        x = self.classifier(x)\n        return x, feats\n\n# %% [markdown]\n# # Main Loop\n\n# %% [code]\ndef inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds,_ = model(imgs)   #output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all\n\n# %% [code]\nif __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        # we'll train fold 0 first\n        if fold == 1: \n\n            print('Inference fold {} started'.format(fold))\n\n            valid_ = train.loc[val_idx,:].reset_index(drop=True)\n            valid_ds = CassavaDataset(valid_, '../input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms(), output_label=False)\n\n            test = pd.DataFrame()\n            test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n            test_ds = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms(), output_label=False)\n\n            val_loader = torch.utils.data.DataLoader(\n                valid_ds, \n                batch_size=CFG['valid_bs'],\n                num_workers=CFG['num_workers'],\n                shuffle=False,\n                pin_memory=False,\n            )\n\n            tst_loader = torch.utils.data.DataLoader(\n                test_ds, \n                batch_size=CFG['valid_bs'],\n                num_workers=CFG['num_workers'],\n                shuffle=False,\n                pin_memory=False,\n            )\n\n            device = torch.device(CFG['device'])\n            model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique()).to(device)\n\n            #val_preds = []\n            tst_preds = []\n\n            #for epoch in range(CFG['epochs']-3):\n            for i, epoch in enumerate(CFG['used_epochs']):    \n                model.load_state_dict(torch.load('../input/resnest-firstfold-898/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))\n\n                with torch.no_grad():\n                    for _ in range(CFG['tta']):\n                        #val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]\n                        tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]\n\n            #val_preds = np.mean(val_preds, axis=0) \n            tst_preds = np.sum(tst_preds, axis=0) \n\n            #print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))\n            #print('fold {} validation accuracy = {:.5f}'.format(fold, (valid_.label.values==np.argmax(val_preds, axis=1)).mean()))\n\n            del model\n            torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnest_preds = tst_preds","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"179eeb01-f6ac-47c3-b592-cecf40e13e1c","_uuid":"3d0ed064-bdce-4008-b361-92a7ddabcd15","execution":{"iopub.execute_input":"2021-01-26T12:30:05.800734Z","iopub.status.busy":"2021-01-26T12:30:05.799924Z","iopub.status.idle":"2021-01-26T12:30:05.803362Z","shell.execute_reply":"2021-01-26T12:30:05.803855Z"},"papermill":{"duration":0.069183,"end_time":"2021-01-26T12:30:05.803983","exception":false,"start_time":"2021-01-26T12:30:05.7348","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"tst_preds = 0.28*eff_preds + 0.36*vit_preds + 0.1*res_preds + 0.17*snapmix_preds + 0.09*resnest_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for i in tst_preds:\n#     i[-1] = 0.975*i[-1]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-01-26T12:30:05.921267Z","iopub.status.busy":"2021-01-26T12:30:05.9205Z","iopub.status.idle":"2021-01-26T12:30:05.924792Z","shell.execute_reply":"2021-01-26T12:30:05.924221Z"},"papermill":{"duration":0.065437,"end_time":"2021-01-26T12:30:05.924892","exception":false,"start_time":"2021-01-26T12:30:05.859455","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(eff_preds,vit_preds,res_preds,snapmix_preds,resnest_preds,'',tst_preds,sep='\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['label'] = np.argmax(tst_preds, axis=1)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3e306fb1-b2f2-428e-894a-5412f4df304f","_uuid":"665f1e8a-114c-47b5-8633-739600dd4258","execution":{"iopub.execute_input":"2021-01-26T12:30:06.041769Z","iopub.status.busy":"2021-01-26T12:30:06.041107Z","iopub.status.idle":"2021-01-26T12:30:06.286082Z","shell.execute_reply":"2021-01-26T12:30:06.284904Z"},"papermill":{"duration":0.305567,"end_time":"2021-01-26T12:30:06.286216","exception":false,"start_time":"2021-01-26T12:30:05.980649","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}