{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":13836,"databundleVersionId":1718836,"sourceType":"competition"},{"sourceId":1490626,"sourceType":"datasetVersion","datasetId":875062},{"sourceId":1760030,"sourceType":"datasetVersion","datasetId":1046169},{"sourceId":2660070,"sourceType":"datasetVersion","datasetId":686792},{"sourceId":8807440,"sourceType":"datasetVersion","datasetId":5297088},{"sourceId":8828323,"sourceType":"datasetVersion","datasetId":5311585},{"sourceId":8232278,"sourceType":"datasetVersion","datasetId":1047930},{"sourceId":183895801,"sourceType":"kernelVersion"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"package_paths = [\n    '/kaggle/input/k/t0ra1107/pytorch-efficientnet-baseline-train-amp-aug',\n    '/kaggle/input/pytorchimagemodels',\n    '/kaggle/input/einops-030/einops-0.3.0-py2.py3-none-any.whl',\n    '/kaggle/input/deit-exp31'\n]\nimport sys\nfor pth in package_paths:\n    sys.path.append(pth)","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:00.638850Z","iopub.execute_input":"2024-07-01T02:53:00.639630Z","iopub.status.idle":"2024-07-01T02:53:00.650750Z","shell.execute_reply.started":"2024-07-01T02:53:00.639596Z","shell.execute_reply":"2024-07-01T02:53:00.649905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom  torch.cuda.amp import autocast, GradScaler\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\nfrom einops import rearrange\nimport timm #from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom\nfrom sklearn.metrics import log_loss","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-01T02:53:00.652032Z","iopub.execute_input":"2024-07-01T02:53:00.652304Z","iopub.status.idle":"2024-07-01T02:53:09.473822Z","shell.execute_reply.started":"2024-07-01T02:53:00.652274Z","shell.execute_reply":"2024-07-01T02:53:09.472857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG = {\n    'fold_num': 5,\n    'seed': 719,\n    'model_weight_dir': '/kaggle/input/deit-exp31/deit-exp31',\n    'img_size': 384,\n    'epochs': 10,\n    'train_bs': 32,\n    'valid_bs': 32,\n    'lr': 1e-4,\n    'dropout_rate': 0.0,\n    'num_workers': 4,\n    'accum_iter': 1, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'tta': 5,\n    'used_epochs': [9, 6, 8, 6, 9],\n    'weights': [1,1,1,1,1]\n}","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:09.475373Z","iopub.execute_input":"2024-07-01T02:53:09.475686Z","iopub.status.idle":"2024-07-01T02:53:09.481251Z","shell.execute_reply.started":"2024-07-01T02:53:09.475661Z","shell.execute_reply":"2024-07-01T02:53:09.480262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"timm version:\", timm.__version__)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:09.482062Z","iopub.execute_input":"2024-07-01T02:53:09.482294Z","iopub.status.idle":"2024-07-01T02:53:09.493057Z","shell.execute_reply.started":"2024-07-01T02:53:09.482274Z","shell.execute_reply":"2024-07-01T02:53:09.492181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\ntrain.head()","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2024-07-01T02:53:09.495431Z","iopub.execute_input":"2024-07-01T02:53:09.495820Z","iopub.status.idle":"2024-07-01T02:53:09.538664Z","shell.execute_reply.started":"2024-07-01T02:53:09.495791Z","shell.execute_reply":"2024-07-01T02:53:09.537792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:09.539904Z","iopub.execute_input":"2024-07-01T02:53:09.540230Z","iopub.status.idle":"2024-07-01T02:53:09.554058Z","shell.execute_reply.started":"2024-07-01T02:53:09.540200Z","shell.execute_reply":"2024-07-01T02:53:09.553183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> We could do stratified validation split in each fold to make each fold's train and validation set looks like the whole train set in target distributions.","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:09.555128Z","iopub.execute_input":"2024-07-01T02:53:09.555430Z","iopub.status.idle":"2024-07-01T02:53:09.567838Z","shell.execute_reply.started":"2024-07-01T02:53:09.555384Z","shell.execute_reply":"2024-07-01T02:53:09.566987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb\n\nimg = get_img('../input/cassava-leaf-disease-classification/train_images/1000015157.jpg')\nplt.imshow(img)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:09.568818Z","iopub.execute_input":"2024-07-01T02:53:09.569109Z","iopub.status.idle":"2024-07-01T02:53:10.007866Z","shell.execute_reply.started":"2024-07-01T02:53:09.569084Z","shell.execute_reply":"2024-07-01T02:53:10.006936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(\n        self, df, data_root, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        \n        img  = get_img(path)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:10.009058Z","iopub.execute_input":"2024-07-01T02:53:10.009396Z","iopub.status.idle":"2024-07-01T02:53:10.021625Z","shell.execute_reply.started":"2024-07-01T02:53:10.009364Z","shell.execute_reply":"2024-07-01T02:53:10.020712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Train\\Validation Image Augmentations","metadata":{}},{"cell_type":"code","source":"from albumentations import (\n    HorizontalFlip, VerticalFlip, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    GaussNoise, MotionBlur, MedianBlur, RandomResizedCrop,\n    RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize\n)\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            CoarseDropout(p=0.5),\n            Cutout(p=0.5),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n  \n        \ndef get_valid_transforms():\n    return Compose([\n            CenterCrop(CFG['img_size'], CFG['img_size'], p=1.),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n\ndef get_inference_transforms():\n    return Compose([\n            RandomResizedCrop(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:10.023230Z","iopub.execute_input":"2024-07-01T02:53:10.023621Z","iopub.status.idle":"2024-07-01T02:53:10.203452Z","shell.execute_reply.started":"2024-07-01T02:53:10.023589Z","shell.execute_reply":"2024-07-01T02:53:10.202511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class CassvaImgClassifier(nn.Module):\n    def __init__(self, n_class, dropout_rate=0.1, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model('tf_efficientnet_b4_ns', pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Sequential(\n            nn.Dropout(dropout_rate),\n            nn.Linear(n_features, n_class, bias=True)\n        )\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:10.206781Z","iopub.execute_input":"2024-07-01T02:53:10.207074Z","iopub.status.idle":"2024-07-01T02:53:10.213462Z","shell.execute_reply.started":"2024-07-01T02:53:10.207051Z","shell.execute_reply":"2024-07-01T02:53:10.212484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CassvaImgClassifierViT(nn.Module):\n    def __init__(self, n_class, dropout_rate=0.1, pretrained=True):\n        super().__init__()\n\n        self.model = timm.create_model('vit_base_patch16_384', pretrained=pretrained)\n        n_features = self.model.head.in_features\n        self.model.head = nn.Sequential(\n            nn.Dropout(dropout_rate),\n            nn.Linear(n_features, n_class, bias=True)\n        )\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:10.214346Z","iopub.execute_input":"2024-07-01T02:53:10.214621Z","iopub.status.idle":"2024-07-01T02:53:10.225245Z","shell.execute_reply.started":"2024-07-01T02:53:10.214599Z","shell.execute_reply":"2024-07-01T02:53:10.224459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CassvaImgClassifierDeiT(nn.Module):\n    def __init__(self, n_class, dropout_rate=0.1, pretrained=True):\n        super().__init__()\n\n        self.model = timm.create_model('deit3_base_patch16_384', pretrained=pretrained)\n        n_features = self.model.head.in_features\n        self.model.head = nn.Sequential(\n            nn.Dropout(dropout_rate),\n            nn.Linear(n_features, n_class, bias=True)\n        )\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:10.226203Z","iopub.execute_input":"2024-07-01T02:53:10.226484Z","iopub.status.idle":"2024-07-01T02:53:10.233533Z","shell.execute_reply.started":"2024-07-01T02:53:10.226463Z","shell.execute_reply":"2024-07-01T02:53:10.232682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Main Loop","metadata":{}},{"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)   #output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:10.234671Z","iopub.execute_input":"2024-07-01T02:53:10.234899Z","iopub.status.idle":"2024-07-01T02:53:10.242651Z","shell.execute_reply.started":"2024-07-01T02:53:10.234880Z","shell.execute_reply":"2024-07-01T02:53:10.241770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    \n    tst_preds = []\n    \n    for fold, (trn_idx, val_idx) in enumerate(folds):\n\n        print('Inference fold {} started'.format(fold))\n        \n        val_preds = []\n\n        valid_ = train.loc[val_idx,:].reset_index(drop=True)\n        valid_ds = CassavaDataset(valid_, '../input/cassava-leaf-disease-classification/train_images/', transforms=get_inference_transforms(), output_label=False)\n\n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms(), output_label=False)\n\n        val_loader = torch.utils.data.DataLoader(\n            valid_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n\n        tst_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n\n        device = torch.device(CFG['device'])\n        # model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique()).to(device)\n        model = CassvaImgClassifierDeiT(5, pretrained=False).to(device)\n\n        #for epoch in range(CFG['epochs']-3):\n        model.load_state_dict(torch.load('{}/fold_{}_{}.pth'.format(CFG['model_weight_dir'], fold, CFG['used_epochs'][fold])))\n\n        with torch.no_grad():\n            for _ in range(CFG['tta']):\n                val_preds += [CFG['weights'][fold]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]\n                tst_preds += [CFG['weights'][fold]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]\n\n        del model\n        torch.cuda.empty_cache()\n\n        val_preds = np.mean(val_preds, axis=0)\n\n        print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))\n        print('fold {} validation accuracy = {:.5f}'.format(fold, (valid_.label.values==np.argmax(val_preds, axis=1)).mean()))\n        \n    tst_preds = np.mean(tst_preds, axis=0)\n","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:10.243759Z","iopub.execute_input":"2024-07-01T02:53:10.244163Z","iopub.status.idle":"2024-07-01T02:53:39.515549Z","shell.execute_reply.started":"2024-07-01T02:53:10.244134Z","shell.execute_reply":"2024-07-01T02:53:39.514351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['label'] = np.argmax(tst_preds, axis=1)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:39.516504Z","iopub.status.idle":"2024-07-01T02:53:39.516843Z","shell.execute_reply.started":"2024-07-01T02:53:39.516682Z","shell.execute_reply":"2024-07-01T02:53:39.516697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-01T02:53:39.518146Z","iopub.status.idle":"2024-07-01T02:53:39.518490Z","shell.execute_reply.started":"2024-07-01T02:53:39.518302Z","shell.execute_reply":"2024-07-01T02:53:39.518314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train part is here: https://www.kaggle.com/khyeh0719/pytorch-efficientnet-baseline-train-amp-aug","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}