{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path = [\n    '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',\n] + sys.path\nsys.path = [\n    '../input/ttach-kaggle/ttach/',\n] + sys.path\nimport ttach as tta","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\n\nimport albumentations as A\nimport cv2\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torch.optim as optim\n\nfrom tqdm.notebook import tqdm\nfrom torch.utils.data import Dataset, DataLoader\nfrom albumentations.pytorch import ToTensorV2\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import KFold, StratifiedKFold\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\nimport warnings  \nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DIR_INPUT = '/kaggle/input/cassava-leaf-disease-classification'\nDIR_WEIGHTS = '/kaggle/input/cassava-pytorch-starter-train'\n\nSEED = 42\nN_FOLDS = 1\nBATCH_SIZE = 16\nSIZE = 512\nCROP = 512\ninit_lr = 5e-5\nn_epochs = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    \n    def __init__(self, df, dataset='train', transforms=None):\n    \n        self.df = df\n        self.transforms=transforms\n        self.dataset=dataset\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, idx):\n        image_src = f'{DIR_INPUT}/{self.dataset}_images/{self.df.loc[idx, \"image_id\"]}'\n        # print(image_src)\n        image = cv2.imread(image_src)\n        #image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if self.transforms:\n            transformed = self.transforms(image=image)\n            images = transformed['image']\n        \n        images = images.astype(np.float32)\n        images /= 255\n        images = images.transpose(2, 0, 1)\n        \n        return torch.tensor(images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\nmodelname=\"efficientnet-b0\"\nmodelname2=\"efficientnet-b2\"\nclass enetv2(nn.Module):\n    def __init__(self, out_dim=1, ModelName=\"efficientnet-b0\"):\n        super(enetv2, self).__init__()\n        self.basemodel = EfficientNet.from_name(ModelName) \n        self.myfc = nn.Linear(self.basemodel._fc.in_features, out_dim)\n        self.basemodel._fc = nn.Identity()        \n            \n    def extract(self, x):\n        return self.basemodel(x)\n\n    def forward(self, x):\n        x = self.basemodel(x)\n        x = self.myfc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transforms_test = A.Compose([\n    A.Resize(height=SIZE, width=SIZE, p=1.0),\n    #ToTensorV2(p=1.0),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv(DIR_INPUT + '/sample_submission.csv')\nsubmission_df.iloc[:, 1] = 0\n\nsubmission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if submission_df.shape[0] == 1:\n    submission_df = pd.DataFrame([{'image_id': '2216849948.jpg', 'label': 0},{'image_id': '2216849948.jpg', 'label': 0}])\n    submission_df.reset_index(drop=True, inplace=True)\n    commit = True\nelse:\n    commit = False\nsubmission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_test = CassavaDataset(df=submission_df, dataset='test', transforms=transforms_test)\ndataloader_test = DataLoader(dataset_test, batch_size=BATCH_SIZE, num_workers=4, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Psuedo-Labeling"},{"metadata":{"trusted":true},"cell_type":"code","source":"submissions = None\ndevice = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device('cpu')\n\nfor i_fold in range(N_FOLDS):\n    model = enetv2(5, modelname2).to(device)\n    model.to(device)\n    \n    checkpoint2 = torch.load(f\"../input/cassavadata/efficientnet-b2_512_final_epoch10_fold0.pth\", map_location=device)\n    model.load_state_dict(checkpoint2)\n    model.eval()\n    \n    transforms = tta.Compose(\n        [\n            tta.HorizontalFlip(),\n        ]\n    )\n    tta_models = []\n    for model in [model]:\n        tta_models.append(tta.ClassificationTTAWrapper(model, transforms))\n    \n    for net in tta_models:\n        test_preds = None\n        for step, batch in enumerate(dataloader_test):\n\n            images = batch\n            images = images.to(device, dtype=torch.float)\n\n            with torch.no_grad():\n                outputs = net(images)\n\n                preds = torch.softmax(outputs, dim=1).data.cpu()\n\n                if test_preds is None:\n                    test_preds = preds\n                else:\n                    test_preds = torch.cat((test_preds, preds), dim=0)\n\n\n        # submission_df[['label']] = test_preds.argmax(test_preds, dim=1)\n        # submission_df.to_csv('submission_fold_{}.csv'.format(i_fold), index=False)\n\n        # logits avg\n        if submissions is None:\n            submissions = test_preds\n        else:\n            submissions += test_preds\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submissions[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pl_df = pd.read_csv(DIR_INPUT + '/sample_submission.csv')\nif pl_df.shape[0] == 1:\n    pl_df = pd.DataFrame([{'image_id': '2216849948.jpg', 'label': 0},{'image_id': '2216849948.jpg', 'label': 0}])\n    pl_df.reset_index(drop=True, inplace=True)\n\npl_df['label'] = torch.argmax(submissions, dim=1)\npl_df[\"pl\"] = np.ones_like(torch.argmax(submissions, dim=1))\npl_df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(DIR_INPUT,\"train.csv\"))\ndf_train[\"pl\"] = np.zeros_like(df_train[\"image_id\"])\n\ndf_train = pd.concat([df_train, pl_df]).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset2(Dataset):\n    \n    def __init__(self, df, dataset='train', transforms=None):\n    \n        self.df = df\n        self.transforms=transforms\n        self.dataset=dataset\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, idx):\n        imageid = self.df.loc[idx, \"image_id\"]\n        label = self.df.loc[idx, \"label\"]\n        dir = self.df.loc[idx, \"pl\"]\n        if dir:\n            image_src = f'{DIR_INPUT}/test_images/{imageid}'\n        else:\n            image_src = f'{DIR_INPUT}/train_images/{imageid}'\n        # print(image_src)\n        image = cv2.imread(image_src)\n        #image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if self.transforms:\n            transformed = self.transforms(image=image)\n            images = transformed['image']\n        \n        images = images.astype(np.float32)\n        images /= 255\n        images = images.transpose(2, 0, 1)\n        #label = row.label\n        \n        return torch.tensor(images), torch.tensor(label)\n    \nimport albumentations\nimport albumentations as A\ntransforms_train = albumentations.Compose([\n    albumentations.ShiftScaleRotate(scale_limit=0.3, rotate_limit=180,p=0.5),\n    A.OneOf([\n        A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2, \n                             val_shift_limit=0.2, p=0.5),\n        A.RandomBrightnessContrast(brightness_limit=0.2, \n                                   contrast_limit=0.2, p=0.5),\n    ],p=0.9),\n    A.Cutout(num_holes=12, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),\n    albumentations.Rotate(p=0.5),\n    albumentations.Transpose(p=0.5),\n    albumentations.VerticalFlip(p=0.5),\n    albumentations.HorizontalFlip(p=0.5),   \n    albumentations.Resize(SIZE, SIZE, p=1.0), \n])\ndataset_train = CassavaDataset2(df=df_train, dataset='test', transforms=transforms_train)\ndataloader_train = DataLoader(dataset_train, batch_size=BATCH_SIZE, num_workers=4, shuffle=True)\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=init_lr)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, n_epochs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Retrain model"},{"metadata":{"trusted":true},"cell_type":"code","source":"scaler = torch.cuda.amp.GradScaler(enabled=False)\ndef train_epoch(loader, optimizer):\n    model.train()\n    train_loss = []\n    bar = tqdm(loader)\n    i = 0\n    for (data, target) in bar:\n        data, target = data.to(device), target.to(device).long()\n        loss_func = criterion\n        optimizer.zero_grad()\n        with torch.cuda.amp.autocast(enabled=False):\n            logits = model(data).squeeze(1)            \n            loss = loss_func(logits, target)\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        optimizer.zero_grad()\n\n        loss_np = loss.detach().cpu().numpy()\n        train_loss.append(loss_np)\n        smooth_loss = sum(train_loss[-100:]) / min(len(train_loss), 100)\n        bar.set_description('loss: %.5f, smth: %.5f' % (loss_np, smooth_loss))\n        i+=1\n        if commit and i==10:\n            break\n    return np.mean(train_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for epoch in range(1, n_epochs+1):\n    torch.cuda.empty_cache() \n    scheduler.step(epoch-1)\n\n    train_loss = train_epoch(dataloader_train , optimizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submissions = None\ndevice = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device('cpu')\n\nfor i_fold in range(N_FOLDS):\n    model.eval()\n    \n    transforms = tta.Compose(\n        [\n            tta.HorizontalFlip(),\n        ]\n    )\n    tta_models = []\n    for model in [model]:\n        tta_models.append(tta.ClassificationTTAWrapper(model, transforms))\n    \n    for net in tta_models:\n        test_preds = None\n        for step, batch in enumerate(dataloader_test):\n\n            images = batch\n            images = images.to(device, dtype=torch.float)\n\n            with torch.no_grad():\n                outputs = net(images)\n\n                preds = torch.softmax(outputs, dim=1).data.cpu()\n\n                if test_preds is None:\n                    test_preds = preds\n                else:\n                    test_preds = torch.cat((test_preds, preds), dim=0)\n\n\n        # submission_df[['label']] = test_preds.argmax(test_preds, dim=1)\n        # submission_df.to_csv('submission_fold_{}.csv'.format(i_fold), index=False)\n\n        # logits avg\n        if submissions is None:\n            submissions = test_preds\n        else:\n            submissions += test_preds\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submissions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['label'] = torch.argmax(submissions, dim=1)\nsubmission_df.to_csv('submission.csv', index=False)\nsubmission_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}