{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Version - 0.4\n - Using GPU\n - EfficientNet b7","metadata":{}},{"cell_type":"code","source":"!pip install pretrainedmodels\n!pip install albumentations\n!pip install --upgrade efficientnet-pytorch","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\n\nimport albumentations as A\nimport cv2\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n\nfrom efficientnet_pytorch import EfficientNet\n\nfrom tqdm.notebook import tqdm\nfrom torch.utils.data import Dataset, DataLoader\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import Rotate \n\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\n\nimport warnings  \nwarnings.filterwarnings('ignore')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Settings","metadata":{}},{"cell_type":"code","source":"# DIR_INPUT = '/kaggle/input/plant-pathology-2020-fgvc7'\nDIR_INPUT = '../input/plant-pathology-2021-fgvc8'\nSEED = 42\nN_FOLDS = 3\nN_EPOCHS = 2\nBATCH_SIZE = 16\nSIZE = 224","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    \"\"\"\n    Seeds basic parameters for reproductibility of results\n    \n    Arguments:\n        seed {int} -- Number of the seed\n    \"\"\"\n    # random.seed(seed)\n    # os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n\nseed_everything(SEED)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ntrain_path = '../input/plant-pathology-2021-fgvc8/train_images'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Dataset","metadata":{}},{"cell_type":"code","source":"class PlantDataset(Dataset):\n    \n    def __init__(self, df, transforms=None):\n        self.df = df\n        self.image_id = df['image'].values\n        self.labels = df['labels'].values\n        self.transforms=transforms\n        \n    def __len__(self):\n        return len(self.labels)\n    \n    def __getitem__(self, idx):\n#         image_src = DIR_INPUT + '/images/' + self.df.loc[idx, 'image_id'] + '.jpg'\n#         # print(image_src)\n#         image = cv2.imread(image_src, cv2.IMREAD_COLOR)\n        image_id = self.image_id[idx]\n        labels = self.labels[idx]\n        image_path = train_path + image_id \n        image = cv2.imread(image_path, cv2.IMREAD_COLOR)\n#         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n#         labels = self.df.loc[idx, [\"scab\", \"healthy\", \"frog_eye_leaf_spot\", \"rust\",\"complex\", \"powdery_mildew\", \"scab frog_eye_leaf_spot\", \"scab frog_eye_leaf_spot complex\",\"frog_eye_leaf_spot complex\", \"rust frog_eye_leaf_spot\", \"rust complex\", \"powdery_mildew complex\"]].values\n#         labels = torch.from_numpy(labels.astype(np.int8))\n#         labels = labels.unsqueeze(-1)\n        \n#         if self.transforms:\n#             transformed = self.transforms(image=image)\n#             image = transformed['image']\n\n        return image, labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Agumentations","metadata":{}},{"cell_type":"code","source":"transforms_train = A.Compose([\n        A.RandomResizedCrop(height=SIZE, width=SIZE, p=1.0),\n        A.Rotate(20),\n        A.Flip(),\n        A.Transpose(),\n    A.Resize(height=SIZE, width=SIZE, p=1.0),\n    A.Normalize(p=1.0),\n    ToTensorV2(p=1.0),\n    ], p=1.0)\n\ntransforms_valid = A.Compose([\n    A.Resize(height=SIZE, width=SIZE, p=1.0),\n    A.Normalize(p=1.0),\n    ToTensorV2(p=1.0),\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# StratifiedKFold","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(DIR_INPUT + '/train.csv')\nlabels_mapping = {\"scab\" : 0, \"healthy\" : 1, \"frog_eye_leaf_spot\" : 2, \"rust\" : 3,\n                 \"complex\" : 4, \"powdery_mildew\" : 5, \"scab frog_eye_leaf_spot\" : 6, \"scab frog_eye_leaf_spot complex\" : 7,\n                 \"frog_eye_leaf_spot complex\" : 8, \"rust frog_eye_leaf_spot\" : 9, \"rust complex\" : 10, \"powdery_mildew complex\" : 11}\ntrain_df['labels'] = train_df['labels'].map(labels_mapping)\ntrain_labels = train_df.iloc[:, 1:].values\nprint(train_labels)\n# Need for the StratifiedKFold split\ntrain_y = train_labels[:, 0] ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df.head()\nprint(train_df)\nprint(train_labels)\nprint(train_y)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folds = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\noof_preds = np.zeros((train_df.shape[0], 4))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PretrainedModels","metadata":{}},{"cell_type":"code","source":"model_name = 'efficientnet-b7'\nmodel = EfficientNet.from_pretrained(model_name, num_classes=4) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DenseCrossEntropy(nn.Module):\n\n    def __init__(self):\n        super(DenseCrossEntropy, self).__init__()\n        \n        \n    def forward(self, logits, labels):\n        logits = logits.float()\n        labels = labels.float()\n        \n        logprobs = F.log_softmax(logits, dim=-1)\n        \n        loss = -labels * logprobs\n        loss = loss.sum(-1)\n\n        return loss.mean()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train for StratifiedKFold","metadata":{}},{"cell_type":"code","source":"def train_one_fold(i_fold, model, criterion, optimizer, lr_scheduler, dataloader_train, dataloader_valid):\n    \n    train_fold_results = []\n\n    for epoch in range(N_EPOCHS):\n\n        print('  Epoch {}/{}'.format(epoch + 1, N_EPOCHS))\n        print('  ' + ('-' * 20))\n\n        model.train()\n        tr_loss = 0\n\n        for step, batch in enumerate(dataloader_train):\n\n            images = batch[0]\n            labels = batch[1]\n\n            images = images.to(device, dtype=torch.float)\n            labels = labels.to(device, dtype=torch.float)\n            \n            outputs = model(images)\n            loss = criterion(outputs, labels.squeeze(-1))                \n            loss.backward()\n\n            tr_loss += loss.item()\n\n            optimizer.step()\n            optimizer.zero_grad()\n\n        # Validate\n        model.eval()\n        \n        val_loss = 0\n        val_preds = None\n        val_labels = None\n\n        for step, batch in enumerate(dataloader_valid):\n\n            images = batch[0]\n            labels = batch[1]\n\n            if val_labels is None:\n                val_labels = labels.clone().squeeze(-1)\n            else:\n                val_labels = torch.cat((val_labels, labels.squeeze(-1)), dim=0)\n\n            images = images.to(device, dtype=torch.float)\n            labels = labels.to(device, dtype=torch.float)\n\n            with torch.no_grad():\n                outputs = model(images)\n\n                loss = criterion(outputs, labels.squeeze(-1))\n                val_loss += loss.item()\n\n                preds = torch.softmax(outputs, dim=1).data.cpu()\n\n                if val_preds is None:\n                    val_preds = preds\n                else:\n                    val_preds = torch.cat((val_preds, preds), dim=0)\n       \n        # if train mode\n        lr_scheduler.step(tr_loss)\n\n        train_fold_results.append({\n            'fold': i_fold,\n            'epoch': epoch,\n            'train_loss': tr_loss / len(dataloader_train),\n            'valid_loss': val_loss / len(dataloader_valid),\n            'valid_score': roc_auc_score(val_labels, val_preds, average='macro'),\n        })\n\n    return val_preds, train_fold_results","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.read_csv(DIR_INPUT + '/sample_submission.csv')\nsubmission_df.iloc[:, 1:] = 0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_test = PlantDataset(df=submission_df, transforms=transforms_valid)\ndataloader_test = DataLoader(dataset_test, batch_size=BATCH_SIZE, num_workers=4, shuffle=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submissions = None\ntrain_results = []\n\nfor i_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df, train_y)):\n    print(\"Fold {}/{}\".format(i_fold + 1, N_FOLDS))\n\n    valid = train_df.iloc[valid_idx]\n    valid.reset_index(drop=True, inplace=True)\n\n    train = train_df.iloc[train_idx]\n    train.reset_index(drop=True, inplace=True)    \n\n    dataset_train = PlantDataset(df=train, transforms=transforms_train)\n    dataset_valid = PlantDataset(df=valid, transforms=transforms_valid)\n\n    dataloader_train = DataLoader(dataset_train, batch_size=BATCH_SIZE, num_workers=4, shuffle=True)\n    dataloader_valid = DataLoader(dataset_valid, batch_size=BATCH_SIZE, num_workers=4, shuffle=False)\n\n    model = model.to(device)\n\n    criterion = DenseCrossEntropy()\n    optimizer = optim.Adam(model.parameters(), lr=0.001)\n    lr_scheduler = optim.lr_scheduler.MultiStepLR(optimizer=optimizer, milestones=[int(N_EPOCHS * 0.5), int(N_EPOCHS * 0.75)], gamma=0.1, last_epoch=-1)\n    \n    val_preds, train_fold_results = train_one_fold(i_fold, model, criterion, optimizer, lr_scheduler, dataloader_train, dataloader_valid)\n    oof_preds[valid_idx, :] = val_preds.numpy()\n    \n    train_results = train_results + train_fold_results\n\n    model.eval()\n    test_preds = None\n\n    for step, batch in enumerate(dataloader_test):\n\n        images = batch[0].to(device, dtype=torch.float)\n\n        with torch.no_grad():\n            outputs = model(images)\n\n            if test_preds is None:\n                test_preds = outputs.data.cpu()\n            else:\n                test_preds = torch.cat((test_preds, outputs.data.cpu()), dim=0)\n    \n    \n    # Save predictions per fold\n    submission_df[['healthy', 'multiple_diseases', 'rust', 'scab']] = torch.softmax(test_preds, dim=1)\n    submission_df.to_csv('submission_fold_{}.csv'.format(i_fold), index=False)\n\n    # logits avg\n    if submissions is None:\n        submissions = test_preds / N_FOLDS\n    else:\n        submissions += test_preds / N_FOLDS\n\nprint(\"5-Folds CV score: {:.4f}\".format(roc_auc_score(train_labels, oof_preds, average='macro')))\n\ntorch.save(model.state_dict(), 'model.pth')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Show Train History","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_training_loss(train_result):\n    plt.figure(figsize=(15,10))\n    plt.subplot(311)\n    train_loss = train_result['train_loss']\n    plt.plot(train_loss.index, train_loss, label = 'train_loss')\n    plt.legend()\n\n    val_loss = train_result['valid_loss']\n    plt.plot(val_loss.index, val_loss, label = 'val_loss')\n    plt.legend()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_valid_score(train_result):\n    plt.figure(figsize=(15,10))\n    plt.subplot(311)\n    valid_score = train_result['valid_score']\n    plt.plot(valid_score.index, valid_score, label = 'valid_score')\n    plt.legend()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_results = pd.DataFrame(train_results)\ntrain_results.head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_training_loss(train_results[train_results['fold'] == 0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_valid_score(train_results[train_results['fold'] == 0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate submission file","metadata":{}},{"cell_type":"code","source":"submission_df[['healthy', 'multiple_diseases', 'rust', 'scab']] = torch.softmax(submissions, dim=1)\nsubmission_df.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}