{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import torch\nimport torchvision\nimport torch.nn.functional as F\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\nimport pandas as pd\nimport numpy as np\nimport gc\nimport os\nimport sys\nimport cv2\nimport time\nimport datetime\nimport warnings\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsys.path.append('../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master')\nfrom efficientnet_pytorch import EfficientNet\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"warnings.simplefilter('ignore')\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(47)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LeafDataset(Dataset):\n    def __init__(self, df: pd.DataFrame, imfolder: str, train: bool = True, transforms = None):\n        \"\"\"\n        Class initialization\n        Args:\n            df (pd.DataFrame): DataFrame with data description\n            imfolder (str): folder with images\n            train (bool): flag of whether a training dataset is being initialized or testing one\n            transforms: image transformation method to be applied\n            \n        \"\"\"\n        self.df = df\n        self.imfolder = imfolder\n        self.transforms = transforms\n        self.train = train\n        \n    def __getitem__(self, index):\n        im_path = os.path.join(self.imfolder, self.df.iloc[index]['image_id'])\n        x = cv2.imread(im_path)\n        x = cv2.cvtColor(x, cv2.COLOR_BGR2RGB)  # GBR2RGB cast\n\n        if self.transforms:\n            x = self.transforms(x)\n            \n        if self.train:\n            y = self.df.iloc[index]['label']\n            return x, y\n        else:\n            return x\n    \n    def __len__(self):\n        return len(self.df)\n    \n    \nclass Net(nn.Module):\n    def __init__(self, arch):\n        super(Net, self).__init__()\n        self.arch = arch\n        if 'ResNet' in str(arch.__class__):\n            self.arch.fc = nn.Linear(in_features=512, out_features=5, bias=True)\n        if 'EfficientNet' in str(arch.__class__):\n            self.arch._fc = nn.Linear(in_features=1280, out_features=5, bias=True)\n        \n    def forward(self, inputs):\n        return self.arch(inputs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')\nsub = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\n# This is to work around Pytorch limitation\n# Because it expects input shape of data to have more than one example\nif len(sub) == 1:\n    sub = pd.concat([sub, sub], ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_transform = torchvision.transforms.Compose([\n    torchvision.transforms.ToPILImage(),\n    torchvision.transforms.Resize((256, 256)),\n    torchvision.transforms.RandomHorizontalFlip(),\n    torchvision.transforms.RandomVerticalFlip(),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])\ntest_transform = torchvision.transforms.Compose([\n    torchvision.transforms.ToPILImage(),\n    torchvision.transforms.Resize((256, 256)),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"architecture = 'efficientnet-b1'  # Model architecture\narch = EfficientNet.from_name(architecture)  # Loading initial weights\nepochs = 12  # Number of epochs to run\nes_patience = 3  # Early Stopping patience - for how many epochs with no improvements to wait\nTTA = 3 # Test Time Augmentation rounds\ndata_folder = '/kaggle/input/cassava-leaf-disease-classification'\noof = np.zeros((len(train_df), 1))  # Out Of Fold predictions\npreds = torch.zeros((len(sub), train_df['label'].nunique()), dtype=torch.float32, device=device)  # Test predictions\nsplits = 4  # number of folds\ntrain_losses = {i+1: [] for i in range(splits)}\ntrain_accs = {i+1: [] for i in range(splits)}\nval_accs = {i+1: [] for i in range(splits)}\n\ntest = LeafDataset(df=sub,\n                   imfolder=os.path.join(data_folder, 'test_images'), \n                   train=False,\n                   transforms=train_transform)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits=splits)\nfor fold, (train_idx, val_idx) in enumerate(skf.split(X=np.zeros(len(train_df)), y=train_df['label']), 1):\n    print('=' * 20, 'Fold', fold, '=' * 20)\n    model_path = f'effnet_b1_fold_{fold}.pth'  # Path and filename to save model to\n    best_val = 0  # Best validation score within this fold\n    patience = es_patience  # Current patience counter\n    arch = EfficientNet.from_name(architecture)\n    model = Net(arch=arch)\n    model = model.to(device)\n    optim = torch.optim.Adam(model.parameters(), lr=0.003)\n    scheduler = ReduceLROnPlateau(optimizer=optim, mode='max', patience=1, verbose=True, factor=0.2)\n    criterion = nn.CrossEntropyLoss()\n    \n    train = LeafDataset(df=train_df.iloc[train_idx].reset_index(drop=True),\n                       imfolder=os.path.join(data_folder, 'train_images'), \n                       train=True,\n                       transforms=train_transform)\n    val = LeafDataset(df=train_df.iloc[val_idx].reset_index(drop=True),\n                       imfolder=os.path.join(data_folder, 'train_images'),\n                       train=True,\n                       transforms=test_transform)\n    \n    train_loader = DataLoader(dataset=train, batch_size=64, shuffle=True, num_workers=2)\n    val_loader = DataLoader(dataset=val, batch_size=16, shuffle=False, num_workers=2)\n    test_loader = DataLoader(dataset=test, batch_size=16, shuffle=False, num_workers=2)\n    \n    for epoch in range(epochs):\n        start_time = time.time()\n        correct = 0\n        epoch_loss = 0\n        model.train()\n        \n        for x, y in train_loader:\n            x = torch.tensor(x, device=device, dtype=torch.float32)\n            y = torch.tensor(y, device=device, dtype=torch.long)\n            optim.zero_grad()\n            z = model(x)\n            loss = criterion(z, y)\n            loss.backward()\n            optim.step()\n            pred = torch.argmax(z, axis=1)\n            correct += (pred.cpu() == y.cpu()).sum().item()\n            epoch_loss += loss.item()\n        train_acc = correct / len(train_idx)  # Train accuracy\n        \n        model.eval()  # switch model to the evaluation mode\n        val_preds = torch.zeros((len(val_idx), 1), dtype=torch.long, device=device)\n        with torch.no_grad():  # Do not calculate gradient since we are only predicting\n            for j, (x_val, y_val) in enumerate(val_loader):\n                x_val = torch.tensor(x_val, device=device, dtype=torch.float32)\n                y_val = torch.tensor(y_val, device=device, dtype=torch.long)\n                z_val = model(x_val)\n                val_pred = torch.argmax(z_val, axis=1)\n                val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val.shape[0]] = val_pred.reshape(-1, 1)\n            val_acc = accuracy_score(train_df.iloc[val_idx]['label'].values, val_preds.cpu())\n            train_time = str(datetime.timedelta(seconds=time.time() - start_time))[:7]\n            train_losses[fold].append(epoch_loss)\n            train_accs[fold].append(train_acc)\n            val_accs[fold].append(val_acc)\n            print(f'[{train_time}] | Epoch: {epoch+1:03} | Loss: {epoch_loss:.4f} | Train acc: {train_acc:.4f} | Val acc: {val_acc:.4f}')\n            scheduler.step(val_acc)\n            \n            if val_acc >= best_val:\n                best_val= val_acc\n                patience = es_patience\n                torch.save(model, model_path)\n            else:\n                patience -= 1\n                if patience == 0:\n                    print(f'Early stopping. Best Val accuracy: {best_val:.4f}')\n                    break\n                    \n    model = torch.load(model_path)  # Loading best model of this fold\n    model.eval()  # switch model to the evaluation mode\n    val_preds = torch.zeros((len(val_idx), 1), dtype=torch.long, device=device)\n    with torch.no_grad():\n        # Predicting on validation set once again to obtain data for OOF\n        for j, (x_val, y_val) in enumerate(val_loader):\n            x_val = torch.tensor(x_val, device=device, dtype=torch.float32)\n            y_val = torch.tensor(y_val, device=device, dtype=torch.long)\n            z_val = model(x_val)\n            val_pred = torch.argmax(z_val, axis=1)\n            val_preds[j*val_loader.batch_size:j*val_loader.batch_size + x_val.shape[0]] = val_pred.reshape(-1, 1)\n        oof[val_idx] = val_preds.cpu().numpy()\n        \n        # Predicting on test set\n        tta_preds = torch.zeros((len(test), train_df['label'].nunique()), dtype=torch.float32, device=device)\n        for _ in range(TTA):\n            for i, x_test in enumerate(test_loader):\n                x_test = torch.tensor(x_test, device=device, dtype=torch.float32)\n                z_test = model(x_test)\n                z_test = torch.softmax(z_test, axis=1)  # we need probabilities to average them later\n                tta_preds[i*test_loader.batch_size:i*test_loader.batch_size + x_test.shape[0]] += z_test\n        preds += tta_preds / TTA\n        \npreds /= skf.n_splits\npreds = torch.argmax(preds, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('OOF score: {:.4f}'.format(accuracy_score(train_df['label'], oof)))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(skf.n_splits, 1, figsize=(10, 4 * skf.n_splits))\nfor i in range(4):\n    axes[i].plot(train_losses[i+1], label='Train loss', color='tab:blue')\n    axes[i].grid();\n    axes[i].legend(fontsize=12, facecolor='white');\n    axes[i].set_ylabel('Loss', fontsize=14);\n    axes[i].set_xlabel('Epoch', fontsize=14);\n    max_ticks = max(len(train_losses[i+1]), len(train_accs[i+1]), len(val_accs[i+1]))\n    axes[i].set_xticks(range(max_ticks));\n    axes[i].set_xticklabels(range(1, max_ticks+1));\n    axes2 = axes[i].twinx()\n    axes2.plot(train_accs[i+1], label='Train accuracy', color='tab:orange')\n    axes2.plot(val_accs[i+1], label='Validation accuracy', color='tab:red')\n    axes2.plot(max(val_accs[i+1]), color='tab:red', marker='o', markersize=10)\n    axes2.legend(fontsize=12, facecolor='white');\n    axes2.set_ylabel('Accuracy', fontsize=14);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['label'] = preds.cpu().numpy()\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}