{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#import libraries\nimport os \nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\nfrom tqdm import tqdm\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, SubsetRandomSampler, Sampler\nfrom torch.optim import SGD, Adam\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, StepLR\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#data directory\nOUTPUT_DIR = './output/'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\nTRAIN_PATH = '../input/cassava-leaf-disease-classification/train_images'\nTEST_PATH = '../input/cassava-leaf-disease-classification/test_images'\ntrain = pd.read_csv('../input/cassava-leaf-disease-classification/train.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Working with data","metadata":{}},{"cell_type":"code","source":"train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g = sns.countplot(train['label'])\ntrain['label'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Here labels are not uniformily distributed. So we need to stratify the data for training.","metadata":{}},{"cell_type":"code","source":"len(train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#setting the devie to cpu or cuda\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#seed everything \ndef seed_everything(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    print('SEEDING IS DONE')\nseed_everything()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Dataset and transformation\nclass TrainDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_id'].values\n        self.labels = df['label'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.file_names[idx]\n        file_path = f'{TRAIN_PATH}/{file_name}'\n        image = cv2.imread(file_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n        label = torch.tensor(self.labels[idx]).long()\n        return image, label\n    \ntrain_transform = A.Compose([\n    A.Resize(512, 512),\n    A.Rotate(limit=30, p = 0.7, border_mode=cv2.BORDER_CONSTANT),\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.9),\n    A.Normalize(\n    mean = [0.485, 0.456, 0.406],\n    std = [0.229, 0.224, 0.225]\n    ),\n    ToTensorV2()\n])\n\n    \nvalid_transform = A.Compose([\n    A.Resize(512, 512),\n    A.Normalize(\n    mean = [0.485, 0.456, 0.406],\n    std = [0.229, 0.224, 0.225]\n    ),\n    ToTensorV2()\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = TrainDataset(train, transform=None)\nfor i in range(4):\n    image, label = train_dataset[i]\n    plt.subplots()\n    plt.imshow(image)\n    plt.title(f'label: {label}')\n    plt.show","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.models as models\nimport torch.nn as nn\nclass MyEnsemble(nn.Module):\n    def __init__(self, modelA, modelB,modelC, modelD, modelE, nb_classes=5):\n        super(MyEnsemble, self).__init__()\n        self.modelA = modelA\n        self.modelB = modelB\n        self.modelC = modelC\n        self.modelD = modelD\n        self.modelE = modelE\n        # Remove last linear layer\n        self.modelA.classifier = nn.Identity()\n        self.modelB.classifier = nn.Identity()\n        self.modelC.classifier = nn.Identity()\n        self.modelD.classifier = nn.Identity()\n        self.modelE.classifier = nn.Identity()\n        # Create new classifier\n        self.classifier = nn.Linear(1280+1280+1408+1536+1792, nb_classes)\n    def forward(self, x):\n        x1 = self.modelA(x.clone())  # clone to make sure x is not changed by inplace methods\n        x1 = x1.view(x1.size(0), -1)\n        x2 = self.modelB(x)\n        x2 = x2.view(x2.size(0), -1)\n        x3 = self.modelC(x)\n        x3 = x3.view(x3.size(0), -1)\n        x4 = self.modelD(x)\n        x4 = x4.view(x4.size(0), -1)\n        x5 = self.modelE(x)\n        x5 = x5.view(x5.size(0), -1)\n        x = torch.cat((x1, x2, x3, x4, x5), dim=1)\n        x = self.classifier(F.relu(x))\n        return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modelA = models.efficientnet_b0(pretrained=True)\nmodelB = models.efficientnet_b1(pretrained=True)\nmodelC = models.efficientnet_b2(pretrained=True)\nmodelD = models.efficientnet_b3(pretrained=True)\nmodelE = models.efficientnet_b4(pretrained=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Freeze these models\nfor param in modelA.parameters():\n    param.requires_grad_(False)\n\nfor param in modelB.parameters():\n    param.requires_grad_(False)\n    \nfor param in modelC.parameters():\n    param.requires_grad_(False)\n    \nfor param in modelD.parameters():\n    param.requires_grad_(False)\n\nfor param in modelE.parameters():\n    param.requires_grad_(False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create ensemble model\nmodel = MyEnsemble(modelA, modelB, modelC, modelD, modelE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating an oof\ndef get_score(y_true, y_pred):\n    return accuracy_score(y_true, y_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cv split using stratifiedkfold\nfolds = train.copy()\nFold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\nepochs = 1\ncount=1\nlearning_rate = 1e-4\ncriterion = nn.CrossEntropyLoss()\noptimizer = Adam(model.parameters(), lr = learning_rate)\nscheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.2, patience=4, verbose=True, eps=1e-6)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor fold, (train_index, val_index) in enumerate(Fold.split(folds, folds['label'])):\n    train_df = folds.loc[train_index].reset_index(drop=True)\n    val_df = folds.loc[val_index].reset_index(drop=True)\n    train_dataset = TrainDataset(train_df, transform = train_transform)\n    valid_dataset = TrainDataset(val_df, transform = valid_transform)\n    train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=4, pin_memory=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=32, shuffle=False, num_workers=4, pin_memory=True)\n\n    with torch.enable_grad():\n        for epoch in range(epochs):\n            model.to(device)\n            model.train()\n            train_loss = 0.0\n            train_accuracy = 0.0\n            loop1 = tqdm(train_loader)\n            for image, label in loop1:\n                image = image.to(device)\n                label = label.to(device)\n            \n                preds = model(image)\n                loss = criterion(preds, label)\n            \n                optimizer.zero_grad()\n                loss.backward()\n            \n                optimizer.step()\n                train_loss += loss.item()\n                train_accuracy += get_score(label.cpu().detach().numpy(), preds.argmax(1).cpu().detach().numpy())\n                \n                loop1.set_description(f'EPOCH: {epoch+1}/{epochs} TRAIN')\n                loop1.set_postfix(TRAIN_AUC = train_accuracy/len(train_loader) , TRAIN_LOSS = train_loss/len(train_loader))\n    \n            with torch.no_grad():\n                valid_loss = 0.0\n                num_corrects = 0.0\n                valid_accuracy = 0.0\n                model.eval()\n                loop2 = tqdm(valid_loader)\n                for x, y in loop2:\n                    x = x.to(device)\n                    y = y.to(device)\n                    output = model(x)\n                    \n                    loss = criterion(output, y)\n                    valid_loss += loss.item()\n                    predictions = output.argmax(1)\n                    \n                    #saving the model\n                    score = get_score(y.cpu().detach().numpy(), output.argmax(1).cpu().detach().numpy())\n                    best_score = 0.\n                    if score > best_score:\n                        best_score = score\n                        torch.save({\n                            'model':model.state_dict()\n                        }, OUTPUT_DIR+f'ensemble_model_fold{fold}_best.path')\n                        \n                    check_point = torch.load(OUTPUT_DIR+f'ensemble_model_fold{fold}_best.path')\n                    \n                    valid_accuracy += get_score(y.cpu().detach().numpy(), output.argmax(1).cpu().detach().numpy())\n                    #creating an oof\n                    if fold == 0:\n                        oof_df = pd.DataFrame(output.cpu().detach().numpy(), columns=['Cassava Bacterial Blight (CBB)', 'Cassava Brown Streak Disease (CBSD)', 'Cassava Green Mottle (CGM)', 'Cassava Mosaic Disease (CMD)', 'Healthy'])\n                        oof_df.loc[:, 'prediction'] = predictions.cpu().detach().numpy()\n                        oof_df.loc[:, 'targets'] = y.cpu().detach().numpy()\n                        oof_df.loc[:, 'fold'] = fold\n                        oof_df.to_csv(OUTPUT_DIR + 'oof_df.csv', index=False)\n                    #progress bar\n                    loop2.set_description(f'EPOCH: {epoch+1}/{epochs} VALID')\n                    loop2.set_postfix(VAL_AUC = valid_accuracy/len(valid_loader) , VAL_LOSS = valid_loss/len(valid_loader))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('./output')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"./output/oof_df.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}