{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n#import os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n #   for filename in filenames:\n  #      print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <u>Libraries</u>"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torchvision\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom torch import nn\n\nfrom PIL import Image\nimport cv2\nimport glob\n\nimport matplotlib.pyplot as plt\nimport os\nimport json\nfrom tqdm.notebook import tqdm\nimport random\n\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, \n    ShiftScaleRotate, CenterCrop, Resize, RandomGamma, RandomShadow, RandomGridShuffle\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\nimport sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport timm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <u> Helper Function </u>"},{"metadata":{},"cell_type":"markdown","source":"# <u> Display the image example </u>"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create the path for the image in the test image folder\nt_path = '../input/cassava-leaf-disease-classification/test_images'\ntest_image_path = os.path.join(t_path, '2216849948.jpg')\n\n# Create PIL image\nim=Image.open(test_image_path)\n\n# From PIL image to tensor\ntensor_im = transforms.ToTensor()(im)\n\n# From tensor to PIL image\ntensor_to_pil = transforms.ToPILImage()(tensor_im)\nprint(\"Image Size :\", tensor_to_pil.size)\nprint(\"Image channels : \" ,tensor_to_pil.mode)\n\nimage = cv2.imread(test_image_path)\nimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <u>Load the train csv file</u>"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_image_path = '../input/cassava-leaf-disease-classification/test_images/'\n\ntrain_csv_path = \"../input/cassava-leaf-disease-classification/train.csv\"\ntrain_df = pd.read_csv(train_csv_path)\n\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## <u> Number occurence of each label </u>\n\nAs we can see, the label 3 is much more present than the others"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## <u>Info of the df</u>\nWe have 21397 images and the type of the labels is int64"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Meanings of the labels"},{"metadata":{"trusted":true},"cell_type":"code","source":"label_file = \"../input/cassava-leaf-disease-classification/label_num_to_disease_map.json\"\n\nlabels = json.load(open(label_file))\nlabels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plan\n\n - Create the Dataset object : \n     - First separate the file names and their labels\n     - __init__() For each of the image id in the dataframe : \n     \n         - create the filename from the 'image_id' column\n         - load it as an image\n         - convert it pil_img = Image.open(img_path) -> The To_tensor transform will transform it into an image for us\n         - Get its label from the 'label' column\n         \n     - Create the __len__() method which will be the number of training examples\n     - Create the __getitem__() method \n - Create the Transform object TO DO\n - Create the DataLoader   \n - Create the CNN object\n - Create loss and optim objects\n - Make the training loop"},{"metadata":{},"cell_type":"markdown","source":"# <u> Dataset </u>\n\nI don't know if it is normal but it takes ages to construct.\n\nIt was because I was loading all the images in the constructor.\n\n"},{"metadata":{},"cell_type":"markdown","source":"\nclass Data(Dataset):\n    def __init__(self, Dataframe, path, Transform):\n        \n        #This function returns the images and their labels\n        def getimages(Dataframe, path):\n            images = list()\n            labels = list()\n            for i in range(len(Dataframe)):\n                print(i/len(Dataframe))\n                name,label = list(zip(train_df.image_id, train_df.label))[i]\n                \n                img_path   = os.path.join(path, name)\n                pil_image  =  Image.open(img_path)\n                \n                images.append(pil_image)\n                labels.append(label)\n                \n            return images, labels\n    \n        self.images, self.labels = getimages(Dataframe, path)\n        \n        self.transform = Transform\n        \n        # In case I forget the images dimensions\n        self.images_dim = self.images[0].size\n      \n    def __len__(self): return len(self.images)\n    \n    def __getitem__(self, index):\n        \n        image = self.images[index]\n        label = self.labels[index]\n        \n        return self.transform(image), label"},{"metadata":{"trusted":true},"cell_type":"code","source":"names_train = train_df['image_id'].values\nlabels_train = train_df['label'].values\n\nnames_test = [name for name in (os.listdir(test_image_path))]\nnames_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# inspired by https://www.kaggle.com/ateplyuk/simplest-starting-code-cassava-leaf-pytorch\n\nclass Data(Dataset):\n    \n    def __init__(self, directory, names, labels, Transform):\n        self.dir = directory\n        self.names = names\n        self.labels = labels\n        self.transform = Transform\n        \n    def __len__(self): return len(self.names)\n    \n    def __getitem__(self, index):\n        \n        \n        #image = cv2.imread(os.path.join(self.dir, self.names[index]))\n        #image = cv2.cvtColor(np.float32(image), cv2.COLOR_BGR2RGB)\n        image = Image.open(os.path.join(self.dir, self.names[index]))       \n        label = self.labels[index]\n        \n        if self.dir == '../input/cassava-leaf-disease-classification/train_images':\n            return self.transform(image), label\n        \n        elif self.dir == '../input/cassava-leaf-disease-classification/test_images/':\n            img_name = self.names[index]\n\n            return index, self.transform(image), img_name\n            \n        else: \n            print('incorrect directory')\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"names_train = train_df['image_id'].values\nlabels_train = train_df['label'].values\n\nproto_names = train_df['image_id'][:5000].values\nproto_labels = train_df['label'][:5000].values\n\nnames_test = [name for name in (os.listdir(test_image_path))]\n\ntraining_path = '../input/cassava-leaf-disease-classification/train_images'\ntest_image_path = '../input/cassava-leaf-disease-classification/test_images/'\n\n\nIM_SIZE = 256\n\np = 0.2\n\n# Test with albumentation\nAlbum = Compose([\n            CenterCrop(int(IM_SIZE/2), int(IM_SIZE/2), p = p),\n            Resize(IM_SIZE, IM_SIZE),\n            Transpose(p = p),\n            HorizontalFlip(p = p),\n            VerticalFlip(p),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=p),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=p),\n            RandomGamma(gamma_limit=(50, 250), p=p), \n            \n            GaussNoise(var_limit=(2500, 3000), mean=0, p = p),\n            IAAAdditiveGaussianNoise(loc=70, scale=(2.5500000000000003, 12.75), p = p),\n            RandomGridShuffle(grid=(2, 2), p = p),\n            OpticalDistortion(distort_limit=0.75, shift_limit=0.75,p = p),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),], p=1.)\n\n\n# Test with torchvision transform api\nTransform = transforms.Compose(\n    [transforms.ToTensor(),\n     transforms.Resize((IM_SIZE, IM_SIZE)),\n     transforms.RandomRotation(90),\n     transforms.RandomRotation(180),\n     transforms.RandomHorizontalFlip(p=0.5),\n     transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])\n\n\n#Train_dataset = Data(directory = training_path, names = proto_names, labels = proto_labels, Transform = Transform)\n#Test_dataset  = Data(directory = test_image_path, names = names_test, labels = None, Transform = Album)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"proto_names.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <u>DataLoader</u>"},{"metadata":{"trusted":true},"cell_type":"code","source":"#batchsize = 16\n\n#train_dl = DataLoader(dataset = Train_dataset, batch_size = batchsize)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <u>CNN</u>"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"class CNN(nn.Module):\n    \n    def __init__(self):\n        super().__init__()\n        \n        self.conv1 = nn.Conv2d(3,12,3,1)\n        self.conv2 = nn.Conv2d(32,24,3,1)\n        self.fc1 = nn.Linear(1016064, 512)\n        self.fc2 = nn.Linear(512, 5)\n        \n    def forward(self,x):\n        x = nn.functional.relu(self.conv1(x))\n        x = torch.flatten(nn.functional.max_pool2d(nn.functional.relu(self.conv2(x)),2),1)\n        print(x.shape)\n        x = self.fc1(x)\n        x = nn.functional.relu(x)\n        x = self.fc2(x)\n        output = nn.functional.log_softmax(x, dim=1)\n        return output\n    \n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loss and Optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"#optimizer = torch.optim.Adam(params = model.parameters() , lr=0.001, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False)\n#criterion = torch.nn.CrossEntropyLoss()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Loop"},{"metadata":{},"cell_type":"markdown","source":"I forgot to divide the training accuracy by the batchsize inside each epoch with resulted in accuracy above 1"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, epochs, train_dl, valid_dl, optimizer, scheduler, criterion, device, batchsize):\n    print('='*20)\n    print('Starting Training')\n    print('='*20)\n    Loss_tracker = []\n    Train_Accuracy_tracker = []\n    Valid_Accuracy_tracker = []\n    \n    for epoch in range(epochs):\n        \n        print('Starting epoch', epoch+1)\n        model.train()\n        Training_loss = 0\n        Training_accuracy = 0\n        Cost = 0\n        \n        \n        Path = 'CassavaAtEpch' + str(epoch)\n        torch.save({\n            'epoch': epoch,\n            'model_state_dict': model.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'cost': Cost}, Path)\n        \n        \n        i = 0\n        for (image, label) in tqdm(train_dl, desc=\"Training\"):\n            i+=1                         \n            image = image.to(device)\n            label = label.to(device)\n            \n            optimizer.zero_grad()\n            logit = model(image)\n            loss = criterion(logit, label)\n            loss.backward()\n            optimizer.step()\n           \n            prediction = torch.argmax(logit,1)\n            \n            Training_accuracy += sum(prediction == label)/batchsize\n            Training_loss += loss.detach().item()\n            \n        Cost = Training_loss/i\n        print(i)\n        print('Epoch : ', epoch+1, 'Loss : ', Cost)\n        print('Epoch : ', epoch+1, 'Accuracy : ', Training_accuracy/i)\n        \n        scheduler.step(Cost)\n        \n        \n        Valid_accuracy = 0\n        model.eval()\n        i = 0\n        for (image, label) in tqdm(valid_dl, desc = 'Evaluating'):\n            i+=1\n            image = image.to(device)\n            label = label.to(device)\n            \n            logit = model(image)\n           \n            prediction = torch.argmax(logit,1)\n            \n            Valid_accuracy += sum(prediction == label)/batchsize            \n        \n        print('Epoch : ', epoch+1, 'Valid Accuracy : ', Valid_accuracy/i) \n        \n        Loss_tracker.append(Cost)\n        Train_Accuracy_tracker.append(Training_accuracy/i)\n        Valid_Accuracy_tracker.append(Valid_accuracy/i)\n        \n        Path = 'CassavaAtEpch' + str(epoch)\n        torch.save({\n            'epoch': epoch,\n            'model_state_dict': model.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'cost': Cost}, Path)\n        \n        \n        \n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training\n\nI first made the mistake of training on the whole dataset which was bad for validation and for evaluation. I decided to use only 5000 images at first to see what would improve the accuracy in less time. Surprisingly, 5000 and 23000 images provided the same performences of 61% accuracy. Then I used the randomsplit function of Pytorch to get a proper validation set.\n\nAfter looking at what other people were saying, the first big problem was the data imbalance. Which I kind of solved by adding weights to my loss function. Now after 10 epochs I have 71% accuracy on the train set and 68% on the validation set and I am happy because it doesn't seem to be overfitting yet and the accuracy can still climb.\n\nAfter going to sleep I reached 80% of accuracy after 30 epochs but unfortunately The process stopped because my computer shutdowned.\n\nThen I tried to add (a lot of) data augmentation with albumentation since everybody is using it but the performence went back to 60%. Maybe I was adding to much, maybe I did not write it the right way. I need to do further research. My get is that my resnet18 model was not deep enough to deal with such variety of augmentation. I should try with a more complex network. Actually, when I went back to my basic augmentation (but kept openning the images with cv2) the performence was very random. There might be something with cv2 I don't get yet."},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#dataset = Data(directory = training_path, names = proto_names, labels = proto_labels, Transform = Transform)\n\ndataset = Data(directory = training_path, names = names_train, labels = labels_train, Transform = Transform)\n\ntrain_dataset, valid_dataset = torch.utils.data.random_split(dataset = dataset, lengths = [18000, 3397])\n\nbatchsize = 12\ntrain_dl = DataLoader(dataset = train_dataset, batch_size = batchsize, shuffle = True, num_workers = 4)\nvalid_dl = DataLoader(dataset = valid_dataset, batch_size = batchsize, shuffle = True, num_workers = 3)\n\n#model = CNN()\n\n#model = torchvision.models.resnet18(pretrained = False)\n#model.fc = nn.Linear(512, 5, bias=True)\n\nmodel = timm.create_model('tf_efficientnet_b5_ns', pretrained = False)\nmodel.classifier = nn.Linear(model.classifier.in_features, 5)\n\n\noptimizer = torch.optim.Adam(params = model.parameters() , lr=0.01, \n                             betas=(0.9, 0.999), eps=1e-08, \n                             weight_decay=0, amsgrad=False)\n\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor = 0.2, patience = 10, verbose = True)\n\nDEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nweight = torch.tensor([1,1,1,0.3,1], device = DEVICE)\ncriterion = torch.nn.CrossEntropyLoss(weight = weight)\n\n\nmodel = model.to(DEVICE)\n\nPATH = '../input/weights/CassavaAtEpch2 (1)'\n\ncheckpoint = torch.load(PATH)\nmodel.load_state_dict(checkpoint['model_state_dict'])\noptimizer.load_state_dict(checkpoint['optimizer_state_dict'])\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor = 0.2, patience = 10, verbose = True)\n\ntrain(model = model, epochs = 30, train_dl = train_dl, \n      valid_dl = valid_dl, optimizer = optimizer ,\n      scheduler = scheduler, criterion = criterion, device = DEVICE, batchsize = batchsize)\n\n#train(model = model, epochs = 3, train_dl = train_dl, \n      valid_dl = valid_dl, optimizer = optimizer ,\n      scheduler = scheduler, criterion = criterion, device = DEVICE, batchsize = batchsize)"},{"metadata":{"trusted":true},"cell_type":"code","source":"0.7007415491700173","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = timm.create_model('tf_efficientnet_b5_ns', pretrained = False)\nmodel.classifier = nn.Linear(model.classifier.in_features, 5)\nPATH = '../input/cassava-last-weights/CassavaAtEpch29'\noptimizer = torch.optim.Adam(params = model.parameters() , lr=0.01, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False)\n\ncheckpoint = torch.load(PATH)\nmodel.load_state_dict(checkpoint['model_state_dict'])\noptimizer.load_state_dict(checkpoint['optimizer_state_dict'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Transform = transforms.Compose(\n    [transforms.ToTensor(),\n     transforms.Resize((IM_SIZE, IM_SIZE)),\n     transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])\n\ntest_dataset = Data(directory = test_image_path, names = names_test, labels = labels_train, Transform = Transform)\n\ntest_dataloader = torch.utils.data.DataLoader(\n        test_dataset, \n        batch_size=1,\n        num_workers=0,\n        shuffle=False,\n        pin_memory=False,\n    )\n\nfor i, img ,img_name in test_dataloader:\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel = model.to(device)\n\nsubmission = pd.DataFrame(columns=('image_id','label'))\nfor i, img ,img_name in test_dataloader:\n        model.eval()\n        img = img.to(device)\n        index = i.item()\n        label = torch.argmax(model(img)).item()\n        submission = submission.append([{'image_id':img_name[0],'label':label}],ignore_index=True)\nsubmission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}