{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        #print(os.path.join(dirname, filename))\n        pass\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-30T08:46:02.106187Z","iopub.execute_input":"2021-05-30T08:46:02.106659Z","iopub.status.idle":"2021-05-30T08:46:15.975829Z","shell.execute_reply.started":"2021-05-30T08:46:02.106608Z","shell.execute_reply":"2021-05-30T08:46:15.974712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make the necessary imports \nimport torch\nfrom torch.utils.data import Dataset, random_split\nfrom torchvision import datasets,models\nimport torchvision\nimport torch.nn as nn\nfrom torchvision.transforms import ToTensor,Compose,RandomCrop,Resize,ToPILImage,RandomResizedCrop,RandomHorizontalFlip,Normalize,ConvertImageDtype\nimport matplotlib.pyplot as plt\nfrom torchvision.io import read_image\nfrom torch.utils.data import DataLoader\n\nimport torch.autograd as autograd         # computation graph\nfrom torch import Tensor                  # tensor node in the computation graph\nimport torch.nn as nn                     # neural networks\nimport torch.nn.functional as F           # layers, activations and more\nimport torch.optim as optim\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2021-05-30T08:46:15.980298Z","iopub.execute_input":"2021-05-30T08:46:15.980652Z","iopub.status.idle":"2021-05-30T08:46:17.302577Z","shell.execute_reply.started":"2021-05-30T08:46:15.980616Z","shell.execute_reply":"2021-05-30T08:46:17.301648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# to add - transforms - resized to 256,256\n# generate dataset class for our custom dataset\nclass CustomImageDataset(Dataset):\n    def __init__(self, annotations_file, img_dir, transform=Compose([Resize((256,256)),RandomResizedCrop(224),ConvertImageDtype(dtype=torch.float),Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n]), target_transform=None):\n        self.img_labels = pd.read_csv(annotations_file)\n        self.img_dir = img_dir\n        self.transform = transform\n        self.target_transform = target_transform\n\n    def __len__(self):\n        return len(self.img_labels)\n\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_dir, self.img_labels.iloc[idx, 0])\n        image = read_image(img_path)\n        label = self.img_labels.iloc[idx, 1]\n        if self.transform:\n            image = self.transform(image)\n        if self.target_transform:\n            label = self.target_transform(label)\n        sample = {\"image\": image, \"label\": label}\n        return image,label\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T08:46:17.305366Z","iopub.execute_input":"2021-05-30T08:46:17.305625Z","iopub.status.idle":"2021-05-30T08:46:17.316644Z","shell.execute_reply.started":"2021-05-30T08:46:17.305595Z","shell.execute_reply":"2021-05-30T08:46:17.315579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset=CustomImageDataset('../input/cassava-leaf-disease-classification/train.csv','../input/cassava-leaf-disease-classification/train_images')\nprint(len(dataset))\nprint(dataset[64])","metadata":{"execution":{"iopub.status.busy":"2021-05-30T08:46:17.318322Z","iopub.execute_input":"2021-05-30T08:46:17.318886Z","iopub.status.idle":"2021-05-30T08:46:17.487412Z","shell.execute_reply.started":"2021-05-30T08:46:17.318846Z","shell.execute_reply":"2021-05-30T08:46:17.486505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split into training and testing sets\ntrain_data,test_data=random_split(dataset, [int(9*len(dataset)/10),int(1*len(dataset)/10)+1 ], generator=torch.Generator().manual_seed(42))\n\n# insert into dataloaders \n# since number of classes are 5 , we will keep batch size 16\ntrain_dataloader=DataLoader(dataset=train_data,batch_size=16,shuffle=True)\ntest_dataloader=DataLoader(dataset=test_data,batch_size=16,shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T08:46:17.488911Z","iopub.execute_input":"2021-05-30T08:46:17.489318Z","iopub.status.idle":"2021-05-30T08:46:17.498388Z","shell.execute_reply.started":"2021-05-30T08:46:17.489276Z","shell.execute_reply":"2021-05-30T08:46:17.497556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import models for ensembling\nnum_models=3 # i plan to do 2 of each type resnet, MobileNet V2,VGG-19 with batch normalization\nmodels_list=[models.shufflenet_v2_x0_5(pretrained=True),models.resnet18(pretrained=True),models.googlenet(pretrained=True)]\nmodels_list=models_list","metadata":{"execution":{"iopub.status.busy":"2021-05-30T08:46:17.499764Z","iopub.execute_input":"2021-05-30T08:46:17.500154Z","iopub.status.idle":"2021-05-30T08:46:18.098112Z","shell.execute_reply.started":"2021-05-30T08:46:17.500113Z","shell.execute_reply":"2021-05-30T08:46:18.097179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# modify last layer\nfor model in models_list:\n    in_ft = model.fc.in_features\n    model.fc = nn.Linear(in_ft,5)\n    model.to(device)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T08:46:18.099477Z","iopub.execute_input":"2021-05-30T08:46:18.099832Z","iopub.status.idle":"2021-05-30T08:46:22.256193Z","shell.execute_reply.started":"2021-05-30T08:46:18.099791Z","shell.execute_reply":"2021-05-30T08:46:22.255293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make 9 splits of train_data\ntraining_splits=random_split(train_data, [int(len(train_data)/3) for i in range(2)]+[int(len(train_data)/3)], generator=torch.Generator().manual_seed(42))","metadata":{"execution":{"iopub.status.busy":"2021-05-30T08:46:22.258677Z","iopub.execute_input":"2021-05-30T08:46:22.259029Z","iopub.status.idle":"2021-05-30T08:46:22.264996Z","shell.execute_reply.started":"2021-05-30T08:46:22.25898Z","shell.execute_reply":"2021-05-30T08:46:22.264102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"split_dataloaders=[]\nfor split in training_splits:\n    dataloader_split=DataLoader(dataset=split,batch_size=16,shuffle=True)\n    split_dataloaders.append(dataloader_split)\nprint(split_dataloaders)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T08:46:22.266696Z","iopub.execute_input":"2021-05-30T08:46:22.267059Z","iopub.status.idle":"2021-05-30T08:46:22.279162Z","shell.execute_reply.started":"2021-05-30T08:46:22.267021Z","shell.execute_reply":"2021-05-30T08:46:22.278002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ndef train(epoch,model,dataloader): \n    model.train()\n    optimizer = optim.Adam(model.parameters(),lr=0.001,betas=(0.9,0.999))\n    losss=nn.CrossEntropyLoss()\n    viz=[]\n    for batch_idx,(data,label) in enumerate(dataloader):\n        \n        data,label=data.to(device),label.to(device)\n        \n        optimizer.zero_grad()\n        output=model(data)\n\n        # we use nll loss becuase we have added logsoftmax layer , or else we would have used cross entropy which combines both\n        loss=losss(output,label)\n        viz.append([batch_idx,loss])\n        loss.backward()\n        optimizer.step()\n        \n        if batch_idx % 100 == 0:\n            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n                    epoch, batch_idx * len(data), len(dataloader.dataset),\n                    100. * batch_idx / len(dataloader), loss.item()))\n    return viz\n\ndef test(model,dataloader):\n    model.eval()\n    with torch.set_grad_enabled(False):\n        size = len(dataloader.dataset)\n        pred_correct=0\n        for batch_id,(data,label) in enumerate(dataloader):\n            data,label=data.to(device),label.to(device)\n            output=model(data)\n            pred_correct += (label == torch.argmax(output,dim=1)).sum()\n  \n    print('the accuracy is {}'.format(pred_correct*100/size))\n  ","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:02:12.590421Z","iopub.execute_input":"2021-05-30T09:02:12.590809Z","iopub.status.idle":"2021-05-30T09:02:12.605085Z","shell.execute_reply.started":"2021-05-30T09:02:12.590773Z","shell.execute_reply":"2021-05-30T09:02:12.603695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# time to train ensemble\nfor epoch in range(0,5):\n    for i in range(len(models_list)):\n        viz=train(epoch,models_list[i],split_dataloaders[i])\n        plt.plot([i[0] for i in viz],[i[1] for i in viz])\n        plt.show()\n        ","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:02:14.473279Z","iopub.execute_input":"2021-05-30T09:02:14.473631Z","iopub.status.idle":"2021-05-30T09:17:05.269066Z","shell.execute_reply.started":"2021-05-30T09:02:14.4736Z","shell.execute_reply":"2021-05-30T09:17:05.265772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ensemble_test(models_list,test_dataloader):\n    size = len(test_dataloader.dataset)\n    pred_correct=0\n    for model in models_list:\n        model.eval()\n        \n    with torch.set_grad_enabled(False):\n        for batch_id,(data,label) in enumerate(test_dataloader):\n            onehot=torch.zeros(len(label),5)\n            data,label,onehot=data.to(device),label.to(device),onehot.to(device)\n            for model in models_list:\n                output=model(data)\n                #predicted_labels=torch.argmax(output,dim=1).to(device)\n                #onehot+=torch.nn.functional.one_hot(predicted_labels,5)\n                onehot+= output\n            batch_pred=torch.argmax(onehot,dim=1)\n            pred_correct += (label == batch_pred).sum()\n        print('accuaracy on the test set for model ensemble is {}'.format(pred_correct*100/size))            ","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:17:10.158133Z","iopub.execute_input":"2021-05-30T09:17:10.158516Z","iopub.status.idle":"2021-05-30T09:17:10.166176Z","shell.execute_reply.started":"2021-05-30T09:17:10.158476Z","shell.execute_reply":"2021-05-30T09:17:10.165314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ensemble_test(models_list,test_dataloader)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:17:10.807894Z","iopub.execute_input":"2021-05-30T09:17:10.808254Z","iopub.status.idle":"2021-05-30T09:17:52.514945Z","shell.execute_reply.started":"2021-05-30T09:17:10.808206Z","shell.execute_reply":"2021-05-30T09:17:52.513442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class testImageDataset(Dataset): # submission\n    def __init__(self, img_dir, transform=Compose([Resize((256,256)),RandomResizedCrop(224),ConvertImageDtype(dtype=torch.float),Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n]), target_transform=None):\n       # self.img_labels = pd.read_csv(annotations_file)\n        self.img_dir = img_dir\n        self.transform = transform\n        self.target_transform = target_transform\n        self.name_list=[]\n        for dirname, _, filenames in os.walk(img_dir):\n            for filename in filenames:\n                self.name_list.append(filename)\n        \n    def __len__(self):\n        return len(self.name_list)\n\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_dir, self.name_list[idx])\n        image = read_image(img_path)\n        #label = self.img_labels.iloc[idx, 1]\n        if self.transform:\n            image = self.transform(image)\n        if self.target_transform:\n            label = self.target_transform(label)\n        #sample = {\"image\": image, \"label\": label}\n        return image,self.name_list[idx]\n","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:28:36.712879Z","iopub.execute_input":"2021-05-30T09:28:36.713252Z","iopub.status.idle":"2021-05-30T09:28:36.721564Z","shell.execute_reply.started":"2021-05-30T09:28:36.713202Z","shell.execute_reply":"2021-05-30T09:28:36.72069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ensemble_sub(models_list,test_dataloader):\n    size = len(test_dataloader.dataset)\n    pred_correct=0\n    for model in models_list:\n        model.eval()\n    df = pd.DataFrame(columns=['image_id','label'])\n    with torch.set_grad_enabled(False):\n        for batch_id,(data,img_path) in enumerate(test_dataloader):\n            onehot=torch.zeros(len(img_path),5)\n            data,onehot=data.to(device),onehot.to(device)\n            for model in models_list:\n                output=model(data)\n                #predicted_labels=torch.argmax(output,dim=1).to(device)\n                #onehot+=torch.nn.functional.one_hot(predicted_labels,5)\n                onehot+= output\n            batch_pred=torch.argmax(onehot,dim=1)\n            #print(img_path[0])\n            for i in range(len(img_path)):\n                df=df.append({'image_id':str(img_path[i]),'label':int(batch_pred[i].item())},ignore_index=True)\n            #pred_correct += (label == batch_pred).sum()\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:31:03.035501Z","iopub.execute_input":"2021-05-30T09:31:03.03587Z","iopub.status.idle":"2021-05-30T09:31:03.045347Z","shell.execute_reply.started":"2021-05-30T09:31:03.035834Z","shell.execute_reply":"2021-05-30T09:31:03.044576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sub=testImageDataset('../input/cassava-leaf-disease-classification/test_images')\nsub_dataloader=DataLoader(dataset=test_sub,batch_size=16,shuffle=False)\ndf=ensemble_sub(models_list,sub_dataloader)\ndf.to_csv('submission.csv',index_label=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-30T09:31:19.089847Z","iopub.execute_input":"2021-05-30T09:31:19.090197Z","iopub.status.idle":"2021-05-30T09:31:19.166914Z","shell.execute_reply.started":"2021-05-30T09:31:19.090163Z","shell.execute_reply":"2021-05-30T09:31:19.166036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}