{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"pip install tez","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-26T18:09:36.965938Z","iopub.execute_input":"2021-08-26T18:09:36.966292Z","iopub.status.idle":"2021-08-26T18:09:44.690686Z","shell.execute_reply.started":"2021-08-26T18:09:36.966258Z","shell.execute_reply":"2021-08-26T18:09:44.689755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install efficientnet-pytorch","metadata":{"execution":{"iopub.status.busy":"2021-08-26T19:26:51.493221Z","iopub.execute_input":"2021-08-26T19:26:51.493535Z","iopub.status.idle":"2021-08-26T19:27:00.391609Z","shell.execute_reply.started":"2021-08-26T19:26:51.493506Z","shell.execute_reply":"2021-08-26T19:27:00.390529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip show tez","metadata":{"execution":{"iopub.status.busy":"2021-08-26T18:09:44.693765Z","iopub.execute_input":"2021-08-26T18:09:44.694027Z","iopub.status.idle":"2021-08-26T18:09:50.431142Z","shell.execute_reply.started":"2021-08-26T18:09:44.693997Z","shell.execute_reply":"2021-08-26T18:09:50.430209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport albumentations #for augmentation\nimport matplotlib.pyplot as plt\nimport pandas as pd\n\nimport tez\nfrom tez.datasets import ImageDataset\nfrom tez.callbacks import EarlyStopping\n\nimport torch\nimport torch.nn as nn\nfrom efficientnet_pytorch import EfficientNet\nfrom torch.nn import functional as F\n\nimport torchvision #required on using pre trained model\n\nfrom sklearn import metrics, model_selection\n\n%matplotlib inline \n#above command shows just below the code and also gets saved in the document","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:24.333229Z","iopub.execute_input":"2021-08-26T20:07:24.333555Z","iopub.status.idle":"2021-08-26T20:07:24.343946Z","shell.execute_reply.started":"2021-08-26T20:07:24.333524Z","shell.execute_reply":"2021-08-26T20:07:24.342898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfx = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:25.493552Z","iopub.execute_input":"2021-08-26T20:07:25.493948Z","iopub.status.idle":"2021-08-26T20:07:25.521745Z","shell.execute_reply.started":"2021-08-26T20:07:25.493916Z","shell.execute_reply":"2021-08-26T20:07:25.52097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfx.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:26.012136Z","iopub.execute_input":"2021-08-26T20:07:26.012459Z","iopub.status.idle":"2021-08-26T20:07:26.021639Z","shell.execute_reply.started":"2021-08-26T20:07:26.01243Z","shell.execute_reply":"2021-08-26T20:07:26.020747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfx.label.value_counts() ","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:26.492099Z","iopub.execute_input":"2021-08-26T20:07:26.492381Z","iopub.status.idle":"2021-08-26T20:07:26.499676Z","shell.execute_reply.started":"2021-08-26T20:07:26.492355Z","shell.execute_reply":"2021-08-26T20:07:26.498573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Biased toward class 3","metadata":{}},{"cell_type":"code","source":"df_train,df_valid = model_selection.train_test_split( \n    dfx,\n    test_size = 0.1,\n    random_state = 42,\n    stratify = dfx.label.values\n)\n\ndf_train = df_train.reset_index(drop=True)\ndf_valid = df_valid.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:27.592365Z","iopub.execute_input":"2021-08-26T20:07:27.592698Z","iopub.status.idle":"2021-08-26T20:07:27.6167Z","shell.execute_reply.started":"2021-08-26T20:07:27.592645Z","shell.execute_reply":"2021-08-26T20:07:27.61595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* We used stratify because there is not equal examples for the various classes wehere we want to classsify so we do it so that the train and test both have equal amount of proportions of the classes (it is used mostly in multi class classification)\n* The reset index will be used to add another column with the index values like 0,1,2.....","metadata":{}},{"cell_type":"code","source":"df_train.head","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:28.532153Z","iopub.execute_input":"2021-08-26T20:07:28.532472Z","iopub.status.idle":"2021-08-26T20:07:28.541422Z","shell.execute_reply.started":"2021-08-26T20:07:28.532442Z","shell.execute_reply":"2021-08-26T20:07:28.540493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_valid.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:28.964152Z","iopub.execute_input":"2021-08-26T20:07:28.964518Z","iopub.status.idle":"2021-08-26T20:07:28.973268Z","shell.execute_reply.started":"2021-08-26T20:07:28.964478Z","shell.execute_reply":"2021-08-26T20:07:28.972477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.image_id.values # .values return the values in that column","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:29.384107Z","iopub.execute_input":"2021-08-26T20:07:29.384391Z","iopub.status.idle":"2021-08-26T20:07:29.390406Z","shell.execute_reply.started":"2021-08-26T20:07:29.384365Z","shell.execute_reply":"2021-08-26T20:07:29.389207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path = \"../input/cassava-leaf-disease-classification/train_images\"","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:29.85193Z","iopub.execute_input":"2021-08-26T20:07:29.852299Z","iopub.status.idle":"2021-08-26T20:07:29.856281Z","shell.execute_reply.started":"2021-08-26T20:07:29.852257Z","shell.execute_reply":"2021-08-26T20:07:29.854962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#now we will make a list to store the address of each image for train and valid\n\ntrain_image_paths = [\n    os.path.join(image_path, x) for x in df_train.image_id.values \n]\n\n\nvalid_image_paths = [\n    os.path.join(image_path, x) for x in df_valid.image_id.values\n]","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:30.412117Z","iopub.execute_input":"2021-08-26T20:07:30.412401Z","iopub.status.idle":"2021-08-26T20:07:30.457022Z","shell.execute_reply.started":"2021-08-26T20:07:30.412375Z","shell.execute_reply":"2021-08-26T20:07:30.456133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Above we used \"list comprehension\" other way of writing the same code is\n\n```\ntrain_image_paths = []\nfor x in df_train.image_id.values:\n    train_image_paths.append(os.path.join(image_path, x)) \n    \n```    ","metadata":{}},{"cell_type":"code","source":"train_image_paths[:5]","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:31.684297Z","iopub.execute_input":"2021-08-26T20:07:31.684609Z","iopub.status.idle":"2021-08-26T20:07:31.692175Z","shell.execute_reply.started":"2021-08-26T20:07:31.68458Z","shell.execute_reply":"2021-08-26T20:07:31.691301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_targets = df_train.label.values\nvalid_targets = df_valid.label.values","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:32.172242Z","iopub.execute_input":"2021-08-26T20:07:32.172546Z","iopub.status.idle":"2021-08-26T20:07:32.177084Z","shell.execute_reply.started":"2021-08-26T20:07:32.172517Z","shell.execute_reply":"2021-08-26T20:07:32.176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* we can write down the torch standard way proceeding forward by first defining our dataset but we will now use dataset method from the tez library for that.","metadata":{}},{"cell_type":"code","source":"train_dataset = ImageDataset(\n    image_paths = train_image_paths,\n    targets = train_targets,\n    augmentations = None \n)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:33.304247Z","iopub.execute_input":"2021-08-26T20:07:33.304566Z","iopub.status.idle":"2021-08-26T20:07:33.311026Z","shell.execute_reply.started":"2021-08-26T20:07:33.304533Z","shell.execute_reply":"2021-08-26T20:07:33.310149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#code copied from the newer example in the tez repo\nTRAIN_BATCH_SIZE = 256\nVALID_BATCH_SIZE = 256\nEPOCHS = 20\nIMAGE_SIZE = 256","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:33.892073Z","iopub.execute_input":"2021-08-26T20:07:33.892381Z","iopub.status.idle":"2021-08-26T20:07:33.89643Z","shell.execute_reply.started":"2021-08-26T20:07:33.892352Z","shell.execute_reply":"2021-08-26T20:07:33.895452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* The above code will return a dictionary with \"image\" and \"targets\" as key ","metadata":{}},{"cell_type":"code","source":"def plot_img(img_dict):\n    \n    image_tensor = img_dict[\"image\"] #extracting the tensor inside the image\n    target = img_dict[\"targets\"] #extracting the target\n    print(target)\n    plt.figure(figsize=(5,5))\n    image = image_tensor.permute(1,2,0) / 255 \n    plt.imshow(image)\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:35.692279Z","iopub.execute_input":"2021-08-26T20:07:35.692596Z","iopub.status.idle":"2021-08-26T20:07:35.699389Z","shell.execute_reply.started":"2021-08-26T20:07:35.692563Z","shell.execute_reply":"2021-08-26T20:07:35.698393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" .permute() rearranges the original tensor according to the desired ordering and returns a new multidimensional rotated tensor","metadata":{}},{"cell_type":"code","source":"plot_img(train_dataset[45])","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:37.512353Z","iopub.execute_input":"2021-08-26T20:07:37.512685Z","iopub.status.idle":"2021-08-26T20:07:37.796257Z","shell.execute_reply.started":"2021-08-26T20:07:37.512633Z","shell.execute_reply":"2021-08-26T20:07:37.795322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#if using tez use albumentations\ntrain_aug = albumentations.Compose(\n    \n    [\n        albumentations.RandomResizedCrop(TRAIN_BATCH_SIZE,TRAIN_BATCH_SIZE), #this is how we want to resize the image\n        albumentations.Transpose(p=0.5), #Transpose the input by swapping rows and columns.\n        albumentations.HorizontalFlip(p=0.5),\n        albumentations.VerticalFlip(p=0.5),\n        albumentations.CoarseDropout(p=0.5)\n    ]\n    \n) \n\n#now we have added the augmentation to the train dataset\n\ntrain_dataset = ImageDataset(\n    image_paths = train_image_paths,\n    targets = train_targets,\n    augmentations = train_aug \n)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:38.272685Z","iopub.execute_input":"2021-08-26T20:07:38.273125Z","iopub.status.idle":"2021-08-26T20:07:38.279719Z","shell.execute_reply.started":"2021-08-26T20:07:38.273081Z","shell.execute_reply":"2021-08-26T20:07:38.278852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#now we do the same for valid\n\nvalid_aug = albumentations.Compose(\n    \n    [\n        albumentations.CenterCrop(TRAIN_BATCH_SIZE,TRAIN_BATCH_SIZE,p=1.0),\n        albumentations.Resize(TRAIN_BATCH_SIZE,TRAIN_BATCH_SIZE),\n        albumentations.Transpose(p=0.5), \n        albumentations.HorizontalFlip(p=0.5),\n        albumentations.VerticalFlip(p=0.5)\n    ]\n    \n) \n\nvalid_dataset = ImageDataset(\n    image_paths = valid_image_paths,\n    targets = valid_targets,\n    augmentations = valid_aug \n)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:39.184362Z","iopub.execute_input":"2021-08-26T20:07:39.184736Z","iopub.status.idle":"2021-08-26T20:07:39.190159Z","shell.execute_reply.started":"2021-08-26T20:07:39.18469Z","shell.execute_reply":"2021-08-26T20:07:39.189022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_img(train_dataset[45])","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:39.904371Z","iopub.execute_input":"2021-08-26T20:07:39.904703Z","iopub.status.idle":"2021-08-26T20:07:40.091555Z","shell.execute_reply.started":"2021-08-26T20:07:39.904671Z","shell.execute_reply":"2021-08-26T20:07:40.090632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LeafModel(tez.Model):\n    def __init__(self,num_classes,pretrained=True):\n        super().__init__()\n        self.effnet = EfficientNet.from_pretrained(\"efficientnet-b3\") #from the new code\n        self.dropout = nn.Dropout(0.1)\n        self.out = nn.Linear(1536,num_classes) #from the new code\n        # it has 1536 input and num classes output\n        self.step_scheduler_after = \"epoch\" # we step it after every batch\n        \n        \n    def loss(self, outputs, targets):\n        if targets is None:\n            return None\n        return nn.CrossEntropyLoss()(outputs,targets) #because it is multi class classification\n    \n    \n    def monitor_metrics(self, outputs, targets):\n        outputs = torch.argmax(outputs, dim=1).cpu().detach().numpy() #max value in each row \n        #we are taking the argmax of the output\n        targets = targets.cpu().detach().numpy()\n        acc =  metrics.accuracy_score(targets, outputs)\n        \n        return {\n            \"accuracy\" : acc\n        }\n    \n    def fetch_optimizer(self):\n        opt = torch.optim.Adam(self.parameters(), lr=3e-4) #now it is inheriting from nn.module\n        return opt\n    \n    def fetch_scheduler(self):\n        \n        sch = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(\n            self.optimizer, T_0=10, T_mult=1, eta_min=1e-6, last_epoch=-1\n        )\n        return sch\n    \n    def forward(self,image,targets=None):\n        ##########\n        batch_size, _, _, _ = image.shape\n\n        x = self.effnet.extract_features(image)\n        x = F.adaptive_avg_pool2d(x, 1).reshape(batch_size, -1)\n        \n        ##########\n        \n        outputs = self.out(self.dropout(x)) #the output is a pass through \"out\"\n        #when using tez the forward function should return three things\n        if targets is not None:\n            loss = self.loss(outputs, targets)\n            mon_metrics =  self.monitor_metrics(outputs, targets)\n            \n            return outputs, loss, mon_metrics\n            \n        return outputs, None, None    #if no targets only return outputs\n        ","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:54.48707Z","iopub.execute_input":"2021-08-26T20:07:54.487388Z","iopub.status.idle":"2021-08-26T20:07:54.500371Z","shell.execute_reply.started":"2021-08-26T20:07:54.48736Z","shell.execute_reply":"2021-08-26T20:07:54.499505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LeafModel(num_classes= dfx.label.nunique(), pretrained = True)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:55.83311Z","iopub.execute_input":"2021-08-26T20:07:55.833425Z","iopub.status.idle":"2021-08-26T20:07:56.06074Z","shell.execute_reply.started":"2021-08-26T20:07:55.833397Z","shell.execute_reply":"2021-08-26T20:07:56.059647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Now we look at our model","metadata":{}},{"cell_type":"code","source":"model","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:57.664616Z","iopub.execute_input":"2021-08-26T20:07:57.664976Z","iopub.status.idle":"2021-08-26T20:07:57.677564Z","shell.execute_reply.started":"2021-08-26T20:07:57.664945Z","shell.execute_reply":"2021-08-26T20:07:57.676544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"es = EarlyStopping(\n    monitor=\"valid_loss\", model_path=\"model.bin\", patience=3, mode=\"min\"\n)\n#early stopping is when our model starts doing bad so we stop it from going fut=rther in epochs\nmodel.fit(\n    train_dataset,\n    valid_dataset=valid_dataset,\n    train_bs=32,\n    valid_bs=64,\n    device=\"cuda\",\n    epochs=10,\n    callbacks=[es],\n    fp16=True,\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T20:07:58.712601Z","iopub.execute_input":"2021-08-26T20:07:58.712968Z","iopub.status.idle":"2021-08-26T21:44:00.772922Z","shell.execute_reply.started":"2021-08-26T20:07:58.712939Z","shell.execute_reply":"2021-08-26T21:44:00.771652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(\"model.bin\")","metadata":{"execution":{"iopub.status.busy":"2021-08-26T21:44:48.490776Z","iopub.execute_input":"2021-08-26T21:44:48.491184Z","iopub.status.idle":"2021-08-26T21:44:49.105898Z","shell.execute_reply.started":"2021-08-26T21:44:48.491147Z","shell.execute_reply":"2021-08-26T21:44:49.10474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}