{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Need your help in figuring out what I'm doing wrong.\n\nHi all, This is not the usual kernel you might have expected. I'm pretty much new to CV and I wanted to participate in this competition to learn CV. \n\nI have been trying out EfficientNet b3 and as a matter of fact other pre-trained variants of EfficientNets as well as other models like ResNet18, 50 too. But one thing is cosistent, were my validation loss does not reduce beyond the first epoch or 2nd epoch, so is my model accuracy too. \nYou can find from the plot below (I moved the cell up after running the entire code below). \n\nI'm actually stuck for quite sometime with this and I am not able to find what's wrong in my code. I have referred other kernels too but could not understand why. \n\nSince this is my first time with CV, I would really appreciate if anyone could point me out were am I going wrong. Please do provide your suggestions too if there is anything else I should try.\n\nNote: The below plot is only for one of the 5 stratified folds, however it's the same scenario for other folds and any models too. "},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"plt.plot(fold_avg_train_loss_history, label='train')\nplt.plot(fold_avg_eval_loss_history, label='eval')\nplt.legend()\n\n# fold_avg_train_loss_history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ! pip3 install ../input/efficientnet-model/dataclasses-0.6-py3-none-any.whl\n# ! pip3 install ../input/efficientnet-model/numpy-1.19.4-cp37-cp37m-manylinux2010_x86_64.whl\n# ! pip3 install ../input/efficientnet-model/torch-1.7.0-cp37-cp37m-manylinux1_x86_64.whl\n# ! pip3 install ../input/efficientnet-model/typing_extensions-3.7.4.3-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kaggle = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if kaggle:\n    #!pip install --no-index --find-links ../input/efficientnet-model/efficientnet_pytorch-0.7.0/dist/ efficientnet-pytorch\n    !pip3 install timm\nelse:\n#     ! pip3 install efficientnet_pytorch\n    ! pip3 install timm\n    ! pip3 install albumentations\n#     ! pip3 install albumentations\n    pass","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nimport os\nimport glob\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\nimport torch\n\nimport matplotlib.pyplot as plt\n\nimport random\nfrom tqdm import tqdm\n\nfrom PIL import Image\n\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import StratifiedKFold\n\n# from efficientnet_pytorch import EfficientNet\n\nimport timm\n\nimport albumentations as A\nfrom albumentations import Compose\nfrom albumentations.pytorch import ToTensorV2\n\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings(action='ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Creating configurations"},{"metadata":{"trusted":true},"cell_type":"code","source":"if kaggle:\n    data_label_path_2020 = \"/kaggle/input/cassava-leaf-disease-classification/train.csv\"\n    train_images_2020_path =  \"/kaggle/input/cassava-leaf-disease-classification/train_images/\"\n    test_images_path =  \"/kaggle/input/cassava-leaf-disease-classification/test_images/\"\n    model_save_path = \"./\"\n    \nelse:\n    data_label_path_2020 = \"../data/train.csv\"\n    \n    train_images_2020_path = \"../data/train_images\"\n    test_images_path = \"../data/test_images\"\n    model_save_path = \"../saved_models/\"\n          \n        \nconfig = {'data_label_path_2020': data_label_path_2020,\n          'train_images_path_2020': train_images_2020_path,\n          'use_2019': False, \n          \n          'test_images_path': test_images_path,\n          'resize_dim': (384, 384),\n          'train_batch_size': 16, \n          'test_batch_size': 64,\n          'num_workers':8, \n          'device':'cuda' if torch.cuda.is_available() else 'cpu', \n          'model_name': 'effnet', \n          'saved_model_folder': model_save_path}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reading the input file¶"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reading the training dataset-2020\ntrain_df_2020 = pd.read_csv(config['data_label_path_2020'])\ntrain_df_2020['images_path'] = config['train_images_path_2020']\ntrain_df_2020['data_year'] = 2020\ntrain_df_2020['train_images_path'] = train_df_2020.images_path + '/' + train_df_2020.image_id\n# train_df_2020 = train_df_2020.sample(100).reset_index(drop=True)\n\nnum_targets = len(train_df_2020.label.unique())\n\ntrain_df = train_df_2020.copy()\n\n# Forming the test dataset\ntest_df = pd.DataFrame()\ntest_df['image_id'] = list(os.listdir(config['test_images_path']))\ntest_df['images_path'] = config['test_images_path']\ntest_df['test_images_path'] = test_df.images_path + '/' + test_df.image_id\n\nprint(train_df.shape, test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_2020","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Stratified KFold"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df_2020['fold'] = -1\n\ntotal_folds = 5\n\nnp.random.seed(100)\nkf = StratifiedKFold(n_splits=total_folds, shuffle=True, random_state=100)\n\nfor i, (train, test) in enumerate(kf.split(train_df_2020, train_df_2020.label)):\n    train_df_2020['fold'][test] = i\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining the Dataset formation class"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset():\n    def __init__(self, dataframe, img_path, resize_dim, data_type, transpose, transforms=None):\n        self.dataframe = dataframe\n        self.img_path = img_path\n        self.resize_dim = resize_dim\n        self.data_type = data_type\n        self.transpose = transpose\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.dataframe)\n    \n    def __getitem__(self, idx):\n        image = Image.open(self.img_path[idx])\n\n        image = np.array(image)\n        \n        if self.transforms:\n            image = self.transforms(image=image)['image']\n        \n        if self.data_type  == 'test':\n            label = 1\n        else:          \n            label = self.dataframe['label'][idx]\n\n        return {'image': image,\n                'label': torch.tensor(label, dtype=torch.long)}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"Image_NET_mean=[0.485, 0.456, 0.406]\nImage_NET_std=[0.229, 0.224, 0.225]\n    \naugmentation = {'train': Compose([A.Resize(config['resize_dim'][0], config['resize_dim'][1], p=1.0),\n#                                   A.RandomResizedCrop(256, 256, p=1.0),\n#                                   A.Transpose(p=0.5),\n                                  A.HorizontalFlip(p=0.5),\n                                  A.VerticalFlip(p=0.5),\n                                  A.RandomRotate90(p=0.5),\n                                  A.ShiftScaleRotate(p=0.5),\n                                  A.Rotate(p=0.5),\n#                                   A.RandomBrightnessContrast(p=0.5),\n#                                   A.RandomShadow(p=0.5),\n                                  A.Normalize(mean=Image_NET_mean, \n                                              std=Image_NET_std, \n                                              max_pixel_value=255.0,\n                                              always_apply=True,\n                                              p=1.0),\n                                  ToTensorV2(p=1.0),\n                                 ], p=1.) , \n                'valid':Compose([A.Resize(config['resize_dim'][0], config['resize_dim'][1], p=1.0),\n                                A.Normalize(mean=Image_NET_mean,\n                                            std=Image_NET_std,\n                                            max_pixel_value=255.0, \n                                            always_apply=True,\n                                            p=1.0),\n                                ToTensorV2(p=1.0),\n                               ], p=1)\n               }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Getting the data from dataclass:\n\ndef get_data(df, resize_dim, data_type, transpose, transforms):\n    data_class = Dataset(dataframe=df,\n                         img_path= df['train_images_path'].values,\n                         resize_dim=resize_dim,\n                         data_type=data_type,\n                         transpose=transpose,\n                         transforms=transforms)\n    return data_class\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining the DataLoader"},{"metadata":{"trusted":true},"cell_type":"code","source":"def define_data_loader(data_class, batch_size, shuffle, num_workers):\n    images_dataloader = DataLoader(data_class, \n                                   batch_size=batch_size, \n                                   shuffle=shuffle,\n                                   num_workers=config['num_workers'])\n    \n    return images_dataloader\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Forming the Classifer model class"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"timm.list_models()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Efficientnet_b3_classifier_for_cassava(nn.Module):\n    def __init__(self, num_labels):\n        super(Efficientnet_b3_classifier_for_cassava, self).__init__()\n        self.num_labels = num_labels\n        \n        ####################\n        self.model = timm.create_model('efficientnet_b3', pretrained=True)\n        for param in self.model.parameters():\n            param.requires_grad = True\n        self.last_layer_size = self.model.classifier.in_features #get the in feature of the last layer\n        self.model.classifier = nn.Linear(self.last_layer_size, self.num_labels)                \n\n\n    def forward(self, x):\n        output = self.model(x)\n        return output\n    ","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"Efficientnet_b3_classifier_for_cassava(num_labels=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\n\ndef setting_seed(seed_no):\n    random.seed(seed_no)\n    np.random.seed(seed_no)\n    torch.manual_seed(seed_no)\n    torch.cuda.manual_seed_all(seed_no)    \n\n    \ndef model_saving(model, epoch):    \n        model_name = config['saved_model_folder'] + 'best_'+ config['model_name'] + 'epoch_' + str(epoch) + \".bin\"\n        torch.save(model.state_dict(), model_name)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_fn(data_loader, model, optimizer, scheduler, params):\n    \n    model.train()\n    setting_seed(seed_no = seed)\n    \n    train_loss  = 0\n    for index, dataset in tqdm(enumerate(data_loader), total = len(data_loader)):\n        image = dataset['image'].to(config['device'], dtype = torch.float)\n        target = dataset['label'].to(config['device'], dtype = torch.long)\n    \n        prediction = model(image)\n        \n        step_loss = criterion(prediction, target)\n        \n        step_loss.sum().backward()\n        optimizer.step()\n#         scheduler.step()\n        optimizer.zero_grad()\n        \n        #torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n        \n        train_loss += step_loss\n        \n    \n    avg_train_loss = train_loss/len(data_loader)\n    \n    return avg_train_loss\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Eval function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def eval_fn(data_loader, model):\n    \n    model.eval()\n    eval_loss = 0\n    \n    actual_output = torch.tensor([]).to(config['device'], dtype=torch.long)\n    predicted_prob = torch.tensor([]).to(config['device'], dtype=torch.float)\n    \n    with torch.no_grad():\n        for index, dataset in tqdm(enumerate(data_loader), total = len(data_loader)):\n            image = dataset['image'].to(config['device'], dtype = torch.float)\n            target = dataset['label'].to(config['device'], dtype = torch.long)\n            \n            prediction = model(image)\n            \n            step_loss = criterion(prediction, target)\n            eval_loss += step_loss\n            \n            actual_output = torch.cat((actual_output, target))\n            predicted_prob = torch.cat((predicted_prob, prediction))\n                        \n            #print(\"Prediction\", prediction.shape, prediction)\n            #print(\"predicted_output\", predicted_output.shape, predicted_output)\n            #print(\"###########################################\")\n            \n        actual_class = np.array(actual_output.detach().cpu())\n        predicted_class = np.argmax(np.array(predicted_prob.detach().cpu()), axis = 1)\n\n        conf_mat = confusion_matrix(actual_class, predicted_class)\n        print(conf_mat)\n\n        avg_eval_loss = eval_loss/len(data_loader)        \n\n    return avg_eval_loss, actual_class, predicted_prob,  predicted_class\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Engine"},{"metadata":{"trusted":true},"cell_type":"code","source":"def training_engine(EPOCHS, train_data, valid_data, patience, lr, fold):\n       \n    setting_seed(seed_no = seed)\n    model = Efficientnet_b3_classifier_for_cassava(num_labels=num_targets)\n    model = nn.DataParallel(model)\n    model.to(config['device'])\n    \n    optimizer_grouped_parameters = model.parameters() #params_2_tune(model)\n    optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr = lr, \n                                 weight_decay=0.00001)\n#     optimizer = torch.optim.SGD(optimizer_grouped_parameters, lr = 0.0005)\n    \n    total_steps = len(train_data) * EPOCHS\n    \n    # Set up the learning rate scheduler\n    # scheduler = transformers.get_linear_schedule_with_warmup(optimizer,\n    #                                                         num_warmup_steps=0, # Default value\n    #                                                         num_training_steps=total_steps)\n\n    # This scheduler will be called after the validation loss is calculated.\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=2, \n                                                           threshold=0.0001, threshold_mode='rel', cooldown=0, \n                                                           min_lr=0, eps=1e-08, verbose=True)\n    \n    best_accuracy=0\n    counter=0\n    \n    fold_avg_train_loss_history = []\n    fold_avg_eval_loss_history = []\n    \n    for epoch in range(1, EPOCHS):    \n    \n        # Training\n        avg_train_loss = train_fn(data_loader = train_data,\n                                  model = model,\n                                  optimizer = optimizer, \n                                  scheduler = scheduler, \n                                  params = optimizer_grouped_parameters)\n\n        # Evaluation\n        avg_eval_loss, actual_class, predicted_prob, predicted_class = eval_fn(data_loader = valid_data,\n                                                                               model = model)\n        \n        acc = accuracy_score(actual_class, predicted_class)\n\n        print(f\"Fold: {fold}, Epoch: {epoch}/{EPOCHS}, train_loss: {avg_train_loss}, eval_loss {avg_eval_loss}, Accuracy: {acc}\")\n\n        if (acc>best_accuracy):\n            best_accuracy = acc\n            best_eval_loss = avg_eval_loss\n            counter = 0\n            \n            final_epoch = epoch\n            \n            print('Saving the model')\n            model_saving(model, epoch=epoch)\n        \n        else:\n            counter+=1\n            print(f\"Accuracy did not improve from the best {best_accuracy}, patience is {counter}/{patience}\")\n            final_epoch = epoch\n            \n            if counter==patience:\n                print(f\"Maximum patience level {patience} reached so exiting the training\")\n                final_epoch = epoch-patience if epoch>patience else epoch\n                \n                break            \n        \n        fold_avg_train_loss_history.append(avg_train_loss.item())\n        fold_avg_eval_loss_history.append(avg_eval_loss.item())\n        \n    return model, actual_class, predicted_prob, predicted_class, \\\n           fold_avg_train_loss_history, fold_avg_eval_loss_history, \\\n           best_accuracy, best_eval_loss, final_epoch\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# K Fold Training"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"fold_summary = {}\nfor i in range(total_folds):\n    current_fold = i+1\n    train_local = train_df_2020.loc[train_df_2020.fold!=i, :].reset_index(drop=True)\n    test_local = train_df_2020.loc[train_df_2020.fold==i, :].reset_index(drop=True)\n    \n    train_local_images = get_data(df=train_local, \n                                  resize_dim=config['resize_dim'], \n                                  data_type='train', \n                                  transpose=True,\n                                  transforms=augmentation['train'])\n\n    test_local_images = get_data(df=test_local,\n                                 resize_dim=config['resize_dim'],\n                                 data_type='train',\n                                 transpose=True,\n                                 transforms=augmentation['valid'])\n    \n    train_local_images_dataloader = define_data_loader(data_class=train_local_images, \n                                                       batch_size=config['train_batch_size'], \n                                                       shuffle=True, \n                                                       num_workers=config['num_workers'])\n\n    test_local_images_dataloader = define_data_loader(data_class=test_local_images, \n                                                      batch_size=config['test_batch_size'], \n                                                      shuffle=False, \n                                                      num_workers=config['num_workers'])\n\n    #print(f\"Training the fold {current_fold}\")\n    # Training on the Local dataset\n    seed = 50\n    model, actual, predicted_prob, predicted_class, fold_avg_train_loss_history, fold_avg_eval_loss_history, best_accuracy, best_eval_loss, final_epoch = training_engine(EPOCHS=10000, \n                                                                                                                                                                            train_data=train_local_images_dataloader, \n                                                                                                                                                                            valid_data=test_local_images_dataloader, \n                                                                                                                                                                            patience=15, \n                                                                                                                                                                            lr=0.0001,\n                                                                                                                                                                            fold=current_fold)\n    \n    \n\n    fold_summary['fold' + str(i+1)] = f\"Fold: {i+1}, Epoch: {final_epoch}, Eval loss : {np.round(best_eval_loss.detach().cpu(), 5)}, Eval accuracy : {np.round(best_accuracy, 5)}\"\n    \n    print(\"\")\n    print(\"######################################################################################\")\n    print(\"################################# Fold results #######################################\")\n    print(\"######################################################################################\")\n    for fold_info in fold_summary.keys():\n        print(fold_summary[fold_info])\n    print(\"\")\n    \n    break","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"conda_pytorch_p36","language":"python","name":"conda_pytorch_p36"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}