{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport os.path as osp\nimport copy\n\nfrom tqdm import tqdm\ntqdm.pandas()\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport pickle\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom torchvision import datasets, models\nfrom torchvision.utils import make_grid\n\nimport os\nimport time\nfrom PIL import Image\nfrom IPython.display import display\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.  Thiết lập giá trị của các tham số cố định\nThiết lập giá trị của các tham số cố định có trong bài:\n1. *num_classes*: tổng số lượng nhãn.\n2. *img_size*: kích thước của ảnh sau quá trình resized bởi DataLoader.\n3. *batch_size*: kích thước mỗi batch.\n4. *device*: accelerator được sử dụng.\n5. *criterion*: hàm mất mát (loss function) được sử dụng.","metadata":{}},{"cell_type":"code","source":"class Config:\n    num_classes = 12\n    img_size = 224\n    batch_size = 64\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    min_lr = 10**-12\n    max_lr = 10\n    pretrained = False\n    criterion = nn.CrossEntropyLoss()\n    epochs = 20","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Chuẩn bị dữ liệu","metadata":{}},{"cell_type":"code","source":"train_image_path = '../input/plant-pathology-2021-fgvc8/train_images'\ntest_image_path = '../input/plant-pathology-2021-fgvc8/test_images'\ntrain_df_path = '../input/plant-pathology-2021-fgvc8/train.csv'\ntest_df_path = '../input/plant-pathology-2021-fgvc8/sample_submission.csv'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/plant-pathology-2021-fgvc8/train.csv\")\ndf_sub = pd.read_csv(\"/kaggle/input/plant-pathology-2021-fgvc8/sample_submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#The number of labels\nlen(df_train.labels.unique())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 12 different labels.","metadata":{}},{"cell_type":"code","source":"#The no.values per label\ndf_train.labels.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nle = LabelEncoder()\ndef encode_label(df):\n    df['encoded_label'] = le.fit_transform(df.labels.values)\n    return df\n\nencode_label(df_train)\n    \n# Lưu từ điển mã hóa\ndf_labels_idx = df_train.loc[df_train.duplicated([\"labels\", \"encoded_label\"])==False]\\\n                [[\"encoded_label\", \"labels\"]].set_index(\"encoded_label\").sort_index()\ndisplay(df_labels_idx)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2 Tạo đường dẫn cho ảnh và phân chia tập huấn luyện, tập thẩm định","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_datapath_list(phase='train', val_size=0.25):\n    if phase in [\"train\", \"val\"]:\n        phase_path = \"train_images\"\n    elif phase in [\"test\"]:\n        phase_path = \"test_images\"\n    else:\n        print(f\"{phase} not in path\")    \n        \n    \"\"\"\n    Use resized training dataset for betting training speed\n    Resized datase from: https://www.kaggle.com/ankursingh12/resized-plant2021\n    \"\"\"\n    if phase == 'train' or phase == 'val': \n        rootpath = \"/kaggle/input/resized-plant2021/img_sz_256/\"\n    else:\n        rootpath = \"/kaggle/input/plant-pathology-2021-fgvc8/test_images/\"\n    \n    target_path = osp.join(rootpath+\"/*.jpg\")\n    path_list = []\n    \n    for path in glob.glob(target_path):\n        path_list.append(path)\n        \n    if phase in [\"train\", \"val\"]:\n        train, val = train_test_split(path_list, test_size=val_size, random_state=0, shuffle=True)\n        if phase == \"train\":\n            path_list = train\n        else:\n            path_list = val\n    \n    return path_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_list = make_datapath_list(phase='train')\nprint(f'The length of training set: {len(train_list)}')\nval_list = make_datapath_list(phase='val')\nprint(f'The length of valuation set: {len(val_list)}')\ntest_list = make_datapath_list(phase='test')\nprint(f'The length of testing set: {len(test_list)}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.3 Augumentation\nNhận thấy chúng ta có khá ít dữ liệu, ...","metadata":{}},{"cell_type":"code","source":"import albumentations as A\nfrom albumentations import Compose\nfrom albumentations.pytorch import ToTensorV2\nimport cv2\n\ntransform = {\n    'train': Compose([\n        A.Rotate(p=0.1, limit=(-85, 80)),\n        A.RandomShadow(\n            num_shadows_lower=2, \n            num_shadows_upper=3, \n            shadow_dimension=3, \n            shadow_roi=(0, 0.7, 0.4, 0.8), \n            p=0.4\n        ),\n        A.ShiftScaleRotate(\n            shift_limit=0.055, \n            scale_limit=0.065, \n            rotate_limit=35, \n            p=0.6\n        ),\n        A.RandomFog(\n            fog_coef_lower=0.2, \n            fog_coef_upper=0.2, \n            alpha_coef=0.2, \n            p=0.3\n        ),\n        A.RGBShift(\n            r_shift_limit=25, \n            g_shift_limit=15, \n            b_shift_limit=15, \n            p=0.3\n        ),\n        A.RandomBrightnessContrast(p=0.3),\n        A.GaussNoise(\n            var_limit=(50, 70),  \n            always_apply=False, \n            p=0.3\n        ),\n        A.Resize(height=Config.img_size, width=Config.img_size),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        ToTensorV2()\n    ]),\n    'val': Compose([\n        A.Resize(Config.img_size, Config.img_size),\n        A.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),            \n        ToTensorV2()\n    ]),\n    'test': Compose([\n        A.Resize(Config.img_size, Config.img_size),\n        A.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.4 Create custom Dataset","metadata":{}},{"cell_type":"code","source":"class PlantDataset(Dataset):\n    \"\"\"\n    Class to create a Dataset\n    \n    Attributes\n    ----------\n    df_train : DataFrame\n        DataFrame containing the image labels.\n    file_list : list\n        A list containing the paths to the images\n    transform : object\n        Instance of the preprocessing class (ImageTransform)\n    phase : 'train' or 'val' or 'test'\n        Specify whether to use train, validation, or test\n    \"\"\"\n    def __init__(self, df_train, file_list, transform=None, phase='train'):\n        self.df_train = df_train\n        self.df_labels_idx = df_labels_idx\n        self.file_list = file_list\n        self.transform = transform[phase]\n        self.phase = phase\n        \n    def __len__(self):\n        \"\"\"\n        Returns the number of images.\n        \"\"\"\n        return len(self.file_list)\n    \n    def __getitem__(self, index):\n        \"\"\"\n        Get data in Tensor format and labels of preprocessed images.\n        \"\"\"\n        \n        # Load the index number image.\n        img_path = self.file_list[index]\n        img = Image.open(img_path)\n        \n        # Preprocessing images\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img_transformed = self.transform(image=img)\n        \n        # image name\n        image_name = img_path[-20:]\n        \n        # Extract the labels\n        if self.phase in [\"train\", \"val\"]:\n            label = df_train.loc[df_train[\"image\"]==image_name][\"encoded_label\"].values[0]\n        elif self.phase in [\"test\"]:\n            label = -1\n        \n        return img_transformed, label, image_name","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = PlantDataset(df_train, train_list, transform=transform, phase='train')\nval_dataset = PlantDataset(df_train, val_list, transform=transform, phase='val')\ntest_dataset = PlantDataset(df_train, test_list, transform=transform, phase='test')\n\nindex = 0\n\nprint(\"【train dataset】\")\nprint(f\"img num : {train_dataset.__len__()}\")\n# print(f\"img : {train_dataset.__getitem__(index)[0].size()}\")\nprint(f\"label : {train_dataset.__getitem__(index)[1]}\")\nprint(f\"image name : {train_dataset.__getitem__(index)[2]}\")\n\nprint(\"\\n【validation dataset】\")\nprint(f\"img num : {val_dataset.__len__()}\")\n# print(f\"img : {val_dataset.__getitem__(index)[0].size()}\")\nprint(f\"label : {val_dataset.__getitem__(index)[1]}\")\nprint(f\"image name : {val_dataset.__getitem__(index)[2]}\")\n\nprint(\"\\n【test dataset】\")\nprint(f\"img num : {test_dataset.__len__()}\")\n# print(f\"img : {test_dataset.__getitem__(index)[0].size()}\")\nprint(f\"label : {test_dataset.__getitem__(index)[1]}\")\nprint(f\"image name : {test_dataset.__getitem__(index)[2]}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## 2.5 Create Dataloader","metadata":{}},{"cell_type":"code","source":"train_dataloader = DataLoader(train_dataset, batch_size=Config.batch_size,shuffle=True)\nval_dataloader = DataLoader(val_dataset, batch_size=Config.batch_size, shuffle=False)\ntest_dataloader = DataLoader(test_dataset, batch_size=Config.batch_size, shuffle=False)\n\n# to Dictionary\ndataloaders_dict = {\"train\": train_dataloader, \"val\": val_dataloader, \"test\": test_dataloader}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Define model","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import f1_score, accuracy_score","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_config = {\n    \"name\": \"2 FCs, 0.0001 Lr, 30 Epochs\",\n    \"classifier\": torch.nn.Sequential(\n                  torch.nn.Linear(1024, 512),\n                  torch.nn.Linear(512, 12)),\n    \"lr\": 0.0001,\n    \"epoch\": 30\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"use_pretrained = True\npretrained_model = models.densenet121(pretrained=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pretrained_model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name, classifier, lr, epoch = model_config.values()\n\npretrained_model.classifier = classifier\n\nprint(f'Model name: {name}')\nprint(pretrained_model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Learning rate finder\nhttps://towardsdatascience.com/the-learning-rate-finder-6618dfcb2025","metadata":{}},{"cell_type":"code","source":"def lr_finder(model, min_lr, max_lr, dataset_lenght=train_dataset.__len__(), \\\n              batch_size=Config.batch_size, criterion=Config.criterion):\n    iter_lrs = [min_lr]\n    iter_losses = []\n    \n    factor = np.exp(np.log(max_lr / min_lr) / (dataset_lenght / batch_size))\n    \n    # Train model with 1 epoch\n    model.to(Config.device)\n    for i, data in tqdm(enumerate(dataloaders_dict['train']), total=len(dataloaders_dict['train'])):\n        \n        optimizer = Adam(model.parameters(), lr=min_lr)\n        \n        # set inputs, labels based on dataloader's batch data\n        inputs = data[0]['image']\n        labels = data[1]\n        inputs = inputs.to(Config.device)\n        labels = labels.to(Config.device)\n\n        #zero the parameter gradients\n        optimizer.zero_grad()\n\n        # forward\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        \n        # backward + optimize only if in training phase\n        loss.backward()\n        optimizer.step()\n                \n        # Update and append next iteration learning rate\n        iter_lrs.append(min_lr)\n        min_lr = min_lr * factor\n        \n        # Append this iteration loss\n        iter_losses.append(np.log(loss.cpu().data.numpy().tolist()))\n        \n    iter_lrs.pop()\n    \n    # Plot loss vs log-scaled learning rate\n    plt.figure(figsize=(10, 7))\n    plot = sns.lineplot(iter_lrs, iter_losses)\n    plot.set(xscale=\"log\", \n             xlabel=\"Learning Rate (log-scale)\", \n             ylabel=\"Training Loss\",\n             title=\"Optimal learning rate is slightly below minimum\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_model = copy.deepcopy(pretrained_model)\nlr_finder(test_model, Config.min_lr, Config.max_lr)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dựa vào biểu đồ, chúng ta có thể chọn learning rate của Adam = 10^-3. ","metadata":{}},{"cell_type":"code","source":"Config.lr = 10**-3\noptimizer = Adam(pretrained_model.parameters(), lr=Config.lr)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Huấn luyện và đánh giá model","metadata":{}},{"cell_type":"markdown","source":"### 4.1 Các biểu đồ của mô hình\n> 1. Loss\n> 2. Accuracy\n> 3. F1","metadata":{}},{"cell_type":"code","source":"def plot_result(train_losses, train_accuracy, train_f1, val_losses, val_accuracy, val_f1):\n    fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(18, 7))\n    ax1.plot(train_losses, label='Train')\n    ax1.plot(val_losses, label='Validation')\n    ax1.set_title('Loss')\n    ax1.legend()\n\n    ax2.plot(train_accuracy, label='Train')\n    ax2.plot(val_accuracy, label='Validation')\n    ax2.set_title('Accuracy')\n    ax2.legend()\n\n    ax3.plot(train_f1, label='Train')\n    ax3.plot(val_f1, label='Validation')\n    ax3.set_title('F1 Score')\n    ax3.legend()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.2 Train model","metadata":{}},{"cell_type":"code","source":"def append_list(list, appended):\n    for el in appended:\n        list.append(el)\n    return list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, criterion, optimizer, num_epochs=Config.epochs):\n    \n    train_losses = []\n    train_accuracy = []\n    train_f1 = []\n\n    val_losses = []\n    val_accuracy = []\n    val_f1 = []\n    \n    print(f\"Devices to be used : {Config.device}\")\n    model.to(Config.device)\n    torch.backends.cudnn.benchmark = True\n    \n    start_time = time.time()\n        \n    for epoch in range(num_epochs):\n        print('Epoch {}/{}'.format(epoch + 1, num_epochs))\n        print('-' * 10)\n        \n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n            \n            epoch_targets = []\n            epoch_predictions = []\n\n            # Iterate over data.\n            for i, data in tqdm(enumerate(dataloaders_dict[phase]), total=len(dataloaders_dict[phase])):\n#                 inputs = np.transpose(data[0]['image'], (0, 3, 1, 2))\n                inputs = data[0]['image']\n                labels = data[1]\n                inputs = inputs.to(Config.device)\n                labels = labels.to(Config.device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    loss = criterion(outputs, labels)\n\n                    _, preds = torch.max(outputs, 1)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                # statistics                \n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n                \n                np_preds = preds.cpu().data.numpy()\n                np_labels = labels.cpu().data.numpy()\n                append_list(epoch_predictions, np_preds)\n                append_list(epoch_targets, np_labels)\n                \n                batch_f1 = f1_score(preds.cpu().data.numpy(), labels.cpu().data.numpy(), average='weighted')\n                \n#                 if i % 50 == 0 and i != 0:\n#                 print(f'Batch: {i}  |  Loss: {loss.item():.4f}   |   F1-score: {batch_f1:.4f}%')         \n\n            epoch_loss = running_loss / len(dataloaders_dict[phase].dataset)\n            epoch_acc = running_corrects.double() / len(dataloaders_dict[phase].dataset)\n            \n            epoch_f1 = f1_score(epoch_predictions, epoch_targets, average='weighted')\n            \n            if phase == 'train':\n                train_losses.append(epoch_loss)\n                train_accuracy.append(epoch_acc)\n                train_f1.append(epoch_f1)\n            else:\n                val_losses.append(epoch_loss)\n                val_accuracy.append(epoch_acc)\n                val_f1.append(epoch_f1)\n    \n            print('{} Loss: {:.4f} Acc: {:.4f} F1_score: {:.4f}'.format('----> ' + phase.capitalize(), epoch_loss, epoch_acc, epoch_f1))\n            \n    print(f'\\nDuration: {time.time() - start_time:.0f} seconds') # print the time elapsed\n    \n    plot_result(train_losses, train_accuracy, train_f1, val_losses, val_accuracy, val_f1)\n    \n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trained_model = train_model(pretrained_model, Config.criterion, optimizer)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.2 Save model","metadata":{}},{"cell_type":"code","source":"def save_model(model, filename):\n    Pkl_Filename = name + \".pkl\"\n\n    with open(Pkl_Filename, 'wb') as file:\n        pickle.dump(model, file)\n        \nsave_model(trained_model, name)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Make prediction","metadata":{}},{"cell_type":"markdown","source":"## 5.2 Create pridictor ","metadata":{}},{"cell_type":"code","source":"class PlantPredictor():\n    \"\"\"\n    Class for predicting labels from output results\n    \n    Attributes\n    ----------\n    df_labels_idx: DataFrame\n        DataFrame that associates INDEX with a label name\n    \"\"\"\n    \n    def __init__(self, model, df_labels_idx, dataloaders_dict):\n        self.model = model\n        self.df_labels_idx = df_labels_idx\n        self.dataloaders_dict = dataloaders_dict\n        self.df_submit = pd.DataFrame()\n        \n    \n    def __predict_max(self, out):\n        \"\"\"\n        Get the label name with the highest probability.\n        \n        Parameters\n        ----------\n        predicted_label_name: str\n            Name of the label with the highest prediction probability\n        \"\"\"\n        maxid = np.argmax(out.detach().numpy(), axis=1)\n        df_predicted_label_name = self.df_labels_idx.iloc[maxid]\n        \n        return df_predicted_label_name\n    \n    def inference(self):\n        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        df_pred_list = []\n        for i, data in enumerate(self.dataloaders_dict['test']):\n            image_name = data[2]\n            self.model.to(device)\n            inputs = data[0]['image']\n            inputs = inputs.to(device)\n            out = self.model(inputs)\n            device = torch.device(\"cpu\")\n            out = out.to(device)\n            df_pred = self.__predict_max(out).reset_index(drop=True)\n            df_pred[\"image\"] = image_name\n            df_pred_list.append(df_pred)\n            \n        self.df_submit = pd.concat(df_pred_list, axis=0)\n        self.df_submit = self.df_submit[[\"image\", \"labels\"]].reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5.3 Make submission","metadata":{}},{"cell_type":"code","source":"predictor = PlantPredictor(trained_model, df_labels_idx, dataloaders_dict)\npredictor.inference()\ndf_submit = predictor.df_submit.copy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submit.to_csv('submission.csv', index=False)\ndf_submit","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}