{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Pytorch VGG-16 Fine tuning\nUsing PyTorch, I fine-tuned the learned weights for the VGG16 network architecture.\nI'm sure there are a lot of things that could be improved, but I hope this will be helpful for everyone implementing this in PyTorch.\nPlease let me know if there's anything I should fix!","metadata":{}},{"cell_type":"code","source":"import glob\nimport os.path as osp\n\nimport pandas as pd\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom tqdm.notebook import tqdm\nfrom PIL import Image\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.utils.data as data\nimport torchvision\nfrom torchvision import models, transforms\nimport cv2\nimport albumentations as A","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! ls /kaggle/input/resized-plant2021/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Constant Config","metadata":{}},{"cell_type":"code","source":"SELECT_MODEL = 'VGG16' # VGG16, AlexNet, DenseNet, submit\nLOAD_PRETRAIN = True\n\nPRETRAIN_PATH = '../input/finetuningmodelzoo/vgg16_final-50epoch_fine_tuning_v1.h' # VGG16\n# PRETRAIN_PATH = '../input/finetuningmodelzoo/alexnet_final200epoch_fine_tuning_v1.h' # AlexNet\n# PRETRAIN_PATH = '../input/finetuningmodelzoo/augment-dense-opensimpleclf-5.h' # DenseNet\nSAVE_WEIGHT_FILENAME = 'augment-vgg16-opensimpleclf-1'\n\nIS_SUBMIT = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CROP_WIDTH = 224\nCROP_HEIGHT = 224 \nSIZE = 256\nMEAN = (0, 0, 0)\nSTD = (1, 1, 1)\nif SELECT_MODEL == 'VGG16' or SELECT_MODEL == 'AlexNet':\n    MEAN = (0.485, 0.456, 0.406)\n    STD = (0.229, 0.224, 0.225)\n\nAGU_PROB = 0.4","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_IMAGE_PATH = '/kaggle/input/resized-plant2021/img_sz_256'\nTEST_IMAGE_PATH = '../input/plant-pathology-2021-fgvc8/test_images'\n# TEST_IMAGE_PATH = '../input/plant-pathology-2021-fgvc8/train_images'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Process","metadata":{}},{"cell_type":"markdown","source":"## Import Data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\ndf_train = pd.read_csv(\"/kaggle/input/plant-pathology-2021-fgvc8/train.csv\")\ndf_sub = pd.read_csv(\"/kaggle/input/plant-pathology-2021-fgvc8/sample_submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Check train","metadata":{}},{"cell_type":"code","source":"display(df_train)\ndisplay(df_train.labels.value_counts())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.size","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d_set = set()\nfor k in df_train.labels.unique():\n    d_set = d_set | set(k.split(\" \"))\nprint(f\"num of labels: {len(d_set)}  {d_set}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Check submission","metadata":{}},{"cell_type":"code","source":"display(df_sub)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Label encoding","metadata":{}},{"cell_type":"code","source":"def to_label(df):\n    \"\"\"\n    Function for Label encoding.\n    \"\"\"\n    le = LabelEncoder()\n    df[\"labels_n\"] = le.fit_transform(df.labels.values)\n    return df\n\ndf_train = to_label(df_train)\ndf_labels_idx = df_train.loc[df_train.duplicated([\"labels\", \"labels_n\"])==False]\\\n                [[\"labels_n\", \"labels\"]].set_index(\"labels_n\").sort_index()\ndisplay(df_labels_idx)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image Path Builder","metadata":{}},{"cell_type":"code","source":"def make_datapath_list(phase=\"train\", val_size=0.25):\n    \"\"\"\n    Function to create a PATH to the data.\n    \n    Parameters\n    ----------\n    phase : 'train' or 'val' or 'test'\n        Specify whether to use Train data or test data.\n    val_size : float\n        Ratio of validation data to train data\n        \n    Returns\n    -------\n    path_lsit : list\n        A list containing the PATH to the data.\n    \"\"\"\n    \n    if phase in [\"train\", \"val\"]:\n        phase_path = \"train_images\"\n    elif phase in [\"test\"]:\n        phase_path = \"test_images\"\n    else:\n        print(f\"{phase} not in path\")\n    rootpath = \"/kaggle/input/plant-pathology-2021-fgvc8/\"\n#     rootpath = \"/kaggle/input/resized-plant2021/img_sz_256/\"\n    target_path = osp.join(TRAIN_IMAGE_PATH , '*.jpg') if  phase in ['train', 'val'] else osp.join(TEST_IMAGE_PATH, \"*.jpg\")\n\n    path_list = []\n    \n    for path in glob.glob(target_path):\n        path_list.append(path)\n        \n    if phase in [\"train\", \"val\"]:\n        train, val = train_test_split(path_list, test_size=val_size, random_state=0, shuffle=True)\n        if phase == \"train\":\n            path_list = train\n        else:\n            path_list = val\n    \n    return path_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_list = make_datapath_list(phase=\"train\")\nprint(f\"train data length : {len(train_list)}\")\nval_list = make_datapath_list(phase=\"val\")\nprint(f\"validation data length : {len(val_list)}\")\ntest_list = make_datapath_list(phase=\"test\")\nprint(f\"test data length : {len(test_list)}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image Augmentation","metadata":{}},{"cell_type":"markdown","source":"* ## Bee Augmentation","metadata":{}},{"cell_type":"code","source":"import random\nimport os\nclass InsectAugmentation:\n    \n    def __init__(self, p):\n        self.p = p\n\n\n    def __call__(self, image, n_insects=2, dark_insect=False, p=0.5, insects_folder='../input/bee-augmentation'):\n        aug_prob = random.random()\n        if aug_prob < self.p:\n            height, width, _ = image.shape  # target image width and height\n            insects_images = [im for im in os.listdir(insects_folder) if 'bee' in im]\n            img_shape = image.shape\n\n            for _ in range(n_insects):\n                insect = cv2.cvtColor(cv2.imread(os.path.join(insects_folder, random.choice(insects_images))), cv2.COLOR_BGR2RGB)\n                insect = cv2.flip(insect, random.choice([-1, 0, 1]))\n                insect = cv2.rotate(insect, random.choice([0, 1, 2]))\n                insect = cv2.resize(insect, (width, height))\n\n                h_height, h_width, _ = insect.shape  # insect image width and height\n                roi_ho = random.randint(0, image.shape[0] - insect.shape[0])\n                roi_wo = random.randint(0, image.shape[1] - insect.shape[1])\n                roi = image[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]\n\n                # Creating a mask and inverse mask \n                img2gray = cv2.cvtColor(insect, cv2.COLOR_BGR2GRAY)\n                ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)\n                #mask_inv = cv2.cvtColor(cv2.bitwise_not(mask),cv2.COLOR_BGR2GRAY)\n                mask_inv = cv2.bitwise_not(mask)\n\n                # Now black-out the area of insect in ROI\n                img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n\n                # Take only region of insect from insect image.\n                if dark_insect:\n                    img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n                    insect_fg = cv2.bitwise_and(img_bg, img_bg, mask=mask)\n                else:\n                    insect_fg = cv2.bitwise_and(insect, insect, mask=mask)\n\n                # Put insect in ROI and modify the target image\n                dst = cv2.add(img_bg, insect_fg, dtype=cv2.CV_64F)\n\n                image[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst\n\n        return {'image': image}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ## Basic Agumentation","metadata":{}},{"cell_type":"code","source":"import albumentations as A\nALBUMENTATION_TRAIN_LIST = [\n    A.SmallestMaxSize(SIZE, p=1),\n    A.RandomCrop(p=1,height = CROP_HEIGHT, width = CROP_WIDTH),\n#     A.RandomSunFlare(p=1),  #REMOVE\n    A.RandomFog(p=AGU_PROB),\n    A.RandomBrightness(p=AGU_PROB),\n    A.Rotate(p=AGU_PROB, limit=90),\n    A.RGBShift(p=AGU_PROB), \n#     A.RandomSnow(p=AGU_PROB), #REMOVE\n    A.HorizontalFlip(p=AGU_PROB), \n    A.VerticalFlip(p=AGU_PROB), \n    A.RandomContrast(limit = 0.5,p = AGU_PROB),\n    A.HueSaturationValue(p=AGU_PROB,hue_shift_limit=20, sat_shift_limit=30, val_shift_limit=50),\n    A.Cutout(p=AGU_PROB),\n    A.Transpose(p=AGU_PROB), \n    A.JpegCompression(p=AGU_PROB),\n    A.CoarseDropout(p=AGU_PROB),\n    A.IAAAdditiveGaussianNoise(loc=0, scale=(2.5500000000000003, 12.75), per_channel=False, p=AGU_PROB),\n    A.IAAAffine(scale=1.0, translate_percent=None, translate_px=None, rotate=0.0, shear=0.0, order=1, cval=0, mode='reflect', p=AGU_PROB),\n    A.IAAAffine(rotate=90., p=AGU_PROB),\n    A.IAAAffine(rotate=180., p=AGU_PROB),\n    A.Normalize(MEAN, STD)\n] if SELECT_MODEL == 'DenseNet' else [\n    A.SmallestMaxSize(SIZE, p=1),\n    A.RandomCrop(p=1,height = CROP_HEIGHT, width = CROP_WIDTH),\n    A.HorizontalFlip(p=AGU_PROB),\n    A.Normalize(MEAN, STD)\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ALBUMENTATION_VALID_LIST = [\n    A.SmallestMaxSize(SIZE, p=1),\n    A.CenterCrop(width = CROP_WIDTH, height = CROP_HEIGHT, p = 1),\n    A.Normalize(MEAN, STD)\n]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# ALBUMENTATION_TEST_LIST = A.Compose([\n#     A.SmallestMaxSize(SIZE, p=1),\n#     A.CenterCrop(width = CROP_WIDTH, height = CROP_HEIGHT, p = 1),\n#     A.Normalize(MEAN, STD)\n# ])\n\n\nif SELECT_MODEL == 'VGG16':\n    ALBUMENTATION_TEST_LIST = transforms.Compose([\n                    transforms.Resize(SIZE),\n                    transforms.CenterCrop(SIZE),\n                    transforms.ToTensor(),\n                    transforms.Normalize(MEAN, STD)\n                ]) \n    print('VGG16 - using torch vision for processing image')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ## Image Transform Class","metadata":{}},{"cell_type":"code","source":"class ImageTransform():\n    \"\"\"\n    Class for image preprocessing.\n    \n    Attributes\n    ----------\n    resize : int\n        224\n    mean : (R, G, B)\n        Average value for each color channel\n    std : (R, G, B)\n        Standard deviation for each color channel\n    \"\"\"\n    \n\n    \n    def __init__(self):\n        self.data_transform = {\n            'train': A.Compose(ALBUMENTATION_TRAIN_LIST),\n            'val': A.Compose(ALBUMENTATION_VALID_LIST),\n            'test': ALBUMENTATION_TEST_LIST\n        }\n        \n        self.bee_transform = InsectAugmentation(p = AGU_PROB)\n\n    def __call__(self, img, phase=\"train\"):\n        \"\"\"\n        Parameters\n        ----------\n        phase: 'train' or 'val' or 'test'\n            Specify the mode of preprocessing\n        \"\"\"\n        if SELECT_MODEL == 'VGG16' and phase == 'test':\n            agu_img = self.data_transform[phase](img)\n            return agu_img\n            \n        agu_img = self.data_transform[phase](image = np.array(img))\n#         agu_img = self.bee_transform(agu_img['image']) # BEE-AUGMENTATION\n        norm_img =  torch.from_numpy(agu_img['image'].transpose(2,0,1))\n        return norm_img\n\nimage_file_path = '/kaggle/input/resized-plant2021/img_sz_256/800113bb65efe69e.jpg'\nimg = Image.open(image_file_path)\n# plt.imshow(img)\ntransform = ImageTransform()\nimg_transformed = transform(img, phase='train')\nplt.imshow(img_transformed.numpy().transpose(1,2,0))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ## Test Augmentation","metadata":{}},{"cell_type":"markdown","source":"+ ### Multiple Augment","metadata":{}},{"cell_type":"code","source":"ROW = 10\nCOL = 10\n\ndef test_agument():\n    image_file_path = '/kaggle/input/resized-plant2021/img_sz_256/800113bb65efe69e.jpg'\n    img = Image.open(image_file_path)\n    transform = ImageTransform()\n    fig = plt.figure(figsize=(20, 20))\n    for i in range(ROW * COL):\n        fig.add_subplot(ROW, COL, i+1)\n        img_transformed = transform(img, phase='train')\n        plt.imshow(img_transformed.numpy().transpose(1,2,0))\n        \ntest_agument()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Dataset Object ","metadata":{}},{"cell_type":"code","source":"class PlantDataset(data.Dataset):\n    \"\"\"\n    Class to create a Dataset\n    \n    Attributes\n    ----------\n    df_train : DataFrame\n        DataFrame containing the image labels.\n    file_list : list\n        A list containing the paths to the images\n    transform : object\n        Instance of the preprocessing class (ImageTransform)\n    phase : 'train' or 'val' or 'test'\n        Specify whether to use train, validation, or test\n    \"\"\"\n    def __init__(self, df_train, file_list, transform=None, phase='train'):\n        self.df_train = df_train\n        self.df_labels_idx = df_labels_idx\n        self.file_list = file_list\n        self.transform = transform\n        self.phase = phase\n   \n    def __len__(self):\n        \"\"\"\n        Returns the number of images.\n        \"\"\"\n        return len(self.file_list)\n    \n    def __getitem__(self, index):\n        \"\"\"\n        Get data in Tensor format and labels of preprocessed images.\n        \"\"\"\n        #print(index)\n        \n        # Load the index number image.\n        img_path = self.file_list[index]\n        img = Image.open(img_path)\n        \n        # Preprocessing images\n        img_transformed = self.transform(img, self.phase)\n        \n        # image name\n        image_name = img_path[-20:]\n        \n        # Extract the labels\n        if self.phase in [\"train\", \"val\"]:\n            label = df_train.loc[df_train[\"image\"]==image_name][\"labels_n\"].values[0]\n        elif self.phase in [\"test\"]:\n            label = -1\n        \n        return img_transformed, label, image_name","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = PlantDataset(df_train, train_list, transform=ImageTransform(), phase='train')\nval_dataset = PlantDataset(df_train, val_list, transform=ImageTransform(), phase='val')\ntest_dataset = PlantDataset(df_train, test_list, transform=ImageTransform(), phase='test')\n\nindex = 0\n\nprint(\"【train dataset】\")\nprint(f\"img num : {train_dataset.__len__()}\")\nprint(f\"img : {train_dataset.__getitem__(index)[0].size()}\")\nprint(f\"label : {train_dataset.__getitem__(index)[1]}\")\nprint(f\"image name : {train_dataset.__getitem__(index)[2]}\")\n\nprint(\"\\n【validation dataset】\")\nprint(f\"img num : {val_dataset.__len__()}\")\nprint(f\"img : {val_dataset.__getitem__(index)[0].size()}\")\nprint(f\"label : {val_dataset.__getitem__(index)[1]}\")\nprint(f\"image name : {val_dataset.__getitem__(index)[2]}\")\n\nprint(\"\\n【test dataset】\")\nprint(f\"img num : {test_dataset.__len__()}\")\nprint(f\"img : {test_dataset.__getitem__(index)[0].size()}\")\nprint(f\"label : {test_dataset.__getitem__(index)[1]}\")\nprint(f\"image name : {test_dataset.__getitem__(index)[2]}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Dataloader Object","metadata":{}},{"cell_type":"code","source":"batch_size = 128\n\n# Create DataLoader\ntrain_dataloader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nval_dataloader = data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\ntest_dataloader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\n# to Dictionary\ndataloaders_dict = {\"train\": train_dataloader, \"val\": val_dataloader, \"test\": test_dataloader}\n\n# Operation check\n#batch_iterator = iter(dataloaders_dict[\"train\"])\n#inputs, labels = next(batch_iterator)\n#print(inputs.size())  # torch.Size([3, 3, 224, 224]) : [batch_size, Channel, H, W]\n#print(labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Network model","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from prettytable import PrettyTable\n\ndef count_parameters(model):\n    table = PrettyTable([\"Modules\", \"Parameters\"])\n    total_params = 0\n    for name, parameter in net.named_parameters():\n        if not parameter.requires_grad: continue\n        param = parameter.numel()\n        table.add_row([name, param])\n        total_params+=param\n    print(table)\n    print(f\"Total Trainable Params: {total_params}\")\n    return total_params","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ## VGG16","metadata":{}},{"cell_type":"markdown","source":"* ### Load Model","metadata":{}},{"cell_type":"code","source":"# ! ls /kaggle/input/pytorch-pretrained-models","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SELECT_MODEL == 'VGG16':\n    # Load the learned VGG-16 model.\n\n    # Create an instance of the VGG-16 model\n    use_pretrained = False\n    net = models.vgg16(pretrained=use_pretrained)\n\n    #save_path = \"/kaggle/working/vgg16_pretrained.h\"\n    #torch.save(net.state_dict(), save_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Change Model's Last Layer","metadata":{}},{"cell_type":"code","source":"if SELECT_MODEL == 'VGG16':\n\n    # Replace the output unit of the last output layer of the VGG-16 model.\n    # out_features 1000 to 12\n    net.classifier[6] = nn.Linear(in_features=4096, out_features=12)\n\n    # Newly created modules have require_grad=True by default\n#     num_features = net.classifier[6].in_features\n\n#     print(list(net.classifier.children()))\n#     features = list(net.classifier.children())[:-1] # Remove last layer\n#     features.extend([\n#         nn.Linear(num_features, 2048),\n#         nn.ReLU(inplace = True),\n#         nn.Dropout(p=0.5, inplace=False),\n#         nn.Linear(2048, 512),\n#         nn.ReLU(inplace = True),\n#         nn.Dropout(p=0.5, inplace=False),\n#         nn.Linear(512, 64),\n#         nn.ReLU(inplace = True),\n#         nn.Dropout(p=0.5, inplace=False),\n#         nn.Linear(64, 12)\n#     ])\n#     net.classifier = nn.Sequential(*features) # Replace the model classifier\n#     print(net)\n\n    # Set to training mode.\n    net.train()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Optimizer","metadata":{}},{"cell_type":"code","source":"if SELECT_MODEL == 'VGG16':\n    for name, param in net.named_parameters():\n        print(name)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SELECT_MODEL == 'VGG16':\n    # Store the parameters to be learned by finetuning in the variable params_to_update.\n    params_to_update_1 = []\n    params_to_update_2 = []\n    params_to_update_3 = []\n    params_to_update_4 = []\n\n    # Specify the parameter name of the layer to be trained.\n\n    update_param_names_1 = [\"features.24.weight\", \"features.24.bias\", \"features.26.weight\", \"features.26.bias\", \"features.28.weight\", \"features.28.bias\"]\n    update_param_names_2 = [\"classifier.0.weight\", \"classifier.0.bias\", \"classifier.3.weight\", \"classifier.3.bias\"]\n    update_param_names_3 = [\"classifier.6.weight\", \"classifier.6.bias\"]\n    update_param_names_4 = [\n        \"features.10.weight\", \"features.10.bias\", \n        \"features.12.weight\", \"features.12.bias\",\n        \"features.14.weight\", \"features.14.bias\", \n        \"features.17.weight\", \"features.17.bias\",\n        \"features.19.weight\", \"features.19.bias\",\n        \"features.21.weight\", \"features.21.bias\"\n    ]\n\n    for name, param in net.named_parameters():\n        if name in update_param_names_1:\n            param.requires_grad = False\n            params_to_update_1.append(param)\n            if param.requires_grad:\n                print(f\"Store in params_to_update_1 : {name}\")\n            else:\n                print(f\"Parameters not to be learned :  {name}\")\n        elif name in update_param_names_2:\n            param.requires_grad = False\n            params_to_update_2.append(param)\n            if param.requires_grad:\n                print(f\"Store in params_to_update_2 : {name}\")\n            else:\n                print(f\"Parameters not to be learned :  {name}\")\n        elif name in update_param_names_3:\n            param.requires_grad = False\n            params_to_update_3.append(param)\n            if param.requires_grad:\n                print(f\"Store in params_to_update_3 : {name}\")\n            else:\n                print(f\"Parameters not to be learned :  {name}\")\n        elif name in update_param_names_4:\n            param.requires_grad = True\n            params_to_update_4.append(param)\n            if param.requires_grad:\n                print(f\"Store in params_to_update_4 : {name}\")\n            else:\n                print(f\"Parameters not to be learned :  {name}\")\n        else:\n            param.requires_grad = False\n            print(f\"Parameters not to be learned :  {name}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SELECT_MODEL == 'VGG16':\n    # Set Optimizer\n    optimizer = optim.SGD([\n        {\"params\": params_to_update_1, \"lr\": 1e-4},\n        {\"params\": params_to_update_2, \"lr\": 1e-4},\n        {\"params\": params_to_update_3, \"lr\": 1e-4}\n    ], momentum=0.9)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ## AlexNet","metadata":{}},{"cell_type":"markdown","source":"* ### Load Model","metadata":{}},{"cell_type":"code","source":"if SELECT_MODEL == 'AlexNet':\n\n    load_path_alexNet = \"/kaggle/input/pytorch-pretrained-models/alexnet-owt-4df8aa71.pth\"\n\n    net = models.alexnet(pretrained=False)\n\n    # Replace the output unit of the last output layer of the VGG-16 model.\n    # out_features 1000 to 12\n    net.classifier[6] = nn.Linear(in_features=4096, out_features=12)\n\n    # Set to training mode.\n    net.train()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Optimizer","metadata":{}},{"cell_type":"code","source":"if SELECT_MODEL == 'AlexNet':\n    for name, param in net.named_parameters():\n        print(name)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SELECT_MODEL == 'AlexNet':\n    # Store the parameters to be learned by finetuning in the variable params_to_update.\n    params_to_update_1 = []\n    params_to_update_2 = []\n    params_to_update_3 = []\n\n    # Specify the parameter name of the layer to be trained.\n    update_param_names_1 = [\"features.0.weight\", \"features.0.bias\",\"features.3.weight\", \"features.3.bias\", \"features.6.weight\", \"features.6.bias\", \"features.8.weight\", \"features.8.bias\", \"features.10.weight\", \"features.10.bias\"]\n    update_param_names_2 = [\"classifier.1.weight\", \"classifier.1.bias\", \"classifier.4.weight\", \"classifier.4.bias\"]\n    update_param_names_3 = [\"classifier.6.weight\", \"classifier.6.bias\"]\n\n\n    for name, param in net.named_parameters():\n        if name in update_param_names_1:\n            param.requires_grad = True\n            params_to_update_1.append(param)\n            if param.requires_grad:\n                print(f\"Store in params_to_update_1 : {name}\")\n            else:\n                print(f\"Parameters not to be learned :  {name}\")\n        elif name in update_param_names_2:\n            param.requires_grad = True\n            params_to_update_2.append(param)\n            if param.requires_grad:\n                print(f\"Store in params_to_update_2 : {name}\")\n            else:\n                print(f\"Parameters not to be learned :  {name}\")\n        elif name in update_param_names_3:\n            param.requires_grad = True\n            params_to_update_3.append(param)\n            if param.requires_grad:\n                print(f\"Store in params_to_update_3 : {name}\")\n            else:\n                print(f\"Parameters not to be learned :  {name}\")\n        else:\n            param.requires_grad = False\n            print(f\"Parameters not to be learned :  {name}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SELECT_MODEL == 'AlexNet':\n    # Set Optimizer\n    optimizer = optim.SGD([\n        {\"params\": params_to_update_1, \"lr\": 5e-4},\n        {\"params\": params_to_update_2, \"lr\": 5e-4},\n        {\"params\": params_to_update_3, \"lr\": 1e-3}\n    ], momentum=0.9)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ## DenseNet","metadata":{}},{"cell_type":"markdown","source":"* ### Load Pretrained Model","metadata":{}},{"cell_type":"code","source":"if SELECT_MODEL == 'DenseNet':\n\n#     load_path = \"../input/pytorch-pretrained-models/densenet161-347e6b360.pth\"\n\n    net = models.densenet161(pretrained= False)\n\n#     if torch.cuda.is_available():\n#         load_weights = torch.load(load_path)\n#         net.load_state_dict(load_weights)\n#     else:\n#         load_weights = torch.load(load_path, map_location={\"cuda:0\": \"cpu\"})\n#         net.load_state_dict(load_weights)","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Modify Model Fit Problem","metadata":{}},{"cell_type":"code","source":"if SELECT_MODEL == 'DenseNet':\n\n    # Replace the output unit of the last output layer of the VGG-16 model.\n    # out_features 1000 to 12\n\n    prev_out_feature = net.classifier.in_features\n    \n#     new_last_layer = [\n#         nn.Linear(prev_out_feature, 2208),\n#         nn.ReLU(inplace = True),\n#         nn.Dropout(p=0.5, inplace=False),\n        \n#         nn.Linear(2208, 1104),\n#         nn.ReLU(inplace = True),\n#         nn.Dropout(p=0.5, inplace=False),\n        \n#         nn.Linear(1104, 552),\n#         nn.ReLU(inplace = True),\n#         nn.Dropout(p=0.2, inplace=False),\n        \n#         nn.Linear(552, 138),\n#         nn.ReLU(inplace = True),\n#         nn.Dropout(p=0.2, inplace=False),\n        \n#         nn.Linear(138, 12),\n#     ]\n    new_last_layer = [\n        nn.Linear(prev_out_feature, 12)\n    ]\n    \n    net.classifier = nn.Sequential(*new_last_layer)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SELECT_MODEL == 'DenseNet':\n    net.classifier.train()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SELECT_MODEL == 'DenseNet':\n#     a = 'features.denseblock3'\n#     print([f'{a}.{name}' for name, _ in net_densenet.features.denseblock3.named_parameters()])\n#     for name, param in net_densenet.named_parameters():\n#         print(name)\n    pass","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* ### Optimizer","metadata":{}},{"cell_type":"code","source":"if SELECT_MODEL == 'DenseNet':\n    # Store the parameters to be learned by finetuning in the variable params_to_update.\n    params_to_update_1 = []\n    params_to_update_2 = []\n    params_to_update_3 = []\n\n    # Specify the parameter name of the layer to be trained.\n    update_param_names_1 = [f'features.denseblock4.{name}' for name, _ in net.features.denseblock4.named_parameters()]\n    update_param_names_2 = [f'features.denseblock3.{name}' for name, _ in net.features.denseblock3.named_parameters()]\n    update_param_names_3 = [f'classifier.{name}' for name, _ in net.classifier.named_parameters()]\n\n\n    for name, param in net.named_parameters():\n        if name in update_param_names_1:\n            param.requires_grad = False\n            params_to_update_1.append(param)\n            if param.requires_grad:\n                print(f\"Store in params_to_update_1 : {name}\")\n#             else:\n#                 print(f\"Parameters not to be learned :  {name}\")\n        elif name in update_param_names_2:\n            param.requires_grad = False\n            params_to_update_2.append(param)\n            if param.requires_grad:\n                print(f\"Store in params_to_update_2 : {name}\")\n#             else:\n#                 print(f\"Parameters not to be learned :  {name}\")\n        elif name in update_param_names_3:\n            param.requires_grad = True\n            params_to_update_3.append(param)\n            if param.requires_grad:\n                print(f\"Store in params_to_update_3 : {name}\")\n#             else:\n#                 print(f\"Parameters not to be learned :  {name}\")\n        else:\n            param.requires_grad = False\n            if param.requires_grad:\n                print(f\"Store in params_to_update_3 : {name}\")\n#             print(f\"Parameters not to be learned :  {name}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SELECT_MODEL == 'DenseNet':\n    # Set Optimizer\n    optimizer = optim.Adam([\n        {\"params\": params_to_update_1, \"lr\": 5e-4},\n        {\"params\": params_to_update_2, \"lr\": 1e-4},\n        {\"params\": params_to_update_3, \"lr\": 1e-4}\n    ], lr = 1e-4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Pretrained Params","metadata":{}},{"cell_type":"code","source":"print(PRETRAIN_PATH)\nif LOAD_PRETRAIN:\n    if torch.cuda.is_available():\n        load_weights = torch.load(PRETRAIN_PATH)\n        net.load_state_dict(load_weights)\n    else:\n        load_weights = torch.load(PRETRAIN_PATH, map_location={\"cuda:0\": \"cpu\"})\n        net.load_state_dict(load_weights)\n    print('loaded pretrained')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define Loss Function","metadata":{}},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Function for model training","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import f1_score, accuracy_score\n\ndef train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):\n    \"\"\"\n    Function for training the model.\n    \n    Parameters\n    ----------\n    net: object\n    dataloaders_dict: dictionary\n    criterion: object\n    optimizer: object\n    num_epochs: int\n    \"\"\"\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    print(f\"Devices to be used : {device}\")\n    net.to(device)\n    torch.backends.cudnn.benchmark = True\n    # loop for epoch\n    train_acc_list = []\n    train_loss_list = []\n    train_f1_score_list = []\n    val_acc_list = []\n    val_loss_list = []\n    val_f1_score_list = []\n    for epoch in range(num_epochs):\n        print(f\"Epoch {epoch+1} / {num_epochs}\")\n        print(\"-------------------------------\")\n        for phase in [\"train\", \"val\"]:\n            if phase == \"train\":\n                net.train()\n            else:\n                net.eval()\n            epoch_loss = 0.0\n            epoch_corrects = 0\n            epoch_f1 = 0\n            #if (epoch == 0) and (phase == \"train\"):\n                #continue\n            epoch_num = 0\n            for inputs, labels, _ in tqdm(dataloaders_dict[phase]):\n                epoch_num += 1\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n                optimizer.zero_grad()\n                with torch.set_grad_enabled(phase == \"train\"):\n                    outputs = net(inputs)\n                    loss = criterion(outputs, labels)\n                    _, preds = torch.max(outputs, 1)\n                    if phase == \"train\":\n                        loss.backward()\n                        optimizer.step()\n                    _loss = loss.item() * inputs.size(0)\n                    _correct = torch.sum(preds == labels.data)\n                    _acc = 1.0 * _correct / len(inputs)\n                    _f1_score = f1_score(preds.cpu().data.numpy(), labels.cpu().data.numpy(), average='weighted')\n                    print(f\"Tracking - {phase} Loss: {_loss:.4f} Acc: {_acc:.4f}  F1-Score: {_f1_score:4f}\")\n                    epoch_loss += _loss\n                    epoch_corrects += _correct\n                    epoch_f1 += _f1_score\n            epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)\n            epoch_acc = epoch_corrects.double() / len(dataloaders_dict[phase].dataset)\n            epoch_f1 = epoch_f1 / epoch_num\n            if phase == \"train\":\n                train_acc_list.append(epoch_acc)\n                train_loss_list.append(epoch_loss)\n                train_f1_score_list.append(epoch_f1)\n            else:\n                net.eval()\n                val_acc_list.append(epoch_acc)\n                val_loss_list.append(epoch_loss)\n                val_f1_score_list.append(epoch_f1)\n            print(f\"{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f} F1-Score: {epoch_f1:4f}\")\n    return train_acc_list, train_loss_list, train_f1_score_list, val_acc_list, val_loss_list, val_f1_score_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Start training ","metadata":{}},{"cell_type":"code","source":"torch.cuda.is_available()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Start Training","metadata":{}},{"cell_type":"code","source":"if not IS_SUBMIT:\n    num_epochs = 10\n    train_acc_list, train_loss_list, train_f1_score_list, val_acc_list, val_loss_list, val_f1_score_list = train_model(net, dataloaders_dict, criterion, optimizer, num_epochs=num_epochs)\n    # Save Model Weight\n    save_path = f\"./{SAVE_WEIGHT_FILENAME}.h\"\n    torch.save(net.state_dict(), save_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not IS_SUBMIT:\n    plt.plot(train_acc_list, label='Train acc')\n    plt.plot(val_acc_list, label='Val acc')\n    plt.legend()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not IS_SUBMIT:\n    plt.plot(train_loss_list, label='Train loss')\n    plt.plot(val_loss_list, label='Val loss')\n    plt.legend()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not IS_SUBMIT:\n    plt.plot(train_f1_score_list, label='Train f1')\n    plt.plot(val_f1_score_list, label='Val f1')\n    plt.legend()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## load weights","metadata":{}},{"cell_type":"code","source":"# load_path = \"/kaggle/working/vgg16_second_fine_tuning_v1.h\"\nif IS_SUBMIT:\n    if torch.cuda.is_available():\n        load_weights = torch.load(PRETRAIN_PATH)\n        net.load_state_dict(load_weights)\n    else:\n        load_weights = torch.load(PRETRAIN_PATH, map_location={\"cuda:0\": \"cpu\"})\n        net.load_state_dict(load_weights)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## inference","metadata":{}},{"cell_type":"code","source":"#batch_iterator = iter(dataloaders_dict[\"val\"])\n#inputs, labels, image_name = next(batch_iterator)\n#print(inputs.size())  # torch.Size([3, 3, 224, 224]) : [batch_size, Channel, H, W]\n#print(labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PlantPredictor():\n    \"\"\"\n    Class for predicting labels from output results\n    \n    Attributes\n    ----------\n    df_labels_idx: DataFrame\n        DataFrame that associates INDEX with a label name\n    \"\"\"\n    \n    def __init__(self, net, df_labels_idx, dataloaders_dict):\n        self.net = net\n        self.df_labels_idx = df_labels_idx\n        self.dataloaders_dict = dataloaders_dict\n        self.df_submit = pd.DataFrame()\n        \n    \n    def __predict_max(self, out):\n        \"\"\"\n        Get the label name with the highest probability.\n        \n        Parameters\n        ----------\n        predicted_label_name: str\n            Name of the label with the highest prediction probability\n        \"\"\"\n        maxid = np.argmax(out.detach().numpy(), axis=1)\n        df_predicted_label_name = self.df_labels_idx.iloc[maxid]\n        \n        return df_predicted_label_name\n    \n    def inference(self):\n        device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n        print(f\"Devices to be used : {device}\")\n        df_pred_list = []\n        for inputs, _, image_name in tqdm(self.dataloaders_dict['test']):\n            device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n            self.net.to(device)\n            inputs = inputs.to(device)\n            out = self.net(inputs)\n            device = torch.device(\"cpu\")\n            out = out.to(device)\n            df_pred = self.__predict_max(out).reset_index(drop=True)\n            df_pred[\"image\"] = image_name\n            df_pred_list.append(df_pred)\n            \n        self.df_submit = pd.concat(df_pred_list, axis=0)\n        self.df_submit = self.df_submit[[\"image\", \"labels\"]].reset_index(drop=True)\n            \n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictor = PlantPredictor(net, df_labels_idx, dataloaders_dict)\npredictor.inference()\n#df_pred = predictor.predict_max(out)\n\n#df_sub.labels = df_pred.labels.reset_index(drop=True)\n#display(df_pred)\n#display(df_sub)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submit = predictor.df_submit.copy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submit","metadata":{}},{"cell_type":"code","source":"df_submit.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}