{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Pytorch VGG-16 Fine tuning\nUsing PyTorch, I fine-tuned the learned weights for the VGG16 network architecture.\nI'm sure there are a lot of things that could be improved, but I hope this will be helpful for everyone implementing this in PyTorch.\nPlease let me know if there's anything I should fix!"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import glob\nimport os.path as osp\n\nimport pandas as pd\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom tqdm.notebook import tqdm\nfrom PIL import Image\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.utils.data as data\nimport torchvision\nfrom torchvision import models, transforms","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Import Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/plant-pathology-2021-fgvc8/train.csv\")\ndf_sub = pd.read_csv(\"/kaggle/input/plant-pathology-2021-fgvc8/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check train"},{"metadata":{"trusted":true},"cell_type":"code","source":"display(df_train)\ndisplay(df_train.labels.value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d_set = set()\nfor k in df_train.labels.unique():\n    d_set = d_set | set(k.split(\" \"))\nprint(f\"num of labels: {len(d_set)}  {d_set}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"display(df_sub)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Label encoding"},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_label(df):\n    \"\"\"\n    Function for Label encoding.\n    \"\"\"\n    le = LabelEncoder()\n    df[\"labels_n\"] = le.fit_transform(df.labels.values)\n    return df\n\ndf_train = to_label(df_train)\ndf_labels_idx = df_train.loc[df_train.duplicated([\"labels\", \"labels_n\"])==False]\\\n                [[\"labels_n\", \"labels\"]].set_index(\"labels_n\").sort_index()\ndisplay(df_labels_idx)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image path"},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_datapath_list(phase=\"train\", val_size=0.25):\n    \"\"\"\n    Function to create a PATH to the data.\n    \n    Parameters\n    ----------\n    phase : 'train' or 'val' or 'test'\n        Specify whether to use Train data or test data.\n    val_size : float\n        Ratio of validation data to train data\n        \n    Returns\n    -------\n    path_lsit : list\n        A list containing the PATH to the data.\n    \"\"\"\n    \n    if phase in [\"train\", \"val\"]:\n        phase_path = \"train_images\"\n    elif phase in [\"test\"]:\n        phase_path = \"test_images\"\n    else:\n        print(f\"{phase} not in path\")\n    rootpath = \"/kaggle/input/plant-pathology-2021-fgvc8/\"\n    target_path = osp.join(rootpath+phase_path+\"/*.jpg\")\n    path_list = []\n    \n    for path in glob.glob(target_path):\n        path_list.append(path)\n        \n    if phase in [\"train\", \"val\"]:\n        train, val = train_test_split(path_list, test_size=val_size, random_state=0, shuffle=True)\n        if phase == \"train\":\n            path_list = train\n        else:\n            path_list = val\n    \n    return path_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_list = make_datapath_list(phase=\"train\")\nprint(f\"train data length : {len(train_list)}\")\nval_list = make_datapath_list(phase=\"val\")\nprint(f\"validation data length : {len(val_list)}\")\ntest_list = make_datapath_list(phase=\"test\")\nprint(f\"test data length : {len(test_list)}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageTransform():\n    \"\"\"\n    Class for image preprocessing.\n    \n    Attributes\n    ----------\n    resize : int\n        224\n    mean : (R, G, B)\n        Average value for each color channel\n    std : (R, G, B)\n        Standard deviation for each color channel\n    \"\"\"\n    \n    def __init__(self, resize, mean, std):\n        self.data_transform = {\n            'train': transforms.Compose([\n                transforms.RandomResizedCrop(\n                    resize, scale=(0.5, 1.0)),\n                transforms.RandomHorizontalFlip(),\n                transforms.ToTensor(),\n                transforms.Normalize(mean, std)\n            ]),\n            'val': transforms.Compose([\n                transforms.Resize(resize),\n                transforms.CenterCrop(resize),\n                transforms.ToTensor(),\n                transforms.Normalize(mean, std)\n            ]),\n            'test': transforms.Compose([\n                transforms.Resize(resize),\n                transforms.CenterCrop(resize),\n                transforms.ToTensor(),\n                transforms.Normalize(mean, std)\n            ])\n        }\n    \n    def __call__(self, img, phase=\"train\"):\n        \"\"\"\n        Parameters\n        ----------\n        phase: 'train' or 'val' or 'test'\n            Specify the mode of preprocessing\n        \"\"\"\n        return self.data_transform[phase](img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test for ImageTransform Class \nimage_file_path = '/kaggle/input/plant-pathology-2021-fgvc8/train_images/800113bb65efe69e.jpg'\nimg = Image.open(image_file_path)\n\nplt.imshow(img)\nplt.show()\n\nsize = 224\nmean = (0.485, 0.456, 0.406)\nstd = (0.229, 0.224, 0.225)\n\ntransform = ImageTransform(size, mean, std)\nimg_transformed = transform(img, phase='train')\nprint(img_transformed.shape)\n\nimg_transformed = img_transformed.numpy().transpose([1, 2, 0])\nimg_transformed = np.clip(img_transformed, 0, 1)\nplt.imshow(img_transformed)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PlantDataset(data.Dataset):\n    \"\"\"\n    Class to create a Dataset\n    \n    Attributes\n    ----------\n    df_train : DataFrame\n        DataFrame containing the image labels.\n    file_list : list\n        A list containing the paths to the images\n    transform : object\n        Instance of the preprocessing class (ImageTransform)\n    phase : 'train' or 'val' or 'test'\n        Specify whether to use train, validation, or test\n    \"\"\"\n    def __init__(self, df_train, file_list, transform=None, phase='train'):\n        self.df_train = df_train\n        self.df_labels_idx = df_labels_idx\n        self.file_list = file_list\n        self.transform = transform\n        self.phase = phase\n        \n    def __len__(self):\n        \"\"\"\n        Returns the number of images.\n        \"\"\"\n        return len(self.file_list)\n    \n    def __getitem__(self, index):\n        \"\"\"\n        Get data in Tensor format and labels of preprocessed images.\n        \"\"\"\n        #print(index)\n        \n        # Load the index number image.\n        img_path = self.file_list[index]\n        img = Image.open(img_path)\n        \n        # Preprocessing images\n        img_transformed = self.transform(img, self.phase)\n        \n        # image name\n        image_name = img_path[-20:]\n        \n        # Extract the labels\n        if self.phase in [\"train\", \"val\"]:\n            label = df_train.loc[df_train[\"image\"]==image_name][\"labels_n\"].values[0]\n        elif self.phase in [\"test\"]:\n            label = -1\n        \n        return img_transformed, label, image_name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = PlantDataset(df_train, train_list, transform=ImageTransform(size, mean, std), phase='train')\nval_dataset = PlantDataset(df_train, val_list, transform=ImageTransform(size, mean, std), phase='val')\ntest_dataset = PlantDataset(df_train, test_list, transform=ImageTransform(size, mean, std), phase='test')\n\nindex = 0\n\nprint(\"【train dataset】\")\nprint(f\"img num : {train_dataset.__len__()}\")\nprint(f\"img : {train_dataset.__getitem__(index)[0].size()}\")\nprint(f\"label : {train_dataset.__getitem__(index)[1]}\")\nprint(f\"image name : {train_dataset.__getitem__(index)[2]}\")\n\nprint(\"\\n【validation dataset】\")\nprint(f\"img num : {val_dataset.__len__()}\")\nprint(f\"img : {val_dataset.__getitem__(index)[0].size()}\")\nprint(f\"label : {val_dataset.__getitem__(index)[1]}\")\nprint(f\"image name : {val_dataset.__getitem__(index)[2]}\")\n\nprint(\"\\n【test dataset】\")\nprint(f\"img num : {test_dataset.__len__()}\")\nprint(f\"img : {test_dataset.__getitem__(index)[0].size()}\")\nprint(f\"label : {test_dataset.__getitem__(index)[1]}\")\nprint(f\"image name : {test_dataset.__getitem__(index)[2]}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataloader"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\n\n# Create DataLoader\ntrain_dataloader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nval_dataloader = data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\ntest_dataloader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\n# to Dictionary\ndataloaders_dict = {\"train\": train_dataloader, \"val\": val_dataloader, \"test\": test_dataloader}\n\n# Operation check\n#batch_iterator = iter(dataloaders_dict[\"train\"])\n#inputs, labels = next(batch_iterator)\n#print(inputs.size())  # torch.Size([3, 3, 224, 224]) : [batch_size, Channel, H, W]\n#print(labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Network model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load the learned VGG-16 model.\n\n# Create an instance of the VGG-16 model\nuse_pretrained = False\nnet = models.vgg16(pretrained=use_pretrained)\n\n#save_path = \"/kaggle/working/vgg16_pretrained.h\"\n#torch.save(net.state_dict(), save_path)\n\nload_path = \"/kaggle/input/d/kuboko/plantpathology2021/vgg16_pretrained.h\"\nif torch.cuda.is_available():\n    load_weights = torch.load(load_path)\n    net.load_state_dict(load_weights)\nelse:\n    load_weights = torch.load(load_path, map_location={\"cuda:0\": \"cpu\"})\n    net.load_state_dict(load_weights)\n\n# Replace the output unit of the last output layer of the VGG-16 model.\n# out_features 1000 to 12\nnet.classifier[6] = nn.Linear(in_features=4096, out_features=12)\n\n# Set to training mode.\nnet.train()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loss function"},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Optimizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Store the parameters to be learned by finetuning in the variable params_to_update.\nparams_to_update_1 = []\nparams_to_update_2 = []\nparams_to_update_3 = []\n\n# Specify the parameter name of the layer to be trained.\nupdate_param_names_1 = [\"features.24.weight\", \"features.24.bias\", \"features.26.weight\", \"features.26.bias\", \"features.28.weight\", \"features.28.bias\"]\nupdate_param_names_2 = [\"classifier.0.weight\", \"classifier.0.bias\", \"classifier.3.weight\", \"classifier.3.bias\"]\nupdate_param_names_3 = [\"classifier.6.weight\", \"classifier.6.bias\"]\n\nfor name, param in net.named_parameters():\n    if name in update_param_names_1:\n        param.requires_grad = True\n        params_to_update_1.append(param)\n        print(f\"Store in params_to_update_1 : {name}\")\n    elif name in update_param_names_2:\n        param.requires_grad = True\n        params_to_update_2.append(param)\n        print(f\"Store in params_to_update_2 : {name}\")\n    elif name in update_param_names_3:\n        param.requires_grad = True\n        params_to_update_3.append(param)\n        print(f\"Store in params_to_update_3 : {name}\")\n    else:\n        param.requires_grad = False\n        print(f\"Parameters not to be learned :  {name}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set Optimizer\noptimizer = optim.SGD([\n    {\"params\": params_to_update_1, \"lr\": 1e-4},\n    {\"params\": params_to_update_2, \"lr\": 5e-4},\n    {\"params\": params_to_update_3, \"lr\": 1e-3}\n], momentum=0.9)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Function for model training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):\n    \"\"\"\n    Function for training the model.\n    \n    Parameters\n    ----------\n    net: object\n    dataloaders_dict: dictionary\n    criterion: object\n    optimizer: object\n    num_epochs: int\n    \"\"\"\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    print(f\"Devices to be used : {device}\")\n    net.to(device)\n    torch.backends.cudnn.benchmark = True\n    # loop for epoch\n    for epoch in range(num_epochs):\n        print(f\"Epoch {epoch+1} / {num_epochs}\")\n        print(\"-------------------------------\")\n        for phase in [\"train\", \"val\"]:\n            if phase == \"train\":\n                net.train()\n            else:\n                net.eval()\n            epoch_loss = 0.0\n            epoch_corrects = 0\n            #if (epoch == 0) and (phase == \"train\"):\n                #continue\n            for inputs, labels, _ in tqdm(dataloaders_dict[phase]):\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n                optimizer.zero_grad()\n                with torch.set_grad_enabled(phase == \"train\"):\n                    outputs = net(inputs)\n                    loss = criterion(outputs, labels)\n                    _, preds = torch.max(outputs, 1)\n                    if phase == \"train\":\n                        loss.backward()\n                        optimizer.step()\n                    epoch_loss += loss.item() * inputs.size(0)\n                    epoch_corrects += torch.sum(preds == labels.data)\n            epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)\n            epoch_acc = epoch_corrects.double() / len(dataloaders_dict[phase].dataset)\n            print(f\"{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Start training "},{"metadata":{"trusted":true},"cell_type":"code","source":"num_epochs = 4\n# train_model(net, dataloaders_dict, criterion, optimizer, num_epochs=num_epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## save weights"},{"metadata":{"trusted":true},"cell_type":"code","source":"#save_path = \"./vgg16_fine_tuning_v1.h\"\n#torch.save(net.state_dict(), save_path)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## load weights"},{"metadata":{"trusted":true},"cell_type":"code","source":"load_path = \"/kaggle/input/d/kuboko/plantpathology2021/vgg16_fine_tuning_v1.h\"\nif torch.cuda.is_available():\n    load_weights = torch.load(load_path)\n    net.load_state_dict(load_weights)\nelse:\n    load_weights = torch.load(load_path, map_location={\"cuda:0\": \"cpu\"})\n    net.load_state_dict(load_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"#batch_iterator = iter(dataloaders_dict[\"val\"])\n#inputs, labels, image_name = next(batch_iterator)\n#print(inputs.size())  # torch.Size([3, 3, 224, 224]) : [batch_size, Channel, H, W]\n#print(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PlantPredictor():\n    \"\"\"\n    Class for predicting labels from output results\n    \n    Attributes\n    ----------\n    df_labels_idx: DataFrame\n        DataFrame that associates INDEX with a label name\n    \"\"\"\n    \n    def __init__(self, net, df_labels_idx, dataloaders_dict):\n        self.net = net\n        self.df_labels_idx = df_labels_idx\n        self.dataloaders_dict = dataloaders_dict\n        self.df_submit = pd.DataFrame()\n        \n    \n    def __predict_max(self, out):\n        \"\"\"\n        Get the label name with the highest probability.\n        \n        Parameters\n        ----------\n        predicted_label_name: str\n            Name of the label with the highest prediction probability\n        \"\"\"\n        maxid = np.argmax(out.detach().numpy(), axis=1)\n        df_predicted_label_name = self.df_labels_idx.iloc[maxid]\n        \n        return df_predicted_label_name\n    \n    def inference(self):\n        device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n        print(f\"Devices to be used : {device}\")\n        df_pred_list = []\n        for inputs, _, image_name in tqdm(self.dataloaders_dict['test']):\n            device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n            self.net.to(device)\n            inputs = inputs.to(device)\n            out = self.net(inputs)\n            device = torch.device(\"cpu\")\n            out = out.to(device)\n            df_pred = self.__predict_max(out).reset_index(drop=True)\n            df_pred[\"image\"] = image_name\n            df_pred_list.append(df_pred)\n            \n        self.df_submit = pd.concat(df_pred_list, axis=0)\n        self.df_submit = self.df_submit[[\"image\", \"labels\"]].reset_index(drop=True)\n            \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictor = PlantPredictor(net, df_labels_idx, dataloaders_dict)\npredictor.inference()\n#df_pred = predictor.predict_max(out)\n\n#df_sub.labels = df_pred.labels.reset_index(drop=True)\n#display(df_pred)\n#display(df_sub)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submit = predictor.df_submit.copy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submit.to_csv(\"/kaggle/working/submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}