{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":13836,"databundleVersionId":1718836,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torcheval\n\n# Import the relevant libraries\nimport os\nimport random\nimport pandas as pd\nfrom PIL import Image\nfrom IPython.display import display\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torchvision\nfrom torchvision import models\nfrom torch.backends import cudnn\nfrom torchvision.datasets import VisionDataset\nfrom torchvision.transforms import v2, InterpolationMode\nfrom torch.utils.data import DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nimport torch.optim as optim\nfrom torcheval.metrics import MulticlassAccuracy\nfrom torch.utils.data import default_collate","metadata":{"execution":{"iopub.status.busy":"2024-04-04T11:19:42.726245Z","iopub.execute_input":"2024-04-04T11:19:42.726603Z","iopub.status.idle":"2024-04-04T11:20:06.936717Z","shell.execute_reply.started":"2024-04-04T11:19:42.726573Z","shell.execute_reply":"2024-04-04T11:20:06.935594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Basic experiment settings\ntorch.manual_seed(13)\n\ncudnn.deterministic = False\ncudnn.benchmark = True\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\n\n# Declare directories\ntrain_dir = \"/kaggle/input/cassava-leaf-disease-classification/train_images/\"\nlabel_dir = \"/kaggle/input/cassava-leaf-disease-classification/train.csv\"\nidx_to_str_dir = \"/kaggle/input/cassava-leaf-disease-classification/label_num_to_disease_map.json\"\n\n# Experiment parameters\nimg_size = 384\nbatch_size = 32\nnum_workers = 4\nnum_epochs = 30 # 20000 / 19257 / 32 ~ 33\nnum_classes = 5","metadata":{"execution":{"iopub.status.busy":"2024-04-04T11:20:06.938824Z","iopub.execute_input":"2024-04-04T11:20:06.939830Z","iopub.status.idle":"2024-04-04T11:20:06.983713Z","shell.execute_reply.started":"2024-04-04T11:20:06.939789Z","shell.execute_reply":"2024-04-04T11:20:06.982633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Choose 16 random images to get an idea of how it looks like (resize needed)\nchosen_files = random.sample(os.listdir(train_dir), 16)\nimages = [Image.open(os.path.join(train_dir, f)).resize((img_size, img_size)) \n          for f in chosen_files]\n\n# Check whether the image is in RGB format\nprint(images[0])\n\n# Create a 4 * 4 grid\ngrid_img = Image.new('RGB', (img_size * 4, img_size * 4))\n\nfor i, img in enumerate(images):\n    row = i // 4\n    col = i % 4\n    grid_img.paste(img, (col * img.width, row * img.height))\n    \ndisplay(grid_img)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T11:20:06.985072Z","iopub.execute_input":"2024-04-04T11:20:06.985472Z","iopub.status.idle":"2024-04-04T11:20:08.316050Z","shell.execute_reply.started":"2024-04-04T11:20:06.985437Z","shell.execute_reply":"2024-04-04T11:20:08.314547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Inspect the structure of the label file\nlabels = pd.read_csv(label_dir)\nprint(labels.head())\n\n# Check the number of images from each class\nlabels['label'].value_counts()\n\n# Split into train and validation by 0.9 to 0.1 ratio with stratification\ntrain_df, test_df = train_test_split(labels, test_size = 0.1, stratify = labels['label'])","metadata":{"execution":{"iopub.status.busy":"2024-04-04T11:20:08.319432Z","iopub.execute_input":"2024-04-04T11:20:08.319928Z","iopub.status.idle":"2024-04-04T11:20:08.378642Z","shell.execute_reply.started":"2024-04-04T11:20:08.319881Z","shell.execute_reply":"2024-04-04T11:20:08.377678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Currently testing vision transformers so check its architecture\nvit_model = models.vit_b_16(weights = models.ViT_B_16_Weights.IMAGENET1K_SWAG_E2E_V1)\n\n# Replace the head with a new linear layer\nvit_model.heads.head = torch.nn.Linear(vit_model.heads.head.in_features, num_classes)\n\n'''\nWhen transferring ViT models to another dataset, we remove the whole head \n(two linear layers) and replace it by a single, zero-initialized linear layer \noutputting the number of classes required by the target dataset. We found \nthis to be a little more robust than simply re-initializing the very last layer.\n-- original paper\n'''\ntorch.nn.init.zeros_(vit_model.heads.head.weight)\ntorch.nn.init.zeros_(vit_model.heads.head.bias)\n\nvit_model.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T11:20:08.379710Z","iopub.execute_input":"2024-04-04T11:20:08.379995Z","iopub.status.idle":"2024-04-04T11:20:12.535518Z","shell.execute_reply.started":"2024-04-04T11:20:08.379970Z","shell.execute_reply":"2024-04-04T11:20:12.534600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create an image dataset\nclass CassavaDataset(VisionDataset):\n    '''Custom dataset for the Cassava data.\n    \n    Args:\n        data_dir: base directory to the images.\n        img_label_df: dataframe containing image-label pairs.\n        transforms: set of transforms to be used.\n    '''\n    def __init__(self, data_dir, img_label_df, transform = None):\n        super().__init__(root = data_dir)\n        \n        self.root = data_dir\n        self.transform = transform\n        self.img_label_df = img_label_df\n        \n    def __getitem__(self, idx):\n        entry = self.img_label_df.iloc[idx]\n        filename, label = entry['image_id'], entry['label']\n        \n        img = Image.open(os.path.join(self.root, filename))\n        if self.transform is not None:\n            img = self.transform(img)\n            \n        return img, label\n    \n    def __len__(self):\n        return len(self.img_label_df)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T11:20:12.536910Z","iopub.execute_input":"2024-04-04T11:20:12.537276Z","iopub.status.idle":"2024-04-04T11:20:12.545002Z","shell.execute_reply.started":"2024-04-04T11:20:12.537244Z","shell.execute_reply":"2024-04-04T11:20:12.543941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare the transformations for model training\ntransforms = v2.Compose([\n    v2.Resize(img_size, interpolation=InterpolationMode.BICUBIC),\n    v2.RandomResizedCrop((img_size, img_size), (0.5, 1)), # Not in original\n    v2.RandomHorizontalFlip(0.4),\n    v2.RandomVerticalFlip(0.4), # Not in original\n    v2.ToImage(),\n    v2.ToDtype(torch.float32, scale = True),\n    v2.Normalize(mean = [0.485, 0.456, 0.406], std = [0.229, 0.224, 0.225])\n])\n\ntest_transforms = v2.Compose([\n    v2.Resize(img_size, interpolation=InterpolationMode.BICUBIC),\n    v2.CenterCrop(img_size),\n    v2.ToImage(),\n    v2.ToDtype(torch.float32, scale = True),\n    v2.Normalize(mean = [0.485, 0.456, 0.406], std = [0.229, 0.224, 0.225])\n])\n\n# Create the relevant datasets\ntrain_dataset = CassavaDataset(train_dir, train_df, transform = transforms)\ntest_dataset = CassavaDataset(train_dir, test_df, transform = test_transforms)\n\n# Mixup in original implementation\nmixup_transform = v2.MixUp(alpha = 0.1, num_classes = num_classes)\n\n# Define a new collate function for mixup\ndef collate_fn(batch):\n    return mixup_transform(*default_collate(batch))\n\n# Create the dataloaders\ntrain_loader = DataLoader(\n    train_dataset,\n    batch_size = batch_size,\n    shuffle = True,\n    num_workers = num_workers,\n    pin_memory = True,\n    collate_fn = collate_fn\n)\n\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size = batch_size,\n    shuffle = True,\n    num_workers = num_workers,\n    pin_memory = True\n)\n\n# Create the ce loss, sgd optimizer and scheduler\ncriterion = torch.nn.CrossEntropyLoss()\noptimizer = optim.SGD(vit_model.parameters(), lr = 6e-3, momentum = 0.9, nesterov=True)\nscheduler = CosineAnnealingLR(optimizer, T_max = num_epochs)\n\n# Metrics for evaluation\nnormalizer = torch.nn.Softmax(dim=1)\naccuracy = MulticlassAccuracy()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T11:20:12.546235Z","iopub.execute_input":"2024-04-04T11:20:12.546587Z","iopub.status.idle":"2024-04-04T11:20:12.588313Z","shell.execute_reply.started":"2024-04-04T11:20:12.546555Z","shell.execute_reply":"2024-04-04T11:20:12.587476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(num_epochs):\n    # Training loop first\n    vit_model.train()\n    \n    for batch_idx, (inputs, labels) in enumerate(train_loader):\n        optimizer.zero_grad()\n        \n        inputs, labels = inputs.to(device), labels.to(device)\n        labels_unmixed = torch.argmax(labels, 1)\n\n        if epoch == 0 and batch_idx == 0:\n            # Checking to ensure it is as expected\n            print(f\"Images has shape: {tuple(inputs.shape)}.\", flush=True)\n            print(f\"Labels has shape: {tuple(labels.shape)}.\", flush=True)\n            \n        outputs = vit_model(inputs)\n        loss = criterion(outputs, labels)\n        \n        preds = normalizer(outputs)\n        accuracy.update(preds, labels_unmixed)\n\n        loss.backward()\n        optimizer.step()\n        \n        # Log every 20 steps\n        if (batch_idx + 1) % 20 == 0:\n            print(\n                f\"TRAINING: Epoch: [{epoch+1}], Batch: [{batch_idx + 1}/{len(train_loader)}], \"\n                + f\"Cls loss: {loss.item():.4f} Accuracy: {accuracy.compute().item():.4f}\"\n            )\n            accuracy.reset()\n                \n    # Update the learning rate\n    scheduler.step()\n    print(scheduler.get_last_lr())\n    \n    vit_model.eval()\n    \n    with torch.no_grad():\n        test_loss = 0\n        \n        for batch_idx, (inputs, labels) in enumerate(test_loader):\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = vit_model(inputs)\n            preds = normalizer(outputs)\n            \n            test_loss += criterion(outputs, labels)\n            accuracy.update(preds, labels)\n            \n        print(\n            f\"VALIDATION: Epoch: [{epoch+1}] Cls loss: {test_loss.item():.4f} Accuracy: {accuracy.compute().item():.4f}\"\n        )\n        accuracy.reset()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T11:20:12.589369Z","iopub.execute_input":"2024-04-04T11:20:12.589676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}