{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"},{"sourceId":8791566,"sourceType":"datasetVersion","datasetId":3993061}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Table of content","metadata":{}},{"cell_type":"markdown","source":"## [I. Intro](#intro)\n## [II. Dataset](#dataset)\n## [III. Data augmentation](#data-augmentation)\n## [IV. Base model selection](#base-model-selection)\n## [V. Hyper parameters tuning](#hpo)\n## [VI. K-fold cross validation](#k-fold_cross_validation)","metadata":{}},{"cell_type":"markdown","source":"# I. Intro <a class=\"anchor\" id=\"intro\"></a>","metadata":{}},{"cell_type":"markdown","source":"This work is shared for beginners who wants a solution for the competition `petals to the metals` from the Kaggle platform using pytorch and regular techniques such as:\n- data augmentation\n- transfer learning\n- hyper parameter search\n- custom learning rate scheduler\n- cross validation\n\nThe entire project can be found on my repo [here](https://github.com/NoeGuedet/Kaggle-efficientnet_v2_l-PyTorch).\n\nWARNING : I am a student and I am still learning ! It took me a couple of month to develop this solution that I am satisfied with. I am not a professionnal by any mean and this work may content some basic error, bad optimization... Contact me and I'll be happy to correct them :)","metadata":{}},{"cell_type":"markdown","source":"Hardware :\n- CPU : TR1920x 12C/24T @ 3.9Ghz\n- RAM : 128Gb DDR4 @ 3200Mhz\n- GPU : RTX 3090 EVGA FTW3","metadata":{}},{"cell_type":"markdown","source":"## Import library","metadata":{}},{"cell_type":"code","source":"!pip install tfrecord","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:43:55.275475Z","iopub.execute_input":"2024-06-26T09:43:55.275966Z","iopub.status.idle":"2024-06-26T09:44:11.901924Z","shell.execute_reply.started":"2024-06-26T09:43:55.275926Z","shell.execute_reply":"2024-06-26T09:44:11.900575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nimport logging\nimport gc\nimport time\nimport random\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nimport optuna\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nimport tfrecord\nimport cv2\n\nimport torch\nimport torchvision.transforms.v2 as transforms\nfrom torch.utils.data import Dataset, DataLoader, ConcatDataset, SubsetRandomSampler\nimport torch.nn as nn\nfrom torch.optim.lr_scheduler import LambdaLR\nfrom torch.optim import Adam\n\nfrom torchvision.models import densenet161, efficientnet_v2_l, vgg19_bn\n\nfrom sklearn.model_selection import KFold","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:44:11.904693Z","iopub.execute_input":"2024-06-26T09:44:11.905059Z","iopub.status.idle":"2024-06-26T09:44:19.344631Z","shell.execute_reply.started":"2024-06-26T09:44:11.905028Z","shell.execute_reply":"2024-06-26T09:44:19.343399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(torch.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:44:19.346049Z","iopub.execute_input":"2024-06-26T09:44:19.346726Z","iopub.status.idle":"2024-06-26T09:44:19.352507Z","shell.execute_reply.started":"2024-06-26T09:44:19.346686Z","shell.execute_reply":"2024-06-26T09:44:19.351397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nprint(f'Device : {DEVICE}')","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:44:19.354112Z","iopub.execute_input":"2024-06-26T09:44:19.354516Z","iopub.status.idle":"2024-06-26T09:44:19.397587Z","shell.execute_reply.started":"2024-06-26T09:44:19.354480Z","shell.execute_reply":"2024-06-26T09:44:19.396438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# II. Dataset <a class=\"anchor\" id=\"dataset\"></a>","metadata":{}},{"cell_type":"markdown","source":"The dataset class is define in the `utility.py` file, with functions inspired by [this notebook]('https://www.kaggle.com/code/adikaboost/transfer-learning-efficientnet-pytorch') (credit to BOOTLEG).","metadata":{}},{"cell_type":"code","source":"DATASET_PATH = \"/kaggle/input/tpu-getting-started/tfrecords-jpeg-512x512\"\n# batch size is limited by the amount of available GPU memory (24Gb) \nBATCH_SIZE = 8\nNUM_CLASSES = 104\nIMG_RESOLUTION = (512, 512)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:44:19.400871Z","iopub.execute_input":"2024-06-26T09:44:19.401190Z","iopub.status.idle":"2024-06-26T09:44:19.409164Z","shell.execute_reply.started":"2024-06-26T09:44:19.401164Z","shell.execute_reply":"2024-06-26T09:44:19.408139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# From https://www.kaggle.com/code/adikaboost/transfer-learning-efficientnet-pytorch\ndef transform_tf_to_df(dataset_path, subset_data):\n    df = pd.DataFrame({\"id\": pd.Series(dtype=\"str\"), \n                       \"class\": pd.Series(dtype=\"int\"), \n                       \"img\": pd.Series(dtype=\"object\")})    \n    tf_files = []\n    \n    for subdir, dirs, files in os.walk(dataset_path):\n        if subdir.split(\"/\")[-1] == subset_data:\n            for file in files:\n                filepath = subdir + os.sep + file\n                tf_files.append(filepath)\n                \n    for tf_file in tf_files:\n        if subset_data == \"test\":\n            loader = tfrecord.tfrecord_loader(tf_file, None, {\"id\": \"byte\", \"image\": \"byte\"})\n        else:\n            loader = tfrecord.tfrecord_loader(tf_file, None, {\"id\": \"byte\",\"image\": \"byte\", \"class\": \"int\"})\n        \n        for record in loader:\n            id_label = record[\"id\"].decode('utf-8')\n            label = record[\"class\"][0].item() if subset_data != \"test\" else None\n            img_bytes = np.frombuffer(record[\"image\"], dtype=np.uint8)\n            img = cv2.imdecode(img_bytes, cv2.IMREAD_COLOR)\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            df.loc[len(df.index)] = [id_label, label, img]\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:44:19.410529Z","iopub.execute_input":"2024-06-26T09:44:19.410942Z","iopub.status.idle":"2024-06-26T09:44:19.424813Z","shell.execute_reply.started":"2024-06-26T09:44:19.410905Z","shell.execute_reply":"2024-06-26T09:44:19.423651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FlowerDataset(Dataset):\n    def __init__(self, dataset_path, subset_data, num_classes=104, transform=None):\n        self.num_classes = num_classes\n        self.transform = transform\n        self.df_data = transform_tf_to_df(dataset_path, subset_data)\n\n    def __len__(self):\n        return self.df_data.shape[0]\n\n    def __getitem__(self, idx):\n        \"Iterable function which applies to each row\"\n        img_id = self.df_data.iloc[idx, 0]\n        label = self.df_data.iloc[idx, 1]\n        image = self.df_data.iloc[idx, 2]\n        image = Image.fromarray(image)\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        y = np.zeros(self.num_classes, dtype=np.float32)\n        y[label] = int(1)\n        return img_id, y, image","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:44:19.426234Z","iopub.execute_input":"2024-06-26T09:44:19.426678Z","iopub.status.idle":"2024-06-26T09:44:19.438946Z","shell.execute_reply.started":"2024-06-26T09:44:19.426640Z","shell.execute_reply":"2024-06-26T09:44:19.437799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = FlowerDataset(DATASET_PATH, 'train', num_classes=NUM_CLASSES)\nval_data = FlowerDataset(DATASET_PATH, 'val', num_classes=NUM_CLASSES)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:44:19.440299Z","iopub.execute_input":"2024-06-26T09:44:19.440633Z","iopub.status.idle":"2024-06-26T09:46:12.008087Z","shell.execute_reply.started":"2024-06-26T09:44:19.440582Z","shell.execute_reply":"2024-06-26T09:46:12.007105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_rows = 2\nn_cols = 3\n\nfor _ in range(0, n_rows):\n    fig, ax = plt.subplots(1, n_cols)\n    for n in range(0, n_cols):    \n        idx = random.randint(1, len(train_data))\n        img = train_data[idx][2]\n        ax[n].imshow(img)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:12.009178Z","iopub.execute_input":"2024-06-26T09:46:12.009476Z","iopub.status.idle":"2024-06-26T09:46:13.513415Z","shell.execute_reply.started":"2024-06-26T09:46:12.009451Z","shell.execute_reply":"2024-06-26T09:46:13.512176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# III. Data augmentation <a class=\"anchor\" id=\"data-augmentation\"></a>","metadata":{}},{"cell_type":"markdown","source":"The data transformation can probably be upgraded to better suit the dataset.","metadata":{}},{"cell_type":"markdown","source":"## Random dropout","metadata":{}},{"cell_type":"markdown","source":"The code below is inspired by this [notebook](https://www.kaggle.com/code/tuckerarrants/kfold-efficientnet-augmentation-s#III.-Augmentation).","metadata":{}},{"cell_type":"code","source":"class RandomImgDropout(object):\n    \"\"\"\n        Apply randomly drops out rectangular regions of an image by setting them to zero. \n\n        Attributes:\n        - p (float): Probability of applying the dropout transformation. Default is 0.5.\n        - dim (int): Dimension of the image (assuming a square image). Default is IMG_RESOLUTION[0].\n        - n_dropout (int): Number of dropout regions to create in the image. Default is 5.\n        - scaled_size (float): Size of the dropout regions as a fraction of the image dimension. Default is 0.1.\n\n        Parameters:\n            img (torch.Tensor): Input image tensor of shape (C, H, W).\n\n        Returns:\n            torch.Tensor: Output image tensor of shape (C, H, W).\n    \"\"\"\n    \n    def __init__(self, p=0.5, dim=IMG_RESOLUTION[0], n_dropout=5, scaled_size=0.1):\n        self.p = p\n        self.dim = dim\n        self.n_dropout = n_dropout\n        self.scaled_size = scaled_size\n            \n    def __call__(self, img):\n        do_tr = torch.rand(1)[0] < self.p\n        \n        if not do_tr:\n            return img\n\n        for _ in range(0, self.n_dropout):\n            x = torch.randint(0, self.dim, ()).type(torch.int32)\n            y = torch.randint(0, self.dim, ()).type(torch.int32)\n            width = torch.tensor(self.scaled_size * self.dim, dtype=torch.int32)\n\n            ya = torch.maximum(y-width//2, torch.tensor(0, dtype=torch.int32))\n            yb = torch.minimum(y+width//2, torch.tensor(self.dim, dtype=torch.int32))\n            xa = torch.maximum(x-width//2, torch.tensor(0, dtype=torch.int32))\n            xb = torch.minimum(x+width//2, torch.tensor(self.dim, dtype=torch.int32))\n\n            img[:, ya:yb, xa:xb] = 0\n            \n        return img","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:13.514945Z","iopub.execute_input":"2024-06-26T09:46:13.515447Z","iopub.status.idle":"2024-06-26T09:46:13.529537Z","shell.execute_reply.started":"2024-06-26T09:46:13.515399Z","shell.execute_reply":"2024-06-26T09:46:13.528361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stats = ((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:13.531014Z","iopub.execute_input":"2024-06-26T09:46:13.531333Z","iopub.status.idle":"2024-06-26T09:46:13.545587Z","shell.execute_reply.started":"2024-06-26T09:46:13.531308Z","shell.execute_reply":"2024-06-26T09:46:13.544525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visual_transform = transforms.Compose([\n    transforms.RandomResizedCrop(size=IMG_RESOLUTION, scale=(0.8, 1)),\n    transforms.RandomEqualize(),\n    transforms.RandomVerticalFlip(),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomApply([transforms.ElasticTransform(alpha=80.0)]),\n    transforms.RandomPerspective(distortion_scale=(0.3), p=0.4),\n    transforms.PILToTensor(),\n    RandomImgDropout(scaled_size=0.12, n_dropout=10),\n    transforms.ToPILImage()\n])\n\n# Same as visual transform but convert image to a normalize tensor at the end\ntrain_transform = transforms.Compose([\n    transforms.RandomResizedCrop(size=IMG_RESOLUTION, scale=(0.8, 1)),\n    transforms.RandomEqualize(),\n    transforms.RandomVerticalFlip(),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomApply([transforms.ElasticTransform(alpha=80.0)]),\n    transforms.RandomPerspective(distortion_scale=(0.3), p=0.4),\n    transforms.PILToTensor(),\n    RandomImgDropout(scaled_size=0.12, n_dropout=10),\n    transforms.ToDtype(torch.float32),\n    transforms.Normalize(*stats,inplace=True)\n])\n\nval_transform = transforms.Compose([\n    transforms.PILToTensor(),\n    transforms.ToDtype(torch.float32),\n    transforms.Normalize(*stats,inplace=True)\n])","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:13.546964Z","iopub.execute_input":"2024-06-26T09:46:13.547319Z","iopub.status.idle":"2024-06-26T09:46:13.612588Z","shell.execute_reply.started":"2024-06-26T09:46:13.547275Z","shell.execute_reply":"2024-06-26T09:46:13.611652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.transform = visual_transform\n\nn_rows = 2\nn_cols = 3\n\nfor _ in range(0, n_rows):\n    fig, ax = plt.subplots(1, n_cols)\n    for n in range(0, n_cols):    \n        idx = random.randint(1, len(train_data))\n        img = train_data[idx][2]\n        ax[n].imshow(img)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-06-26T09:46:13.614507Z","iopub.execute_input":"2024-06-26T09:46:13.615040Z","iopub.status.idle":"2024-06-26T09:46:16.450789Z","shell.execute_reply.started":"2024-06-26T09:46:13.614998Z","shell.execute_reply":"2024-06-26T09:46:16.449734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.transform = train_transform\nval_data.transform = val_transform","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:16.458355Z","iopub.execute_input":"2024-06-26T09:46:16.458811Z","iopub.status.idle":"2024-06-26T09:46:16.463480Z","shell.execute_reply.started":"2024-06-26T09:46:16.458781Z","shell.execute_reply":"2024-06-26T09:46:16.462443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## (Optionnal) Benchmark of the CPU with the dataloader","metadata":{}},{"cell_type":"markdown","source":"The goal of this benchmark is to use an optimal number of thread to process the dataset and avoid a potential bottleneck of the CPU 'feeding' the GPU to slowly.","metadata":{}},{"cell_type":"markdown","source":"The cell below have been converted to raw format to avoid accidentaly running the benchmark at each restart of the notebook. ","metadata":{}},{"cell_type":"code","source":"# My cpu is 12 Cores 24 Threads. I avoid using all the threads which make the system crash.\n# torch.get_num_thread() return only the number of cores, not the actual number of threads.\n# n_workers = [i for i in range(4, 24, 2)]\n# n_workers","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:16.465064Z","iopub.execute_input":"2024-06-26T09:46:16.465964Z","iopub.status.idle":"2024-06-26T09:46:16.473650Z","shell.execute_reply.started":"2024-06-26T09:46:16.465927Z","shell.execute_reply":"2024-06-26T09:46:16.472516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# time_score = []\n\n# for n in n_workers:\n#     data = DataLoader(train_data, batch_size=BATCH_SIZE, num_workers=n)\n#     start_time = time.time()\n#     for img_id, label, img in data:\n#         pass\n#     end_time = time.time()\n#     time_score.append(end_time - start_time)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:16.475033Z","iopub.execute_input":"2024-06-26T09:46:16.475577Z","iopub.status.idle":"2024-06-26T09:46:16.484885Z","shell.execute_reply.started":"2024-06-26T09:46:16.475550Z","shell.execute_reply":"2024-06-26T09:46:16.483802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, ax = plt.subplots()\n\n# ax.bar(n_workers, time_score)\n\n# ax.set_ylabel('Time (in seconds)')\n# ax.set_xlabel('Number of workers')\n# ax.set_title('AVG time to process the dataset per number of workers')\n\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:16.486105Z","iopub.execute_input":"2024-06-26T09:46:16.486492Z","iopub.status.idle":"2024-06-26T09:46:16.496914Z","shell.execute_reply.started":"2024-06-26T09:46:16.486463Z","shell.execute_reply":"2024-06-26T09:46:16.495664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As shown in the plot below, the optimal number of workers for this hardware is 16 (fast enough without to much power consumption from using all the threads).\n\n![CPU Benchmark](./cpu_benchmark.png \"AVG time to process the dataset per number of workers\")","metadata":{}},{"cell_type":"code","source":"# N_WORKERS = 16\n# Only 2 available CPU on kaggle\nN_WORKERS = 2","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:16.498398Z","iopub.execute_input":"2024-06-26T09:46:16.498781Z","iopub.status.idle":"2024-06-26T09:46:16.506417Z","shell.execute_reply.started":"2024-06-26T09:46:16.498753Z","shell.execute_reply":"2024-06-26T09:46:16.505298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(train_data, batch_size=BATCH_SIZE, num_workers=N_WORKERS, shuffle=True, drop_last=True)\nval_loader = DataLoader(val_data, batch_size=BATCH_SIZE, num_workers=N_WORKERS, shuffle=True, drop_last=True)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:16.507903Z","iopub.execute_input":"2024-06-26T09:46:16.508316Z","iopub.status.idle":"2024-06-26T09:46:16.516319Z","shell.execute_reply.started":"2024-06-26T09:46:16.508277Z","shell.execute_reply":"2024-06-26T09:46:16.515159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# IV. Base model selection <a class=\"anchor\" id=\"base-model-selection\"></a>","metadata":{}},{"cell_type":"markdown","source":"## Training function","metadata":{}},{"cell_type":"code","source":"def model_trainer(model, criterion, optimizer, train_loader, val_loader=None, epochs=0, scheduler=None, device='cpu', show_progress=False, trial=None):\n    # Move the model to the specified device\n    model.to(device)\n    \n    history = {\n        'train_loss': [],\n        'train_accuracy': []\n    }\n    \n    if val_loader is not None:\n        history['val_loss'] = []\n        history['val_accuracy'] = []\n    \n    epochs_loop = tqdm(range(0, epochs), desc=\"Epoch\", leave=True) if show_progress else range(0, epochs)\n    for epoch in epochs_loop:        \n        model.train()\n        running_loss = 0.0\n        correct = 0\n        total = 0\n\n        # training loop\n        training_loop = tqdm(train_loader, leave=False, desc=\"Training\") if show_progress else train_loader\n        for _, labels, inputs in training_loop:\n            inputs, labels = inputs.to(device), labels.to(device)\n            \n            # Zero the parameter gradients\n            optimizer.zero_grad()\n            \n            # Forward pass\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            \n            # Backward pass and optimize\n            loss.backward()\n            optimizer.step()\n            \n            # Calculate statistics\n            running_loss += loss.item()\n            \n            _, predicted = torch.max(outputs.data, 1)\n            _, y_class = torch.max(labels.data, 1)\n\n            total += labels.size(0)\n            correct += (predicted == y_class).sum().item()\n\n            if show_progress:\n                training_loop.set_postfix(training_loss=running_loss / (training_loop.n + 1), training_accuracy=correct / total)\n            \n        # Calculate average loss and accuracy for the epoch\n        train_epoch_loss = running_loss / len(train_loader)\n        train_epoch_accuracy = correct / total\n\n        if show_progress:\n            epochs_loop.set_postfix(train_loss=train_epoch_loss, train_accuracy=train_epoch_accuracy)\n            \n        history['train_loss'].append(train_epoch_loss)\n        history['train_accuracy'].append(train_epoch_accuracy)\n        \n        if val_loader is not None:\n            model.eval()\n            val_loss = 0.0\n            val_correct = 0\n            val_total = 0\n\n            val_loop = tqdm(val_loader, leave=False, desc=\"Validating\") if show_progress else val_loader\n            \n            with torch.no_grad():\n                for _, labels, inputs in val_loop:\n                    inputs, labels = inputs.to(device), labels.to(device)\n                    \n                    outputs = model(inputs)\n                    loss = criterion(outputs, labels)\n                    \n                    val_loss += loss.item()\n                    _, predicted = torch.max(outputs.data, 1)\n                    _, y_class = torch.max(labels.data, 1)\n                    \n                    val_total += labels.size(0)\n                    val_correct += (predicted == y_class).sum().item()\n\n                    if show_progress:\n                        val_loop.set_postfix(val_loss=val_loss / (val_loop.n + 1), val_accuracy=val_correct / val_total)\n            \n            val_epoch_loss = val_loss / len(val_loader)\n            val_epoch_accuracy = val_correct / val_total\n\n            if show_progress:\n                epochs_loop.set_postfix(train_loss=train_epoch_loss, val_loss=val_epoch_loss, train_accuracy=train_epoch_accuracy,  val_accuracy=val_epoch_accuracy)\n            \n            history['val_loss'].append(val_epoch_loss)\n            history['val_accuracy'].append(val_epoch_accuracy)\n\n        if scheduler is not None:\n            scheduler.step()\n\n        if trial is not None:\n            rep_acc = val_epoch_accuracy if val_loader is not None else train_epoch_accuracy\n            trial.report(rep_acc, epoch)\n            \n            if trial.should_prune():\n                raise optuna.exceptions.TrialPruned()\n    \n    return history","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:16.517995Z","iopub.execute_input":"2024-06-26T09:46:16.518334Z","iopub.status.idle":"2024-06-26T09:46:16.541797Z","shell.execute_reply.started":"2024-06-26T09:46:16.518294Z","shell.execute_reply":"2024-06-26T09:46:16.540488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model benchmark","metadata":{}},{"cell_type":"markdown","source":"Now, the selection of the best base model can be done. For that, some preselected CNN will be benchmarked and the one with the best accuracy after a 10 epoch of training will be used as the base model.\n\nHere are all the models that will be tested:\n- DenseNet161\n- EfficientNetV2-L\n- VGG-19_BN","metadata":{}},{"cell_type":"code","source":"# replace the final layer of each model to a new one with the right amount of classes\ndensenet_model = densenet161(weights='DEFAULT')\ndensenet_model.classifier = nn.LazyLinear(out_features=NUM_CLASSES)\n\nefficient_model = efficientnet_v2_l(weights='DEFAULT')\nefficient_model.classifier[-1] = nn.LazyLinear(out_features=NUM_CLASSES)\n\nvgg_model = vgg19_bn(weights='DEFAULT')\nvgg_model.classifier[-1] = nn.LazyLinear(out_features=NUM_CLASSES)\n\nmodels = {\n    'densenet161': densenet_model,\n    'efficientnet_v2_l': efficient_model,\n    'vgg19_bn': vgg_model\n}","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:16.543450Z","iopub.execute_input":"2024-06-26T09:46:16.543994Z","iopub.status.idle":"2024-06-26T09:46:37.698003Z","shell.execute_reply.started":"2024-06-26T09:46:16.543957Z","shell.execute_reply":"2024-06-26T09:46:37.697026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The cell below has been converted to raw format to avoid accidentally running the benchmark at each restart of the notebook.","metadata":{}},{"cell_type":"code","source":"# models_benchmark = {}\n\n# epochs = 10\n# lr = 0.001\n# criterion = nn.CrossEntropyLoss()\n\n# benchmark_train_loader = DataLoader(train_data, batch_size=BATCH_SIZE, num_workers=12, shuffle=True)\n# benchmark_val_loader = DataLoader(val_data, batch_size=BATCH_SIZE, num_workers=12, shuffle=False)\n\n# for model_name, model in models.items():\n#     optimizer = Adam(model.parameters(), lr=lr)\n\n#     print(f\"#### {model_name} ####\")\n#     history = model_trainer(\n#         model,\n#         criterion,\n#         optimizer,\n#         benchmark_train_loader,\n#         val_loader=benchmark_val_loader,\n#         epochs=epochs,\n#         device=DEVICE\n#     )\n\n#     for metric, data in history.items():\n#         models_benchmark[model_name + '_' + metric] = data\n    \n#     # free gpu memory for the next trial\n#     model.cpu()\n#     del model\n#     gc.collect()\n#     torch.cuda.empty_cache()\n\n# models_benchmark_df = pd.DataFrame.from_dict(models_benchmark)\n# models_benchmark_df.to_csv('./models_benchmark.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:37.699328Z","iopub.execute_input":"2024-06-26T09:46:37.699747Z","iopub.status.idle":"2024-06-26T09:46:37.705964Z","shell.execute_reply.started":"2024-06-26T09:46:37.699709Z","shell.execute_reply":"2024-06-26T09:46:37.704634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_benchmark_df = pd.read_csv('/kaggle/input/petals-to-the-metals-model/models_benchmark.csv')","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:37.707310Z","iopub.execute_input":"2024-06-26T09:46:37.707623Z","iopub.status.idle":"2024-06-26T09:46:37.752887Z","shell.execute_reply.started":"2024-06-26T09:46:37.707577Z","shell.execute_reply":"2024-06-26T09:46:37.751675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_benchmark_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:37.754478Z","iopub.execute_input":"2024-06-26T09:46:37.754933Z","iopub.status.idle":"2024-06-26T09:46:37.780592Z","shell.execute_reply.started":"2024-06-26T09:46:37.754896Z","shell.execute_reply":"2024-06-26T09:46:37.779451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for model_name in models.keys():\n    print(f\"#### {model_name} stats ####\")\n    fig, ax = plt.subplots(1, 2)\n    fig.set_size_inches(15, 4)\n\n    # Plotting loss\n    ax[0].set_ylim([0, 5])\n    ax[0].plot(models_benchmark_df[model_name + '_train_loss'], label='Train Loss')\n    ax[0].plot(models_benchmark_df[model_name + '_val_loss'], label='Val Loss')\n    ax[0].set_ylabel('Loss')\n    ax[0].set_xlabel('Epoch')\n    ax[0].set_title('Loss')\n    ax[0].legend()\n\n    # Plotting accuracy\n    ax[1].set_ylim([0, 1])\n    ax[1].plot(models_benchmark_df[model_name + '_train_accuracy'], label='Train Accuracy')\n    ax[1].plot(models_benchmark_df[model_name + '_val_accuracy'], label='Val Accuracy')\n    ax[1].set_ylabel('Accuracy')\n    ax[1].set_xlabel('Epoch')\n    ax[1].set_title('Accuracy')\n    ax[1].legend()\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:37.782347Z","iopub.execute_input":"2024-06-26T09:46:37.782895Z","iopub.status.idle":"2024-06-26T09:46:39.426995Z","shell.execute_reply.started":"2024-06-26T09:46:37.782854Z","shell.execute_reply":"2024-06-26T09:46:39.425760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The efficient V2 large model is the one which perform the best with a better accuracy than the others one after 10 epochs of training. It converge faster then the densenet 161 while still having a validation loss smaller than the training loss which mean that there is still room for even better performance.\n\nAll the model do not overfit at all, which can suggest that the data augmentation is ok.","metadata":{}},{"cell_type":"markdown","source":"# V. Hyper parameters tuning <a class=\"anchor\" id=\"hpo\"></a>","metadata":{}},{"cell_type":"markdown","source":"Now that the base model has been selected (efficient V2 large), I am going to proceed to a hyper parameters search to improve the base model. The goal of this search will be to find a good classifier architecture with the right amount of hidden and the right amount of dropout and perceptrons per hidden layer.","metadata":{}},{"cell_type":"markdown","source":"The library used for this HPO is optuna, which is simple to work with.\nWe only need to create an objective function that will ask for hyperparameters variables and return a score. The framework will try to optimize the score of the objective function by giving better combinations of variables values.","metadata":{}},{"cell_type":"markdown","source":"## 1. Custom classifier function","metadata":{}},{"cell_type":"markdown","source":"This function will take in input some hyper parameters variables and return a Sequential container with the applied inputs.","metadata":{}},{"cell_type":"code","source":"def get_custom_classifier(linear_layers: list, dropout_layers: list, bn_layers: list):\n    layers = []\n    for out_features, dropout_rate, is_bn in zip(linear_layers, dropout_layers, bn_layers):\n        layers.append(nn.Dropout(p=dropout_rate))\n        layers.append(nn.LazyLinear(out_features))\n        if is_bn : layers.append(nn.LazyBatchNorm1d())\n        layers.append(nn.ReLU())\n        \n    layers.append(nn.LazyLinear(NUM_CLASSES))\n    layers.append(nn.Softmax(dim=1))\n    return nn.Sequential(*layers)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:39.428574Z","iopub.execute_input":"2024-06-26T09:46:39.428977Z","iopub.status.idle":"2024-06-26T09:46:39.437156Z","shell.execute_reply.started":"2024-06-26T09:46:39.428946Z","shell.execute_reply":"2024-06-26T09:46:39.436059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Objective function && data retrieval","metadata":{}},{"cell_type":"code","source":"def objective(trial):     \n    model = efficientnet_v2_l(weights='DEFAULT')\n    \n    n_hidden_layers = trial.suggest_int('num_hidden_layers', 0, 3)\n    linear_layers = []\n    dropout_layers = []\n    bn_layers = []\n    \n    for i in range(0, n_hidden_layers):\n        linear_layers.append(trial.suggest_int(f'l{i}_out_features', 128, 1024, step=32))\n        dropout_layers.append(trial.suggest_float(f'l{i}_dropout_rate', 0, 0.6))\n        bn_layers.append(trial.suggest_categorical(f'l{i}_is_bn', [True, False]))\n\n    model.classifier = get_custom_classifier(linear_layers, dropout_layers, bn_layers)\n\n    criterion = nn.CrossEntropyLoss()\n    optimizer = Adam(model.parameters(), lr=1e-5)\n    \n    hist = model_trainer(model, criterion, optimizer, train_loader, val_loader=val_loader, epochs=15, device=DEVICE, show_progress=False, trial=trial)\n\n    model.cpu()\n    del model, criterion, optimizer\n    gc.collect()\n    torch.cuda.empty_cache()\n\n    return hist['val_accuracy'][-1]","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:39.439155Z","iopub.execute_input":"2024-06-26T09:46:39.439526Z","iopub.status.idle":"2024-06-26T09:46:39.587295Z","shell.execute_reply.started":"2024-06-26T09:46:39.439496Z","shell.execute_reply":"2024-06-26T09:46:39.586105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Let the calculation begin ! ","metadata":{}},{"cell_type":"markdown","source":"I actually wasn't able to run the hyper parameters search within the jupyterlab interface as it requires to keep the tab open to maintain the connection and the kernel would die after a certain amount of time (I don't know why).\n\nHere is how I did :\n- Convert the notebook to a .py file using this command line : `jupyter nbconvert --to script petals_to_the_metals.ipynb`\n- Rename the file : `mv ./petals_to_the_metals.py ./hyper_parameter_tuning.py`\n- Clear the code to just run the HPO and save the result within `HPO.csv` file and add logging info to track the process\n- Start the HPO with this command line : `nohup python hyperparameter_tuning.py > hyper_parameter_tuning.log 2>&1 &` which will start a new background process that will not stop on the jupyterlab session closing","metadata":{}},{"cell_type":"code","source":"# study = optuna.create_study(direction=\"maximize\")\n# study.optimize(objective, n_trials=100, gc_after_trial=True, show_progress_bar=False)\n# HPO_df = study.trials_dataframe()\n# HPO_df.to_csv('./HPO.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:39.588475Z","iopub.execute_input":"2024-06-26T09:46:39.588802Z","iopub.status.idle":"2024-06-26T09:46:39.600586Z","shell.execute_reply.started":"2024-06-26T09:46:39.588775Z","shell.execute_reply":"2024-06-26T09:46:39.599519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HPO_df = pd.read_csv('/kaggle/input/petals-to-the-metals-model/HPO.csv').sort_values('value', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:39.601792Z","iopub.execute_input":"2024-06-26T09:46:39.602081Z","iopub.status.idle":"2024-06-26T09:46:39.638317Z","shell.execute_reply.started":"2024-06-26T09:46:39.602057Z","shell.execute_reply":"2024-06-26T09:46:39.637450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HPO_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:39.639552Z","iopub.execute_input":"2024-06-26T09:46:39.639857Z","iopub.status.idle":"2024-06-26T09:46:39.661468Z","shell.execute_reply.started":"2024-06-26T09:46:39.639832Z","shell.execute_reply":"2024-06-26T09:46:39.660331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The best parameters for the classifier seems to be a single hidden layer with 1024 perceptronsand with a dropout of ~0.38. Using batch normalization does not seem to have a significant impact.","metadata":{}},{"cell_type":"code","source":"hidden_layers = [1024] \ndropout_layers = [0.38] \nbn_layers = [False]","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:39.663152Z","iopub.execute_input":"2024-06-26T09:46:39.663910Z","iopub.status.idle":"2024-06-26T09:46:39.670525Z","shell.execute_reply.started":"2024-06-26T09:46:39.663868Z","shell.execute_reply":"2024-06-26T09:46:39.669393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# VI. K-fold cross validation <a class=\"anchor\" id=\"k-fold_cross_validation\"></a>","metadata":{}},{"cell_type":"markdown","source":"## Learning rate scheduler","metadata":{}},{"cell_type":"markdown","source":"### Features extractor","metadata":{}},{"cell_type":"markdown","source":"Since we are using a pretrained model, it is important to have a specific learning rate scheduler shich will update the pretrained weights slowly at the beginning to avoid overshooting or getting stuck in suboptimal points.","metadata":{}},{"cell_type":"markdown","source":"Here is the learning rate stratgey that will be use :\n1. warmup stage : adjust to the new dataset without making large, potentially harmful updates\n2. aggressive learning rate at the middle of the training to make the model converge faster\n3. decrease the learning rate to better find a local minima\n\nThe code below is taken from [this notebook](https://www.kaggle.com/code/tuckerarrants/kfold-efficientnet-augmentation-s#IV.-Model-Training).","metadata":{}},{"cell_type":"code","source":"# since the model is pretrained and the batch size is small, a small lr is better\nhead_lr_start = 1e-5\nhead_lr_min = 1e-5\nhead_lr_max = 1e-4\nhead_lr_rampup_epochs = 5\nhead_lr_sustain_epoch = 0\nhead_lr_decay = .8\n\ndef custom_head_lr_scheduler(epoch):\n    if epoch < head_lr_rampup_epochs:\n        return (head_lr_max - head_lr_start) / head_lr_rampup_epochs * epoch + head_lr_start\n        \n    elif epoch < head_lr_rampup_epochs + head_lr_sustain_epoch:\n        return head_lr_max\n        \n    else:\n        return (head_lr_max - head_lr_min) * head_lr_decay**(epoch - head_lr_rampup_epochs - head_lr_sustain_epoch) + head_lr_min","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:39.671847Z","iopub.execute_input":"2024-06-26T09:46:39.672305Z","iopub.status.idle":"2024-06-26T09:46:39.682926Z","shell.execute_reply.started":"2024-06-26T09:46:39.672274Z","shell.execute_reply":"2024-06-26T09:46:39.681817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate learning rates for each epoch\nlearning_rates = [custom_head_lr_scheduler(epoch) for epoch in range(0, 30)]\n\n# Plot the learning rate schedule\nplt.figure(figsize=(10, 6))\nplt.plot(range(0, 30), learning_rates, marker='o')\nplt.title('Custom Learning Rate Schedule - Head')\nplt.xlabel('Epoch')\nplt.ylabel('Learning Rate')\nplt.grid(True)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:39.684563Z","iopub.execute_input":"2024-06-26T09:46:39.685336Z","iopub.status.idle":"2024-06-26T09:46:40.016485Z","shell.execute_reply.started":"2024-06-26T09:46:39.685294Z","shell.execute_reply":"2024-06-26T09:46:40.015440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Classifier","metadata":{}},{"cell_type":"markdown","source":"The classifier is made of one hidden layer and the final layer and is trained from scratch (no pretraining).\nA warmup cooldown strategy is not necessary for non-pretrained weights so a decay of the learning time over epoch will be used.","metadata":{}},{"cell_type":"code","source":"# same for the classifier, low lr because of small batch size\nclr_lr_max = 1e-4\nclr_lr_min = 1e-6\nclr_lr_decay = 0.8\n\ndef custom_clr_lr_scheduler(epoch):\n    lr = (clr_lr_max - clr_lr_min) * clr_lr_decay**(epoch) + clr_lr_min\n    return lr","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:40.018051Z","iopub.execute_input":"2024-06-26T09:46:40.018790Z","iopub.status.idle":"2024-06-26T09:46:40.024924Z","shell.execute_reply.started":"2024-06-26T09:46:40.018749Z","shell.execute_reply":"2024-06-26T09:46:40.023792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate learning rates for each epoch\nlearning_rates = [custom_clr_lr_scheduler(epoch) for epoch in range(0, 30)]\n\n# Plot the learning rate schedule\nplt.figure(figsize=(10, 6))\nplt.plot(range(0, 30), learning_rates, marker='o')\nplt.title('Custom Learning Rate Schedule - Classifier')\nplt.xlabel('Epoch')\nplt.ylabel('Learning Rate')\nplt.grid(True)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:40.026397Z","iopub.execute_input":"2024-06-26T09:46:40.027109Z","iopub.status.idle":"2024-06-26T09:46:40.366065Z","shell.execute_reply.started":"2024-06-26T09:46:40.027080Z","shell.execute_reply":"2024-06-26T09:46:40.364859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Early stopping","metadata":{}},{"cell_type":"markdown","source":"From this [stackoverflow topic](https://stackoverflow.com/questions/71998978/early-stopping-in-pytorch).","metadata":{}},{"cell_type":"code","source":"# Early stop class based of the trend of a given loss\nclass EarlyStopper:\n    def __init__(self, patience=1, min_delta=0):\n        self.patience = patience\n        self.min_delta = min_delta\n        self.counter = 0\n        self.min_validation_loss = float('inf')\n        self.early_stop = False\n\n    def step(self, validation_loss):\n        if validation_loss < self.min_validation_loss:\n            self.min_validation_loss = validation_loss\n            self.counter = 0\n        elif validation_loss > (self.min_validation_loss + self.min_delta):\n            self.counter += 1\n            if self.counter >= self.patience:\n                self.early_stop = True","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:40.367427Z","iopub.execute_input":"2024-06-26T09:46:40.367789Z","iopub.status.idle":"2024-06-26T09:46:40.375794Z","shell.execute_reply.started":"2024-06-26T09:46:40.367760Z","shell.execute_reply":"2024-06-26T09:46:40.374458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training function with early stopping","metadata":{}},{"cell_type":"markdown","source":"Since the dataloader dynamically change from a folder to another during training, I had to pass the dataset as an argument of the training function and to dynamically change the transform attribute of the dataset. ","metadata":{}},{"cell_type":"code","source":"def es_training(model, optimizer, criterion, dataset, train_loader, val_loader, epochs, patience, min_delta, scheluder = None, device='cpu'):\n    early_stopper = EarlyStopper(patience, min_delta)\n    model.to(device)\n\n    history = {\n        'train_loss': [],\n        'train_accuracy': [],\n        'val_loss': [],\n        'val_accuracy': []\n    }\n\n    for epoch in range(0, epochs):        \n        model.train()\n        # change the data augmentation to match the training stage\n        dataset.transform = train_transform\n        train_loss = 0.0\n        train_correct = 0\n        train_total = 0\n        \n        for _, labels, inputs in train_loader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            # Zero the parameter gradients\n            optimizer.zero_grad()\n            # Forward pass\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            # Backward pass and optimize\n            loss.backward()\n            optimizer.step()\n            # Calculate statistics            \n            _, predicted = torch.max(outputs.data, 1)\n            _, y_class = torch.max(labels.data, 1)\n            train_loss += loss.item()\n            train_total += labels.size(0)\n            train_correct += (predicted == y_class).sum().item()\n\n        # Update lr\n        if scheduler is not None:\n            scheduler.step()\n            \n        # Calculate average training loss and accuracy for the epoch\n        train_epoch_loss = train_loss / len(train_loader)\n        train_epoch_accuracy = train_correct / train_total\n\n        model.eval()\n        # change the data augmentation to match the evaluating stage\n        dataset.transform = val_transform\n        val_loss = 0.0\n        val_correct = 0\n        val_total = 0\n\n        with torch.no_grad():\n            for _, labels, inputs in val_loader:\n                inputs, labels = inputs.to(device), labels.to(device)\n                # Forward pass\n                outputs = model(inputs)\n                loss = criterion(outputs, labels)\n                # Calculate statistics\n                val_loss += loss.item()\n                _, predicted = torch.max(outputs.data, 1)\n                _, y_class = torch.max(labels.data, 1)\n                val_total += labels.size(0)\n                val_correct += (predicted == y_class).sum().item()\n\n        # Calculate average validation loss and accuracy for the epoch\n        val_epoch_loss = val_loss / len(val_loader)\n        val_epoch_accuracy = val_correct / val_total\n\n        # update history\n        history['train_loss'].append(train_epoch_loss)\n        history['train_accuracy'].append(train_epoch_accuracy)\n        history['val_loss'].append(val_epoch_loss)\n        history['val_accuracy'].append(val_epoch_accuracy)\n\n        print(\n            f'Epoch {epoch} completed',\n            f'training loss = {train_epoch_loss}',\n            f'training accuracy = {train_epoch_accuracy}',\n            f'val loss = {val_epoch_loss}',\n            f'val accuracy = {val_epoch_accuracy}',\n            sep=' | '\n        )\n\n        # check for early stop\n        early_stopper.step(val_epoch_loss)\n        if early_stopper.early_stop:\n            print('-'*5 + f'Early stop trigger --> stopping training' + '-'*5)\n            break\n\n    return history","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:40.377130Z","iopub.execute_input":"2024-06-26T09:46:40.378043Z","iopub.status.idle":"2024-06-26T09:46:40.398078Z","shell.execute_reply.started":"2024-06-26T09:46:40.378012Z","shell.execute_reply":"2024-06-26T09:46:40.397029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Defining K-fold variables","metadata":{}},{"cell_type":"code","source":"def get_model():\n    model = efficientnet_v2_l(weights='DEFAULT')\n    \n    # Unfreeze all layers\n    for param in model.parameters():\n        param.requires_grad = True\n    \n    model.classifier = get_custom_classifier(hidden_layers, dropout_layers, bn_layers)\n        \n    return model","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:40.399409Z","iopub.execute_input":"2024-06-26T09:46:40.399751Z","iopub.status.idle":"2024-06-26T09:46:40.413365Z","shell.execute_reply.started":"2024-06-26T09:46:40.399725Z","shell.execute_reply":"2024-06-26T09:46:40.412305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_state = 3210\nk_folds = 5\nkfold = KFold(n_splits=k_folds, shuffle=True, random_state=random_state)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:40.414734Z","iopub.execute_input":"2024-06-26T09:46:40.415057Z","iopub.status.idle":"2024-06-26T09:46:40.426740Z","shell.execute_reply.started":"2024-06-26T09:46:40.415031Z","shell.execute_reply":"2024-06-26T09:46:40.425679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"k_models = [get_model() for _ in range(0, k_folds)]","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:40.430261Z","iopub.execute_input":"2024-06-26T09:46:40.430627Z","iopub.status.idle":"2024-06-26T09:46:56.466807Z","shell.execute_reply.started":"2024-06-26T09:46:40.430577Z","shell.execute_reply":"2024-06-26T09:46:56.465561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the training and validation dataset are no longer needed\ndel train_data, val_data\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:56.475572Z","iopub.execute_input":"2024-06-26T09:46:56.475933Z","iopub.status.idle":"2024-06-26T09:46:56.693042Z","shell.execute_reply.started":"2024-06-26T09:46:56.475905Z","shell.execute_reply":"2024-06-26T09:46:56.691680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This simplify the loading of the entire dataset\n\n!mkdir /kaggle/input/tpu-getting-started/tfrecords-jpeg-512x512/train+val\n!cp -r /kaggle/input/tpu-getting-started/tfrecords-jpeg-512x512/train /kaggle/input/tpu-getting-started/tfrecords-jpeg-512x512/train+val\n!cp -r /kaggle/input/tpu-getting-started/tfrecords-jpeg-512x512/val /kaggle/input/tpu-getting-started/tfrecords-jpeg-512x512/train+val","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:46:56.694345Z","iopub.execute_input":"2024-06-26T09:46:56.694727Z","iopub.status.idle":"2024-06-26T09:47:01.100430Z","shell.execute_reply.started":"2024-06-26T09:46:56.694697Z","shell.execute_reply":"2024-06-26T09:47:01.098929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = FlowerDataset(DATASET_PATH, 'train+val')","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:47:01.102332Z","iopub.execute_input":"2024-06-26T09:47:01.102749Z","iopub.status.idle":"2024-06-26T09:47:01.115192Z","shell.execute_reply.started":"2024-06-26T09:47:01.102714Z","shell.execute_reply":"2024-06-26T09:47:01.114079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## K-fold training","metadata":{}},{"cell_type":"markdown","source":"Same problem as the hyper parameter search, when running the code below, after some times the kernel and the session is shutting down automatically which stop the training.\n\nHere is how I managed it :\n\n1. Convert the notebook to a .py file using this command line : `jupyter nbconvert --to script petals_to_the_metals.ipynb`\n2. Rename the file : `mv ./petals_to_the_metals.py ./k-fold_training.py`\n3. Clear the code to just run the training and save the result within the `k-fold_training_histories.csv` file and add logging info to track the process\n4. Start the training with this command line : `nohup python k-fold_training.py > k-fold_training.log 2>&1 &` which will start a new background process that will not stop on the jupyterlab session closing","metadata":{}},{"cell_type":"code","source":"# Early stop variables\nes_patience = 5\nes_min_delta = 0.02","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:47:01.116632Z","iopub.execute_input":"2024-06-26T09:47:01.117036Z","iopub.status.idle":"2024-06-26T09:47:01.124933Z","shell.execute_reply.started":"2024-06-26T09:47:01.117005Z","shell.execute_reply":"2024-06-26T09:47:01.123739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\nepochs = 50\nhistories = {}","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:47:01.126218Z","iopub.execute_input":"2024-06-26T09:47:01.126573Z","iopub.status.idle":"2024-06-26T09:47:01.136892Z","shell.execute_reply.started":"2024-06-26T09:47:01.126541Z","shell.execute_reply":"2024-06-26T09:47:01.135733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The cell below has been converted to raw format to avoid accidentally running the training at each restart of the notebook.","metadata":{}},{"cell_type":"code","source":"# for fold, (train_ids, val_ids) in enumerate(kfold.split(dataset)):\n#     print(f'###### FOLD {fold} ######')\n#     # Sample elements randomly from a given list of ids, no replacement.\n#     train_subsampler = torch.utils.data.SubsetRandomSampler(train_ids)\n#     val_subsampler = torch.utils.data.SubsetRandomSampler(val_ids)\n\n#     # Define data loaders for training and testing data in this fold\n#     train_loader = torch.utils.data.DataLoader(dataset, batch_size=BATCH_SIZE, num_workers=N_WORKERS, sampler=train_subsampler)\n#     val_loader = torch.utils.data.DataLoader(dataset, batch_size=BATCH_SIZE, num_workers=N_WORKERS, sampler=val_subsampler)\n\n#     model = k_models[fold]\n#     optimizer = Adam([\n#                         {'params': model.features.parameters()},\n#                         {'params': model.classifier.parameters()}\n#                     ], lr=1)\n    \n#     scheduler = LambdaLR(optimizer, lr_lambda=[custom_head_lr_scheduler, custom_clr_lr_scheduler])\n    \n#     history = es_training(\n#                     model, \n#                     optimizer, \n#                     criterion,\n#                     dataset,\n#                     train_loader, \n#                     val_loader, \n#                     epochs, \n#                     patience=es_patience, \n#                     min_delta=es_min_delta, \n#                     scheluder = scheduler, \n#                     device=DEVICE\n#                 )\n    \n#     histories[f'model_f-{fold}_train_accuracy'] = history['train_accuracy']\n#     histories[f'model_f-{fold}_val_accuracy'] = history['val_accuracy']\n#     histories[f'model_f-{fold}_train_loss'] = history['train_loss']\n#     histories[f'model_f-{fold}_val_loss'] = history['val_loss']\n    \n#     save_path = f'./models/model_f-{fold}.pth'\n#     torch.save(model.state_dict(), save_path)\n\n#     # free GPU memory\n#     model.cpu()\n#     del scheduler, optimizer, model\n#     gc.collect()\n#     torch.cuda.empty_cache()\n\n# # Convert dict to dataframe with all the key not having the same size :\n# # https://stackoverflow.com/questions/38446457/filling-dict-with-na-values-to-allow-conversion-to-pandas-dataframe\n# histories = pd.DataFrame.from_dict(histories, orient='index').T\n# histories.to_csv('k-fold_training_histories.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:47:01.138207Z","iopub.execute_input":"2024-06-26T09:47:01.138559Z","iopub.status.idle":"2024-06-26T09:47:01.148923Z","shell.execute_reply.started":"2024-06-26T09:47:01.138530Z","shell.execute_reply":"2024-06-26T09:47:01.147798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"histories = pd.read_csv('/kaggle/input/petals-to-the-metals-model/k-fold_training_histories.csv')","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:47:01.150481Z","iopub.execute_input":"2024-06-26T09:47:01.150926Z","iopub.status.idle":"2024-06-26T09:47:01.172576Z","shell.execute_reply.started":"2024-06-26T09:47:01.150889Z","shell.execute_reply":"2024-06-26T09:47:01.171116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Models metrics","metadata":{}},{"cell_type":"code","source":"for i in range(0, k_folds):\n    print(f\"####### MODEL FOLD {i} #######\")\n    train_acc = histories[f'model_f-{i}_train_accuracy']\n    val_acc = histories[f'model_f-{i}_val_accuracy']\n    train_loss = histories[f'model_f-{i}_train_loss']\n    val_loss = histories[f'model_f-{i}_val_loss']\n\n    fig, ax = plt.subplots(1, 2)\n    fig.set_size_inches(15, 4)\n\n    # Plotting loss\n    ax[0].set_ylim([0, 5])\n    ax[0].plot(train_loss, label='Train Loss')\n    ax[0].plot(val_loss, label='Val Loss')\n    ax[0].set_ylabel('Loss')\n    ax[0].set_xlabel('Epoch')\n    ax[0].set_title('Loss')\n    ax[0].legend()\n\n    # Plotting accuracy\n    ax[1].set_ylim([0, 1])\n    ax[1].plot(train_acc, label='Train Accuracy')\n    ax[1].plot(val_acc, label='Val Accuracy')\n    ax[1].set_ylabel('Accuracy')\n    ax[1].set_xlabel('Epoch')\n    ax[1].set_title('Accuracy')\n    ax[1].legend()\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:47:01.173975Z","iopub.execute_input":"2024-06-26T09:47:01.174325Z","iopub.status.idle":"2024-06-26T09:47:04.052071Z","shell.execute_reply.started":"2024-06-26T09:47:01.174296Z","shell.execute_reply":"2024-06-26T09:47:04.049837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The model converge very fast (~3 epochs) and have an accuracy of ~0.95 accross all the fold. A better learning rate strategy and a better data augmentation could help the model converging slower but with a better generalisation ? ","metadata":{}},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"markdown","source":"Loading the test dataset and dataloader","metadata":{}},{"cell_type":"code","source":"# free memory as `dataset` is no longer needed\ndel dataset\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:47:04.053387Z","iopub.execute_input":"2024-06-26T09:47:04.053826Z","iopub.status.idle":"2024-06-26T09:47:04.352728Z","shell.execute_reply.started":"2024-06-26T09:47:04.053788Z","shell.execute_reply":"2024-06-26T09:47:04.351578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = FlowerDataset(DATASET_PATH, 'test', transform=val_transform)\n# the batch size can drastically be raise up since the model will not be trained, more GPU memory will be available.\ntest_dataloader = DataLoader(test_data, batch_size=128, num_workers=N_WORKERS)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:47:04.354587Z","iopub.execute_input":"2024-06-26T09:47:04.354980Z","iopub.status.idle":"2024-06-26T09:47:57.198538Z","shell.execute_reply.started":"2024-06-26T09:47:04.354951Z","shell.execute_reply":"2024-06-26T09:47:57.197637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loading the previously trained weights","metadata":{}},{"cell_type":"code","source":"for i in range(0, k_folds):\n    path = f'/kaggle/input/petals-to-the-metals-model/model_f-{i}.pth'\n    k_models[i].load_state_dict(torch.load(path))","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:47:57.200129Z","iopub.execute_input":"2024-06-26T09:47:57.200536Z","iopub.status.idle":"2024-06-26T09:48:21.953391Z","shell.execute_reply.started":"2024-06-26T09:47:57.200500Z","shell.execute_reply":"2024-06-26T09:48:21.952429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = []\npreds = []\n\n# Make predictions with each model and average them\nwith torch.no_grad():\n    for sample_id, _, inputs in test_dataloader:\n        inputs = inputs.to(DEVICE)\n        ids.extend(sample_id)\n        mean_logps = []\n\n        for model in k_models:\n            model.to(DEVICE)\n            model.eval()\n            outputs = model(inputs)\n            mean_logps.append(outputs)\n            # free GPU memory\n            model.cpu()\n            del model\n            gc.collect()\n        \n        mean_logp = torch.mean(torch.stack(mean_logps), dim=0)\n        preds.extend(torch.argmax(mean_logp, dim=1).tolist())","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:48:21.954998Z","iopub.execute_input":"2024-06-26T09:48:21.955436Z","iopub.status.idle":"2024-06-26T10:06:37.581413Z","shell.execute_reply.started":"2024-06-26T09:48:21.955397Z","shell.execute_reply":"2024-06-26T10:06:37.580385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'id': ids, 'label': preds})\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-26T10:06:37.582730Z","iopub.execute_input":"2024-06-26T10:06:37.583046Z","iopub.status.idle":"2024-06-26T10:06:37.598711Z","shell.execute_reply.started":"2024-06-26T10:06:37.583021Z","shell.execute_reply":"2024-06-26T10:06:37.597786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('/kaggle/working/submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T10:06:37.599775Z","iopub.execute_input":"2024-06-26T10:06:37.600053Z","iopub.status.idle":"2024-06-26T10:06:37.636711Z","shell.execute_reply.started":"2024-06-26T10:06:37.600032Z","shell.execute_reply":"2024-06-26T10:06:37.635933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final accuracy","metadata":{}},{"cell_type":"markdown","source":"The final accuracy is 0.967 :\n\n![Final Accuracy](./final_acc.png)","metadata":{}}]}