{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":8066583,"sourceType":"datasetVersion","datasetId":4732842}],"dockerImageVersionId":30674,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Hello fellow Kagglers,\n\nThis notebook demonstrates the training strategy using EfficientVit and is a work in progress.\n\nInference notebook and further updates will follow soon!\n\n[dataset](https://www.kaggle.com/code/markwijkhuizen/birdclef-2024-eda-preprocessed-dataset)","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/mit-han-lab/efficientvit.git","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:30:25.291395Z","iopub.execute_input":"2024-04-15T17:30:25.291654Z","iopub.status.idle":"2024-04-15T17:30:28.573102Z","shell.execute_reply.started":"2024-04-15T17:30:25.291630Z","shell.execute_reply":"2024-04-15T17:30:28.571741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -qq onnxsim gdown lovely-tensors","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:30:28.575591Z","iopub.execute_input":"2024-04-15T17:30:28.576089Z","iopub.status.idle":"2024-04-15T17:30:42.893390Z","shell.execute_reply.started":"2024-04-15T17:30:28.576055Z","shell.execute_reply":"2024-04-15T17:30:42.892192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gdown\n\nurl = 'https://drive.google.com/uc?id=1q5y0YbN08O4ToUBK8RfZSDKp-s1y5_44'\noutput = 'l1-r224.pt'  # Replace with the correct file extension if known\ngdown.download(url, output, quiet=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:30:43.844663Z","iopub.execute_input":"2024-04-15T17:30:43.845094Z","iopub.status.idle":"2024-04-15T17:30:46.797291Z","shell.execute_reply.started":"2024-04-15T17:30:43.845063Z","shell.execute_reply":"2024-04-15T17:30:46.796393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.insert(0,'/kaggle/working/efficientvit')\nfrom efficientvit.cls_model_zoo import create_cls_model\nimport lovely_tensors as lt\nlt.monkey_patch()\n\nimport numpy as np\nimport pandas as pd\nimport imageio.v3 as imageio\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm.notebook import tqdm\nfrom torchvision import transforms\n\nimport torch\nimport torchmetrics\nimport timm\nimport pickle\nimport psutil\nimport time\nimport os","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:35:21.033430Z","iopub.execute_input":"2024-04-15T17:35:21.033795Z","iopub.status.idle":"2024-04-15T17:35:21.040575Z","shell.execute_reply.started":"2024-04-15T17:35:21.033763Z","shell.execute_reply":"2024-04-15T17:35:21.039533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class Config:\n    # Dataset\n    HEIGHT = 128\n    WIDTH = 320\n    ROOT_FOLDER = '/kaggle/input/birdclef-2024-dataset'\n    # Training\n    BATCH_SIZE = 16\n    N_EPOCHS = 20\n    # Model\n    BACKBONE = 'efficientvit_b1.r288_in1k'\n    # Learning Rate Scheduler\n    LR_MAX = 3e-4\n    WEIGHT_DECAY = 0.00\n    # Others\n    SEED = 42\n    IS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\n    \nCONFIG = Config()","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:35:21.721099Z","iopub.execute_input":"2024-04-15T17:35:21.721767Z","iopub.status.idle":"2024-04-15T17:35:21.727040Z","shell.execute_reply.started":"2024-04-15T17:35:21.721740Z","shell.execute_reply":"2024-04-15T17:35:21.726037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sample Submission","metadata":{}},{"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/birdclef-2024/sample_submission.csv')\n\n# Set labels\nCONFIG.LABELS = sample_submission.columns[1:]\nCONFIG.N_CLASSES = len(CONFIG.LABELS)\nprint(f'# classes: {CONFIG.N_CLASSES}')\n\ndisplay(sample_submission.head())","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:35:22.742219Z","iopub.execute_input":"2024-04-15T17:35:22.743063Z","iopub.status.idle":"2024-04-15T17:35:22.787469Z","shell.execute_reply.started":"2024-04-15T17:35:22.743034Z","shell.execute_reply":"2024-04-15T17:35:22.786661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"code","source":"# Load Spectrogram PNG Bytes\nwith open(f'{CONFIG.ROOT_FOLDER}/X.pkl', 'rb') as file:\n    X = pickle.load(file)\n    \n# Load Labels\nwith open(f'{CONFIG.ROOT_FOLDER}/y.pkl', 'rb') as file:\n    y = pickle.load(file)\n    \nCONFIG.N_SAMPLES = len(y)\nCONFIG.N_STEPS_PER_EPOCH = CONFIG.N_SAMPLES // CONFIG.BATCH_SIZE\nCONFIG.N_STEPS = CONFIG.N_STEPS_PER_EPOCH * CONFIG.N_EPOCHS\n\nprint(f'N_SAMPLES: {CONFIG.N_SAMPLES:,}')\nprint(f'N_STEPS_PER_EPOCH: {CONFIG.N_STEPS_PER_EPOCH:,}')\nprint(f'N_STEPS: {CONFIG.N_STEPS:,}')","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:39:55.254859Z","iopub.execute_input":"2024-04-15T17:39:55.255800Z","iopub.status.idle":"2024-04-15T17:39:58.998740Z","shell.execute_reply.started":"2024-04-15T17:39:55.255760Z","shell.execute_reply":"2024-04-15T17:39:58.997718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transforms","metadata":{}},{"cell_type":"code","source":"# Training Augmentations\nTRAIN_TRANSFORMS = A.Compose([\n        A.RandomBrightnessContrast(brightness_limit=0.10, contrast_limit=0.10, p=0.50),\n        A.ImageCompression(quality_lower=75, quality_upper=100, p=0.5),\n    ])","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:39:59.000393Z","iopub.execute_input":"2024-04-15T17:39:59.000716Z","iopub.status.idle":"2024-04-15T17:39:59.006725Z","shell.execute_reply.started":"2024-04-15T17:39:59.000662Z","shell.execute_reply":"2024-04-15T17:39:59.005854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self, X, y, transforms=None):\n        self.X = X\n        self.y = y\n        self.keys = tuple(X.keys())\n        self.transforms = transforms\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, index):\n        # Acquire Key\n        key = self.keys[index]\n        # Read Decode PNG Spectrogram\n        spec = imageio.imread(self.X[key])\n        std_global = spec.std()\n        # Random Offset\n        _, W = spec.shape\n        if W < CONFIG.WIDTH: # Pad\n            spec = np.pad(spec, ((0,0), (0,CONFIG.WIDTH-W)))\n        elif W > CONFIG.WIDTH: # Crop\n            offset = np.random.randint(0, W-CONFIG.WIDTH)\n            spec = spec[:,offset:offset+CONFIG.WIDTH]\n        \n        label = self.y[key]\n        \n        return spec, label","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:40:02.620926Z","iopub.execute_input":"2024-04-15T17:40:02.621288Z","iopub.status.idle":"2024-04-15T17:40:02.630063Z","shell.execute_reply.started":"2024-04-15T17:40:02.621249Z","shell.execute_reply":"2024-04-15T17:40:02.629120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train\ntrain_dataset = MyDataset(X, y, TRAIN_TRANSFORMS)\n\ntrain_dataloader = DataLoader(\n        train_dataset,\n        batch_size=CONFIG.BATCH_SIZE,\n        shuffle=True,\n        drop_last=True,\n        num_workers=psutil.cpu_count(),\n    )\ntrain_dataloader_iter = iter(train_dataloader)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:40:03.120550Z","iopub.execute_input":"2024-04-15T17:40:03.121303Z","iopub.status.idle":"2024-04-15T17:40:03.806726Z","shell.execute_reply.started":"2024-04-15T17:40:03.121275Z","shell.execute_reply":"2024-04-15T17:40:03.804560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example batch\nX_batch, y_batch = next(train_dataloader_iter)\n# X_batch\nprint(f'X_batch shape: {X_batch.shape}, dtype: {X_batch.dtype}')\nprint(f'X_batch min: {X_batch.min():.3f}, max: {X_batch.max():.3f}')\nprint(f'X_batch µ: {X_batch.float().mean():.3f}, σ: {X_batch.float().std():.3f}')\n# Label\nprint(f'y_batch shape: {y_batch.shape}, dtype: {y_batch.dtype}')\nprint(f'y_batch min: {y_batch.min()}, max: {y_batch.max()}')","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:40:03.816637Z","iopub.execute_input":"2024-04-15T17:40:03.820034Z","iopub.status.idle":"2024-04-15T17:40:03.901186Z","shell.execute_reply.started":"2024-04-15T17:40:03.819989Z","shell.execute_reply":"2024-04-15T17:40:03.900414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Batch","metadata":{}},{"cell_type":"code","source":"# Plot a training batch\ndef plot_batch(nrows=8, ncols=2):\n    fig, axes = plt.subplots(nrows, ncols, figsize=(ncols*6, nrows*4))\n    for r in range(nrows):\n        for c in range(ncols):\n            idx = (r * ncols) + c\n            # Denormalize Image\n            axes[r,c].imshow(X_batch[idx])\n            axes[r,c].set_title(f'shape: {X_batch[idx].numpy().shape}, label: {y_batch[idx]}')\n    plt.show()\n    \nplot_batch()","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:40:04.568031Z","iopub.execute_input":"2024-04-15T17:40:04.568686Z","iopub.status.idle":"2024-04-15T17:40:08.548512Z","shell.execute_reply.started":"2024-04-15T17:40:04.568639Z","shell.execute_reply":"2024-04-15T17:40:08.547166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def create_model():\n    model = create_cls_model(name=\"l1\", weight_url=\"l1-r224.pt\")\n\n    old_dict = model.backbone.stages[0].op_list[0].conv.state_dict()\n    old_dict['weight'] = model.backbone.stages[0].op_list[0].conv.state_dict()['weight'][:,0,:,:].unsqueeze(1)\n    model.backbone.stages[0].op_list[0].conv = torch.nn.Conv2d(1, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n    model.backbone.stages[0].op_list[0].conv.load_state_dict(old_dict)\n\n    old_dict = model.head.op_list[3].linear.state_dict()\n    old_dict['weight'] = old_dict['weight'][:182,:]\n    old_dict['bias'] = old_dict['bias'][:182]\n    model.head.op_list[3].linear = torch.nn.Linear(in_features=3200, out_features=182, bias=True)\n    model.head.op_list[3].linear.load_state_dict(old_dict)\n    return model\n    \nclass Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        # ImageNet Normalize Input\n#         self.normalize = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n        self.normalize = transforms.Normalize([0.485], [0.229])\n        \n        # Backbone\n        self.backbone = create_model()\n        \n    def forward(self, inputs):\n        # Go From HxW → 3xHxW\n        inputs = inputs.unsqueeze(1)#.expand(-1, 3, -1, -1)\n        # Normalize [0-255] → [0-1]\n        inputs = inputs.float() / 255\n        # Normalize\n        inputs = self.normalize(inputs)\n        \n        return self.backbone(inputs)","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:41:39.502479Z","iopub.execute_input":"2024-04-15T17:41:39.503091Z","iopub.status.idle":"2024-04-15T17:41:39.513781Z","shell.execute_reply.started":"2024-04-15T17:41:39.503061Z","shell.execute_reply":"2024-04-15T17:41:39.512941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create new Model\ndef count_parameters(model):\n    return sum([p.numel() for p in model.parameters()])\n\nmodel = Model().cuda()\n\n# Number of parameters\nprint(f'# Model Parameters: {count_parameters(model):,}')\n\n# Forward pass\nwith torch.no_grad():\n    # Put inputs on GPU\n    print(X_batch.shape)\n    outputs = model(X_batch.cuda())\n    print(f'outputs shape: {outputs.shape}, min: {outputs.min():.3f}, max: {outputs.max():.3f}')\n    print(f'µ: {outputs.mean():.3f}, σ: {outputs.std():.3f}')","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:41:39.732035Z","iopub.execute_input":"2024-04-15T17:41:39.732509Z","iopub.status.idle":"2024-04-15T17:41:41.078924Z","shell.execute_reply.started":"2024-04-15T17:41:39.732483Z","shell.execute_reply":"2024-04-15T17:41:41.078022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Rate Scheduler","metadata":{}},{"cell_type":"code","source":"# Get the learning rate scheduler\ndef get_lr_scheduler(optimizer):\n    return torch.optim.lr_scheduler.OneCycleLR(\n        optimizer=optimizer,\n        max_lr=CONFIG.LR_MAX,\n        total_steps=CONFIG.N_STEPS,\n        pct_start=0.10,\n        anneal_strategy='cos',\n        div_factor=1e3,\n        final_div_factor=1e4,\n    )","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:41:54.793073Z","iopub.execute_input":"2024-04-15T17:41:54.793443Z","iopub.status.idle":"2024-04-15T17:41:54.798654Z","shell.execute_reply.started":"2024-04-15T17:41:54.793412Z","shell.execute_reply":"2024-04-15T17:41:54.797666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Learning Rate Scheduler\ndef plot_lr_scheduler():\n    lr_scheduler = get_lr_scheduler(torch.optim.Adam(model.parameters()))\n    lrs  = []\n    for step in range(CONFIG.N_STEPS):\n        lrs.append(lr_scheduler.get_last_lr())\n        lr_scheduler.step()\n    # Plot Learning Rate\n    plt.figure(figsize=(12,5))\n    plt.title('Learning Rate Schedule')\n    plt.xticks(np.arange(0, CONFIG.N_STEPS+1, CONFIG.N_STEPS_PER_EPOCH), range(CONFIG.N_EPOCHS+1))\n    plt.xlim(0, CONFIG.N_STEPS)\n    plt.ylim(0, CONFIG.LR_MAX*1.1)\n    plt.xlabel('Epoch')\n    plt.ylabel('Learning Rate')\n    plt.plot(lrs)\n    plt.grid()\n    plt.show()\n    # Reset Learning Rate Scheduler\n    lr_scheduler._step_count = 0\n    lr_scheduler.last_epoch = 0\n\nplot_lr_scheduler()","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:42:00.282728Z","iopub.execute_input":"2024-04-15T17:42:00.283091Z","iopub.status.idle":"2024-04-15T17:42:00.892948Z","shell.execute_reply.started":"2024-04-15T17:42:00.283064Z","shell.execute_reply":"2024-04-15T17:42:00.892006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# Average meter to keep track of metrics/loss during training\nclass AverageMeter(object):\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val):\n        self.sum += val.sum()\n        self.count += val.numel()\n        # Average is simply the sum divided by the count\n        self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:42:05.589483Z","iopub.execute_input":"2024-04-15T17:42:05.589825Z","iopub.status.idle":"2024-04-15T17:42:05.595861Z","shell.execute_reply.started":"2024-04-15T17:42:05.589797Z","shell.execute_reply":"2024-04-15T17:42:05.594737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loss\nloss_fn = nn.CrossEntropyLoss()\n# Optimizer\noptimizer = torch.optim.AdamW(\n    params=model.parameters(),\n    lr=CONFIG.LR_MAX,\n    weight_decay=CONFIG.WEIGHT_DECAY,\n)\n# Learning Rate Scheduler\nLR_SCHEDULER = get_lr_scheduler(optimizer)\n# Metrics\nLOSS = AverageMeter()\nACC = torchmetrics.Accuracy(task='multiclass', num_classes=CONFIG.N_CLASSES).cuda()\nROC_AUC = torchmetrics.AUROC(task='multiclass', num_classes=CONFIG.N_CLASSES).cuda()","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:42:06.080452Z","iopub.execute_input":"2024-04-15T17:42:06.080819Z","iopub.status.idle":"2024-04-15T17:42:06.091569Z","shell.execute_reply.started":"2024-04-15T17:42:06.080789Z","shell.execute_reply":"2024-04-15T17:42:06.090665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(CONFIG.N_EPOCHS):\n    # Reset Metrics\n    LOSS.reset()\n    ACC.reset()\n    ROC_AUC.reset()\n    # Put model in training mode\n    model.train()\n    # Iterate Over Training Dataloader\n    for step, (X_batch, y_true) in enumerate(train_dataloader):\n        # Put batch on GPU\n        X_batch = X_batch.cuda()\n        y_true = y_true.cuda()\n        # Step Time\n        t_start = time.perf_counter()\n        # Forward Pass\n        y_pred = model(X_batch)\n        # Loss\n        loss = loss_fn(y_pred, y_true)\n        # Update Loss Metrics\n        LOSS.update(loss)\n        # Compute Gradients\n        loss.backward()\n        # Backward Pass\n        optimizer.step()\n        # Zero Out Gradients\n        optimizer.zero_grad()\n        # Update Metrics\n        ACC.update(y_pred.softmax(dim=1), y_true)\n        ROC_AUC.update(y_pred.softmax(dim=1), y_true)\n        # Logs\n        if not CONFIG.IS_INTERACTIVE and (step + 1) == CONFIG.N_STEPS_PER_EPOCH:\n            print(\n                f'EPOCH {epoch+1:02d} {step+1:04d}/{CONFIG.N_STEPS_PER_EPOCH} | ' +\n                f'loss: {LOSS.avg:.4f}, ACC: {ACC.compute():.3f}, ROC_AUC: {ROC_AUC.compute():.3f}, ' +\n                f'step: {(time.perf_counter()-t_start):.3f}s, lr: {LR_SCHEDULER.get_last_lr()[0]:.2e}',\n            )\n        elif CONFIG.IS_INTERACTIVE:\n            print(\n                f'EPOCH {epoch+1:02d} {step+1:04d}/{CONFIG.N_STEPS_PER_EPOCH} | ' +\n                f'loss: {LOSS.avg:.4f}, ACC: {ACC.compute():.3f}, ROC_AUC: {ROC_AUC.compute():.3f}, ' +\n                f'step: {(time.perf_counter()-t_start):.3f}s, lr: {LR_SCHEDULER.get_last_lr()[0]:.2e}'\n                , end='\\n' if (step + 1) == CONFIG.N_STEPS_PER_EPOCH else ' ' * 10 + '\\r', flush=True,\n            )\n        # Learning Rate Scheduler Step\n        LR_SCHEDULER.step()\n\n# Save entire model object\ntorch.save(model, 'model.pth')","metadata":{"execution":{"iopub.status.busy":"2024-04-15T17:42:07.053754Z","iopub.execute_input":"2024-04-15T17:42:07.054104Z","iopub.status.idle":"2024-04-15T17:42:28.301133Z","shell.execute_reply.started":"2024-04-15T17:42:07.054076Z","shell.execute_reply":"2024-04-15T17:42:28.299728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}