{"metadata":{"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":70203,"databundleVersionId":8068726,"sourceType":"competition"},{"sourceId":8648388,"sourceType":"datasetVersion","datasetId":5180154}],"dockerImageVersionId":30733,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":418.546697,"end_time":"2024-06-10T06:32:14.452580","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-06-10T06:25:15.905883","version":"2.5.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"1bf6bdf0a6244955887ceddb3d5a7577":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_271ed561d5534d92a0d1638c66d41c7e","placeholder":"​","style":"IPY_MODEL_5ba80312ae0c43ea996a76e94a18e665","value":" 36.5M/36.5M [00:00&lt;00:00, 81.8MB/s]"}},"1e7840a8a8964f5d8b18ff05e7b43b60":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_698723e13d1f43408893cc67568b8024","IPY_MODEL_f8b9b43a095a49a68500312f05699ec6","IPY_MODEL_1bf6bdf0a6244955887ceddb3d5a7577"],"layout":"IPY_MODEL_65c9ce73ac8b4058909de7ee21887156"}},"271ed561d5534d92a0d1638c66d41c7e":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5ba80312ae0c43ea996a76e94a18e665":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"65c9ce73ac8b4058909de7ee21887156":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"698723e13d1f43408893cc67568b8024":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_6bcfb9cb39ed4016b5e55aa1d3f04fdf","placeholder":"​","style":"IPY_MODEL_fd13b9b96bcd465c9cc25dd8063f168b","value":"model.safetensors: 100%"}},"6bcfb9cb39ed4016b5e55aa1d3f04fdf":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"89fa9d2ca24a460197416d74b238a1c5":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"a94b8a585e0e460babb04234698804df":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f8b9b43a095a49a68500312f05699ec6":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_a94b8a585e0e460babb04234698804df","max":36494688,"min":0,"orientation":"horizontal","style":"IPY_MODEL_89fa9d2ca24a460197416d74b238a1c5","value":36494688}},"fd13b9b96bcd465c9cc25dd8063f168b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport imageio.v3 as imageio\nimport albumentations as A\n\n\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm.notebook import tqdm\nfrom torchvision import transforms\n\nimport torch\nimport torchmetrics\nimport timm\nimport psutil\nimport time\n\nimport librosa\nimport cv2\nimport pickle\nimport lzma\nimport os","metadata":{"papermill":{"duration":13.10278,"end_time":"2024-06-10T06:25:32.033166","exception":false,"start_time":"2024-06-10T06:25:18.930386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:27:22.578170Z","iopub.execute_input":"2024-06-10T07:27:22.578515Z","iopub.status.idle":"2024-06-10T07:27:22.584829Z","shell.execute_reply.started":"2024-06-10T07:27:22.578490Z","shell.execute_reply":"2024-06-10T07:27:22.583806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config():\n\n    # Competition Root Folder\n    ROOT_FOLDER = '/kaggle/input/birdclef-2024'\n    DATA_FOLDER = '/kaggle/input/birdclef-24-birbdetect'\n\n    # Dataset\n    HEIGHT = 128\n    WIDTH = 320\n\n    # Training\n    BATCH_SIZE = 16\n    N_EPOCHS = 20\n    # Model\n    BACKBONE = 'wide_resnet50_2.tv2_in1k'\n    # Learning Rate Scheduler\n    LR_MAX = 3e-4\n    WEIGHT_DECAY = 0.00\n    # Others\n    SEED = 42\n    # IS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\n    IS_INTERACTIVE = True\n    \nCONFIG = Config()","metadata":{"papermill":{"duration":0.014083,"end_time":"2024-06-10T06:25:32.054436","exception":false,"start_time":"2024-06-10T06:25:32.040353","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:27:30.399967Z","iopub.execute_input":"2024-06-10T07:27:30.400334Z","iopub.status.idle":"2024-06-10T07:27:30.405972Z","shell.execute_reply.started":"2024-06-10T07:27:30.400306Z","shell.execute_reply":"2024-06-10T07:27:30.405046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv(f'{CONFIG.ROOT_FOLDER}/sample_submission.csv')\n\n# Set labels\nCONFIG.LABELS = sample_submission.columns[1:]\nCONFIG.N_LABELS = len(CONFIG.LABELS)\nCONFIG.N_CLASSES = len(CONFIG.LABELS)","metadata":{"papermill":{"duration":0.028878,"end_time":"2024-06-10T06:25:32.089393","exception":false,"start_time":"2024-06-10T06:25:32.060515","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:27:32.834664Z","iopub.execute_input":"2024-06-10T07:27:32.835520Z","iopub.status.idle":"2024-06-10T07:27:32.857916Z","shell.execute_reply.started":"2024-06-10T07:27:32.835490Z","shell.execute_reply":"2024-06-10T07:27:32.856973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load Spectrogram PNG Bytes\nwith open(f'{CONFIG.DATA_FOLDER}/temp_X.pkl', 'rb') as file:\n    X = pickle.load(file)\n    \n# Load Labels\nwith open(f'{CONFIG.DATA_FOLDER}/temp_y.pkl', 'rb') as file:\n    y = pickle.load(file)\n    \n\nCONFIG.N_SAMPLES = len(y)\nCONFIG.N_STEPS_PER_EPOCH = CONFIG.N_SAMPLES // CONFIG.BATCH_SIZE\nCONFIG.N_STEPS = CONFIG.N_STEPS_PER_EPOCH * CONFIG.N_EPOCHS\n\nprint(f'N_SAMPLES: {CONFIG.N_SAMPLES:,}')\nprint(f'N_STEPS_PER_EPOCH: {CONFIG.N_STEPS_PER_EPOCH:,}')\nprint(f'N_STEPS: {CONFIG.N_STEPS:,}')","metadata":{"papermill":{"duration":38.38519,"end_time":"2024-06-10T06:26:10.480695","exception":false,"start_time":"2024-06-10T06:25:32.095505","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:27:35.085359Z","iopub.execute_input":"2024-06-10T07:27:35.085713Z","iopub.status.idle":"2024-06-10T07:28:09.964975Z","shell.execute_reply.started":"2024-06-10T07:27:35.085685Z","shell.execute_reply":"2024-06-10T07:28:09.964028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Augmentations\nTRAIN_TRANSFORMS = A.Compose([\n        A.RandomBrightnessContrast(brightness_limit=0.10, contrast_limit=0.10, p=0.50),\n        A.ImageCompression(quality_lower=75, quality_upper=100, p=0.5),\n    ])","metadata":{"papermill":{"duration":0.014005,"end_time":"2024-06-10T06:26:10.500878","exception":false,"start_time":"2024-06-10T06:26:10.486873","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:28:21.378567Z","iopub.execute_input":"2024-06-10T07:28:21.378932Z","iopub.status.idle":"2024-06-10T07:28:21.384617Z","shell.execute_reply.started":"2024-06-10T07:28:21.378906Z","shell.execute_reply":"2024-06-10T07:28:21.383511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self, X, y, transforms=None):\n        self.X = X\n        self.y = y\n        self.keys = tuple(X.keys())\n        self.transforms = transforms\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, index):\n        # Acquire Key\n        key = self.keys[index]\n        # Read Decode PNG Spectrogram\n        spec = imageio.imread(self.X[key])\n        std_global = spec.std()\n        # Random Offset\n        _, W = spec.shape\n        if W < CONFIG.WIDTH: # Pad\n            spec = np.pad(spec, ((0,0), (0,CONFIG.WIDTH-W)))\n        elif W > CONFIG.WIDTH: # Crop\n            offset = np.random.randint(0, W-CONFIG.WIDTH)\n            spec = spec[:,offset:offset+CONFIG.WIDTH]\n        \n        label = self.y[key]\n        \n        return spec, label","metadata":{"papermill":{"duration":0.016401,"end_time":"2024-06-10T06:26:10.523181","exception":false,"start_time":"2024-06-10T06:26:10.506780","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:28:26.470878Z","iopub.execute_input":"2024-06-10T07:28:26.471670Z","iopub.status.idle":"2024-06-10T07:28:26.479687Z","shell.execute_reply.started":"2024-06-10T07:28:26.471636Z","shell.execute_reply":"2024-06-10T07:28:26.478823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train\ntrain_dataset = MyDataset(X, y, TRAIN_TRANSFORMS)\n\ntrain_dataloader = DataLoader(\n        train_dataset,\n        batch_size=CONFIG.BATCH_SIZE,\n        shuffle=True,\n        drop_last=True,\n        # num_workers=psutil.cpu_count(),\n        num_workers=0,\n    )\ntrain_dataloader_iter = iter(train_dataloader)","metadata":{"papermill":{"duration":0.03257,"end_time":"2024-06-10T06:26:10.561714","exception":false,"start_time":"2024-06-10T06:26:10.529144","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:28:34.195670Z","iopub.execute_input":"2024-06-10T07:28:34.196030Z","iopub.status.idle":"2024-06-10T07:28:34.213686Z","shell.execute_reply.started":"2024-06-10T07:28:34.196003Z","shell.execute_reply":"2024-06-10T07:28:34.212963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example batch\nX_batch, y_batch = next(train_dataloader_iter)\n# X_batch\nprint(f'X_batch shape: {X_batch.shape}, dtype: {X_batch.dtype}')\nprint(f'X_batch min: {X_batch.min():.3f}, max: {X_batch.max():.3f}')\nprint(f'X_batch µ: {X_batch.float().mean():.3f}, σ: {X_batch.float().std():.3f}')\n# Label\nprint(f'y_batch shape: {y_batch.shape}, dtype: {y_batch.dtype}')\nprint(f'y_batch min: {y_batch.min()}, max: {y_batch.max()}')","metadata":{"papermill":{"duration":0.175647,"end_time":"2024-06-10T06:26:10.743488","exception":false,"start_time":"2024-06-10T06:26:10.567841","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:28:39.015713Z","iopub.execute_input":"2024-06-10T07:28:39.016513Z","iopub.status.idle":"2024-06-10T07:28:39.155195Z","shell.execute_reply.started":"2024-06-10T07:28:39.016483Z","shell.execute_reply":"2024-06-10T07:28:39.154197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot a training batch\ndef plot_batch(nrows=8, ncols=2):\n    fig, axes = plt.subplots(nrows, ncols, figsize=(ncols*6, nrows*4))\n    for r in range(nrows):\n        for c in range(ncols):\n            idx = (r * ncols) + c\n            # Denormalize Image\n            axes[r,c].imshow(X_batch[idx])\n            axes[r,c].set_title(f'shape: {X_batch[idx].numpy().shape}, label: {y_batch[idx]}')\n    plt.show()\n    \nplot_batch()","metadata":{"papermill":{"duration":3.669616,"end_time":"2024-06-10T06:26:14.419784","exception":false,"start_time":"2024-06-10T06:26:10.750168","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:28:48.316737Z","iopub.execute_input":"2024-06-10T07:28:48.317579Z","iopub.status.idle":"2024-06-10T07:28:51.793945Z","shell.execute_reply.started":"2024-06-10T07:28:48.317549Z","shell.execute_reply":"2024-06-10T07:28:51.793016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#TODO: REMOVE\n# Function to search for timm models\ndef search_timm_model(query):\n    search_result = [n for n in timm.list_models(pretrained=True) if query in n]\n    for i, name in enumerate(search_result):\n        print(f'{i:02d} | {name}')\n        \nsearch_timm_model('wide_resnet50_2')","metadata":{"papermill":{"duration":0.053645,"end_time":"2024-06-10T06:26:14.505939","exception":false,"start_time":"2024-06-10T06:26:14.452294","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:29:06.515826Z","iopub.execute_input":"2024-06-10T07:29:06.516471Z","iopub.status.idle":"2024-06-10T07:29:06.534714Z","shell.execute_reply.started":"2024-06-10T07:29:06.516440Z","shell.execute_reply":"2024-06-10T07:29:06.533742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#TODO: Check and remove\n# Count model parameters\ndef count_parameters(model):\n    return sum([p.numel() for p in model.parameters()])","metadata":{"papermill":{"duration":0.038402,"end_time":"2024-06-10T06:26:14.574266","exception":false,"start_time":"2024-06-10T06:26:14.535864","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:29:18.183582Z","iopub.execute_input":"2024-06-10T07:29:18.183951Z","iopub.status.idle":"2024-06-10T07:29:18.188565Z","shell.execute_reply.started":"2024-06-10T07:29:18.183922Z","shell.execute_reply":"2024-06-10T07:29:18.187649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        # ImageNet Normalize Input\n        self.normalize = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n        \n        # Backbone\n        self.backbone = timm.create_model(\n                CONFIG.BACKBONE,\n                pretrained=True,\n                num_classes=CONFIG.N_CLASSES,\n            )\n        \n    def forward(self, inputs):\n        # Go From HxW → 3xHxW\n        inputs = inputs.unsqueeze(1).expand(-1, 3, -1, -1)\n        # Normalize [0-255] → [0-1]\n        inputs = inputs.float() / 255\n        # Normalize\n        inputs = self.normalize(inputs)\n        \n        return self.backbone(inputs)","metadata":{"ExecuteTime":{"end_time":"2024-06-08T18:20:18.394597Z","start_time":"2024-06-08T18:20:18.384642Z"},"papermill":{"duration":0.041455,"end_time":"2024-06-10T06:26:14.646095","exception":false,"start_time":"2024-06-10T06:26:14.604640","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:29:21.215151Z","iopub.execute_input":"2024-06-10T07:29:21.215529Z","iopub.status.idle":"2024-06-10T07:29:21.223127Z","shell.execute_reply.started":"2024-06-10T07:29:21.215497Z","shell.execute_reply":"2024-06-10T07:29:21.222017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create new Model\nmodel = Model().cuda()\n\n# Number of parameters\nprint(f'# Model Parameters: {count_parameters(model):,}')\n\n# Forward pass\nwith torch.no_grad():\n    # Put inputs on GPU\n    outputs = model(X_batch.cuda())\n    print(f'outputs shape: {outputs.shape}, min: {outputs.min():.3f}, max: {outputs.max():.3f}')\n    print(f'µ: {outputs.mean():.3f}, σ: {outputs.std():.3f}')","metadata":{"ExecuteTime":{"end_time":"2024-06-08T18:22:21.221749Z","start_time":"2024-06-08T18:22:20.851371Z"},"papermill":{"duration":2.420257,"end_time":"2024-06-10T06:26:17.097355","exception":false,"start_time":"2024-06-10T06:26:14.677098","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:30:26.962080Z","iopub.execute_input":"2024-06-10T07:30:26.962427Z","iopub.status.idle":"2024-06-10T07:30:31.774752Z","shell.execute_reply.started":"2024-06-10T07:30:26.962402Z","shell.execute_reply":"2024-06-10T07:30:31.773754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the learning rate scheduler\ndef get_lr_scheduler(optimizer):\n    return torch.optim.lr_scheduler.OneCycleLR(\n        optimizer=optimizer,\n        max_lr=CONFIG.LR_MAX,\n        total_steps=CONFIG.N_STEPS,\n        pct_start=0.10,\n        anneal_strategy='cos',\n        div_factor=1e3,\n        final_div_factor=1e4,\n    )","metadata":{"ExecuteTime":{"end_time":"2024-06-08T18:23:26.938532Z","start_time":"2024-06-08T18:23:26.933909Z"},"papermill":{"duration":0.040211,"end_time":"2024-06-10T06:26:17.169173","exception":false,"start_time":"2024-06-10T06:26:17.128962","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:31:00.677224Z","iopub.execute_input":"2024-06-10T07:31:00.677581Z","iopub.status.idle":"2024-06-10T07:31:00.682978Z","shell.execute_reply.started":"2024-06-10T07:31:00.677552Z","shell.execute_reply":"2024-06-10T07:31:00.682021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot Learning Rate Scheduler\ndef plot_lr_scheduler():\n    lr_scheduler = get_lr_scheduler(torch.optim.Adam(model.parameters()))\n    lrs  = []\n    for step in range(CONFIG.N_STEPS):\n        lrs.append(lr_scheduler.get_last_lr())\n        lr_scheduler.step()\n    # Plot Learning Rate\n    plt.figure(figsize=(12,5))\n    plt.title('Learning Rate Schedule')\n    plt.xticks(np.arange(0, CONFIG.N_STEPS+1, CONFIG.N_STEPS_PER_EPOCH), range(CONFIG.N_EPOCHS+1))\n    plt.xlim(0, CONFIG.N_STEPS)\n    plt.ylim(0, CONFIG.LR_MAX*1.1)\n    plt.xlabel('Epoch')\n    plt.ylabel('Learning Rate')\n    plt.plot(lrs)\n    plt.grid()\n    plt.show()\n    # Reset Learning Rate Scheduler\n    lr_scheduler._step_count = 0\n    lr_scheduler.last_epoch = 0\n\nplot_lr_scheduler()","metadata":{"ExecuteTime":{"end_time":"2024-06-08T18:23:28.852698Z","start_time":"2024-06-08T18:23:28.793070Z"},"papermill":{"duration":0.317456,"end_time":"2024-06-10T06:26:17.517229","exception":false,"start_time":"2024-06-10T06:26:17.199773","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:31:03.804007Z","iopub.execute_input":"2024-06-10T07:31:03.804616Z","iopub.status.idle":"2024-06-10T07:31:04.020317Z","shell.execute_reply.started":"2024-06-10T07:31:03.804587Z","shell.execute_reply":"2024-06-10T07:31:04.019383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Average meter to keep track of metrics/loss during training\nclass AverageMeter(object):\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val):\n        self.sum += val.sum()\n        self.count += val.numel()\n        # Average is simply the sum divided by the count\n        self.avg = self.sum / self.count","metadata":{"papermill":{"duration":0.040814,"end_time":"2024-06-10T06:26:17.590351","exception":false,"start_time":"2024-06-10T06:26:17.549537","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:31:08.438657Z","iopub.execute_input":"2024-06-10T07:31:08.439033Z","iopub.status.idle":"2024-06-10T07:31:08.445174Z","shell.execute_reply.started":"2024-06-10T07:31:08.439004Z","shell.execute_reply":"2024-06-10T07:31:08.444099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loss\nloss_fn = nn.CrossEntropyLoss()\n# Optimizer\noptimizer = torch.optim.AdamW(\n    params=model.parameters(),\n    lr=CONFIG.LR_MAX,\n    weight_decay=CONFIG.WEIGHT_DECAY,\n)\n# Learning Rate Scheduler\nLR_SCHEDULER = get_lr_scheduler(optimizer)\n# Metrics\nLOSS = AverageMeter()\nACC = torchmetrics.Accuracy(task='multiclass', num_classes=CONFIG.N_CLASSES).cuda()\nROC_AUC = torchmetrics.AUROC(task='multiclass', num_classes=CONFIG.N_CLASSES).cuda()","metadata":{"papermill":{"duration":0.045146,"end_time":"2024-06-10T06:26:17.667303","exception":false,"start_time":"2024-06-10T06:26:17.622157","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:31:12.088850Z","iopub.execute_input":"2024-06-10T07:31:12.089170Z","iopub.status.idle":"2024-06-10T07:31:12.099575Z","shell.execute_reply.started":"2024-06-10T07:31:12.089147Z","shell.execute_reply":"2024-06-10T07:31:12.098672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(CONFIG.N_EPOCHS):\n    # Reset Metrics\n    LOSS.reset()\n    ACC.reset()\n    ROC_AUC.reset()\n    # Put model in training mode\n    model.train()\n    # Iterate Over Training Dataloader\n    for step, (X_batch, y_true) in enumerate(train_dataloader):\n        # Put batch on GPU\n        X_batch = X_batch.cuda()\n        y_true = y_true.cuda()\n        # Step Time\n        t_start = time.perf_counter()\n        # Forward Pass\n        y_pred = model(X_batch)\n        # Loss\n        loss = loss_fn(y_pred, y_true)\n        # Update Loss Metrics\n        LOSS.update(loss)\n        # Compute Gradients\n        loss.backward()\n        # Backward Pass\n        optimizer.step()\n        # Zero Out Gradients\n        optimizer.zero_grad()\n        # Update Metrics\n        ACC.update(y_pred.softmax(dim=1), y_true)\n        ROC_AUC.update(y_pred.softmax(dim=1), y_true)\n        # Logs\n        if not CONFIG.IS_INTERACTIVE and (step + 1) == CONFIG.N_STEPS_PER_EPOCH:\n            print(\n                f'EPOCH {epoch+1:02d} {step+1:04d}/{CONFIG.N_STEPS_PER_EPOCH} | ' +\n                f'loss: {LOSS.avg:.4f}, ACC: {ACC.compute():.3f}, ROC_AUC: {ROC_AUC.compute():.3f}, ' +\n                f'step: {(time.perf_counter()-t_start):.3f}s, lr: {LR_SCHEDULER.get_last_lr()[0]:.2e}',\n            )\n        elif CONFIG.IS_INTERACTIVE:\n            print(\n                f'EPOCH {epoch+1:02d} {step+1:04d}/{CONFIG.N_STEPS_PER_EPOCH} | ' +\n                f'loss: {LOSS.avg:.4f}, ACC: {ACC.compute():.3f}, ROC_AUC: {ROC_AUC.compute():.3f}, ' +\n                f'step: {(time.perf_counter()-t_start):.3f}s, lr: {LR_SCHEDULER.get_last_lr()[0]:.2e}'\n                , end='\\n' if (step + 1) == CONFIG.N_STEPS_PER_EPOCH else ' ' * 10 + '\\r', flush=True,\n            )\n        # Learning Rate Scheduler Step\n        LR_SCHEDULER.step()\n\n# Save entire model object\ntorch.save(model, 'model.pth')","metadata":{"papermill":{"duration":353.632472,"end_time":"2024-06-10T06:32:11.332329","exception":false,"start_time":"2024-06-10T06:26:17.699857","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-06-10T07:31:15.006915Z","iopub.execute_input":"2024-06-10T07:31:15.007259Z","iopub.status.idle":"2024-06-10T07:41:34.036088Z","shell.execute_reply.started":"2024-06-10T07:31:15.007231Z","shell.execute_reply":"2024-06-10T07:41:34.035262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.133636,"end_time":"2024-06-10T06:32:11.610586","exception":false,"start_time":"2024-06-10T06:32:11.476950","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}