{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.13"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59093,"databundleVersionId":7469972,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm==0.6.13 huggingface_hub kaggle pynvml -Uqq\n!jupyter notebook --ServerApp.iopub_data_rate_limit=1.0e10\n!jupyter notebook --ServerApp.iopub_msg_rate_limit=1.0e10\n# !kaggle datasets download -d vishalbakshi/hms-hbac-training-spectrogram-images\n# zipfile.ZipFile('hms-hbac-training-spectrogram-images.zip').extractall('hms-hbac-training-spectrogram-images')","metadata":{"execution":{"iopub.execute_input":"2024-03-28T23:10:47.691366Z","iopub.status.busy":"2024-03-28T23:10:47.690672Z","iopub.status.idle":"2024-03-28T23:10:53.446884Z","shell.execute_reply":"2024-03-28T23:10:53.446185Z","shell.execute_reply.started":"2024-03-28T23:10:47.691332Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nimport timm\nimport gc\n\nfrom fastai.vision.all import *\nfrom fastcore.parallel import *\n\n#path = Path('/kaggle/input/hms-hbac-training-spectrogram-images/train_spectrograms')\npath = Path('/notebooks/hms-hbac-training-spectrogram-images/train_spectrograms')\npath.ls()","metadata":{"execution":{"iopub.execute_input":"2024-03-28T23:10:25.825605Z","iopub.status.busy":"2024-03-28T23:10:25.825409Z","iopub.status.idle":"2024-03-28T23:10:30.714361Z","shell.execute_reply":"2024-03-28T23:10:30.713629Z","shell.execute_reply.started":"2024-03-28T23:10:25.825589Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Background","metadata":{}},{"cell_type":"markdown","source":"In this notebook I'll train large versions of the top 9 models that I picked from the previous submissions (3 from each family: `convnext`, `vit` and `swinv2`).\n\nHere are the training results:","metadata":{}},{"cell_type":"markdown","source":"|arch|item method|item size|batch size|epochs|name|TTA Validation Error Rate|\n|:-:|:-:|:-:|:-:|:-:|:-:|:-:|\n|convnext_large_in22k|squish|(311, 400)|None|5|1|0.3507|\n|convnext_large_in22k|crop|(400, 311)|None|5|3|0.3520|\n|convnext_large_in22k|squish|(400, 311)|None|5|4|0.3534\n|vit_large_patch16_224|crop|256 --> (400, 311) --> (320, 512)|RandomResizedCropGPU(size=224, min_scale=1.0)|3 -> 5 -> 7|AW|0.3678\n|vit_large_patch16_224|pad|256 --> 400 --> (320, 512)|aug_transforms(size=224, min_scale=0.75)|3 -> 5 -> 7|BB|0.3476\n|vit_large_patch16_224|squish|256 --> (400, 311) --> (320, 512)|aug_transforms(size=224, min_scale=0.75)|3 -> 5 -> 7|AU|0.3538\n|swinv2_large_window12_192_22k|crop|(400, 311)|aug_transforms(size=192, min_scale=0.75)|8|J|0.3736\n|swinv2_large_window12_192_22k|crop|(400, 311)|RandomResizedCropGPU(size=192, min_scale=1.0)|8|AB|0.3502\n|swinv2_large_window12_192_22k|squish|256 --> (400, 311) --> (320, 512)|aug_transforms(size=192, min_scale=0.75)|3 --> 5 --> 8|AU|0.3417","metadata":{}},{"cell_type":"markdown","source":"## Train and Export","metadata":{}},{"cell_type":"markdown","source":"I'll make sure to remove the `seed` since I want the models to train and validate on different splits from each other.","metadata":{}},{"cell_type":"code","source":"def progressive_resizing(fn, sizes, method, batch, arch, epochs, pad_mode=PadMode.Reflection):\n    dls = ImageDataLoaders.from_folder(\n        path, \n        valid_pct=0.2, \n        item_tfms=Resize(sizes[0], method=method, pad_mode=pad_mode),\n        batch_tfms=batch,\n        bs=64//4)\n    \n    cbs = GradientAccumulation(64)\n    learn = vision_learner(dls, arch, metrics=error_rate, cbs=cbs).to_fp16()\n    learn.fine_tune(epochs[0], 0.01)\n\n    dls = ImageDataLoaders.from_folder(\n            path, \n            valid_pct=0.2, \n            item_tfms=Resize(sizes[1], method=method, pad_mode=pad_mode),\n            batch_tfms=batch,\n            bs=64//4)\n\n    learn.dls = dls\n    learn.fine_tune(epochs[1], 0.01)\n\n    dls = ImageDataLoaders.from_folder(\n            path, \n            valid_pct=0.2, \n            item_tfms=Resize(sizes[2], method=method, pad_mode=pad_mode),\n            batch_tfms=batch,\n            bs=64//4)\n\n    learn.dls = dls\n    learn.fine_tune(epochs[2], 0.01)\n    print(error_rate(*learn.tta(dl=dls.valid)))\n    learn.save(fn, with_opt=False)","metadata":{"execution":{"iopub.execute_input":"2024-03-28T23:11:07.910769Z","iopub.status.busy":"2024-03-28T23:11:07.910055Z","iopub.status.idle":"2024-03-28T23:11:07.917320Z","shell.execute_reply":"2024-03-28T23:11:07.916668Z","shell.execute_reply.started":"2024-03-28T23:11:07.910739Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(fn, arch, item, batch, epochs, accum=4):        \n    dls = ImageDataLoaders.from_folder(\n        path, \n        valid_pct=0.2, \n        item_tfms=item,\n        batch_tfms=batch,\n        bs=64//accum)\n    \n    cbs = GradientAccumulation(64) if accum else []\n    learn = vision_learner(dls, arch, metrics=error_rate, cbs=cbs).to_fp16()\n    learn.fine_tune(epochs, 0.01)\n    print(error_rate(*learn.tta(dl=dls.valid)))\n    learn.save(fn, with_opt=False)\n","metadata":{"execution":{"iopub.execute_input":"2024-03-28T23:11:04.795061Z","iopub.status.busy":"2024-03-28T23:11:04.794275Z","iopub.status.idle":"2024-03-28T23:11:04.800568Z","shell.execute_reply":"2024-03-28T23:11:04.799724Z","shell.execute_reply.started":"2024-03-28T23:11:04.795033Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ndef report_gpu():\n    print(torch.cuda.list_gpu_processes())\n    gc.collect()\n    torch.cuda.empty_cache()","metadata":{"execution":{"iopub.execute_input":"2024-03-28T23:10:59.509565Z","iopub.status.busy":"2024-03-28T23:10:59.509229Z","iopub.status.idle":"2024-03-28T23:10:59.513507Z","shell.execute_reply":"2024-03-28T23:10:59.512943Z","shell.execute_reply.started":"2024-03-28T23:10:59.509506Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"report_gpu()","metadata":{"execution":{"iopub.execute_input":"2024-03-28T23:11:01.018475Z","iopub.status.busy":"2024-03-28T23:11:01.017845Z","iopub.status.idle":"2024-03-28T23:11:01.155825Z","shell.execute_reply":"2024-03-28T23:11:01.155023Z","shell.execute_reply.started":"2024-03-28T23:11:01.018446Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### convnext_small_in22k","metadata":{}},{"cell_type":"code","source":"# timm.list_models('swinv2*')","metadata":{"execution":{"iopub.execute_input":"2024-03-28T15:18:00.826610Z","iopub.status.busy":"2024-03-28T15:18:00.825949Z","iopub.status.idle":"2024-03-28T15:18:00.829737Z","shell.execute_reply":"2024-03-28T15:18:00.829113Z","shell.execute_reply.started":"2024-03-28T15:18:00.826575Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arch='convnext_large_in22k'","metadata":{"execution":{"iopub.execute_input":"2024-03-28T23:11:31.311947Z","iopub.status.busy":"2024-03-28T23:11:31.311156Z","iopub.status.idle":"2024-03-28T23:11:31.316120Z","shell.execute_reply":"2024-03-28T23:11:31.315130Z","shell.execute_reply.started":"2024-03-28T23:11:31.311913Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arch","metadata":{"execution":{"iopub.execute_input":"2024-03-28T23:11:32.783528Z","iopub.status.busy":"2024-03-28T23:11:32.782914Z","iopub.status.idle":"2024-03-28T23:11:32.787867Z","shell.execute_reply":"2024-03-28T23:11:32.787348Z","shell.execute_reply.started":"2024-03-28T23:11:32.783505Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model 1","metadata":{}},{"cell_type":"code","source":"train(\n    fn=f\"/notebooks/hms_hbac_{arch}_1\",\n    arch=arch, \n    item=Resize((311,400), method='squish'), \n    batch=None,\n    epochs=5)","metadata":{"execution":{"iopub.execute_input":"2024-03-28T15:54:40.971492Z","iopub.status.busy":"2024-03-28T15:54:40.970956Z","iopub.status.idle":"2024-03-28T16:18:38.808111Z","shell.execute_reply":"2024-03-28T16:18:38.807390Z","shell.execute_reply.started":"2024-03-28T15:54:40.971470Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model 3","metadata":{}},{"cell_type":"code","source":"train(\n    fn=f\"/notebooks/hms_hbac_{arch}_3\",\n    arch=arch, \n    item=Resize((400,311)), \n    batch=None,\n    epochs=5)","metadata":{"execution":{"iopub.execute_input":"2024-03-28T16:20:57.490150Z","iopub.status.busy":"2024-03-28T16:20:57.489854Z","iopub.status.idle":"2024-03-28T16:44:58.228854Z","shell.execute_reply":"2024-03-28T16:44:58.228189Z","shell.execute_reply.started":"2024-03-28T16:20:57.490123Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model 4","metadata":{}},{"cell_type":"code","source":"train(\n    fn=f\"/notebooks/hms_hbac_{arch}_4\",\n    arch=arch, \n    item=Resize((400,311), method='squish'), \n    batch=None,\n    epochs=5)","metadata":{"execution":{"iopub.execute_input":"2024-03-28T23:15:01.312368Z","iopub.status.busy":"2024-03-28T23:15:01.311708Z","iopub.status.idle":"2024-03-28T23:38:35.213666Z","shell.execute_reply":"2024-03-28T23:38:35.212918Z","shell.execute_reply.started":"2024-03-28T23:15:01.312333Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"report_gpu()","metadata":{"execution":{"iopub.execute_input":"2024-03-28T23:38:35.215378Z","iopub.status.busy":"2024-03-28T23:38:35.215181Z","iopub.status.idle":"2024-03-28T23:38:35.683214Z","shell.execute_reply":"2024-03-28T23:38:35.682346Z","shell.execute_reply.started":"2024-03-28T23:38:35.215331Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### vit_large_patch16_224","metadata":{}},{"cell_type":"code","source":"arch='vit_large_patch16_224'","metadata":{"execution":{"iopub.execute_input":"2024-03-28T23:38:35.684703Z","iopub.status.busy":"2024-03-28T23:38:35.684478Z","iopub.status.idle":"2024-03-28T23:38:35.688045Z","shell.execute_reply":"2024-03-28T23:38:35.687487Z","shell.execute_reply.started":"2024-03-28T23:38:35.684685Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arch","metadata":{"execution":{"iopub.execute_input":"2024-03-28T23:38:35.689779Z","iopub.status.busy":"2024-03-28T23:38:35.689584Z","iopub.status.idle":"2024-03-28T23:38:35.693510Z","shell.execute_reply":"2024-03-28T23:38:35.693068Z","shell.execute_reply.started":"2024-03-28T23:38:35.689755Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model AW","metadata":{}},{"cell_type":"code","source":"progressive_resizing(\n    fn=\"/notebooks/hms_hbac_{arch}_AW\", \n    sizes=[256,(400,311),(320,512)], \n    method='crop', \n    batch=RandomResizedCropGPU(size=224, min_scale=1.0), \n    arch=arch, \n    epochs=[3,5,7], \n    pad_mode=PadMode.Reflection)","metadata":{"execution":{"iopub.execute_input":"2024-03-28T23:38:35.694316Z","iopub.status.busy":"2024-03-28T23:38:35.694123Z","iopub.status.idle":"2024-03-29T00:27:59.075908Z","shell.execute_reply":"2024-03-29T00:27:59.075241Z","shell.execute_reply.started":"2024-03-28T23:38:35.694300Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"report_gpu()","metadata":{"execution":{"iopub.execute_input":"2024-03-29T00:27:59.078373Z","iopub.status.busy":"2024-03-29T00:27:59.078084Z","iopub.status.idle":"2024-03-29T00:27:59.560880Z","shell.execute_reply":"2024-03-29T00:27:59.560393Z","shell.execute_reply.started":"2024-03-29T00:27:59.078350Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model BB","metadata":{}},{"cell_type":"code","source":"progressive_resizing(\n    fn=\"/notebooks/hms_hbac_{arch}_BB\", \n    sizes=[256, 400, (320,512)], \n    method=ResizeMethod.Pad, \n    batch=aug_transforms(size=224, min_scale=0.75), \n    arch=arch, \n    epochs=[3,5,7], \n    pad_mode=PadMode.Zeros)","metadata":{"execution":{"iopub.execute_input":"2024-03-29T00:27:59.562232Z","iopub.status.busy":"2024-03-29T00:27:59.561796Z","iopub.status.idle":"2024-03-29T01:17:54.660241Z","shell.execute_reply":"2024-03-29T01:17:54.659448Z","shell.execute_reply.started":"2024-03-29T00:27:59.562199Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"report_gpu()","metadata":{"execution":{"iopub.execute_input":"2024-03-29T01:17:54.661839Z","iopub.status.busy":"2024-03-29T01:17:54.661636Z","iopub.status.idle":"2024-03-29T01:17:55.164943Z","shell.execute_reply":"2024-03-29T01:17:55.164400Z","shell.execute_reply.started":"2024-03-29T01:17:54.661820Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model AU","metadata":{}},{"cell_type":"code","source":"progressive_resizing(\n    fn=\"/notebooks/hms_hbac_{arch}_AU\", \n    sizes=[256, (400,311), (320,512)], \n    method='squish', \n    batch=aug_transforms(size=224, min_scale=0.75), \n    arch=arch, \n    epochs=[3,5,7], \n    pad_mode=PadMode.Reflection)","metadata":{"execution":{"iopub.execute_input":"2024-03-29T01:17:55.166234Z","iopub.status.busy":"2024-03-29T01:17:55.166011Z","iopub.status.idle":"2024-03-29T02:07:52.921262Z","shell.execute_reply":"2024-03-29T02:07:52.920375Z","shell.execute_reply.started":"2024-03-29T01:17:55.166216Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"report_gpu()","metadata":{"execution":{"iopub.execute_input":"2024-03-29T02:07:52.924085Z","iopub.status.busy":"2024-03-29T02:07:52.923517Z","iopub.status.idle":"2024-03-29T02:07:53.448859Z","shell.execute_reply":"2024-03-29T02:07:53.448199Z","shell.execute_reply.started":"2024-03-29T02:07:52.924062Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## swinv2_large_window12_192_22k","metadata":{}},{"cell_type":"code","source":"arch = 'swinv2_large_window12_192_22k'","metadata":{"execution":{"iopub.execute_input":"2024-03-29T03:04:15.839687Z","iopub.status.busy":"2024-03-29T03:04:15.839017Z","iopub.status.idle":"2024-03-29T03:04:15.842563Z","shell.execute_reply":"2024-03-29T03:04:15.841906Z","shell.execute_reply.started":"2024-03-29T03:04:15.839663Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arch","metadata":{"execution":{"iopub.execute_input":"2024-03-29T03:04:51.295228Z","iopub.status.busy":"2024-03-29T03:04:51.294548Z","iopub.status.idle":"2024-03-29T03:04:51.299291Z","shell.execute_reply":"2024-03-29T03:04:51.298617Z","shell.execute_reply.started":"2024-03-29T03:04:51.295202Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model J","metadata":{}},{"cell_type":"code","source":"train(\n    fn=f\"/notebooks/hms_hbac_{arch}_J\",\n    arch=arch, \n    item=Resize((400,311)), \n    batch=aug_transforms(size=192, min_scale=0.75),\n    epochs=8)","metadata":{"execution":{"iopub.execute_input":"2024-03-29T03:07:29.701864Z","iopub.status.busy":"2024-03-29T03:07:29.701125Z","iopub.status.idle":"2024-03-29T03:32:03.429025Z","shell.execute_reply":"2024-03-29T03:32:03.428297Z","shell.execute_reply.started":"2024-03-29T03:07:29.701837Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"report_gpu()","metadata":{"execution":{"iopub.execute_input":"2024-03-29T03:32:03.431408Z","iopub.status.busy":"2024-03-29T03:32:03.430609Z","iopub.status.idle":"2024-03-29T03:32:03.850747Z","shell.execute_reply":"2024-03-29T03:32:03.850107Z","shell.execute_reply.started":"2024-03-29T03:32:03.431382Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model AB","metadata":{}},{"cell_type":"code","source":"train(\n    fn=f\"/notebooks/hms_hbac_{arch}_AB\",\n    arch=arch, \n    item=Resize((400,311)), \n    batch=RandomResizedCropGPU(size=192, min_scale=1.0),\n    epochs=8)","metadata":{"execution":{"iopub.execute_input":"2024-03-29T03:32:03.854649Z","iopub.status.busy":"2024-03-29T03:32:03.854390Z","iopub.status.idle":"2024-03-29T03:55:42.516907Z","shell.execute_reply":"2024-03-29T03:55:42.516238Z","shell.execute_reply.started":"2024-03-29T03:32:03.854632Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"report_gpu()","metadata":{"execution":{"iopub.execute_input":"2024-03-29T05:09:18.245506Z","iopub.status.busy":"2024-03-29T05:09:18.244972Z","iopub.status.idle":"2024-03-29T05:09:18.831377Z","shell.execute_reply":"2024-03-29T05:09:18.830407Z","shell.execute_reply.started":"2024-03-29T05:09:18.245474Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model AU","metadata":{}},{"cell_type":"code","source":"progressive_resizing(\n    fn=f\"/notebooks/hms_hbac_{arch}_AU\", \n    sizes=[256, (400,311), (320,512)], \n    method='squish', \n    batch=aug_transforms(size=192, min_scale=0.75), \n    arch=arch, \n    epochs=[3,5,8], \n    pad_mode=PadMode.Reflection)","metadata":{"execution":{"iopub.execute_input":"2024-03-29T04:18:10.937171Z","iopub.status.busy":"2024-03-29T04:18:10.936867Z","iopub.status.idle":"2024-03-29T05:08:26.451583Z","shell.execute_reply":"2024-03-29T05:08:26.450786Z","shell.execute_reply.started":"2024-03-29T04:18:10.937150Z"}},"execution_count":null,"outputs":[]}]}