{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **ConvNeXt-xlarge with Gem Pooling**\n\npreviously tried `convnext_base_in22ft1k` which scored 0.360\nhence now trying out xlarge","metadata":{}},{"cell_type":"code","source":"!pip install timm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-13T19:57:05.097530Z","iopub.execute_input":"2022-08-13T19:57:05.097941Z","iopub.status.idle":"2022-08-13T19:57:20.667148Z","shell.execute_reply.started":"2022-08-13T19:57:05.097910Z","shell.execute_reply":"2022-08-13T19:57:20.666077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torchvision.transforms import functional\nimport torch.nn.functional as F\nfrom typing import Optional,cast\nfrom collections import OrderedDict","metadata":{"execution":{"iopub.status.busy":"2022-08-13T19:49:32.891133Z","iopub.execute_input":"2022-08-13T19:49:32.892092Z","iopub.status.idle":"2022-08-13T19:49:35.213957Z","shell.execute_reply.started":"2022-08-13T19:49:32.891982Z","shell.execute_reply":"2022-08-13T19:49:35.212891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from timm import create_model, list_models","metadata":{"execution":{"iopub.status.busy":"2022-08-13T19:57:20.669275Z","iopub.execute_input":"2022-08-13T19:57:20.669679Z","iopub.status.idle":"2022-08-13T19:57:22.025729Z","shell.execute_reply.started":"2022-08-13T19:57:20.669642Z","shell.execute_reply":"2022-08-13T19:57:22.024169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## All ConvNeXt models","metadata":{}},{"cell_type":"code","source":"list_models('convnext*',pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T08:23:08.764041Z","iopub.execute_input":"2022-08-10T08:23:08.764424Z","iopub.status.idle":"2022-08-10T08:23:08.777848Z","shell.execute_reply.started":"2022-08-10T08:23:08.764390Z","shell.execute_reply":"2022-08-10T08:23:08.776661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### from [Summary of some pretrained models](https://www.kaggle.com/competitions/google-universal-image-embedding/discussion/340043):\n`convnext_xlarge_in22k` scored 0.385\nlet's check if replacing the usual final embed layer `AdaptiveAvgPool1d` with `GeM1d` will improve the score or not","metadata":{}},{"cell_type":"markdown","source":"## GemPool: [https://amaarora.github.io/2020/08/30/gempool.html](https://amaarora.github.io/2020/08/30/gempool.html)","metadata":{}},{"cell_type":"code","source":"class GeM1d(nn.Module):\n    def __init__(self, p=3, eps=1e-6, size=64):\n        super(GeM1d, self).__init__()\n        self.p = nn.Parameter(torch.ones(1)*p)\n        self.eps = eps\n        self.size = 64\n\n    def forward(self, x):\n        return self.gem(x, p=self.p, eps=self.eps)\n        \n    def gem(self, x, p: nn.Parameter, eps: float):\n        return F.adaptive_avg_pool1d(x.clamp(min=eps).pow(p), (self.size)).pow(1./p)\n        \n    def __repr__(self):\n        return self.__class__.__name__ + \\\n                '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + \\\n                ', ' + 'eps=' + str(self.eps) + ')'","metadata":{"execution":{"iopub.status.busy":"2022-08-13T20:00:16.914931Z","iopub.execute_input":"2022-08-13T20:00:16.915382Z","iopub.status.idle":"2022-08-13T20:00:16.925262Z","shell.execute_reply.started":"2022-08-13T20:00:16.915344Z","shell.execute_reply":"2022-08-13T20:00:16.923910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ModelGeM(nn.Module):\n    def __init__(self, model_name='convnext_xlarge_in22k', size=224):\n        super(ModelGeM, self).__init__()\n        self.model_name = model_name\n        self.input_size = (size,size)\n        \n        self.feature_extractor = create_model(self.model_name, \n                                              pretrained=True, \n                                              num_classes = 0\n                                             )\n        self.feature_extractor.eval()\n        \n        self.flatten = nn.Flatten()\n#         self.embed = nn.AdaptiveAvgPool1d(64)\n        self.embed = GeM1d()\n        \n        \n    def forward(self,x):\n        x = functional.resize(x, self.input_size)\n        x = x/255.0\n        x = functional.normalize(x,\n            mean = [0.485, 0.456, 0.406], \n            std = [0.229, 0.224, 0.225]\n        )\n        x = self.feature_extractor(x)\n        x = self.embed(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-08-13T20:05:20.267337Z","iopub.execute_input":"2022-08-13T20:05:20.267883Z","iopub.status.idle":"2022-08-13T20:05:20.279918Z","shell.execute_reply.started":"2022-08-13T20:05:20.267837Z","shell.execute_reply":"2022-08-13T20:05:20.278888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = torch.rand(1,3,224,224)\ngemm = ModelGeM()\ngemm(x).shape","metadata":{"execution":{"iopub.status.busy":"2022-08-13T20:05:23.207006Z","iopub.execute_input":"2022-08-13T20:05:23.208415Z","iopub.status.idle":"2022-08-13T20:05:25.509188Z","shell.execute_reply.started":"2022-08-13T20:05:23.208354Z","shell.execute_reply":"2022-08-13T20:05:25.507896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_model = torch.jit.script(ModelGeM())","metadata":{"execution":{"iopub.status.busy":"2022-08-10T08:24:51.968566Z","iopub.execute_input":"2022-08-10T08:24:51.969286Z","iopub.status.idle":"2022-08-10T08:25:01.350400Z","shell.execute_reply.started":"2022-08-10T08:24:51.969249Z","shell.execute_reply":"2022-08-10T08:25:01.348892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_model.save('saved_model.pt')","metadata":{"execution":{"iopub.status.busy":"2022-08-10T08:25:01.352274Z","iopub.execute_input":"2022-08-10T08:25:01.353142Z","iopub.status.idle":"2022-08-10T08:25:03.222055Z","shell.execute_reply.started":"2022-08-10T08:25:01.353101Z","shell.execute_reply":"2022-08-10T08:25:03.220820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from zipfile import ZipFile\n\nwith ZipFile('submission.zip','w') as zip:           \n    zip.write('saved_model.pt', arcname='saved_model.pt')","metadata":{"execution":{"iopub.status.busy":"2022-08-10T08:25:03.227110Z","iopub.execute_input":"2022-08-10T08:25:03.227903Z","iopub.status.idle":"2022-08-10T08:25:08.097573Z","shell.execute_reply.started":"2022-08-10T08:25:03.227846Z","shell.execute_reply":"2022-08-10T08:25:08.096610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T08:25:08.098995Z","iopub.execute_input":"2022-08-10T08:25:08.099916Z","iopub.status.idle":"2022-08-10T08:25:08.380496Z","shell.execute_reply.started":"2022-08-10T08:25:08.099881Z","shell.execute_reply":"2022-08-10T08:25:08.379634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference\nto check if saved model is working as expected -- had issues when I replaced the head in the timm model","metadata":{}},{"cell_type":"code","source":"model = torch.jit.load('saved_model.pt')","metadata":{"execution":{"iopub.status.busy":"2022-08-10T08:25:08.383417Z","iopub.execute_input":"2022-08-10T08:25:08.383881Z","iopub.status.idle":"2022-08-10T08:25:11.980688Z","shell.execute_reply.started":"2022-08-10T08:25:08.383846Z","shell.execute_reply":"2022-08-10T08:25:11.979754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = torch.rand(1,3,224,224)\nmodel(x).shape","metadata":{"execution":{"iopub.status.busy":"2022-08-10T08:25:11.982597Z","iopub.execute_input":"2022-08-10T08:25:11.983496Z","iopub.status.idle":"2022-08-10T08:25:15.770322Z","shell.execute_reply.started":"2022-08-10T08:25:11.983427Z","shell.execute_reply":"2022-08-10T08:25:15.768819Z"},"trusted":true},"execution_count":null,"outputs":[]}]}