{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ftfy regex tqdm","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:01:19.483371Z","iopub.execute_input":"2022-08-09T17:01:19.484822Z","iopub.status.idle":"2022-08-09T17:01:31.845572Z","shell.execute_reply.started":"2022-08-09T17:01:19.484585Z","shell.execute_reply":"2022-08-09T17:01:31.843722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install git+https://github.com/openai/CLIP.git","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:01:31.854278Z","iopub.execute_input":"2022-08-09T17:01:31.854816Z","iopub.status.idle":"2022-08-09T17:01:47.695754Z","shell.execute_reply.started":"2022-08-09T17:01:31.854758Z","shell.execute_reply":"2022-08-09T17:01:47.694072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:01:47.699520Z","iopub.execute_input":"2022-08-09T17:01:47.700020Z","iopub.status.idle":"2022-08-09T17:01:59.942337Z","shell.execute_reply.started":"2022-08-09T17:01:47.699968Z","shell.execute_reply":"2022-08-09T17:01:59.940573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torchinfo","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:01:59.947731Z","iopub.execute_input":"2022-08-09T17:01:59.948090Z","iopub.status.idle":"2022-08-09T17:02:12.849114Z","shell.execute_reply.started":"2022-08-09T17:01:59.948054Z","shell.execute_reply":"2022-08-09T17:02:12.847442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport clip\nfrom clip.clip import _download, _MODELS\nimport timm\nimport torch\nimport torch.nn as nn\nfrom torchinfo import summary\nfrom torchvision import transforms\n\nclip_code = 'ViT-L/14@336px'\n\nmodel_path = _download(_MODELS[clip_code], os.path.expanduser(\"~/.cache/clip\"))\nwith open(model_path, 'rb') as opened_file:\n    clip_vit = torch.jit.load(opened_file, map_location=\"cuda:0\").visual.eval()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:02:52.220718Z","iopub.execute_input":"2022-08-09T17:02:52.221274Z","iopub.status.idle":"2022-08-09T17:03:00.642689Z","shell.execute_reply.started":"2022-08-09T17:02:52.221226Z","shell.execute_reply":"2022-08-09T17:03:00.641120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clip.available_models()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:03:10.856600Z","iopub.execute_input":"2022-08-09T17:03:10.857193Z","iopub.status.idle":"2022-08-09T17:03:10.871255Z","shell.execute_reply.started":"2022-08-09T17:03:10.857151Z","shell.execute_reply":"2022-08-09T17:03:10.869225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torchvision import models\nfrom torchvision import transforms\nfrom torchvision.transforms.functional import InterpolationMode","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:03:11.369074Z","iopub.execute_input":"2022-08-09T17:03:11.369575Z","iopub.status.idle":"2022-08-09T17:03:11.377629Z","shell.execute_reply.started":"2022-08-09T17:03:11.369508Z","shell.execute_reply":"2022-08-09T17:03:11.376013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:03:11.380853Z","iopub.execute_input":"2022-08-09T17:03:11.381787Z","iopub.status.idle":"2022-08-09T17:03:11.389663Z","shell.execute_reply.started":"2022-08-09T17:03:11.381738Z","shell.execute_reply":"2022-08-09T17:03:11.388279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:03:11.392047Z","iopub.execute_input":"2022-08-09T17:03:11.393082Z","iopub.status.idle":"2022-08-09T17:03:11.401766Z","shell.execute_reply.started":"2022-08-09T17:03:11.393038Z","shell.execute_reply":"2022-08-09T17:03:11.400204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n#model, preprocess = clip.load('RN50x64', device, jit=True)\n\n#model, preprocess = clip.load('ViT-L/14', device, jit=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:03:11.428835Z","iopub.execute_input":"2022-08-09T17:03:11.429758Z","iopub.status.idle":"2022-08-09T17:03:11.437423Z","shell.execute_reply.started":"2022-08-09T17:03:11.429710Z","shell.execute_reply":"2022-08-09T17:03:11.435905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:03:11.441987Z","iopub.execute_input":"2022-08-09T17:03:11.442391Z","iopub.status.idle":"2022-08-09T17:03:11.593636Z","shell.execute_reply.started":"2022-08-09T17:03:11.442362Z","shell.execute_reply":"2022-08-09T17:03:11.591982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class VitModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n        #self.vit_model, _ = clip.load('RN50x64', device, jit=True)\n        self.vit_model = clip_vit\n        self.pool = nn.AdaptiveAvgPool1d(64)\n        \n    def forward(self, x):\n        x = x.to(torch.device('cuda:0'))\n        #x = transforms.functional.resize(x,size=[448, 448], interpolation=transforms.InterpolationMode.BICUBIC)\n        x = transforms.functional.resize(x,size=[336, 336], interpolation=transforms.InterpolationMode.BICUBIC)\n        x = x/255.0\n        x = transforms.functional.normalize(x,\n                                            mean=[0.48145466, 0.4578275, 0.40821073], \n                                            std=[0.26862954, 0.26130258, 0.27577711])\n\n        x = self.vit_model.encode_image(x)\n        return self.pool(x)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:03:11.595705Z","iopub.execute_input":"2022-08-09T17:03:11.596733Z","iopub.status.idle":"2022-08-09T17:03:11.608168Z","shell.execute_reply.started":"2022-08-09T17:03:11.596688Z","shell.execute_reply":"2022-08-09T17:03:11.606352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_image_model():\n    model = VitModel().to(torch.device('cuda:0'))\n    model = model.eval()\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:03:11.610390Z","iopub.execute_input":"2022-08-09T17:03:11.611764Z","iopub.status.idle":"2022-08-09T17:03:11.625618Z","shell.execute_reply.started":"2022-08-09T17:03:11.611696Z","shell.execute_reply":"2022-08-09T17:03:11.624139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model = load_image_model()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:03:11.629174Z","iopub.execute_input":"2022-08-09T17:03:11.630058Z","iopub.status.idle":"2022-08-09T17:03:11.639391Z","shell.execute_reply.started":"2022-08-09T17:03:11.630013Z","shell.execute_reply":"2022-08-09T17:03:11.637876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class myVitModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.feature_extractor = clip_vit\n        self.avg = nn.AdaptiveAvgPool1d(64)\n\n    def forward(self, x):\n        x1 = transforms.functional.resize(x, size=[336, 336]) / 255.0\n        x1 = transforms.functional.normalize(x1, mean=[0.48145466, 0.4578275, 0.40821073], std=[0.26862954, 0.26130258, 0.27577711])\n        x1 = self.avg(self.feature_extractor(x1.half()))\n\n        return x1\n\nmodel = myVitModel()\nmodel.eval()\n\nsaved_model = torch.jit.script(model)\nsaved_model.save('saved_model.pt')\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:03:11.641290Z","iopub.execute_input":"2022-08-09T17:03:11.642837Z","iopub.status.idle":"2022-08-09T17:03:14.774814Z","shell.execute_reply.started":"2022-08-09T17:03:11.642779Z","shell.execute_reply":"2022-08-09T17:03:14.772624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_input_size = (1, 3, 480, 640)  # the model should work with any input_size\nsummary(model, input_size=test_input_size)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:06:03.956919Z","iopub.execute_input":"2022-08-09T17:06:03.957351Z","iopub.status.idle":"2022-08-09T17:06:19.337028Z","shell.execute_reply.started":"2022-08-09T17:06:03.957317Z","shell.execute_reply":"2022-08-09T17:06:19.335350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_input_size = (1, 3, 4545, 2324)  # the model should work with any input_size\nsummary(model, input_size=test_input_size)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:08:29.383317Z","iopub.execute_input":"2022-08-09T17:08:29.383790Z","iopub.status.idle":"2022-08-09T17:08:45.013244Z","shell.execute_reply.started":"2022-08-09T17:08:29.383756Z","shell.execute_reply":"2022-08-09T17:08:45.011652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from zipfile import ZipFile\n\nwith ZipFile('submission.zip','w') as zip:\n    zip.write('saved_model.pt', arcname='saved_model.pt')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T17:08:54.233008Z","iopub.execute_input":"2022-08-09T17:08:54.233470Z","iopub.status.idle":"2022-08-09T17:08:57.582624Z","shell.execute_reply.started":"2022-08-09T17:08:54.233435Z","shell.execute_reply":"2022-08-09T17:08:57.581229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}