{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm pyarrow","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:39:11.350757Z","iopub.execute_input":"2024-03-12T06:39:11.351736Z","iopub.status.idle":"2024-03-12T06:39:19.236328Z","shell.execute_reply.started":"2024-03-12T06:39:11.351696Z","shell.execute_reply":"2024-03-12T06:39:19.235258Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim import lr_scheduler\nimport torch.optim as optim\nimport timm\nimport torchvision\nfrom torchvision import datasets, transforms\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nfrom torchvision.transforms import InterpolationMode\n# importing tensorfow to read .tfrec files\nimport matplotlib.pyplot as plt\nimport numpy as np # linear algebra\nimport pandas as pd # data p|rocessing, CSV file I/O (e.g. pd.read_csv)\nimport glob\n\nimport io","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:47:42.326106Z","iopub.execute_input":"2024-03-12T06:47:42.326495Z","iopub.status.idle":"2024-03-12T06:47:43.504033Z","shell.execute_reply.started":"2024-03-12T06:47:42.326465Z","shell.execute_reply":"2024-03-12T06:47:43.503161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## I was facing the trouble with configuring TPU and Pytorch XLA, my notebokk felt bad about using Pytorch XLA and i dunno why. IMO its all about using tensorflow and Pytorch XLA in one run. Kind of incompatibility. So i Solved this by: \n* I preprocessed .tfrec files with tf.\n* Create dataloaders and save them with torch.save\n* Re import all libs and moduls without tf.\n### I hope, it would be usefull to someone :)\n\n","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:39:38.243064Z","iopub.execute_input":"2024-03-12T06:39:38.243424Z","iopub.status.idle":"2024-03-12T06:39:49.557606Z","shell.execute_reply.started":"2024-03-12T06:39:38.243394Z","shell.execute_reply":"2024-03-12T06:39:49.556500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = glob.glob('/kaggle/input/tpu-getting-started/*/train/*.tfrec')\nval_files = glob.glob('/kaggle/input/tpu-getting-started/*/val/*.tfrec')\ntest_files = glob.glob('/kaggle/input/tpu-getting-started/*/test/*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:39:49.559870Z","iopub.execute_input":"2024-03-12T06:39:49.560589Z","iopub.status.idle":"2024-03-12T06:39:49.672336Z","shell.execute_reply.started":"2024-03-12T06:39:49.560541Z","shell.execute_reply":"2024-03-12T06:39:49.671428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cheking how the xla is working\nimport torch\nt = torch.randn(2, 2, device=xm.xla_device())\nprint(t.device)\nprint(t)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:47:48.703751Z","iopub.execute_input":"2024-03-12T06:47:48.704326Z","iopub.status.idle":"2024-03-12T06:48:01.420749Z","shell.execute_reply.started":"2024-03-12T06:47:48.704289Z","shell.execute_reply":"2024-03-12T06:48:01.419692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_feature_description = {\n    'class': tf.io.FixedLenFeature([], tf.int64),\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\n\ndef _parse_image_function(example_proto):\n    # Parse the input tf.Example proto using the dictionary above.\n    return tf.io.parse_single_example(example_proto, train_feature_description)\n\ntrain_ids = []\ntrain_class = []\ntrain_images = []\n\nfor i in train_files:\n    train_image_dataset = tf.data.TFRecordDataset(i)\n\n    train_image_dataset = train_image_dataset.map(_parse_image_function)\n\n    ids = [str(id_features['id'].numpy())[2:-1] for id_features in train_image_dataset] # [2:-1] is done to remove b' from 1st and 'from last in train id names\n    train_ids = train_ids + ids\n\n    classes = [int(class_features['class'].numpy()) for class_features in train_image_dataset]\n    train_class = train_class + classes\n\n    images = [image_features['image'].numpy() for image_features in train_image_dataset]\n    train_images = train_images + images","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:41:14.126203Z","iopub.execute_input":"2024-03-12T06:41:14.127477Z","iopub.status.idle":"2024-03-12T06:41:47.076481Z","shell.execute_reply.started":"2024-03-12T06:41:14.127429Z","shell.execute_reply":"2024-03-12T06:41:47.075436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_feature_description = {\n    'class': tf.io.FixedLenFeature([], tf.int64),\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\n\ndef _parse_image_function(example_proto):\n  # Parse the input tf.Example proto using the dictionary above.\n  return tf.io.parse_single_example(example_proto, val_feature_description)\n\nval_ids = []\nval_class = []\nval_images = []\n\nfor i in val_files:\n    val_image_dataset = tf.data.TFRecordDataset(i)\n\n    val_image_dataset = val_image_dataset.map(_parse_image_function)\n\n    ids = [str(image_features['id'].numpy())[2:-1] for image_features in val_image_dataset]\n    val_ids += ids\n\n    classes = [int(image_features['class'].numpy()) for image_features in val_image_dataset]\n    val_class += classes \n\n    images = [image_features['image'].numpy() for image_features in val_image_dataset]\n    val_images += images","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:41:47.078179Z","iopub.execute_input":"2024-03-12T06:41:47.078579Z","iopub.status.idle":"2024-03-12T06:42:12.199487Z","shell.execute_reply.started":"2024-03-12T06:41:47.078547Z","shell.execute_reply":"2024-03-12T06:42:12.198466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_feature_description = {\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\n\ndef _parse_image_function_test(example_proto):\n    return tf.io.parse_single_example(example_proto, test_feature_description)\n\ntest_ids = []\ntest_images = []\nfor i in test_files:\n    test_image_dataset = tf.data.TFRecordDataset(i)\n    \n    test_image_dataset = test_image_dataset.map(_parse_image_function_test)\n\n    ids = [str(id_features['id'].numpy())[2:-1] for id_features in test_image_dataset]\n    test_ids = test_ids + ids\n\n    images = [image_features['image'].numpy() for image_features in test_image_dataset]\n    test_images = test_images + images","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:42:12.200633Z","iopub.execute_input":"2024-03-12T06:42:12.200925Z","iopub.status.idle":"2024-03-12T06:42:45.278552Z","shell.execute_reply.started":"2024-03-12T06:42:12.200895Z","shell.execute_reply":"2024-03-12T06:42:45.277550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestPetalDataset(Dataset):\n    def __init__(self, ids, images, transform):\n        self.ids = ids\n        self.images = images\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.ids)\n    \n    def __getitem__(self, idx):\n        image = self.images[idx]\n        image = Image.open(io.BytesIO(image))\n        trans_image = self.transform(image)\n        id_s = self.ids[idx]\n        return trans_image, id_s","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:48:05.595659Z","iopub.execute_input":"2024-03-12T06:48:05.596056Z","iopub.status.idle":"2024-03-12T06:48:16.059823Z","shell.execute_reply.started":"2024-03-12T06:48:05.596025Z","shell.execute_reply":"2024-03-12T06:48:16.058749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### If you are creating transform with dictionary, like in the example below, you cant change transform in the direct way in your dataloader in the future. So i created one more template dataset","metadata":{}},{"cell_type":"code","source":"class PetalDataset(Dataset):\n    def __init__(self, targets, ids, images, transform, valid=True):\n        self.targets = targets\n        self.ids = ids\n        self.images = images\n        self.transform = transform\n        self.valid = valid\n        transforms\n    def __len__(self):\n        return len(self.ids)\n    \n    def __getitem__(self, idx):\n        image = self.images[idx]\n        image = Image.open(io.BytesIO(image))\n        if self.valid:\n            trans_image = self.transform['valid'](image)\n        else:\n            trans_image = self.transform['train'](image)\n        target = self.targets[idx]\n        id_s = self.ids[idx]\n        \n        return int(target), id_s, trans_image","metadata":{"execution":{"iopub.status.busy":"2024-03-11T10:24:00.525500Z","iopub.execute_input":"2024-03-11T10:24:00.525904Z","iopub.status.idle":"2024-03-11T10:24:00.725155Z","shell.execute_reply.started":"2024-03-11T10:24:00.525859Z","shell.execute_reply":"2024-03-11T10:24:00.723961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PetalDatasetTemplate(Dataset):\n    def __init__(self, targets, ids, images, transform):\n        self.targets = targets\n        self.ids = ids\n        self.images = images\n        self.transform = transform\n        transforms\n    def __len__(self):\n        return len(self.ids)\n    \n    def __getitem__(self, idx):\n        image = self.images[idx]\n        image = Image.open(io.BytesIO(image))\n        trans_image = self.transform(image)\n        target = self.targets[idx]\n        id_s = self.ids[idx]\n        \n        return int(target), id_s, trans_image","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:48:23.002249Z","iopub.execute_input":"2024-03-12T06:48:23.003315Z","iopub.status.idle":"2024-03-12T06:48:23.008633Z","shell.execute_reply.started":"2024-03-12T06:48:23.003274Z","shell.execute_reply":"2024-03-12T06:48:23.007777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_re.default_cfg","metadata":{"execution":{"iopub.status.busy":"2024-03-09T08:17:12.983640Z","iopub.execute_input":"2024-03-09T08:17:12.984075Z","iopub.status.idle":"2024-03-09T08:17:12.989854Z","shell.execute_reply.started":"2024-03-09T08:17:12.984035Z","shell.execute_reply":"2024-03-09T08:17:12.989115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### All models in timm have prefereble transforms. You can check it in the defauld_cfg method. But be careful with input size.","metadata":{}},{"cell_type":"code","source":"# # vit32 and vit16 are the same\n# vit_32_transform = transforms.Compose([\n#     transforms.Resize(size=(224, 224), interpolation=InterpolationMode.BICUBIC),\n#     transforms.CenterCrop(size = int(224 * 0.9)),\n#     transforms.ToTensor(),\n#     transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))\n# ])\n# vit32_dict_transforms = {\n#     'valid': vit_32_transform,\n#     'train': vit_32_transform\n# }\n# convnext transform\nconvnext_transform = transforms.Compose([\n    transforms.Resize((224/0.875), interpolation=InterpolationMode.BICUBIC),\n    transforms.CenterCrop(size = int(224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))\n])\nconvnext_dict_transforms = {\n    'valid': convnext_transform,\n    'train': convnext_transform\n}\n# # levit256 transform\n# levit256_transform = transforms.Compose([\n#     transforms.Resize(size=(224, 224), interpolation=InterpolationMode.BICUBIC),\n#     transforms.CenterCrop(size = int(224 * 0.9)),\n#     transforms.ToTensor(),\n#     transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))\n# ])\n# levit256_dict_transforms = {\n#     'valid': levit256_transform,\n#     'train': levit256_transform\n# }\n# # levit256 transform\n# resnet26d_train_transform = transforms.Compose([\n#     transforms.Resize(size=(224, 224), interpolation=InterpolationMode.BICUBIC),\n#     transforms.CenterCrop(size = int(224 * 0.875)),\n#     transforms.ToTensor(),\n#     transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))\n# ])\n# resnet26d_valid_transform = transforms.Compose([\n#     transforms.Resize(size=(288, 288), interpolation=InterpolationMode.BICUBIC),\n#     transforms.CenterCrop(size = int(224 * 0.95)),\n#     transforms.ToTensor(),  \n#     transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))\n# ])\n# resnet26d_dict_transforms = {\n#     'valid': resnet26d_valid_transform,\n#     'train': resnet26d_train_transform\n# }","metadata":{"execution":{"iopub.status.busy":"2024-03-11T10:19:26.159590Z","iopub.execute_input":"2024-03-11T10:19:26.159973Z","iopub.status.idle":"2024-03-11T10:19:26.166372Z","shell.execute_reply.started":"2024-03-11T10:19:26.159942Z","shell.execute_reply":"2024-03-11T10:19:26.165654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_transform = transforms.Compose([\n    transforms.RandomResizedCrop(size=(224, 224), antialias=True),\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])\n])\n\ntrain_transform = transforms.Compose([\n    transforms.Resize(224),\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\ndict_transforms = {\n    'valid': valid_transform,\n    'train': train_transform\n}\n\ntest_transform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:42:45.299133Z","iopub.execute_input":"2024-03-12T06:42:45.299385Z","iopub.status.idle":"2024-03-12T06:42:45.307340Z","shell.execute_reply.started":"2024-03-12T06:42:45.299358Z","shell.execute_reply":"2024-03-12T06:42:45.306647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = PetalDatasetTemplate(targets=train_class, \n    ids=train_ids, \n    images=train_images, \n    transform=train_transform\n)\nval_dataset = PetalDatasetTemplate(targets=val_class,\n    ids=val_ids, \n    images=val_images, \n    transform=valid_transform\n)\ntest_dataset = TestPetalDataset(ids=test_ids, images=test_images, transform=test_transform)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:42:45.308258Z","iopub.execute_input":"2024-03-12T06:42:45.308542Z","iopub.status.idle":"2024-03-12T06:42:45.318313Z","shell.execute_reply.started":"2024-03-12T06:42:45.308508Z","shell.execute_reply":"2024-03-12T06:42:45.317664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loaders = DataLoader(train_dataset, batch_size=128, num_workers=8, shuffle=True, pin_memory=True)\nval_loaders = DataLoader(val_dataset, batch_size=128, num_workers=8, shuffle=True, pin_memory=True)\ntest_loaders = DataLoader(test_dataset, batch_size=128, num_workers=4, shuffle=True, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:42:45.319205Z","iopub.execute_input":"2024-03-12T06:42:45.319438Z","iopub.status.idle":"2024-03-12T06:42:45.327009Z","shell.execute_reply.started":"2024-03-12T06:42:45.319414Z","shell.execute_reply":"2024-03-12T06:42:45.326418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(train_loaders, 'trainloader.pth')\ntorch.save(val_loaders, 'validloader.pth')\ntorch.save(test_loaders, 'testloader.pth')","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:42:45.327846Z","iopub.execute_input":"2024-03-12T06:42:45.328094Z","iopub.status.idle":"2024-03-12T06:44:32.693518Z","shell.execute_reply.started":"2024-03-12T06:42:45.328068Z","shell.execute_reply":"2024-03-12T06:44:32.692546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### And after this moment, you are Free of tensorflow influence. Congratulations!\n### IMPORTANT NOTE: execute torch DataSets classes after your xla import, otherwise you cant load your loaders :) \nYou can go and import xla now","metadata":{}},{"cell_type":"code","source":"import torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:47:32.562981Z","iopub.execute_input":"2024-03-12T06:47:32.563483Z","iopub.status.idle":"2024-03-12T06:47:38.815082Z","shell.execute_reply.started":"2024-03-12T06:47:32.563450Z","shell.execute_reply":"2024-03-12T06:47:38.814173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = torch.load('trainloader.pth')\nvalid_loader = torch.load('validloader.pth')\ntest_loader = torch.load('testloader.pth')","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:48:32.260176Z","iopub.execute_input":"2024-03-12T06:48:32.260946Z","iopub.status.idle":"2024-03-12T06:49:29.762758Z","shell.execute_reply.started":"2024-03-12T06:48:32.260906Z","shell.execute_reply":"2024-03-12T06:49:29.761754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x, y, z = next(iter(train_loader))\nx1, y1, z1 = next(iter(valid_loader))","metadata":{"execution":{"iopub.status.busy":"2024-03-08T08:08:47.255048Z","iopub.execute_input":"2024-03-08T08:08:47.255409Z","iopub.status.idle":"2024-03-08T08:08:47.278260Z","shell.execute_reply.started":"2024-03-08T08:08:47.255376Z","shell.execute_reply":"2024-03-08T08:08:47.277390Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### You even, can change your transform, if you want to.","metadata":{}},{"cell_type":"code","source":"train_loader.dataset.transform = train_transform\nvalid_loader.dataset.transform = valid_transform","metadata":{"execution":{"iopub.status.busy":"2024-03-11T10:39:57.687738Z","iopub.execute_input":"2024-03-11T10:39:57.688252Z","iopub.status.idle":"2024-03-11T10:39:57.693098Z","shell.execute_reply.started":"2024-03-11T10:39:57.688186Z","shell.execute_reply":"2024-03-11T10:39:57.692054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Lets take a look on the vision models.","metadata":{}},{"cell_type":"markdown","source":"### I take the info from course of Jeremy Howard, you can take a look, its really interesting :)\n\nhttps://www.kaggle.com/code/jhoward/the-best-vision-models-for-fine-tuning","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nurl = 'https://gist.githubusercontent.com/jph00/959aaf8695e723246b5e21f3cd5deb02/raw/sweep.csv'\nvision_df = pd.read_csv(url)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vision_df['family'] = vision_df.model_name.str.extract('^([a-z]+?(?:v2)?)(?:\\d|_|$)')\nvision_df.loc[vision_df.family=='swinv2', 'family'] = 'swin'\npt_all = vision_df.pivot_table(values=['error_rate','fit_time','GPU_mem'], index=['dataset', 'family', 'model_name'],\n                        aggfunc=np.min).reset_index()\npt_all['score'] = pt_all.error_rate*(pt_all.fit_time+80)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pt_all","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pt = pt_all.sort_values('score').reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Here we can see model with minimal gpu usage and small fit time\n### (the fastest and smallest model with great results)","metadata":{}},{"cell_type":"code","source":"pt_all.query(\"(GPU_mem < 2.7) & (fit_time < 25)\").sort_values('error_rate').head(10).reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## *we should separate pets dataset and planet, cuz results are quite different.*","metadata":{}},{"cell_type":"markdown","source":"## About average models based on pets dataset","metadata":{}},{"cell_type":"code","source":"pt_all[pt_all.dataset=='pets'].query(\"(GPU_mem < 2.7) & (fit_time < 100)\").sort_values('error_rate').head(10).reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pt_all[pt_all.dataset=='pets'].query(\"(GPU_mem<1.6) & (fit_time<85)\").sort_values(\"error_rate\").head(15).reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### To be honest, I'm a bit disappointed that well-known models like EfficientNet and ResNet are no longer as \"top-tier.\", I guess it is what it is","metadata":{}},{"cell_type":"markdown","source":"### You can tweak this options, to choose what you need, depends on your system and goals.\nAnd its funny, also..","metadata":{}},{"cell_type":"code","source":"# Models i chose from vision dataframe.\nmodels = {\n    'model_vit_small_patch32_224': 'vit_small_patch32_224',\n    'model_vit_small_patch16_224': 'vit_small_patch16_224',\n    'model_convnext_tiny_in22k': 'convnext_tiny_in22k',\n    'model_levit_256': 'levit_256',\n    'model_resnet26d': 'resnet26d'\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avail_pretrained_models = timm.list_models(pretrained=True)\navail_pretrained_models[:10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1 = timm.create_model('vit_small_patch32_224', pretrained=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1.default_cfg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We can check under the hood of the model\nBy defalut, there are input block, main blocks, and fc layers","metadata":{}},{"cell_type":"code","source":"model1.named_modules","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Also, you can check models name by name of its family \n( vit_small_patch32_224 - vit )","metadata":{}},{"cell_type":"code","source":"vit_models = timm.list_models('*vit*')\nlen(vit_models), vit_models[:10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_levit256 = timm.create_model('levit_256', pretrained=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_levit256.default_cfg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_resnet26d = timm.create_model('resnet26d', pretrained=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm","metadata":{"execution":{"iopub.status.busy":"2024-03-09T10:01:50.171797Z","iopub.execute_input":"2024-03-09T10:01:50.172634Z","iopub.status.idle":"2024-03-09T10:01:50.176293Z","shell.execute_reply.started":"2024-03-09T10:01:50.172597Z","shell.execute_reply":"2024-03-09T10:01:50.175515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We can compare configurations of those models","metadata":{}},{"cell_type":"markdown","source":"We have 104 labels, so we should change last fc layer(head), specifically - out_features to 104\nBut here is the trick, our models have different classifier name, so do it accurately","metadata":{}},{"cell_type":"code","source":"# Creating pretrained models with predefined number of classes (104)\nmodel_vit_small_patch32_224 = timm.create_model('vit_base_patch32_224', pretrained=True, num_classes=104)\n# model_vit_small_patch16_224 = timm.create_model('vit_small_patch16_224', pretrained=True, num_classes=104)\n# model_convnext_tiny_in22k_2 = timm.create_model('convnext_tiny.fb_in22k', pretrained=True, num_classes=104)\n# model_levit_256 = timm.create_model('levit_256', pretrained=True, num_classes=104)\n# model_resnet26d = timm.create_model('resnet26d', pretrained=True, num_classes=104)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:50:00.511590Z","iopub.execute_input":"2024-03-12T06:50:00.512079Z","iopub.status.idle":"2024-03-12T06:50:09.863952Z","shell.execute_reply.started":"2024-03-12T06:50:00.512044Z","shell.execute_reply":"2024-03-12T06:50:09.863125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check number of out features\nprint(\n    model_vit_small_patch32_224.head.out_features,\n    model_vit_small_patch16_224.head.out_features,\n    model_convnext_tiny_in22k.head.fc.out_features,\n    model_levit_256.head.linear.out_features,\n    model_resnet26d.fc.out_features\n)\n","metadata":{"execution":{"iopub.status.busy":"2024-03-09T07:43:43.232447Z","iopub.execute_input":"2024-03-09T07:43:43.232777Z","iopub.status.idle":"2024-03-09T07:43:43.237321Z","shell.execute_reply.started":"2024-03-09T07:43:43.232750Z","shell.execute_reply":"2024-03-09T07:43:43.236681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_vit_small_patch32_224.default_cfg","metadata":{"execution":{"iopub.status.busy":"2024-03-11T12:33:16.935209Z","iopub.execute_input":"2024-03-11T12:33:16.935623Z","iopub.status.idle":"2024-03-11T12:33:16.942967Z","shell.execute_reply.started":"2024-03-11T12:33:16.935590Z","shell.execute_reply":"2024-03-11T12:33:16.942028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [\n    model_vit_small_patch32_224,\n    model_vit_small_patch16_224,\n    model_convnext_tiny_in22k, \n    model_levit_256, \n    model_resnet26d\n]","metadata":{"execution":{"iopub.status.busy":"2024-03-09T07:00:01.943690Z","iopub.execute_input":"2024-03-09T07:00:01.943954Z","iopub.status.idle":"2024-03-09T07:02:58.422709Z","shell.execute_reply.started":"2024-03-09T07:00:01.943926Z","shell.execute_reply":"2024-03-09T07:02:58.421765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### If you are using gpu, try this cell","metadata":{}},{"cell_type":"code","source":"!pip install GPUtil\nfrom GPUtil import showUtilization as gpu_usage\nfrom numba import cuda\n\ndef free_gpu_cache():\n    print(\"Initial GPU Usage\")\n    gpu_usage()                             \n    \n    torch.cuda.empty_cache()\n\n    cuda.select_device(0)\n    cuda.close()\n    cuda.select_device(0)\n\n    print(\"GPU Usage after emptying the cache\")\n    gpu_usage()\n\nfree_gpu_cache()","metadata":{"execution":{"iopub.status.busy":"2024-03-09T07:26:31.040689Z","iopub.execute_input":"2024-03-09T07:26:31.041095Z","iopub.status.idle":"2024-03-09T07:27:43.807336Z","shell.execute_reply.started":"2024-03-09T07:26:31.041061Z","shell.execute_reply":"2024-03-09T07:27:43.806026Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from timm.optim.optim_factory import create_optimizer\nfrom timm import create_model \nimport torch\nimport torch.optim as optim \nimport timm\nfrom timm.data import create_dataset, create_loader\nimport numpy as np \nfrom matplotlib import pyplot as plt\nimport torchvision\nimport torch.nn as nn\nfrom tqdm import tqdm\nimport logging \nfrom timm.optim import optim_factory\nlogging.getLogger().setLevel(logging.INFO)\nlosses_dict = {}","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:50:18.523496Z","iopub.execute_input":"2024-03-12T06:50:18.523876Z","iopub.status.idle":"2024-03-12T06:50:18.536290Z","shell.execute_reply.started":"2024-03-12T06:50:18.523844Z","shell.execute_reply":"2024-03-12T06:50:18.535550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_fn = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:50:19.660814Z","iopub.execute_input":"2024-03-12T06:50:19.661390Z","iopub.status.idle":"2024-03-12T06:50:19.665687Z","shell.execute_reply.started":"2024-03-12T06:50:19.661358Z","shell.execute_reply":"2024-03-12T06:50:19.664716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AverageMeter:\n    \"\"\"\n    Computes and stores the average and current value\n    \"\"\"\n\n    def __init__(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:50:19.872441Z","iopub.execute_input":"2024-03-12T06:50:19.872715Z","iopub.status.idle":"2024-03-12T06:50:19.878155Z","shell.execute_reply.started":"2024-03-12T06:50:19.872687Z","shell.execute_reply":"2024-03-12T06:50:19.877205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from types import SimpleNamespace\nfrom timm.optim.optim_factory import create_optimizer\nlosses_dict = {}\nargs = SimpleNamespace()\nargs.weight_decay = 0 \nargs.lr = 1e-4\nargs.momentum = 0.9","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:50:22.019708Z","iopub.execute_input":"2024-03-12T06:50:22.020544Z","iopub.status.idle":"2024-03-12T06:50:22.024904Z","shell.execute_reply.started":"2024-03-12T06:50:22.020511Z","shell.execute_reply":"2024-03-12T06:50:22.024116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(args, loader, model, loss_fn = nn.CrossEntropyLoss(), num_epochs=6, **optim_kwargs):\n    logging.info(f\"\\nCreated training model: {model.__class__.__name__}\")\n    \n    train_accuracy = []\n    epoch_losses = []\n    # Try Learning rate (AdamW): 0.008 and 0.02\n    # optimizer = optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0)\n    # optimizer = optim.Adam(model.parameters(), lr=1e-4, weight_decay=0)\n    # optimizer = optim.NAdam(model.parameters(), lr=1e-4, weight_decay=0)\n    # optimizer = optim.Adam(model.parameters(), lr=1e-4, weight_decay=0)\n#     optimizer = optim.SGD(model.parameters(), lr=1e-4, momentum=0.9, weight_decay=0)\n#     scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\n    optimizer = optim_factory.create_optimizer(args, model, **optim_kwargs)\n    logging.info(f\"created optimizer: {optimizer.__class__.__name__}\")\n    for epoch in range(num_epochs):\n        logging.info(f\"\\nTraining Epoch {epoch + 1}/{num_epochs}\")\n        \n        total_correct = 0\n        total_samples = 0\n        losses = []\n        loss_avg = AverageMeter()\n        \n        device = xm.xla_device()\n        WRAPPED_MODEL = xmp.MpModelWrapper(model)\n        model = WRAPPED_MODEL.to(device)\n        #model.to(device)\n        \n#         model = model.cuda()\n        mp_device_loader = pl.MpDeviceLoader(loader, device)\n        tk0 = tqdm(enumerate(mp_device_loader), total=len(mp_device_loader))\n        for i, (classes, _, inputs) in tk0:\n            inputs = inputs.to(device)\n            classes = classes.to(device)\n#             inputs = inputs.cuda()\n#             classes = classes.cuda()\n            preds = model(inputs)\n            loss = loss_fn(preds, classes)\n            loss.backward()\n#             xm.optimizer_step(optimizer)\n#             xm.mark_step()\n            optimizer.step()\n            optimizer.zero_grad()\n            loss_avg.update(loss.item(), loader.batch_size)\n            losses.append(loss_avg.avg)\n            tk0.set_postfix(loss=loss.item())\n            \n            # Compute accuracy\n            _, predicted = torch.max(preds, 1)\n            total_correct += (predicted == classes).sum().item()\n            total_samples += classes.size(0)\n            \n#         scheduler.step()\n        \n        epoch_accuracy = total_correct / total_samples\n        logging.info(f\"Epoch {epoch + 1} Accuracy: {epoch_accuracy}\")\n        epoch_losses.append(losses)\n        train_accuracy.append(epoch_accuracy)\n#         print(\"Epoch: \", epoch+1, \", accuracy is \", epoch_accuracy)\n    \n    return epoch_losses, train_accuracy","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:50:29.994479Z","iopub.execute_input":"2024-03-12T06:50:29.995132Z","iopub.status.idle":"2024-03-12T06:50:30.004750Z","shell.execute_reply.started":"2024-03-12T06:50:29.995098Z","shell.execute_reply":"2024-03-12T06:50:30.003991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate(model, loader, num_epochs=1, loss_fn=nn.CrossEntropyLoss()):\n    \"\"\"\n    This function validates the performance of the model on validation data for multiple epochs.\n    \"\"\"\n    logging.info(f\"\\nCreated validating model: {model.__class__.__name__}\")\n    \n    \n    epoch_losses = []  # Создаем список для хранения потерь по эпохам\n    valid_accuracy = []  # Создаем список для хранения точности по эпохам\n    \n    for epoch in range(num_epochs):\n        logging.info(f\"\\nValidating Epoch {epoch + 1}/{num_epochs}\")\n        \n        total_correct = 0\n        total_samples = 0\n        losses = []\n        loss_avg = AverageMeter()\n\n#         model = model.cuda()\n        \n        device = xm.xla_device()\n        WRAPPED_MODEL = xmp.MpModelWrapper(model)\n        model = WRAPPED_MODEL.to(device)\n       # model.to(device)\n        model.eval()\n#         print(f'Validating for epoch {epoch + 1}...')\n        mp_device_loader = pl.MpDeviceLoader(loader, device)\n        with torch.no_grad():\n            tk0 = tqdm(enumerate(mp_device_loader), total=len(mp_device_loader))\n            for i, (classes, _, inputs) in tk0:\n                #inputs, classes = inputs.cuda(), classes.cuda()\n                inputs, classes = inputs.to(device), classes.to(device)\n                preds = model(inputs)\n                loss = loss_fn(preds, classes)\n                loss_avg.update(loss.item(), loader.batch_size)\n                losses.append(loss_avg.avg)\n                tk0.set_postfix(loss=loss.item())\n                \n                _, predicted = torch.max(preds, 1)\n                total_correct += (predicted == classes).sum().item()\n                total_samples += classes.size(0)\n\n        epoch_accuracy = total_correct / total_samples\n        logging.info(f\"Epoch {epoch + 1} Accuracy: {epoch_accuracy}\")\n        epoch_losses.append(losses)\n        valid_accuracy.append(epoch_accuracy)\n#         print(\"Epoch: \", epoch+1, \", accuracy is \", epoch_accuracy)\n    return epoch_losses, valid_accuracy","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:50:39.391060Z","iopub.execute_input":"2024-03-12T06:50:39.391383Z","iopub.status.idle":"2024-03-12T06:50:39.400720Z","shell.execute_reply.started":"2024-03-12T06:50:39.391358Z","shell.execute_reply":"2024-03-12T06:50:39.399996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6 epochs is enough, after seventh epoch, loss would be increasing","metadata":{}},{"cell_type":"code","source":"for opt in ['SGD']:\n    args.opt = opt\n    traon_loss_timm, train_accuracy_timm = train(args, train_loader, model_vit_small_patch32_224)\n    val_losses_timm, accuracy_val_timm = validate(model_vit_small_patch32_224, valid_loader)","metadata":{"execution":{"iopub.status.busy":"2024-03-11T12:06:04.272770Z","iopub.execute_input":"2024-03-11T12:06:04.273165Z","iopub.status.idle":"2024-03-11T12:30:54.175545Z","shell.execute_reply.started":"2024-03-11T12:06:04.273127Z","shell.execute_reply":"2024-03-11T12:30:54.174597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for opt in ['SGD']:\n    args.opt = opt\n    traon_loss_timm, train_accuracy_timm = train(args, train_loader, model_vit_small_patch32_224)\n    val_losses_timm, accuracy_val_timm = validate(model_vit_small_patch32_224, valid_loader)","metadata":{"execution":{"iopub.status.busy":"2024-03-12T06:54:26.998292Z","iopub.execute_input":"2024-03-12T06:54:26.998653Z","iopub.status.idle":"2024-03-12T07:09:43.093628Z","shell.execute_reply.started":"2024-03-12T06:54:26.998621Z","shell.execute_reply":"2024-03-12T07:09:43.092347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for opt in ['AdamaP']:\n    args.opt = opt\n    traon_loss_timm, train_accuracy_timm = train(args, train_loader, model_vit_small_patch32_224)\n    val_losses_timm, accuracy_val_timm = validate(model_vit_small_patch32_224, valid_loader)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs, ids = next(iter(test_loader))","metadata":{"execution":{"iopub.status.busy":"2024-03-12T07:10:35.547106Z","iopub.execute_input":"2024-03-12T07:10:35.548061Z","iopub.status.idle":"2024-03-12T07:10:38.063562Z","shell.execute_reply.started":"2024-03-12T07:10:35.548008Z","shell.execute_reply":"2024-03-12T07:10:38.062297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs.shape\nprint(f\"Length of test_dataloader dataloader: {len(test_loader)} batches of {128}\")\ninputs, ids  = next(iter(test_loader))","metadata":{"execution":{"iopub.status.busy":"2024-03-12T07:12:27.619231Z","iopub.execute_input":"2024-03-12T07:12:27.620270Z","iopub.status.idle":"2024-03-12T07:12:30.172582Z","shell.execute_reply.started":"2024-03-12T07:12:27.620230Z","shell.execute_reply":"2024-03-12T07:12:30.171135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Computing predictions...')\nprobabilities = model_vit_small_patch32_224.predict(inputs)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Generating submission.csv file...')\n\n# Get image ids from test set and convert to unicode\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\n# Write the submission file\nnp.savetxt(\n    'submission.csv',\n    np.rec.fromarrays([test_ids, predictions]),\n    fmt=['%s', '%d'],\n    delimiter=',',\n    header='id,label',\n    comments='',\n)\n\n# Look at the first few predictions\n!head submission.csv","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_data = []\n\nwith torch.no_grad():\n    for images, ids in test_loader:\n        device = xm.xla_device()\n        model_vit_small_patch32_224.to(device)\n\n        # Переведите в режим оценки (inference mode)\n        model_vit_small_patch32_224.eval()\n        images = images.to(device)\n        y_preds = model_vit_small_patch32_224(images)\n        y_preds = y_preds.argmax(dim=1)\n        for img_id, y_pred in zip(ids, y_preds.cpu()):\n            submission_data.append({'id': img_id, 'label': y_pred.item()})\nsubmission_df = pd.DataFrame(submission_data)\nsubmission_df.to_csv('submission.csv', index=False)  ","metadata":{"execution":{"iopub.status.busy":"2024-03-12T07:23:22.412795Z","iopub.execute_input":"2024-03-12T07:23:22.413844Z","iopub.status.idle":"2024-03-12T07:24:25.533388Z","shell.execute_reply.started":"2024-03-12T07:23:22.413800Z","shell.execute_reply":"2024-03-12T07:24:25.532255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for images, ids in test_loader:\n    print(images.shape)\n    break","metadata":{"execution":{"iopub.status.busy":"2024-03-12T07:19:33.615539Z","iopub.execute_input":"2024-03-12T07:19:33.615884Z","iopub.status.idle":"2024-03-12T07:19:36.194338Z","shell.execute_reply.started":"2024-03-12T07:19:33.615855Z","shell.execute_reply":"2024-03-12T07:19:36.192950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-03-12T07:24:47.263010Z","iopub.execute_input":"2024-03-12T07:24:47.263362Z","iopub.status.idle":"2024-03-12T07:24:47.274661Z","shell.execute_reply.started":"2024-03-12T07:24:47.263335Z","shell.execute_reply":"2024-03-12T07:24:47.273802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.info()","metadata":{"execution":{"iopub.status.busy":"2024-03-12T07:28:27.187744Z","iopub.execute_input":"2024-03-12T07:28:27.188139Z","iopub.status.idle":"2024-03-12T07:28:27.201107Z","shell.execute_reply.started":"2024-03-12T07:28:27.188110Z","shell.execute_reply":"2024-03-12T07:28:27.200098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}