{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":6799,"databundleVersionId":4225553,"sourceType":"competition"}],"dockerImageVersionId":30665,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport os\nimport torch.nn.functional as F\n\nfrom tqdm import tqdm\nfrom torchvision.io import read_image, ImageReadMode\nfrom torchvision.datasets import ImageFolder\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader, Dataset, ConcatDataset","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-03-22T01:16:50.373035Z","iopub.execute_input":"2024-03-22T01:16:50.373282Z","iopub.status.idle":"2024-03-22T01:16:56.62246Z","shell.execute_reply.started":"2024-03-22T01:16:50.37326Z","shell.execute_reply":"2024-03-22T01:16:56.621437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nimage_dir = \"/kaggle/input/imagenet-object-localization-challenge/ILSVRC/Data/CLS-LOC\"","metadata":{"execution":{"iopub.status.busy":"2024-03-22T01:16:56.624436Z","iopub.execute_input":"2024-03-22T01:16:56.625146Z","iopub.status.idle":"2024-03-22T01:16:56.678171Z","shell.execute_reply.started":"2024-03-22T01:16:56.625112Z","shell.execute_reply":"2024-03-22T01:16:56.677217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchvision.models import vgg19, VGG19_Weights\n\nvgg19_weights = VGG19_Weights.IMAGENET1K_V1\nvgg19_model = vgg19(weights=vgg19_weights)\nvgg19_transform = vgg19_weights.transforms()","metadata":{"execution":{"iopub.status.busy":"2024-03-22T01:16:56.67969Z","iopub.execute_input":"2024-03-22T01:16:56.680014Z","iopub.status.idle":"2024-03-22T01:17:07.366693Z","shell.execute_reply.started":"2024-03-22T01:16:56.679989Z","shell.execute_reply":"2024-03-22T01:17:07.365467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImagenetTrainClassDataset(Dataset):\n    def __init__(self, path:str, class_id:int, transform):\n        assert path.split('/')[-1] == 'train'\n        super().__init__()\n        class_names = sorted(os.listdir(path))\n        self.class_name = class_names[class_id]\n        self.class_path = path + '/' + self.class_name\n        \n        self.img_names = sorted(os.listdir(self.class_path))\n        self.transform = transform\n    \n    def __getitem__(self, idx):\n        img_path = self.class_path + '/' + self.img_names[idx]\n        image = read_image(img_path, ImageReadMode.RGB)\n        return self.transform(image)\n    \n    def __len__(self):\n        return len(self.img_names)\n    \nclass ImagenetTestDataset(Dataset):\n    def __init__(self, path:str, transform):\n        assert path.split('/')[-1] == 'test'\n        super().__init__()\n        \n        self.path = path\n        self.img_names = sorted(os.listdir(self.path))\n        self.transform = transform\n    \n    def __getitem__(self, idx):\n        img_path = self.path + '/' + self.img_names[idx]\n        image = read_image(img_path, ImageReadMode.RGB)\n        return self.transform(image)\n    \n    def __len__(self):\n        return len(self.img_names)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**TRAIN SET**","metadata":{}},{"cell_type":"code","source":"train_path = image_dir + '/train'\n\nchunk_id = 0\nchunk_size = 1000 # number of classes in one chunk\nsubsets = []\nname_list = []\n\nfor i in range(chunk_id * chunk_size, (chunk_id+1) * chunk_size):\n    class_subset = ImagenetTrainClassDataset(train_path, class_id=i, transform=vgg19_transform)\n    subsets.append(class_subset)\n    name_list += class_subset.img_names\n\nname_list = [name.split('.')[0] for name in name_list] # remove JPEG extension\nsubset = ConcatDataset(subsets)\n# ------------------------------------------------------------ #  \ntrain_dataloader = DataLoader(subset, batch_size=100, shuffle=False, num_workers=2)\n# ------------------------------------------------------------ #  \ntrain_probs = torch.empty((len(subset), 1000), dtype=torch.float32)\n\nvgg19_model = vgg19_model.to(device)\nvgg19_model.eval()\nwith torch.no_grad():\n    for i, images in tqdm(enumerate(train_dataloader)):\n        images = images.to(device)\n        logits = vgg19_model(images)\n        probs = F.softmax(logits, dim=1)\n        train_probs[i*100: i*100 + probs.size(0)] = probs.detach().cpu()\n# ------------------------------------------------------------ #  \noutput = {\n    'probs': train_probs,\n    'img_names': name_list\n}\n\ntorch.save(output, f'vgg19_v1_train_{chunk_id}.pth')\ndel train_probs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_path = image_dir + '/train'\n#\n#chunk_id = 0\n#chunk_size = 1000 # number of classes in one chunk\n#subsets = []\n#name_list = []\n\n#for i in range(chunk_id * chunk_size, (chunk_id+1) * chunk_size):\n#    class_subset = ImagenetTrainClassDataset(train_path, class_id=i, transform=vgg11_transform)\n#    subsets.append(class_subset)\n#    name_list += class_subset.img_names\n\n#name_list = [name.split('.')[0] for name in name_list] # remove JPEG extension\n#subset = ConcatDataset(subsets)\n# ------------------------------------------------------------ #  \n#train_dataloader = DataLoader(subset, batch_size=100, shuffle=False, num_workers=2)\n# ------------------------------------------------------------ #  \n#train_probs = torch.empty((len(subset), 1000), dtype=torch.float32)\n\n#vgg11_model = vgg11_model.to(device)\n#vgg11_model.eval()\n#with torch.no_grad():\n#    for i, images in tqdm(enumerate(train_dataloader)):\n#        images = images.to(device)\n#        logits = vgg11_model(images)\n#        probs = F.softmax(logits, dim=1)\n#        train_probs[i*100: i*100 + probs.size(0)] = probs.detach().cpu()\n# ------------------------------------------------------------ #  \n#output = {\n#    'probs': train_probs,\n#    'img_names': name_list\n#}\n#\n#torch.save(output, f'vgg11_v1_train_{chunk_id}.pth')\n#del train_probs","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**TEST SET**","metadata":{}},{"cell_type":"code","source":"test_path = image_dir + '/test'\ndataset = ImagenetTestDataset(path=test_path, transform=vgg19_transform)\nname_list = [name.split('.')[0].split('_')[-1] for name in dataset.img_names]\n# ------------------------------------------------------------ #   \ntest_dataloader = DataLoader(dataset, batch_size=100, shuffle=False, num_workers=2)\n# ------------------------------------------------------------ #   \ntest_probs = torch.empty((len(dataset), 1000), dtype=torch.float32)\nvgg19_model = vgg19_model.to(device)\nvgg19_model.eval()\nwith torch.no_grad():\n    for i, images in tqdm(enumerate(test_dataloader)):\n        images = images.to(device)\n        logits = vgg19_model(images)\n        probs = F.softmax(logits, dim=1)\n        test_probs[i*100: i*100 + probs.size(0)] = probs.detach().cpu()\n# ------------------------------------------------------------ #        \noutput = {\n    'probs': test_probs,\n    'img_names': name_list\n}\n# ------------------------------------------------------------ #   \ntorch.save(output, f'vgg19_v1_test.pth')\ndel test_probs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_path = image_dir + '/test'\n#dataset = ImagenetTestDataset(path=test_path, transform=vgg11_transform)\n#name_list = [name.split('.')[0].split('_')[-1] for name in dataset.img_names]\n# ------------------------------------------------------------ #   \n#test_dataloader = DataLoader(dataset, batch_size=100, shuffle=False, num_workers=2)\n# ------------------------------------------------------------ #   \n#test_probs = torch.empty((len(dataset), 1000), dtype=torch.float32)\n#vgg11_model = vgg11_model.to(device)\n#vgg11_model.eval()\n#with torch.no_grad():\n#    for i, images in tqdm(enumerate(test_dataloader)):\n#        images = images.to(device)\n#        logits = vgg11_model(images)\n#        probs = F.softmax(logits, dim=1)\n#        test_probs[i*100: i*100 + probs.size(0)] = probs.detach().cpu()\n# ------------------------------------------------------------ #        \n#output = {\n#    'probs': test_probs,\n#    'img_names': name_list\n#}\n# ------------------------------------------------------------ #   \n#torch.save(output, f'vgg11_v1_test.pth')\n#del test_probs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_path = image_dir + '/test'\n#dataset = ImagenetTestDataset(path=test_path, transform=vgg13_transform)\n#name_list = [name.split('.')[0].split('_')[-1] for name in dataset.img_names]\n# ------------------------------------------------------------ #   \n#test_dataloader = DataLoader(dataset, batch_size=100, shuffle=False, num_workers=2)\n# ------------------------------------------------------------ #   \n#test_probs = torch.empty((len(dataset), 1000), dtype=torch.float32)\n#vgg13_model = vgg13_model.to(device)\n#vgg13_model.eval()\n#with torch.no_grad():\n#    for i, images in tqdm(enumerate(test_dataloader)):\n#        images = images.to(device)\n#        logits = vgg13_model(images)\n#        probs = F.softmax(logits, dim=1)\n#        test_probs[i*100: i*100 + probs.size(0)] = probs.detach().cpu()\n# ------------------------------------------------------------ #        \n#output = {\n#    'probs': test_probs,\n#    'img_names': name_list\n#}\n# ------------------------------------------------------------ #   \n#torch.save(output, f'vgg13_v1_test.pth')\n#del test_probs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_path = image_dir + '/test'\n#dataset = ImagenetTestDataset(path=test_path, transform=vgg16_transform)\n#name_list = [name.split('.')[0].split('_')[-1] for name in dataset.img_names]\n# ------------------------------------------------------------ #   \n#test_dataloader = DataLoader(dataset, batch_size=100, shuffle=False, num_workers=2)\n# ------------------------------------------------------------ #   \n#test_probs = torch.empty((len(dataset), 1000), dtype=torch.float32)\n#vgg16_model = vgg16_model.to(device)\n#vgg16_model.eval()\n#with torch.no_grad():\n#    for i, images in tqdm(enumerate(test_dataloader)):\n#        images = images.to(device)\n#        logits = vgg16_model(images)\n#        probs = F.softmax(logits, dim=1)\n#        test_probs[i*100: i*100 + probs.size(0)] = probs.detach().cpu()\n# ------------------------------------------------------------ #        \n#output = {\n#    'probs': test_probs,\n#    'img_names': name_list\n#}\n# ------------------------------------------------------------ #   \n#torch.save(output, f'vgg16_v1_test.pth')\n#del test_probs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**END**","metadata":{}}]}