{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":6799,"databundleVersionId":4225553,"sourceType":"competition"}],"dockerImageVersionId":30664,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -U faiss-cpu","metadata":{"execution":{"iopub.status.busy":"2024-04-04T04:09:28.523435Z","iopub.execute_input":"2024-04-04T04:09:28.523833Z","iopub.status.idle":"2024-04-04T04:09:43.212822Z","shell.execute_reply.started":"2024-04-04T04:09:28.523803Z","shell.execute_reply":"2024-04-04T04:09:43.211758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport os\nimport faiss\nimport numpy as np\n\nfrom tqdm import tqdm\nfrom torchvision.io import read_image, ImageReadMode\nfrom torchvision.datasets import ImageFolder\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader\nfrom matplotlib import pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-04T04:09:59.612601Z","iopub.execute_input":"2024-04-04T04:09:59.612942Z","iopub.status.idle":"2024-04-04T04:09:59.618683Z","shell.execute_reply.started":"2024-04-04T04:09:59.612917Z","shell.execute_reply":"2024-04-04T04:09:59.617421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_dir = \"/kaggle/input/imagenet-object-localization-challenge/ILSVRC/Data/CLS-LOC\"\ntransform = transforms.Compose([\n    transforms.Resize((224, 224))\n])\nprint(transform)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T04:10:07.341539Z","iopub.execute_input":"2024-04-04T04:10:07.342352Z","iopub.status.idle":"2024-04-04T04:10:07.347828Z","shell.execute_reply.started":"2024-04-04T04:10:07.342322Z","shell.execute_reply":"2024-04-04T04:10:07.346724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = image_dir + \"/test/\"\ntest_img_names = os.listdir(test_path)\ntest_img_names.sort()\n\na = 40000\nb = 50000\n\ncentroid_images = torch.empty((10000, 3, 224, 224))\nfor idx, name in tqdm(enumerate(test_img_names[a:b])):\n    img_name = test_path + name\n    image = read_image(img_name, ImageReadMode.RGB)\n    centroid_images[idx] = transform(image)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T04:10:15.996634Z","iopub.execute_input":"2024-04-04T04:10:15.99701Z","iopub.status.idle":"2024-04-04T04:13:08.008599Z","shell.execute_reply.started":"2024-04-04T04:10:15.996982Z","shell.execute_reply":"2024-04-04T04:13:08.00743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"centroid_images = centroid_images.reshape(10000, -1).detach().numpy()","metadata":{"execution":{"iopub.status.busy":"2024-04-04T04:13:08.477592Z","iopub.execute_input":"2024-04-04T04:13:08.477914Z","iopub.status.idle":"2024-04-04T04:13:08.485487Z","shell.execute_reply.started":"2024-04-04T04:13:08.477887Z","shell.execute_reply":"2024-04-04T04:13:08.484546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"centroid_images.shape","metadata":{"execution":{"iopub.status.busy":"2024-04-04T04:13:29.732632Z","iopub.execute_input":"2024-04-04T04:13:29.733218Z","iopub.status.idle":"2024-04-04T04:13:29.74074Z","shell.execute_reply.started":"2024-04-04T04:13:29.733187Z","shell.execute_reply":"2024-04-04T04:13:29.739566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import defaultdict\n\nunlabeled_group = defaultdict(lambda: list())","metadata":{"execution":{"iopub.status.busy":"2024-04-04T04:35:37.534238Z","iopub.execute_input":"2024-04-04T04:35:37.534904Z","iopub.status.idle":"2024-04-04T04:35:37.541222Z","shell.execute_reply.started":"2024-04-04T04:35:37.53486Z","shell.execute_reply":"2024-04-04T04:35:37.539807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d = centroid_images.shape[1]\nindex = faiss.IndexFlatL2(d)\nindex.add(centroid_images)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T04:16:06.964424Z","iopub.execute_input":"2024-04-04T04:16:06.964817Z","iopub.status.idle":"2024-04-04T04:16:13.213825Z","shell.execute_reply.started":"2024-04-04T04:16:06.964788Z","shell.execute_reply":"2024-04-04T04:16:13.212904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unlabeled_img_names = test_img_names[:a] + test_img_names[b:]# 90,000 unlabeled images\nchunk_size = 10000\n\ncluster_list = np.array([], dtype=int)\nfor chunk_id in tqdm(range(90000 // chunk_size)):\n    chunk_unlabeled_images = torch.empty((chunk_size, 3, 224, 224))\n    # load image chunk\n    for idx, name in enumerate(unlabeled_img_names[chunk_id*chunk_size : (chunk_id+1) * chunk_size]):\n        img_name = test_path + name\n        image = read_image(img_name, ImageReadMode.RGB)\n        chunk_unlabeled_images[idx] = transform(image)\n\n    chunk_unlabeled_images = chunk_unlabeled_images.reshape(chunk_size, -1).detach().numpy()\n    _, I = index.search(chunk_unlabeled_images, 1)\n    cluster_list = np.append(cluster_list, I.reshape(-1))","metadata":{"execution":{"iopub.status.busy":"2024-04-04T04:28:53.106216Z","iopub.execute_input":"2024-04-04T04:28:53.106613Z","iopub.status.idle":"2024-04-04T04:30:14.322797Z","shell.execute_reply.started":"2024-04-04T04:28:53.106583Z","shell.execute_reply":"2024-04-04T04:30:14.321063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert len(cluster_list) == 90000","metadata":{"execution":{"iopub.status.busy":"2024-04-04T04:30:55.566667Z","iopub.execute_input":"2024-04-04T04:30:55.567049Z","iopub.status.idle":"2024-04-04T04:30:55.573529Z","shell.execute_reply.started":"2024-04-04T04:30:55.567021Z","shell.execute_reply":"2024-04-04T04:30:55.572385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, cluster_id in enumerate(cluster_list):\n    unlabeled_group[cluster_id.item()].append(idx)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T04:35:45.531732Z","iopub.execute_input":"2024-04-04T04:35:45.532124Z","iopub.status.idle":"2024-04-04T04:35:45.537454Z","shell.execute_reply.started":"2024-04-04T04:35:45.532094Z","shell.execute_reply":"2024-04-04T04:35:45.536635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\n\nwith open(f'unlabeled_group.json', 'w') as f:\n    json.dump(unlabeled_group, f, indent=2)","metadata":{"execution":{"iopub.status.busy":"2024-04-04T04:35:46.469643Z","iopub.execute_input":"2024-04-04T04:35:46.470214Z","iopub.status.idle":"2024-04-04T04:35:46.47703Z","shell.execute_reply.started":"2024-04-04T04:35:46.470186Z","shell.execute_reply":"2024-04-04T04:35:46.476126Z"},"trusted":true},"execution_count":null,"outputs":[]}]}