{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! pip install --upgrade pip\n! pip install fastdup\n! pip install wurlitzer\n","metadata":{"_uuid":"2a48c44f-dd8a-4f08-96e7-3e7b697059ea","_cell_guid":"6d6efb8b-1dd5-4bbd-8538-59cd0a95abcf","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:06.980699Z","iopub.execute_input":"2022-09-24T19:19:06.981404Z","iopub.status.idle":"2022-09-24T19:19:35.811483Z","shell.execute_reply.started":"2022-09-24T19:19:06.981367Z","shell.execute_reply":"2022-09-24T19:19:35.810301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import urllib.request\nimport zipfile\nimport fastdup\nimport os\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler\nfrom copy import deepcopy\nfrom community import community_louvain\nimport networkx as nx\nimport pickle\nimport sys\nfrom PIL import Image\nimport random\nfrom collections import Counter\nfrom copy import deepcopy\nfrom sklearn import preprocessing\nfrom glob import glob\nimport pickle5\nfastdup.__version__","metadata":{"_uuid":"8849b558-5130-4022-941e-04aa1a82c058","_cell_guid":"7f32382a-f6b7-439e-9bee-8174f9caa0fb","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.81555Z","iopub.execute_input":"2022-09-24T19:19:35.815887Z","iopub.status.idle":"2022-09-24T19:19:35.829394Z","shell.execute_reply.started":"2022-09-24T19:19:35.815855Z","shell.execute_reply":"2022-09-24T19:19:35.828481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load_ext wurlitzer","metadata":{"_uuid":"26ecd2cf-23d0-41b0-9b26-040c70d3a5b7","_cell_guid":"564fdb87-34ec-4c16-a297-427e68f3399a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.831741Z","iopub.execute_input":"2022-09-24T19:19:35.8321Z","iopub.status.idle":"2022-09-24T19:19:35.845373Z","shell.execute_reply.started":"2022-09-24T19:19:35.832071Z","shell.execute_reply":"2022-09-24T19:19:35.843877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GET ImageNet","metadata":{"_uuid":"ba43d077-985a-45a3-9122-f285811bcd92","_cell_guid":"f5e003e0-a093-419e-a622-9fd458955d73","trusted":true}},{"cell_type":"code","source":"# # download kaggle.json from kaggle-user-profile \n# !rm -r ~/.kaggle\n# !mkdir ~/.kaggle\n# !mv ./kaggle.json ~/.kaggle/\n# !chmod 600 ~/.kaggle/kaggle.json\n# !kaggle datasets list","metadata":{"_uuid":"9fa9a581-6aee-4e92-af5a-87f5e1425fe8","_cell_guid":"396524ca-c735-4d6d-9d92-d725e6715f71","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.84797Z","iopub.execute_input":"2022-09-24T19:19:35.848332Z","iopub.status.idle":"2022-09-24T19:19:35.858755Z","shell.execute_reply.started":"2022-09-24T19:19:35.848298Z","shell.execute_reply":"2022-09-24T19:19:35.857835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '/kaggle/input/imagenet-object-localization-challenge'","metadata":{"_uuid":"2cefd7b6-580b-48b8-b7d0-c870773ebc5d","_cell_guid":"a4cd427a-1b9c-46fb-a466-b827270174e3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.860089Z","iopub.execute_input":"2022-09-24T19:19:35.860709Z","iopub.status.idle":"2022-09-24T19:19:35.8723Z","shell.execute_reply.started":"2022-09-24T19:19:35.860674Z","shell.execute_reply":"2022-09-24T19:19:35.871165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rows = []\n# for r in open('/kaggle/input/imagenet-object-localization-challenge/LOC_synset_mapping.txt', 'rt'):\n#     class_id, synset = r.split(' ', 1)\n#     synset = [x.strip() for x in synset.split(',')]\n#     rows.append(pd.Series({\n#         'class_id': class_id,\n#         'synset': synset\n#     }))\n# df_synset = pd.DataFrame(rows)","metadata":{"_uuid":"e27cd725-fbf5-49cd-b904-f778553b2a3c","_cell_guid":"045edb9c-c761-4af5-9ece-51c51dd4627c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.87392Z","iopub.execute_input":"2022-09-24T19:19:35.874624Z","iopub.status.idle":"2022-09-24T19:19:35.883203Z","shell.execute_reply.started":"2022-09-24T19:19:35.874588Z","shell.execute_reply":"2022-09-24T19:19:35.882106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # READ FROM FILES\n# df_train = pd.DataFrame()\n# df_train['fname'] = glob(f'{data_dir}/ILSVRC/Data/CLS-LOC/train/**/*')\n# df_train['label'] = df_train['fname'].apply(lambda x: x.split(os.sep)[-2])\n# df_train['ImageId'] = df_train['fname'].apply(lambda x: x.split(os.sep)[-1].split('.')[0])\n# df_train['synset'] = df_train['label'].apply(lambda x: df_synset.query(f'class_id==\"{x}\"').iloc[0]['synset'])\n# df_train.to_pickle('/kaggle/input/imagenet-data-annot/kaggle/input/imagenet_train_full.pkl')","metadata":{"_uuid":"88653bfc-2749-4dee-87c1-2732b88ec8b3","_cell_guid":"794e07a2-3e45-4a82-8afd-28cbca192abb","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.884564Z","iopub.execute_input":"2022-09-24T19:19:35.885082Z","iopub.status.idle":"2022-09-24T19:19:35.894567Z","shell.execute_reply.started":"2022-09-24T19:19:35.885047Z","shell.execute_reply":"2022-09-24T19:19:35.893565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"eceb66b4-1a45-4328-b5a3-c81ee50920c7","_cell_guid":"69d7fad3-886d-4fdc-9e89-18fde626d79c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.895911Z","iopub.execute_input":"2022-09-24T19:19:35.896595Z","iopub.status.idle":"2022-09-24T19:19:35.904598Z","shell.execute_reply.started":"2022-09-24T19:19:35.896559Z","shell.execute_reply":"2022-09-24T19:19:35.903656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop images - FastDup free version allows up to 999,999 images\n# from sklearn.model_selection import train_test_split\n\n# df_train, drop = train_test_split(df_train, test_size=0.21946241200405567)\n# for i, row in drop.iterrows():\n#     os.remove(row.fname)\n# df_train.to_pickle('/kaggle/input/imagenet-data-annot/imagenet_train_1000000.pkl')","metadata":{"_uuid":"1caddb85-69e9-4714-8dd2-b749ba22d34d","_cell_guid":"9a6a1a45-7b51-49af-881e-252f58f30935","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.906139Z","iopub.execute_input":"2022-09-24T19:19:35.906615Z","iopub.status.idle":"2022-09-24T19:19:35.914982Z","shell.execute_reply.started":"2022-09-24T19:19:35.906581Z","shell.execute_reply":"2022-09-24T19:19:35.914027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ DATA CSV\n# df_train = pd.read_csv('ImageNet/LOC_train_solution.csv')\n# df_train['label'] = ddf_trainf['PredictionString'].apply(lambda x: x.split()[0])\n# df_train['fname'] = df_train['ImageId'].apply(lambda x: f'ImageNet/ILSVRC/Data/CLS-LOC/train/{x.split(\"_\")[0]}/{x}.JPEG')\n# df_train['synset'] = df_train['label'].apply(lambda x: df_synset.query(f'class_id==\"{x}\"').iloc[0]['synset'])\n# df_train = df_train.drop('PredictionString', axis=1)","metadata":{"_uuid":"d4044508-9a80-4840-9c21-45d6732e0c30","_cell_guid":"ef9182c9-c40c-4104-a5ce-f3272f82647c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.919576Z","iopub.execute_input":"2022-09-24T19:19:35.920444Z","iopub.status.idle":"2022-09-24T19:19:35.926829Z","shell.execute_reply.started":"2022-09-24T19:19:35.920417Z","shell.execute_reply":"2022-09-24T19:19:35.926139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# READ DATA CSV\n# df_val = pd.read_csv('ImageNet/LOC_val_solution.csv')\n# df_val['label'] = df_val['PredictionString'].apply(lambda x: x.split()[0])\n# df_val['fname'] = df_val['ImageId'].apply(lambda x: f'ImageNet/ILSVRC/Data/CLS-LOC/val/{x}.JPEG')\n# df_val['synset'] = df_val['label'].apply(lambda x: df_synset.query(f'class_id==\"{x}\"').iloc[0]['synset'])\n# df_val = df_val.drop('PredictionString', axis=1)\n# df_val.to_pickle('/kaggle/input/imagenet_val.pkl')","metadata":{"_uuid":"ebd94acc-0a75-4492-b312-37c2939bbaf2","_cell_guid":"dfc85821-ae68-4b38-a060-92cb0d5bc938","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.927545Z","iopub.execute_input":"2022-09-24T19:19:35.927831Z","iopub.status.idle":"2022-09-24T19:19:35.939836Z","shell.execute_reply.started":"2022-09-24T19:19:35.927806Z","shell.execute_reply":"2022-09-24T19:19:35.939087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run FastDup","metadata":{"_uuid":"b0d30c5e-564f-4f1c-8cb8-99acb2ade8da","_cell_guid":"6300d160-7dd7-46e2-ac55-999bcd3f9311","trusted":true}},{"cell_type":"code","source":"# train_dir = f'{data_dir}/ILSVRC/Data/CLS-LOC/train'\n# val_dir = f'{data_dir}/ILSVRC/Data/CLS-LOC/test'","metadata":{"_uuid":"b81ecb26-3143-4c19-95ff-31eae7d3f096","_cell_guid":"f32d84b2-1611-4e8c-951b-f5e23df9821d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.941522Z","iopub.execute_input":"2022-09-24T19:19:35.942092Z","iopub.status.idle":"2022-09-24T19:19:35.951378Z","shell.execute_reply.started":"2022-09-24T19:19:35.942052Z","shell.execute_reply":"2022-09-24T19:19:35.950558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rm -r out","metadata":{"_uuid":"cab270fd-3169-43dc-aa1d-bf4b87a98d47","_cell_guid":"7d55f853-d738-4f88-965f-922d18fee5b2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.952199Z","iopub.execute_input":"2022-09-24T19:19:35.952505Z","iopub.status.idle":"2022-09-24T19:19:35.964714Z","shell.execute_reply.started":"2022-09-24T19:19:35.952478Z","shell.execute_reply":"2022-09-24T19:19:35.963693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with open('files_list.txt', 'wt') as f: \n#     f.write(','.join(df_train.fname.to_list()))","metadata":{"_uuid":"51a51b8f-9ee7-462f-89be-2ad09eb516d7","_cell_guid":"2f6d40a1-0035-405a-89b1-a81d74e08595","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.967312Z","iopub.execute_input":"2022-09-24T19:19:35.967594Z","iopub.status.idle":"2022-09-24T19:19:35.975345Z","shell.execute_reply.started":"2022-09-24T19:19:35.967564Z","shell.execute_reply":"2022-09-24T19:19:35.974363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fastdup.run('files_list.txt',\n#             work_dir='out',\n#             compute='cpu',\n#             nearest_neighbors_k=10, \n#             threshold=0.8)","metadata":{"_uuid":"2f38b1dc-25d7-4db5-861f-7f3bdaf14da9","_cell_guid":"40819bcc-a198-4fe1-afad-e7311033bcbf","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.976758Z","iopub.execute_input":"2022-09-24T19:19:35.9774Z","iopub.status.idle":"2022-09-24T19:19:35.986316Z","shell.execute_reply.started":"2022-09-24T19:19:35.977363Z","shell.execute_reply":"2022-09-24T19:19:35.985445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_index = pd.read_csv('out/atrain_stats.csv')\n# df_cc = pd.read_csv('out/connected_components.csv')\n# df_cc['n_componnents'] = df_cc.groupby('component_id')['component_id'].transform(\"count\")\n# df_feat = pd.read_csv('out/atrain_features.dat.csv')\n# df_similarity = pd.read_csv('out/similarity.csv')\n# df_similarity['distance'] = (df_similarity.distance - df_similarity.distance.min()) / (df_similarity.distance.max() - df_similarity.distance.min())","metadata":{"_uuid":"ad21e76b-ec6d-421e-81ea-3fa1d6af712c","_cell_guid":"cf23d5ec-1797-4776-af3a-369f68e447db","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.987552Z","iopub.execute_input":"2022-09-24T19:19:35.988198Z","iopub.status.idle":"2022-09-24T19:19:35.996481Z","shell.execute_reply.started":"2022-09-24T19:19:35.988161Z","shell.execute_reply":"2022-09-24T19:19:35.995779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"e3b6757f-130f-4f74-8c3b-59109ce070ab","_cell_guid":"9c339564-9b1e-4776-a93c-0551a3dd2890","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Community Detection","metadata":{"_uuid":"582a4dac-fe53-42b7-aa2e-640114b94a78","_cell_guid":"cbfd36f7-142d-4923-ad0f-55d58b0c1879","trusted":true}},{"cell_type":"code","source":"# # Graph & community detection - might take a while\n\n# # build graoh \n# gnx = nx.Graph()\n# for i, row in df_similarity.iterrows():\n#     gnx.add_edge(row['from'], row['to'], weight=row['distance'])\n    \n# # run community detection \n# partition = community_louvain.best_partition(gnx)\n# df_partition = pd.DataFrame([{'node': node, 'partition': p} for node, p in partition.items()])\n# for node, node_data in gnx.nodes(data=True):\n#     node_data['partition'] = partition[node]\n# pickle.dump(gnx, open('imagenaet_gnx', 'wb'))\n\n# # updatae data df\n# df_train['partition'] = df_train.fname.apply(lambda x: gnx.nodes.get(x, {'partition': -1})['partition'])\n# df_train.to_pickle('/kaggle/input/imagenet-data-annot/imagenet_train_1000000_partition.pkl')","metadata":{"_uuid":"ca42df61-5a3d-46c0-a527-adad6a34852e","_cell_guid":"620d3f74-b573-4410-92c6-2e27d303b76b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:35.997197Z","iopub.execute_input":"2022-09-24T19:19:35.997471Z","iopub.status.idle":"2022-09-24T19:19:36.006475Z","shell.execute_reply.started":"2022-09-24T19:19:35.997445Z","shell.execute_reply":"2022-09-24T19:19:36.005747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"01c9118a-1d3f-411c-aca2-d1cdc7271d9a","_cell_guid":"54e9a3f6-4c2d-4f38-8a4f-1ceffa64c427","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clone SimCLR","metadata":{"_uuid":"01362aef-f50d-4130-9bb9-27997b75fb8e","_cell_guid":"211db862-f8df-4a53-9fff-5d407a82cc87","trusted":true}},{"cell_type":"code","source":"! git clone https://github.com/sthalles/SimCLR.git","metadata":{"_uuid":"111030cf-7517-47c5-a4bf-393e974ac4ad","_cell_guid":"5aab73b7-8b06-435f-9e04-3a220a665b33","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:36.007396Z","iopub.execute_input":"2022-09-24T19:19:36.008846Z","iopub.status.idle":"2022-09-24T19:19:36.978514Z","shell.execute_reply.started":"2022-09-24T19:19:36.00881Z","shell.execute_reply":"2022-09-24T19:19:36.977401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append('SimCLR/')\nimport torch\nimport torch.backends.cudnn as cudnn\nfrom torchvision import models\nfrom models.resnet_simclr import ResNetSimCLR\nfrom simclr import SimCLR","metadata":{"_uuid":"1b87f858-8d52-4eff-9266-759b63f43c64","_cell_guid":"e2fef403-5d7c-4db3-ad4d-885e906e1e22","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:36.980505Z","iopub.execute_input":"2022-09-24T19:19:36.980839Z","iopub.status.idle":"2022-09-24T19:19:37.369057Z","shell.execute_reply.started":"2022-09-24T19:19:36.98081Z","shell.execute_reply":"2022-09-24T19:19:37.368094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"a8fc7c59-0d85-4bea-bba3-6b8a9bfd9767","_cell_guid":"5c3f6a9f-b61f-41f6-9f05-b9afc15e7a0c","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset & Sampler","metadata":{"_uuid":"b179daa3-a1ab-4078-908f-7c9f5755fe2d","_cell_guid":"951af1e6-6153-4e19-93af-4755479c5534","trusted":true}},{"cell_type":"code","source":"from torch.utils.data import Dataset\nfrom torch.utils.data.dataset import T_co\nfrom torchvision import transforms\nfrom torchvision.transforms import transforms\nfrom torchvision.transforms.functional import to_tensor\nfrom data_aug.gaussian_blur import GaussianBlur\nfrom torchvision import transforms, datasets\nfrom data_aug.view_generator import ContrastiveLearningViewGenerator\n\n\nclass ImageNetDataset(Dataset):\n    def __init__(self, df, transform=True, n_transforms=2):\n        self._data_df = df\n        if 'partition' in df:\n            self._partition_to_index = self._create_partition_to_index()\n        \n        # SimCLR transforms\n        color_jitter = transforms.ColorJitter(0.8, 0.8, 0.8, 0.2)\n        if transform:\n            self._data_transforms = ContrastiveLearningViewGenerator(transforms.Compose([transforms.RandomResizedCrop(size=128),\n                                              transforms.RandomHorizontalFlip(),\n                                              transforms.RandomApply([color_jitter], p=0.8),\n                                              transforms.RandomGrayscale(p=0.2),\n                                              GaussianBlur(kernel_size=30),\n                                              transforms.ToTensor()]), n_transforms)\n        else:\n            self._data_transforms = transforms.ToTensor()\n        \n    # for sampler \n    def partition_to_index(self):\n        return self._partition_to_index\n    \n    # for sampler \n    def _create_partition_to_index(self):\n        partition_to_index = {p: [] for p in self._data_df.partition}\n        for i, row in self._data_df.iterrows():\n            partition_to_index[row.partition].append(i)\n        for p, index_list in partition_to_index.items():\n            random.shuffle(index_list)\n        return partition_to_index\n\n    def __getitem__(self, index: int) -> T_co:\n        instance = self._data_df.loc[index]\n        im = Image.open(instance.fname).convert('RGB')\n#         im = square_crop_resize(im, 512)\n        return self._data_transforms(im), instance.label_\n\n    def __len__(self):\n        return len(self._data_df)\n    \n    \nclass CommunitySampler():\n    def __init__(self, partition_to_index, batch_size):\n        self._partition_to_index = partition_to_index\n        self._batch_size = batch_size\n        self._len = sum([len(l) for k, l in partition_to_index.items()])\n        \n    def _pick_batch(self, partition_to_index):\n    \n        # sample keys \n        if self._batch_size > len(partition_to_index):\n            sample_keys = list(partition_to_index.keys())\n        else:\n            sample_keys = list(random.sample(partition_to_index.keys(), batch_size))\n\n        # take first element from each list \n        chosen_list = []\n        for k in sample_keys:\n            index_list = partition_to_index[k]\n            chosen_list.append(index_list[0])\n\n            # delete taken item\n            if len(index_list) == 1:\n                del partition_to_index[k]\n            else:\n                partition_to_index[k] = index_list[1:]\n        return chosen_list\n\n    def __iter__(self):\n        partition_to_index = deepcopy(self._partition_to_index)\n\n        while len(partition_to_index) > 0:\n            for instance_index in self._pick_batch(partition_to_index):\n                yield instance_index\n            \n    def __len__(self):\n        return self._len","metadata":{"_uuid":"aaf396c5-ee6c-47d0-b450-567dc659caeb","_cell_guid":"ea3a358d-7c7d-40bb-94a7-f9e410714530","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:19:37.370359Z","iopub.execute_input":"2022-09-24T19:19:37.370712Z","iopub.status.idle":"2022-09-24T19:19:37.394373Z","shell.execute_reply.started":"2022-09-24T19:19:37.370671Z","shell.execute_reply":"2022-09-24T19:19:37.393346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-09-24T19:24:38.432599Z","iopub.execute_input":"2022-09-24T19:24:38.43332Z","iopub.status.idle":"2022-09-24T19:24:38.82439Z","shell.execute_reply.started":"2022-09-24T19:24:38.433281Z","shell.execute_reply":"2022-09-24T19:24:38.82336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_pickle('/kaggle/input/imagenet-data-annot/imagenet_train_1000000_partition.pkl')\n# df_val = pd.read_pickle('/kaggle/input/imagenet-data-annot/kaggle/input/imagenet_val.pkl')","metadata":{"_uuid":"b8bef48f-6df7-4509-a5b7-050449c4b5d0","_cell_guid":"72ee1309-a7ed-4dcd-9972-99483f5890b8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:25:24.860693Z","iopub.execute_input":"2022-09-24T19:25:24.861323Z","iopub.status.idle":"2022-09-24T19:25:25.976132Z","shell.execute_reply.started":"2022-09-24T19:25:24.861285Z","shell.execute_reply":"2022-09-24T19:25:25.974262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_mapper = preprocessing.LabelEncoder().fit(df_train.label)\ndf_train['label_'] = label_mapper.transform(df_train.label)\n# df_val['label_'] = label_mapper.transform(df_val.label)","metadata":{"execution":{"iopub.status.busy":"2022-09-24T19:25:25.979144Z","iopub.execute_input":"2022-09-24T19:25:25.980374Z","iopub.status.idle":"2022-09-24T19:25:26.277306Z","shell.execute_reply.started":"2022-09-24T19:25:25.980333Z","shell.execute_reply":"2022-09-24T19:25:26.275091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = ImageNetDataset(df_train.sample(999936)) # error if df_train does not divide in batch size\n# val_dataset = ImageNetDataset(df_val, transform=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-24T19:25:26.279702Z","iopub.execute_input":"2022-09-24T19:25:26.281593Z","iopub.status.idle":"2022-09-24T19:26:20.407307Z","shell.execute_reply.started":"2022-09-24T19:25:26.281541Z","shell.execute_reply":"2022-09-24T19:26:20.406289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run SimCLR","metadata":{"_uuid":"654c0883-3bf7-49bc-b10c-c3ee7ecf7968","_cell_guid":"9474b465-12ed-4784-ad4a-0e0c327b0c6c","trusted":true}},{"cell_type":"code","source":"args = pd.Series({\n    'device': torch.device('cuda'),\n    'workers': 16,\n    'seed': None,\n    'log_every_n_steps': 100,\n    'temperature': 0.07,\n    'n_views': 2,\n    'gpu_index': 0,\n    'fp16_precision': True,\n    'epochs': 300,  #200\n    'disable_cuda': True,\n    'batch_size': 256,\n    'weight_decay': 1e-4,\n    'lr': 3e-4, \n    'out_dim': 256,\n    'arch': 'resnet50'\n})\n\n# if GPU\ncudnn.deterministic = True\ncudnn.benchmark = True","metadata":{"_uuid":"5b388b6b-eec1-4acd-bc45-7c4177191c37","_cell_guid":"6188f430-ed77-480a-801e-4f559b71b0ea","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:26:20.409389Z","iopub.execute_input":"2022-09-24T19:26:20.409682Z","iopub.status.idle":"2022-09-24T19:26:20.452028Z","shell.execute_reply.started":"2022-09-24T19:26:20.409655Z","shell.execute_reply":"2022-09-24T19:26:20.450845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = torch.utils.data.DataLoader(\n    train_dataset, \n    batch_size=args['batch_size'], \n    num_workers=args['workers'],\n    pin_memory=True,\n    sampler=CommunitySampler(train_dataset.partition_to_index(), args['batch_size']))\n\n# val_loader = torch.utils.data.DataLoader(\n#     val_dataset, \n#     batch_size=args['batch_size'], \n#     num_workers=args['workers'], \n#     pin_memory=True, \n#     collate_fn=square_collate)","metadata":{"_uuid":"5b10d6d8-4c2a-460c-9e04-d7571506450d","_cell_guid":"7da3c522-e3e2-4a19-a65f-4a2561b74d66","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:26:20.454664Z","iopub.execute_input":"2022-09-24T19:26:20.455787Z","iopub.status.idle":"2022-09-24T19:26:20.467073Z","shell.execute_reply.started":"2022-09-24T19:26:20.455746Z","shell.execute_reply":"2022-09-24T19:26:20.466104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ResNetSimCLR(base_model=args['arch'], out_dim=args['out_dim'])\noptimizer = torch.optim.Adam(model.parameters(), args['lr'], weight_decay=args['weight_decay'])\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=len(train_loader), eta_min=0, last_epoch=-1)","metadata":{"_uuid":"dfabded3-5c71-4a45-8aea-167074fcd0d5","_cell_guid":"48c1885a-fadd-460b-9c0d-d877913630a8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:26:20.468564Z","iopub.execute_input":"2022-09-24T19:26:20.468933Z","iopub.status.idle":"2022-09-24T19:26:21.064294Z","shell.execute_reply.started":"2022-09-24T19:26:20.468897Z","shell.execute_reply":"2022-09-24T19:26:21.063302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  It’s a no-op if the 'gpu_index' argument is a negative integer or None.\n\nwith torch.cuda.device(args.gpu_index):\n    args_ = deepcopy(args)\n    args_['lr'] = 1e-1\n    args_['epochs'] = 10\n    simclr = SimCLR(model=model, optimizer=optimizer, scheduler=scheduler, args=args_)\n    simclr.train(train_loader)\n    \nwith torch.cuda.device(args.gpu_index):\n    simclr = SimCLR(model=model, optimizer=optimizer, scheduler=scheduler, args=args)\n    simclr.train(train_loader)","metadata":{"_uuid":"9d49b646-814e-443e-a7cd-4e6738aae64c","_cell_guid":"b6749f66-4040-46b4-b298-177d62ae7fa7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-09-24T19:26:21.065444Z","iopub.execute_input":"2022-09-24T19:26:21.066598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model.state_dict(), 'model_parameters.pt')","metadata":{"_uuid":"e9883673-1b24-43af-8b25-a7ca8627e631","_cell_guid":"c487beb8-e2b1-4458-9890-bf50e894757b","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Eval on Cifar10","metadata":{"_uuid":"0cefa1ed-9803-44ed-b423-ae986979b092","_cell_guid":"99dc6d98-e55a-441f-b203-ad6e1d42d8f6","trusted":true}},{"cell_type":"code","source":"import torch\nimport sys\nimport numpy as np\nimport os\nimport yaml\nimport matplotlib.pyplot as plt\nimport torchvision\nimport pandas as pd","metadata":{"_uuid":"dbe8b5c1-b27d-4bfb-9b05-c0f909c882d6","_cell_guid":"46086359-710b-4d74-af0a-78ff065dba30","collapsed":false,"id":"YUemQib7ZE4D","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install gdown","metadata":{"_uuid":"3a588e18-15cc-433f-af4f-71bc5f2dc266","_cell_guid":"042732b2-c958-4a8e-a567-a32c83a46dbf","collapsed":false,"id":"WSgRE1CcLqdS","outputId":"400bebff-26ad-4956-f100-30a055de2ed4","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folder_name = 'resnet50'\nfile_id = 'model_parameters.pt'\nprint(folder_name, file_id)","metadata":{"_uuid":"3b0bfe7d-2e91-4c40-94e0-9421a6c24cc1","_cell_guid":"6636a284-1cef-4bac-9d3f-a307dca36c54","collapsed":false,"id":"G7YMxsvEZMrX","outputId":"622532d9-d61d-4f0d-82e8-4de9c7786e6e","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# download and extract model files\nos.system('gdown https://drive.google.com/uc?id={}'.format(file_id))\nos.system('unzip {}'.format(folder_name))\n!ls","metadata":{"_uuid":"d58f3c70-cf98-4d86-9852-ffae8e44cd53","_cell_guid":"c2384b12-4b14-4213-9afc-325b8aa85507","collapsed":false,"id":"PWZ8fet_YoJm","outputId":"33f11065-e590-4c02-c4c6-f59a2731e300","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader\nimport torchvision.transforms as transforms\nfrom torchvision import datasets","metadata":{"_uuid":"b60752cc-2b31-48e2-b72d-b30a3de807f9","_cell_guid":"ed4c2168-c270-4417-a1c7-840d88bf036b","collapsed":false,"id":"3_nypQVEv-hn","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(\"Using device:\", device)","metadata":{"_uuid":"95a6faa3-030e-4600-83c3-5881593cd603","_cell_guid":"673f5b17-9dac-452c-a2de-77039cedc5ae","collapsed":false,"id":"lDfbL3w_Z0Od","outputId":"f3d8a195-5acc-4025-f432-10a9dba8cc45","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_stl10_data_loaders(download, shuffle=False, batch_size=256):\n  train_dataset = datasets.STL10('./data', split='train', download=download,\n                                  transform=transforms.ToTensor())\n\n  train_loader = DataLoader(train_dataset, batch_size=batch_size,\n                            num_workers=0, drop_last=False, shuffle=shuffle)\n  \n  test_dataset = datasets.STL10('./data', split='test', download=download,\n                                  transform=transforms.ToTensor())\n\n  test_loader = DataLoader(test_dataset, batch_size=2*batch_size,\n                            num_workers=10, drop_last=False, shuffle=shuffle)\n  return train_loader, test_loader\n\ndef get_cifar10_data_loaders(download, shuffle=False, batch_size=256):\n  train_dataset = datasets.CIFAR10('./data', train=True, download=download,\n                                  transform=transforms.ToTensor())\n\n  train_loader = DataLoader(train_dataset, batch_size=batch_size,\n                            num_workers=0, drop_last=False, shuffle=shuffle)\n  \n  test_dataset = datasets.CIFAR10('./data', train=False, download=download,\n                                  transform=transforms.ToTensor())\n\n  test_loader = DataLoader(test_dataset, batch_size=2*batch_size,\n                            num_workers=10, drop_last=False, shuffle=shuffle)\n  return train_loader, test_loader","metadata":{"_uuid":"7ab4d855-4d2a-4507-90f2-f65aa40cf451","_cell_guid":"db158c75-4d79-426c-ad16-655f2efaebb3","collapsed":false,"id":"BfIPl0G6_RrT","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = pd.Series({\n          'device': torch.device('cuda'),\n          'workers': 64,\n          'seed': None,\n          'log_every_n_steps': 100,\n          'temperature': 0.07,\n          'n_views': 2,\n          'gpu_index': 0,\n          'fp16_precision': True,\n          'epochs': 200, #200,\n          'disable_cuda': True,\n          'batch_size': 256,\n          'weight_decay': 1e-4,\n          'lr': 3e-4,\n          'out_dim': 128,\n          'arch': 'resnet18',\n          'dataset_name': 'cifar10'\n    })","metadata":{"_uuid":"6223964a-fef1-42b9-9d05-bb0808ad7131","_cell_guid":"276acfac-1f51-4bfb-bc0b-b9c98e2d723e","collapsed":false,"id":"6N8lYkbmDTaK","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config.arch == 'resnet18':\n  model = torchvision.models.resnet18(pretrained=False, num_classes=10).to(device)\nelif config.arch == 'resnet50':\n  model = torchvision.models.resnet50(pretrained=False, num_classes=10).to(device)","metadata":{"_uuid":"69039baf-b5b0-49ca-ab07-655210dbf9e3","_cell_guid":"0a8adf7b-ed3f-4f79-85a1-d9c383cd57aa","collapsed":false,"id":"a18lPD-tIle6","outputId":"150c57cb-8b19-4570-d4ce-e230c545e64d","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"state_dict = torch.load('model_parameters.pt', map_location=device)\n# state_dict = checkpoint['state_dict']\n\nfor k in list(state_dict.keys()):\n\n  if k.startswith('backbone.'):\n    if k.startswith('backbone') and not k.startswith('backbone.fc'):\n      # remove prefix\n      state_dict[k[len(\"backbone.\"):]] = state_dict[k]\n  del state_dict[k]","metadata":{"_uuid":"c585ceab-a54b-4e30-9a68-189b051f3919","_cell_guid":"4cbd3517-97b3-48a1-9bca-e92008332dcf","collapsed":false,"id":"4AIfgq41GuTT","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"log = model.load_state_dict(state_dict, strict=False)\nassert log.missing_keys == ['fc.weight', 'fc.bias']","metadata":{"_uuid":"93164a14-87eb-4cbd-a20e-251a31288b4f","_cell_guid":"79c7c737-b3d0-49fb-b154-f5320085e9a7","collapsed":false,"id":"VVjA83PPJYWl","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config.dataset_name == 'cifar10':\n  train_loader, test_loader = get_cifar10_data_loaders(download=True)\nelif config.dataset_name == 'stl10':\n  train_loader, test_loader = get_stl10_data_loaders(download=True)\nprint(\"Dataset:\", config.dataset_name)","metadata":{"_uuid":"196acf03-ff75-47f8-b2e0-dab685d948cd","_cell_guid":"958fc23e-6b99-4359-8b30-89cd8e34f282","collapsed":false,"id":"_GC0a14uWRr6","outputId":"c985d862-c189-4a12-be3b-485603579925","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# freeze all layers but the last fc\nfor name, param in model.named_parameters():\n    if name not in ['fc.weight', 'fc.bias']:\n        param.requires_grad = False\n\nparameters = list(filter(lambda p: p.requires_grad, model.parameters()))\nassert len(parameters) == 2  # fc.weight, fc.bias","metadata":{"_uuid":"97c0ed9b-3359-4f72-8298-a83550ac07b2","_cell_guid":"5c312b1c-b6c2-442c-9740-94d42266c691","collapsed":false,"id":"pYT_KsM0Mnnr","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = torch.optim.Adam(model.parameters(), lr=3e-4, weight_decay=0.0008)\ncriterion = torch.nn.CrossEntropyLoss().to(device)","metadata":{"_uuid":"e323f916-100c-428c-b082-c854b1e5ee04","_cell_guid":"54db4073-77dc-4c48-9d16-8c1f1f8ca3b9","collapsed":false,"id":"aPVh1S_eMRDU","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def accuracy(output, target, topk=(1,)):\n    \"\"\"Computes the accuracy over the k top predictions for the specified values of k\"\"\"\n    with torch.no_grad():\n        maxk = max(topk)\n        batch_size = target.size(0)\n\n        _, pred = output.topk(maxk, 1, True, True)\n        pred = pred.t()\n        correct = pred.eq(target.view(1, -1).expand_as(pred))\n\n        res = []\n        for k in topk:\n            correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True)\n            res.append(correct_k.mul_(100.0 / batch_size))\n        return res","metadata":{"_uuid":"99278dac-84f8-4ea9-b198-4a9bad3e2d20","_cell_guid":"76e82cc3-a735-4ed4-9afe-082eb769bd73","collapsed":false,"id":"edr6RhP2PdVq","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 100\nfor epoch in range(epochs):\n  top1_train_accuracy = 0\n  for counter, (x_batch, y_batch) in enumerate(train_loader):\n    x_batch = x_batch.to(device)\n    y_batch = y_batch.to(device)\n\n    logits = model(x_batch)\n    loss = criterion(logits, y_batch)\n    top1 = accuracy(logits, y_batch, topk=(1,))\n    top1_train_accuracy += top1[0]\n\n    optimizer.zero_grad()\n    loss.backward()\n    optimizer.step()\n\n  top1_train_accuracy /= (counter + 1)\n  top1_accuracy = 0\n  top5_accuracy = 0\n  for counter, (x_batch, y_batch) in enumerate(test_loader):\n    x_batch = x_batch.to(device)\n    y_batch = y_batch.to(device)\n\n    logits = model(x_batch)\n  \n    top1, top5 = accuracy(logits, y_batch, topk=(1,5))\n    top1_accuracy += top1[0]\n    top5_accuracy += top5[0]\n  \n  top1_accuracy /= (counter + 1)\n  top5_accuracy /= (counter + 1)\n  print(f\"Epoch {epoch}\\tTop1 Train accuracy {top1_train_accuracy.item()}\\tTop1 Test accuracy: {top1_accuracy.item()}\\tTop5 test acc: {top5_accuracy.item()}\")","metadata":{"_uuid":"1b0f6b10-62cc-451e-ac1b-7c946a36fa96","_cell_guid":"f7f08756-8d5f-41ab-b64a-f0996d6e57af","collapsed":false,"id":"qOder0dAMI7X","outputId":"b197db68-7789-44b5-ddc5-bdafa0a8d3f1","jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"1d48f7f3-10bb-4922-9b5f-4dccdf9d9c03","_cell_guid":"073cef54-0688-4cdf-ac7d-156bb5ba5824","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"a1559266-4fbc-40db-a6ac-265d3dcbe1d5","_cell_guid":"1034e4f2-1b68-4564-af90-74976e37fd8d","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}