{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n\ndf = pd.read_csv('/kaggle/input/data-vc-2023-maybe-final/final_annot.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-05T16:00:30.430132Z","iopub.execute_input":"2023-07-05T16:00:30.430533Z","iopub.status.idle":"2023-07-05T16:00:32.08742Z","shell.execute_reply.started":"2023-07-05T16:00:30.430501Z","shell.execute_reply":"2023-07-05T16:00:32.086649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!lscpu | grep -vE \"Vulnerability\"","metadata":{"execution":{"iopub.status.busy":"2023-07-05T16:22:31.012067Z","iopub.execute_input":"2023-07-05T16:22:31.012855Z","iopub.status.idle":"2023-07-05T16:22:32.069438Z","shell.execute_reply.started":"2023-07-05T16:22:31.012815Z","shell.execute_reply":"2023-07-05T16:22:32.068028Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade gdown","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:41:10.402608Z","iopub.execute_input":"2023-07-06T01:41:10.40371Z","iopub.status.idle":"2023-07-06T01:41:25.648411Z","shell.execute_reply.started":"2023-07-06T01:41:10.403673Z","shell.execute_reply":"2023-07-06T01:41:25.647038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!gdown --fuzzy https://drive.google.com/file/d/10dqY0FGB2E7p3kc_6Pcqr5kWesv6kAfJ/view","metadata":{"execution":{"iopub.status.busy":"2023-07-05T16:27:08.120457Z","iopub.execute_input":"2023-07-05T16:27:08.120883Z","iopub.status.idle":"2023-07-05T16:27:12.489505Z","shell.execute_reply.started":"2023-07-05T16:27:08.12084Z","shell.execute_reply":"2023-07-05T16:27:12.488018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_71k = pd.read_csv('/kaggle/working/71k_img_pairs_2.csv', names=['i1', 'i2'])","metadata":{"execution":{"iopub.status.busy":"2023-07-05T16:29:30.225278Z","iopub.execute_input":"2023-07-05T16:29:30.225785Z","iopub.status.idle":"2023-07-05T16:29:30.358156Z","shell.execute_reply.started":"2023-07-05T16:29:30.225743Z","shell.execute_reply":"2023-07-05T16:29:30.356985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_71k)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T16:32:06.018216Z","iopub.execute_input":"2023-07-05T16:32:06.018624Z","iopub.status.idle":"2023-07-05T16:32:06.026005Z","shell.execute_reply.started":"2023-07-05T16:32:06.018594Z","shell.execute_reply":"2023-07-05T16:32:06.024828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set_ = set(df_71k['i1'].tolist() + df_71k['i2'].tolist())","metadata":{"execution":{"iopub.status.busy":"2023-07-05T16:31:22.441962Z","iopub.execute_input":"2023-07-05T16:31:22.44237Z","iopub.status.idle":"2023-07-05T16:31:22.468621Z","shell.execute_reply.started":"2023-07-05T16:31:22.442336Z","shell.execute_reply":"2023-07-05T16:31:22.467684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_71k['same'] = (df_71k['i1'] == df_71k['i2'])","metadata":{"execution":{"iopub.status.busy":"2023-07-05T16:35:49.86585Z","iopub.execute_input":"2023-07-05T16:35:49.866253Z","iopub.status.idle":"2023-07-05T16:35:49.882855Z","shell.execute_reply.started":"2023-07-05T16:35:49.866221Z","shell.execute_reply":"2023-07-05T16:35:49.881698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_71k[df_71k['same']]","metadata":{"execution":{"iopub.status.busy":"2023-07-05T16:53:05.476775Z","iopub.execute_input":"2023-07-05T16:53:05.477163Z","iopub.status.idle":"2023-07-05T16:53:05.495733Z","shell.execute_reply.started":"2023-07-05T16:53:05.477133Z","shell.execute_reply":"2023-07-05T16:53:05.494602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set_)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T16:31:34.805002Z","iopub.execute_input":"2023-07-05T16:31:34.805423Z","iopub.status.idle":"2023-07-05T16:31:34.812725Z","shell.execute_reply.started":"2023-07-05T16:31:34.805387Z","shell.execute_reply":"2023-07-05T16:31:34.811472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!gdown --fuzzy https://drive.google.com/file/d/1xTqCQ7r_pW4LiQA_9scJYJBs2Gze_jUe/view?usp=sharing","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:41:25.650866Z","iopub.execute_input":"2023-07-06T01:41:25.651227Z","iopub.status.idle":"2023-07-06T01:41:30.874511Z","shell.execute_reply.started":"2023-07-06T01:41:25.651195Z","shell.execute_reply":"2023-07-06T01:41:30.873023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ndf_70k = pd.read_csv('pairs (1).txt', names=['i1', 'i2'])","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:41:38.144501Z","iopub.execute_input":"2023-07-06T01:41:38.145009Z","iopub.status.idle":"2023-07-06T01:41:38.287102Z","shell.execute_reply.started":"2023-07-06T01:41:38.144969Z","shell.execute_reply":"2023-07-06T01:41:38.286016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set__ = set(df_70k['i1'].tolist() + df_70k['i2'].tolist())","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:41:39.287589Z","iopub.execute_input":"2023-07-06T01:41:39.287999Z","iopub.status.idle":"2023-07-06T01:41:39.337831Z","shell.execute_reply.started":"2023-07-06T01:41:39.287965Z","shell.execute_reply":"2023-07-06T01:41:39.336843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_70k['same'] = (df_70k['i1'] == df_70k['i2'])","metadata":{"execution":{"iopub.status.busy":"2023-07-05T16:52:26.258019Z","iopub.execute_input":"2023-07-05T16:52:26.258519Z","iopub.status.idle":"2023-07-05T16:52:26.281735Z","shell.execute_reply.started":"2023-07-05T16:52:26.258481Z","shell.execute_reply":"2023-07-05T16:52:26.280893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_70k['same'].sum()","metadata":{"execution":{"iopub.status.busy":"2023-07-05T16:52:43.768785Z","iopub.execute_input":"2023-07-05T16:52:43.76932Z","iopub.status.idle":"2023-07-05T16:52:43.776365Z","shell.execute_reply.started":"2023-07-05T16:52:43.769289Z","shell.execute_reply":"2023-07-05T16:52:43.775487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_70k","metadata":{"execution":{"iopub.status.busy":"2023-07-05T16:55:57.657781Z","iopub.execute_input":"2023-07-05T16:55:57.658207Z","iopub.status.idle":"2023-07-05T16:55:57.67155Z","shell.execute_reply.started":"2023-07-05T16:55:57.658176Z","shell.execute_reply":"2023-07-05T16:55:57.670626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/data-vc-2023-maybe-final/final_annot.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:41:43.326821Z","iopub.execute_input":"2023-07-06T01:41:43.327198Z","iopub.status.idle":"2023-07-06T01:41:44.961529Z","shell.execute_reply.started":"2023-07-06T01:41:43.327168Z","shell.execute_reply":"2023-07-06T01:41:44.960499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set___ = set([df[f'ref{i+1}'].tolist() for i in range(20)])","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:41:44.963254Z","iopub.execute_input":"2023-07-06T01:41:44.963591Z","iopub.status.idle":"2023-07-06T01:41:45.032311Z","shell.execute_reply.started":"2023-07-06T01:41:44.963563Z","shell.execute_reply":"2023-07-06T01:41:45.030659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_ref = []\nfor i in range(20):\n    full_ref.extend(df[f'ref{i+1}'].tolist())","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:42:08.715029Z","iopub.execute_input":"2023-07-06T01:42:08.715402Z","iopub.status.idle":"2023-07-06T01:42:08.747412Z","shell.execute_reply.started":"2023-07-06T01:42:08.715374Z","shell.execute_reply":"2023-07-06T01:42:08.746199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(full_ref)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:42:15.439992Z","iopub.execute_input":"2023-07-06T01:42:15.440379Z","iopub.status.idle":"2023-07-06T01:42:15.450556Z","shell.execute_reply.started":"2023-07-06T01:42:15.44035Z","shell.execute_reply":"2023-07-06T01:42:15.449427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set___ = set(full_ref+df_70k['i1'].tolist() + df_70k['i2'].tolist())","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:42:17.509033Z","iopub.execute_input":"2023-07-06T01:42:17.509445Z","iopub.status.idle":"2023-07-06T01:42:17.663753Z","shell.execute_reply.started":"2023-07-06T01:42:17.509411Z","shell.execute_reply":"2023-07-06T01:42:17.662459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set___)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:42:22.975446Z","iopub.execute_input":"2023-07-06T01:42:22.97585Z","iopub.status.idle":"2023-07-06T01:42:22.983631Z","shell.execute_reply.started":"2023-07-06T01:42:22.975818Z","shell.execute_reply":"2023-07-06T01:42:22.98284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir imgnet","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:42:27.415979Z","iopub.execute_input":"2023-07-06T01:42:27.416401Z","iopub.status.idle":"2023-07-06T01:42:28.506933Z","shell.execute_reply.started":"2023-07-06T01:42:27.416369Z","shell.execute_reply":"2023-07-06T01:42:28.505036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nimport os\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed\n\n\ndef move_file(x):\n#     cls_name = x.split('/')[0]\n#     os.makedirs('imgnet/'+cls_name, exist_ok=True)\n    ff = '/kaggle/input/imagenet-object-localization-challenge/ILSVRC/Data/CLS-LOC/train/'+x\n    !zip -g xxx.zip $ff\n#     shutil.copy('/kaggle/input/imagenet-object-localization-challenge/ILSVRC/Data/CLS-LOC/train/'+x, \n#                'imgnet/'+x)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T17:57:36.556108Z","iopub.execute_input":"2023-07-05T17:57:36.556604Z","iopub.status.idle":"2023-07-05T17:57:36.564803Z","shell.execute_reply.started":"2023-07-05T17:57:36.556539Z","shell.execute_reply":"2023-07-05T17:57:36.563263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ll = list(set___)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:42:47.846025Z","iopub.execute_input":"2023-07-06T01:42:47.847037Z","iopub.status.idle":"2023-07-06T01:42:47.879133Z","shell.execute_reply.started":"2023-07-06T01:42:47.847003Z","shell.execute_reply":"2023-07-06T01:42:47.877822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in tqdm(range(0, len(ll), 1000)):\n    l__ = ll[i:min(i+1000, len(ll))]\n    ff = ' '.join(['/kaggle/input/imagenet-object-localization-challenge/ILSVRC/Data/CLS-LOC/train/'+x for x in l__])\n    !zip -q imgnet.zip $ff\n#Parallel(n_jobs=-1)(delayed(move_file)(x) for x in tqdm(set___))","metadata":{"execution":{"iopub.status.busy":"2023-07-05T18:07:42.789191Z","iopub.execute_input":"2023-07-05T18:07:42.789658Z","iopub.status.idle":"2023-07-05T18:21:56.610647Z","shell.execute_reply.started":"2023-07-05T18:07:42.789612Z","shell.execute_reply":"2023-07-05T18:21:56.608826Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(2443883/1024/1024)","metadata":{"execution":{"iopub.status.busy":"2023-07-05T17:14:09.891943Z","iopub.execute_input":"2023-07-05T17:14:09.892308Z","iopub.status.idle":"2023-07-05T17:14:09.899523Z","shell.execute_reply.started":"2023-07-05T17:14:09.89228Z","shell.execute_reply":"2023-07-05T17:14:09.898381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf imgnet","metadata":{"execution":{"iopub.status.busy":"2023-07-05T18:05:21.267856Z","iopub.execute_input":"2023-07-05T18:05:21.270601Z","iopub.status.idle":"2023-07-05T18:05:23.262971Z","shell.execute_reply.started":"2023-07-05T18:05:21.270405Z","shell.execute_reply":"2023-07-05T18:05:23.261183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm imgnet.zip","metadata":{"execution":{"iopub.status.busy":"2023-07-05T18:22:05.802491Z","iopub.execute_input":"2023-07-05T18:22:05.803106Z","iopub.status.idle":"2023-07-05T18:22:06.941216Z","shell.execute_reply.started":"2023-07-05T18:22:05.803044Z","shell.execute_reply":"2023-07-05T18:22:06.939762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm /kaggle/working/ziPeHZDa","metadata":{"execution":{"iopub.status.busy":"2023-07-05T18:06:47.573773Z","iopub.execute_input":"2023-07-05T18:06:47.57421Z","iopub.status.idle":"2023-07-05T18:06:48.634819Z","shell.execute_reply.started":"2023-07-05T18:06:47.57417Z","shell.execute_reply":"2023-07-05T18:06:48.633081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm /kaggle/working/ziHbkjZr","metadata":{"execution":{"iopub.status.busy":"2023-07-05T18:07:19.624726Z","iopub.execute_input":"2023-07-05T18:07:19.625228Z","iopub.status.idle":"2023-07-05T18:07:20.66821Z","shell.execute_reply.started":"2023-07-05T18:07:19.625182Z","shell.execute_reply":"2023-07-05T18:07:20.666775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm /kaggle/working/ziLLVZt7","metadata":{"execution":{"iopub.status.busy":"2023-07-05T18:07:25.391054Z","iopub.execute_input":"2023-07-05T18:07:25.391482Z","iopub.status.idle":"2023-07-05T18:07:26.454013Z","shell.execute_reply.started":"2023-07-05T18:07:25.391442Z","shell.execute_reply":"2023-07-05T18:07:26.451909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm /kaggle/working/zi1XmbUc","metadata":{"execution":{"iopub.status.busy":"2023-07-05T18:22:18.32224Z","iopub.execute_input":"2023-07-05T18:22:18.322714Z","iopub.status.idle":"2023-07-05T18:22:19.398168Z","shell.execute_reply.started":"2023-07-05T18:22:18.322669Z","shell.execute_reply":"2023-07-05T18:22:19.3967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile\nimport os\nfrom tqdm.autonotebook import tqdm\n\ndef zip_paths(paths_list, zip_filename):\n    \"\"\"\n    Compress a list of paths into a zip archive\n    \n    Args:\n    paths_list (list): A list of paths to be compressed\n    zip_filename (str): The name of the zip archive to be created\n    \n    Returns:\n    None\n    \"\"\"\n    with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as zip_file:\n        for path in tqdm(paths_list, leave=True):\n            # Get the filename from the full path\n            cls_name = path.split('/')[-2]\n            filename = os.path.basename(path)\n            # Add the file to the zip archive\n            zip_file.write('/kaggle/input/imagenet-object-localization-challenge/ILSVRC/Data/CLS-LOC/train/'+path, \n                           cls_name+\"/\"+filename)\n\n        #print(f'Zip archive created: {zip_filename}')\n\n# Example usage\n#paths = ['/path/to/file1.txt', '/path/to/file2.txt', '/path/to/file3.txt']\nzip_filename = 'imgnet.zip'\nzip_paths(ll, zip_filename)","metadata":{"execution":{"iopub.status.busy":"2023-07-06T01:42:53.832147Z","iopub.execute_input":"2023-07-06T01:42:53.832552Z","iopub.status.idle":"2023-07-06T02:32:24.815283Z","shell.execute_reply.started":"2023-07-06T01:42:53.832521Z","shell.execute_reply":"2023-07-06T02:32:24.813674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf imgnet/","metadata":{"execution":{"iopub.status.busy":"2023-07-06T02:32:34.654474Z","iopub.execute_input":"2023-07-06T02:32:34.654992Z","iopub.status.idle":"2023-07-06T02:32:35.775799Z","shell.execute_reply.started":"2023-07-06T02:32:34.654955Z","shell.execute_reply":"2023-07-06T02:32:35.774146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}