{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-13T12:04:51.805715Z","iopub.execute_input":"2022-05-13T12:04:51.806394Z","iopub.status.idle":"2022-05-13T12:04:51.837803Z","shell.execute_reply.started":"2022-05-13T12:04:51.806250Z","shell.execute_reply":"2022-05-13T12:04:51.837017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image as pil_image\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-05-13T12:48:56.708468Z","iopub.execute_input":"2022-05-13T12:48:56.708833Z","iopub.status.idle":"2022-05-13T12:48:56.714707Z","shell.execute_reply.started":"2022-05-13T12:48:56.708796Z","shell.execute_reply":"2022-05-13T12:48:56.713418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import hashlib\nfrom tqdm import tqdm\nDATA_FOLDER = \"../input/hotelid-2022-train-images-256x256/\"\nIMAGE_FOLDER = DATA_FOLDER + \"images/\"\nOUTPUT_FOLDER = \"\"\n\ntrain_df = pd.read_csv(os.path.join(DATA_FOLDER, 'train.csv'))\ntrain_df['hash'] = np.zeros(44703)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-13T12:30:42.761953Z","iopub.execute_input":"2022-05-13T12:30:42.763051Z","iopub.status.idle":"2022-05-13T12:30:42.824030Z","shell.execute_reply.started":"2022-05-13T12:30:42.762985Z","shell.execute_reply":"2022-05-13T12:30:42.822882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import hashlib\n\nhashes = [];\nfor record in tqdm(train_df['image_id']):\n    with open(IMAGE_FOLDER+record, \"rb\") as f:\n        hash = hashlib.sha256(f.read()).hexdigest()\n    hashes.append(hash)\ntrain_df['hash'] = hashes","metadata":{"execution":{"iopub.status.busy":"2022-05-13T12:30:45.491255Z","iopub.execute_input":"2022-05-13T12:30:45.491886Z","iopub.status.idle":"2022-05-13T12:31:14.249133Z","shell.execute_reply.started":"2022-05-13T12:30:45.491848Z","shell.execute_reply":"2022-05-13T12:31:14.248190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"items = len(train_df['image_id'])\nuniques = train_df['hash'].nunique()\ntrain_df = train_df.reset_index()\ngrouped_hash = train_df.groupby('hash')\n","metadata":{"execution":{"iopub.status.busy":"2022-05-13T12:32:56.131529Z","iopub.execute_input":"2022-05-13T12:32:56.132471Z","iopub.status.idle":"2022-05-13T12:32:56.167389Z","shell.execute_reply.started":"2022-05-13T12:32:56.132427Z","shell.execute_reply":"2022-05-13T12:32:56.166286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_hash_count = grouped_hash['hash'].count()\ndups = grouped_hash_count[grouped_hash_count > 1]","metadata":{"execution":{"iopub.status.busy":"2022-05-13T13:03:53.281069Z","iopub.execute_input":"2022-05-13T13:03:53.282292Z","iopub.status.idle":"2022-05-13T13:03:53.294355Z","shell.execute_reply.started":"2022-05-13T13:03:53.282242Z","shell.execute_reply":"2022-05-13T13:03:53.293325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(dups.head())\nduplicate_ids = train_df[train_df['hash'].isin(dups.index)]\nprint(duplicate_ids.sort_values(by=['hash','hotel_id']))\nprint(duplicate_ids.head())","metadata":{"execution":{"iopub.status.busy":"2022-05-13T13:04:50.469061Z","iopub.execute_input":"2022-05-13T13:04:50.469413Z","iopub.status.idle":"2022-05-13T13:04:50.495914Z","shell.execute_reply.started":"2022-05-13T13:04:50.469381Z","shell.execute_reply":"2022-05-13T13:04:50.494857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_img1_id = duplicate_ids['image_id'].iloc[0]\ntest_img1_hotel = duplicate_ids['hotel_id'].iloc[0]\nprint(test_img1_hotel)\ntest_img2_id = duplicate_ids['image_id'].iloc[1]\ntest_img2_hotel = duplicate_ids['hotel_id'].iloc[1]\nprint(test_img2_hotel)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T13:05:26.893956Z","iopub.execute_input":"2022-05-13T13:05:26.894932Z","iopub.status.idle":"2022-05-13T13:05:26.901937Z","shell.execute_reply.started":"2022-05-13T13:05:26.894882Z","shell.execute_reply":"2022-05-13T13:05:26.900924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_image = np.array(pil_image.open('../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/train_images/'+str(test_img1_hotel)+'/'+str(test_img1_id))).astype(np.uint8)\nplt.figure(figsize=(6,6))\nplt.imshow(test_image)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T13:05:27.632921Z","iopub.execute_input":"2022-05-13T13:05:27.633266Z","iopub.status.idle":"2022-05-13T13:05:27.993767Z","shell.execute_reply.started":"2022-05-13T13:05:27.633234Z","shell.execute_reply":"2022-05-13T13:05:27.992820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_image = np.array(pil_image.open('../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/train_images/'+str(test_img2_hotel)+'/'+str(test_img2_id))).astype(np.uint8)\nplt.figure(figsize=(6,6))\nplt.imshow(test_image)","metadata":{"execution":{"iopub.status.busy":"2022-05-13T13:05:28.347557Z","iopub.execute_input":"2022-05-13T13:05:28.347900Z","iopub.status.idle":"2022-05-13T13:05:28.701750Z","shell.execute_reply.started":"2022-05-13T13:05:28.347864Z","shell.execute_reply":"2022-05-13T13:05:28.701107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_dups = items - uniques\nprint(n_dups)\nprint(duplicate_ids[\"hash\"].nunique())\nprint(duplicate_ids.head())","metadata":{"execution":{"iopub.status.busy":"2022-05-13T13:12:08.788683Z","iopub.execute_input":"2022-05-13T13:12:08.788991Z","iopub.status.idle":"2022-05-13T13:12:08.801642Z","shell.execute_reply.started":"2022-05-13T13:12:08.788962Z","shell.execute_reply":"2022-05-13T13:12:08.800606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicates_sorted_by_hash = duplicate_ids.sort_values(by=['hash']).groupby('hash')\nprint(duplicates_sorted_by_hash.head()) ","metadata":{"execution":{"iopub.status.busy":"2022-05-13T13:18:04.105345Z","iopub.execute_input":"2022-05-13T13:18:04.106203Z","iopub.status.idle":"2022-05-13T13:18:04.123063Z","shell.execute_reply.started":"2022-05-13T13:18:04.106155Z","shell.execute_reply":"2022-05-13T13:18:04.122238Z"},"trusted":true},"execution_count":null,"outputs":[]}]}