{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# II. PREPROCESSING\n- identify and remove duplicate images from train & test sets\n- convert remaining images to smaller dimensions (256 & 512 versions) for quicker training.\n- save the images in jpg format for feeding into experiments.\n\n# Summary of Findings\n- 181 pairs of duplicate images exist.\n- 74 groups containing the same image exist.\n- The bounding boxes for the same or similar image are not consistent.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"!conda install gdcm -c conda-forge -y","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-10T12:54:00.546343Z","iopub.execute_input":"2021-08-10T12:54:00.547219Z","iopub.status.idle":"2021-08-10T12:55:04.603278Z","shell.execute_reply.started":"2021-08-10T12:54:00.547019Z","shell.execute_reply":"2021-08-10T12:55:04.602161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport datetime\nimport gc\nimport glob\nimport imagehash\nimport math\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nimport PIL\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport sys\nimport tqdm","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-10T12:55:15.288466Z","iopub.execute_input":"2021-08-10T12:55:15.288812Z","iopub.status.idle":"2021-08-10T12:55:15.790021Z","shell.execute_reply.started":"2021-08-10T12:55:15.288777Z","shell.execute_reply":"2021-08-10T12:55:15.788793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Parameters","metadata":{}},{"cell_type":"code","source":"base_path = '../input/siim-covid19-detection'","metadata":{"execution":{"iopub.status.busy":"2021-08-10T12:55:24.876318Z","iopub.execute_input":"2021-08-10T12:55:24.876696Z","iopub.status.idle":"2021-08-10T12:55:24.880948Z","shell.execute_reply.started":"2021-08-10T12:55:24.876659Z","shell.execute_reply":"2021-08-10T12:55:24.879811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{}},{"cell_type":"code","source":"def read_dicom_image(image_file, voi_lut=True, fix_monochrome=True):\n    \"\"\"\n    Reads a dicom image from a file an returns a numpy array.\n    References: https://www.kaggle.com/trungthanhnguyen0502/eda-vinbigdata-chest-x-ray-abnormalities\n    Args:\n        image_file:\n        voi_lut:\n        fix_monochrome:\n\n    Returns:\n\n    \"\"\"\n    dicom = pydicom.read_file(image_file)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data\n\n\ndef string2boxes(string):\n    strings = string.split()\n    if strings[0].lower() == 'none':\n        return []\n    else:\n        return [{'class': strings[idx],\n                 'conf': float(strings[idx+1]),\n                 'x1': float(strings[idx+2]),\n                 'y1': float(strings[idx+3]),\n                 'x2': float(strings[idx+4]),\n                 'y2': float(strings[idx+5]),\n                 } for idx in range(0, len(strings), 6)]\n\n\ndef plot_image(image, boxes=None, size=(5,5), title=None, columns=4):\n    def plot_img(image, boxes=None, title=None):\n        if isinstance(image, str):\n            image_id = os.path.splitext(os.path.split(image)[1])[0]\n            df = df_image.loc[df_image['id'] == image_id + '_image']\n            boxes = string2boxes(df['label'].iloc[0]) if len(df) > 0 else None\n            image = read_dicom_image(image)\n        image = np.stack([image] * 3, axis=-1)\n        if boxes is not None:\n            for box in boxes:\n                image = cv2.rectangle(image, (int(box['x1']), int(box['y1'])), (int(box['x2']), int(box['y2'])), [0, 255, 0], 10)\n        plt.axis('on')\n        plt.imshow(image, cmap='gray')\n        if title is not None:\n            plt.title(title)\n\n    plt.figure(figsize=size)\n    if isinstance(image, list):\n        num = len(image)\n        columns = min(columns, num)\n        rows = math.ceil(num / columns)\n\n        for index, single_image in enumerate(image):\n            plt.subplot(rows, columns, index + 1)\n            plot_img(single_image, boxes=boxes, title=title[index])\n    else:\n        plot_img(image, boxes=boxes, title=title)\n    plt.show()\n\n\ndef images_find_duplicates(image_files, threshold=0.9):\n    \"\"\"\n    Function to find duplicates in images.\n    References: https://www.kaggle.com/appian/let-s-find-out-duplicate-images-with-imagehash\n    Args:\n        image_files:\n        threshold:\n\n    Returns:\n\n    \"\"\"\n    funcs = [imagehash.average_hash, imagehash.phash, imagehash.dhash, imagehash.whash]\n    image_ids = image_files\n    hashes = []\n    for file in tqdm.tqdm(image_files):\n        image = PIL.Image.fromarray(read_dicom_image(file))\n        hashes.append(np.array([f(image).hash for f in funcs]).reshape(256))\n    hashes_all = np.array(hashes)\n\n    # Comparisons without Pytorch\n    sim_list = []\n    for i in tqdm.tqdm(range(hashes_all.shape[0])):\n        sim_list.append(np.sum(hashes_all[i] == hashes_all, axis=1)/256)\n\n    # nxn-matrix of similarities (n = # of images), upper triangular matrix\n    similarities = np.triu(np.array(sim_list), 1)\n\n    idx_pair = np.where(similarities > threshold)\n    df_pairs = pd.DataFrame({'image1': [image_ids[i] for i in list(idx_pair[0])],\n                             'image2': [image_ids[i] for i in list(idx_pair[1])],\n                             'similarity': [similarities[i1, i2] for i1, i2 in zip(idx_pair[0], idx_pair[1])]})\n\n    idx_group = np.zeros(len(image_files))\n    group_id = 1\n    for i1, i2 in zip(idx_pair[0], idx_pair[1]):\n        if idx_group[i1] == 0 and idx_group[i2] == 0:\n            idx_group[i1] = group_id\n            idx_group[i2] = group_id\n            group_id += 1\n        elif idx_group[i1] != 0 and idx_group[i2] == 0:\n            idx_group[i2] = idx_group[i1]\n        elif idx_group[i1] == 0 and idx_group[i2] != 0:\n            idx_group[i1] = idx_group[i2]\n        elif idx_group[i1] != 0 and idx_group[i2] != 0 and idx_group[i1] != idx_group[i2]:\n            common_id = min(idx_group[i1], idx_group[i2])\n            idx_group[idx_group == idx_group[i1]] = common_id\n            idx_group[idx_group == idx_group[i2]] = common_id\n\n    group_list = []\n    for i in range(1, group_id + 1):\n        group_ids = list(np.where(idx_group == i)[0])\n        if len(group_ids) > 0:\n            group_list.append([image_ids[j] for j in group_ids])\n\n    return df_pairs, group_list\n\n\ndef print_group_info(i, df_group):\n    print(f'\\nGroup {i+1}')\n    print(f'Number of unique studies:       {len(df_group[\"study_id\"].unique())}')\n    print(f'Number of unique study labels:  {len(df_group[\"study_label\"].unique())}\\n')\n    print(df_group[['image_id', 'num_boxes', 'study_id', 'study_label']])\n    plot_image(list(df_group['image_file']), size=(20, 10), title=list(df_group['image_id']), columns=8)    \n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-10T12:55:28.821007Z","iopub.execute_input":"2021-08-10T12:55:28.821328Z","iopub.status.idle":"2021-08-10T12:55:28.849546Z","shell.execute_reply.started":"2021-08-10T12:55:28.821298Z","shell.execute_reply":"2021-08-10T12:55:28.848446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Find Duplicates\nRead files and search for duplicates.","metadata":{}},{"cell_type":"code","source":"train_files = sorted(glob.glob(os.path.join(base_path, 'train/*/*/*.dcm')))\nprint(f'Number of training files: {len(train_files)}')\ndf_image = pd.read_csv(os.path.join(base_path, 'train_image_level.csv'))\ndf_study = pd.read_csv(os.path.join(base_path, 'train_study_level.csv'))\ndf_study['study_label'] = df_study.apply(lambda r: ', '.join([df_study.columns[i] for i in range(1, 5) if r[i] > 0]), axis=1)\n\ndf_pairs, group_list = images_find_duplicates(train_files, threshold=0.95)\nprint(f'\\nNumber of duplicate pairs: {len(df_pairs)}')\nprint(f'Number of duplicate groups: {len(group_list)}')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-10T12:55:37.170290Z","iopub.execute_input":"2021-08-10T12:55:37.170637Z","iopub.status.idle":"2021-08-10T15:00:56.150781Z","shell.execute_reply.started":"2021-08-10T12:55:37.170590Z","shell.execute_reply":"2021-08-10T15:00:56.149730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_group_list = []\ndf_pairs.to_csv('pairs.csv')\n\nwith open('duplicates.csv', 'w') as text_file:\n    for i, group in enumerate(group_list):\n        group_ids = [os.path.splitext(os.path.basename(file))[0] + '_image' for file in group]\n        df_group_ids = pd.DataFrame({'id': group_ids, 'image_file': group})\n        df_group = df_group_ids.merge(df_image, on='id').sort_values('id')\n        df_group['study_id'] = df_group['StudyInstanceUID'] + '_study'\n        df_group['num_boxes'] = df_group.apply(lambda r: len(string2boxes(r['label'])), axis=1)\n        df_group = df_group.merge(df_study, left_on='study_id', right_on='id')\n        df_group = df_group.rename(columns={'id_x': 'image_id'})\n        df_group_list.append(df_group)\n        text_file.write(','.join(group_ids) + '\\n')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-10T15:07:10.152703Z","iopub.execute_input":"2021-08-10T15:07:10.153196Z","iopub.status.idle":"2021-08-10T15:07:10.916944Z","shell.execute_reply.started":"2021-08-10T15:07:10.153164Z","shell.execute_reply":"2021-08-10T15:07:10.916294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Duplicate Images with Bounding Boxes in More than 1 Image","metadata":{}},{"cell_type":"code","source":"for index, df_group in enumerate(df_group_list):\n    if len(df_group[df_group['num_boxes'] > 0]) > 1:\n        print_group_info(index, df_group)","metadata":{"execution":{"iopub.status.busy":"2021-08-10T15:07:54.214708Z","iopub.execute_input":"2021-08-10T15:07:54.215009Z","iopub.status.idle":"2021-08-10T15:07:57.101092Z","shell.execute_reply.started":"2021-08-10T15:07:54.214982Z","shell.execute_reply":"2021-08-10T15:07:57.100276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Duplicate Images with More Than 1 Study Label","metadata":{}},{"cell_type":"code","source":"for index, df_group in enumerate(df_group_list):\n    if len(df_group['study_label'].unique()) > 1:\n        print_group_info(index, df_group)","metadata":{"execution":{"iopub.status.busy":"2021-08-10T15:08:03.276634Z","iopub.execute_input":"2021-08-10T15:08:03.276984Z","iopub.status.idle":"2021-08-10T15:08:05.032123Z","shell.execute_reply.started":"2021-08-10T15:08:03.276951Z","shell.execute_reply":"2021-08-10T15:08:05.031386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Duplicate Images from More than 1 Study","metadata":{}},{"cell_type":"code","source":"with open('duplicates_study.csv', 'w') as text_file:\n    for index, df_group in enumerate(df_group_list):\n        if len(df_group['study_id'].unique()) > 1:\n            print_group_info(index, df_group)\n            text_file.write(','.join(list(df_group['study_id'].unique())) + '\\n')","metadata":{"execution":{"iopub.status.busy":"2021-08-10T15:08:11.722089Z","iopub.execute_input":"2021-08-10T15:08:11.722416Z","iopub.status.idle":"2021-08-10T15:08:24.279581Z","shell.execute_reply.started":"2021-08-10T15:08:11.722386Z","shell.execute_reply":"2021-08-10T15:08:24.278764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Full List of Duplicate Images","metadata":{}},{"cell_type":"code","source":"for index, df_group in enumerate(df_group_list):\n    print_group_info(index, df_group)","metadata":{"execution":{"iopub.status.busy":"2021-08-10T15:08:34.494745Z","iopub.execute_input":"2021-08-10T15:08:34.495079Z","iopub.status.idle":"2021-08-10T15:13:14.301097Z","shell.execute_reply.started":"2021-08-10T15:08:34.495048Z","shell.execute_reply":"2021-08-10T15:13:14.300255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get list of duplicate images for dropping\nimages_to_drop=set()\nimages_to_keep=set()\nfor index1, df_group in enumerate(df_group_list):\n    for index2, row in df_group.iterrows():\n        if(index2==0):\n            images_to_keep.add(row['image_id'])\n            continue\n        images_to_drop.add(row['image_id'])\n\n#print(\"drop: \", len(images_to_drop))\n#print(\"keep: \", len(images_to_keep))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ndups_df = pd.read_csv('../input/duplicates-1/duplicates_1.csv')\ndups_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2021-08-10T17:10:55.264133Z","iopub.execute_input":"2021-08-10T17:10:55.264449Z","iopub.status.idle":"2021-08-10T17:10:55.294579Z","shell.execute_reply.started":"2021-08-10T17:10:55.264423Z","shell.execute_reply":"2021-08-10T17:10:55.293733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_to_drop","metadata":{"execution":{"iopub.status.busy":"2021-08-10T16:10:50.414341Z","iopub.execute_input":"2021-08-10T16:10:50.414694Z","iopub.status.idle":"2021-08-10T16:10:50.421719Z","shell.execute_reply.started":"2021-08-10T16:10:50.414663Z","shell.execute_reply":"2021-08-10T16:10:50.420663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}