{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# De-duplicating labels\n\nAs noted by [@ptran1203](https://www.kaggle.com/ptran1203) in this [discussion](https://www.kaggle.com/competitions/hubmap-hacking-the-human-vasculature/discussion/416380), there are some duplicated annotations in the provided `polygons.jsonl` annotations file.  \n\nThis notebook deduplicates those labels. \n\nIt turns out that there are 822 duplicate annotations out of a total of 17518, so about 6.5% of the labels are duplicated. ","metadata":{}},{"cell_type":"code","source":"import json","metadata":{"execution":{"iopub.status.busy":"2023-06-11T15:03:45.171388Z","iopub.execute_input":"2023-06-11T15:03:45.171999Z","iopub.status.idle":"2023-06-11T15:03:45.177589Z","shell.execute_reply.started":"2023-06-11T15:03:45.171963Z","shell.execute_reply":"2023-06-11T15:03:45.176298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read .jsonl file and convert it to a list of dicts\n# The dicts contain IDs, class names and segmentation masks\n# from https://www.kaggle.com/code/leonidkulyk/eda-hubmap-hhv-interactive-annotations\nwith open('/kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl', 'r') as json_file:\n    json_list = list(json_file)\n    \ntiles_dicts = []\nfor json_str in json_list:\n    tiles_dicts.append(json.loads(json_str))","metadata":{"execution":{"iopub.status.busy":"2023-06-11T15:03:56.338832Z","iopub.execute_input":"2023-06-11T15:03:56.339239Z","iopub.status.idle":"2023-06-11T15:03:59.657143Z","shell.execute_reply.started":"2023-06-11T15:03:56.339193Z","shell.execute_reply":"2023-06-11T15:03:59.656213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_dicts = []\nnum_dupes = 0\ntotal_annotations = 0\nfor tile in tiles_dicts: \n    unique_annotations = []\n    for annotation in tile['annotations']:\n        total_annotations += 1\n        if annotation not in unique_annotations:\n            unique_annotations.append(annotation)\n        else:\n            num_dupes += 1\n            \n    \n    \n    cleaned_dicts.append({'id': tile['id'], 'annotations': unique_annotations})\n\nprint(f'Number of duplicates removed: {num_dupes}')\nprint(f'Total annotations: {total_annotations}')","metadata":{"execution":{"iopub.status.busy":"2023-06-11T15:03:59.659429Z","iopub.execute_input":"2023-06-11T15:03:59.660788Z","iopub.status.idle":"2023-06-11T15:03:59.844849Z","shell.execute_reply.started":"2023-06-11T15:03:59.660742Z","shell.execute_reply":"2023-06-11T15:03:59.843784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(tiles_dicts)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T14:53:31.659882Z","iopub.execute_input":"2023-06-11T14:53:31.660205Z","iopub.status.idle":"2023-06-11T14:53:31.666324Z","shell.execute_reply.started":"2023-06-11T14:53:31.660179Z","shell.execute_reply":"2023-06-11T14:53:31.665128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(cleaned_dicts)","metadata":{"execution":{"iopub.status.busy":"2023-06-11T14:53:31.667905Z","iopub.execute_input":"2023-06-11T14:53:31.668199Z","iopub.status.idle":"2023-06-11T14:53:31.679843Z","shell.execute_reply.started":"2023-06-11T14:53:31.668174Z","shell.execute_reply":"2023-06-11T14:53:31.678828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('cleaned_polygons.jsonl', 'w') as jsonl_file:\n    \n    for c_dict in cleaned_dicts:\n        json.dump(c_dict, jsonl_file, separators=(',', ':'))\n        jsonl_file.write('\\n')","metadata":{"execution":{"iopub.status.busy":"2023-06-11T14:56:35.346944Z","iopub.execute_input":"2023-06-11T14:56:35.347408Z","iopub.status.idle":"2023-06-11T14:56:47.418090Z","shell.execute_reply.started":"2023-06-11T14:56:35.347368Z","shell.execute_reply":"2023-06-11T14:56:47.417031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -lh /kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl","metadata":{"execution":{"iopub.status.busy":"2023-06-11T14:56:47.419975Z","iopub.execute_input":"2023-06-11T14:56:47.420335Z","iopub.status.idle":"2023-06-11T14:56:48.436357Z","shell.execute_reply.started":"2023-06-11T14:56:47.420305Z","shell.execute_reply":"2023-06-11T14:56:48.434916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -lh cleaned_polygons.jsonl","metadata":{"execution":{"iopub.status.busy":"2023-06-11T14:56:48.438172Z","iopub.execute_input":"2023-06-11T14:56:48.438630Z","iopub.status.idle":"2023-06-11T14:56:49.448355Z","shell.execute_reply.started":"2023-06-11T14:56:48.438594Z","shell.execute_reply":"2023-06-11T14:56:49.447007Z"},"trusted":true},"execution_count":null,"outputs":[]}]}