{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":10338,"databundleVersionId":862042,"sourceType":"competition"},{"sourceId":7604865,"sourceType":"datasetVersion","datasetId":4427334}],"dockerImageVersionId":30648,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ultralytics\n# !pip install -e git+https://github.com/troobadure/ultralytics@shibarinu\\#egg=ultralytics\n# !pip install -e git+https://github.com/ultralytics/ultralytics\\#egg=ultralytics --exists-action s","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2025-02-20T05:30:02.209861Z","iopub.execute_input":"2025-02-20T05:30:02.210222Z","iopub.status.idle":"2025-02-20T05:30:12.835486Z","shell.execute_reply.started":"2025-02-20T05:30:02.210191Z","shell.execute_reply":"2025-02-20T05:30:12.834670Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport random\nimport shutil\n\nimport cv2\nimport pydicom\nfrom PIL import Image\n\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\n\nfrom ultralytics import YOLO\nimport wandb\n\nnp.random.seed(1337)","metadata":{"execution":{"iopub.status.busy":"2025-02-20T05:30:12.836649Z","iopub.execute_input":"2025-02-20T05:30:12.836884Z","iopub.status.idle":"2025-02-20T05:30:18.157448Z","shell.execute_reply.started":"2025-02-20T05:30:12.836863Z","shell.execute_reply":"2025-02-20T05:30:18.156845Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"try:\n    shutil.rmtree('/kaggle/working/')\nexcept:\n    pass","metadata":{"execution":{"iopub.status.busy":"2025-02-20T05:30:18.159633Z","iopub.execute_input":"2025-02-20T05:30:18.160196Z","iopub.status.idle":"2025-02-20T05:30:18.164615Z","shell.execute_reply.started":"2025-02-20T05:30:18.160165Z","shell.execute_reply":"2025-02-20T05:30:18.163654Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CSV_FILE = '../input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv'\nTRAIN_SRC_DIR = '../input/rsna-pneumonia-detection-challenge/stage_2_train_images/'\nTEST_SRC_DIR = '../input/rsna-pneumonia-detection-challenge/stage_2_test_images/'\nDATASET_DIR = './dataset/'\nTEST_IMG_DIR = 'test_images/'\n\nos.mkdir(DATASET_DIR)\nos.mkdir(DATASET_DIR + 'images/')\nos.mkdir(DATASET_DIR + 'images/train/')\nos.mkdir(DATASET_DIR + 'images/val/')\nos.mkdir(DATASET_DIR + 'images/test/')\nos.mkdir(DATASET_DIR + 'labels/')\nos.mkdir(DATASET_DIR + 'labels/train/')\nos.mkdir(DATASET_DIR + 'labels/val/')\nos.mkdir(DATASET_DIR + 'labels/test/')\nos.mkdir(TEST_IMG_DIR)","metadata":{"execution":{"iopub.status.busy":"2025-02-20T05:30:18.165569Z","iopub.execute_input":"2025-02-20T05:30:18.165877Z","iopub.status.idle":"2025-02-20T05:30:18.174030Z","shell.execute_reply.started":"2025-02-20T05:30:18.165847Z","shell.execute_reply":"2025-02-20T05:30:18.173400Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prepare images and labels for training","metadata":{}},{"cell_type":"code","source":"annotations = pd.read_csv(CSV_FILE)\nprint(annotations.info())\nannotations.head()","metadata":{"execution":{"iopub.status.busy":"2025-02-20T05:30:18.174970Z","iopub.execute_input":"2025-02-20T05:30:18.175260Z","iopub.status.idle":"2025-02-20T05:30:18.262812Z","shell.execute_reply.started":"2025-02-20T05:30:18.175234Z","shell.execute_reply":"2025-02-20T05:30:18.262053Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"positive_annotations = annotations[annotations.Target == 1]\nnegative_annotations = annotations[annotations.Target == 0]\n\nprint(positive_annotations['patientId'].drop_duplicates().shape[0])\nprint(negative_annotations['patientId'].drop_duplicates().shape[0])\nprint(negative_annotations['patientId'].shape[0])\n\nnegative_sample = negative_annotations.sample(600)\nnegative_sample['patientId'].shape[0]\n\nannotations = pd.concat([positive_annotations, negative_sample])\nprint(annotations.shape)\nannotations.head()","metadata":{"execution":{"iopub.status.busy":"2025-02-20T05:30:18.263896Z","iopub.execute_input":"2025-02-20T05:30:18.264485Z","iopub.status.idle":"2025-02-20T05:30:18.288293Z","shell.execute_reply.started":"2025-02-20T05:30:18.264450Z","shell.execute_reply":"2025-02-20T05:30:18.287563Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"patient_id_series = annotations.patientId.drop_duplicates()\nprint('Number of images:', patient_id_series.size)\n\ntrain_series, val_series = train_test_split(patient_id_series, test_size=0.1, random_state=42)\nprint('Train set number:', len(train_series))\nprint('Validation set number:', len(val_series))","metadata":{"execution":{"iopub.status.busy":"2025-02-20T05:30:18.289249Z","iopub.execute_input":"2025-02-20T05:30:18.289545Z","iopub.status.idle":"2025-02-20T05:30:18.297575Z","shell.execute_reply.started":"2025-02-20T05:30:18.289519Z","shell.execute_reply":"2025-02-20T05:30:18.296677Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for patient_id in tqdm(train_series):\n    src_path = TRAIN_SRC_DIR + patient_id + '.dcm'\n    dcm_data = pydicom.dcmread(src_path)\n    image_array = dcm_data.pixel_array\n    image = Image.fromarray(image_array)\n    image.save(DATASET_DIR + 'images/train/' + patient_id + '.jpg')\nprint('Images moved to train folder:', len(os.listdir(DATASET_DIR + 'images/train/')))\n    \nfor patient_id in tqdm(val_series):\n    src_path = TRAIN_SRC_DIR + patient_id + '.dcm'\n    dcm_data = pydicom.dcmread(src_path)\n    image_array = dcm_data.pixel_array\n    image = Image.fromarray(image_array)\n    image.save(DATASET_DIR + 'images/val/' + patient_id + '.jpg')\nprint('Images moved to val folder:', len(os.listdir(DATASET_DIR + 'images/val/')))","metadata":{"execution":{"iopub.status.busy":"2025-02-20T05:30:18.298810Z","iopub.execute_input":"2025-02-20T05:30:18.299172Z","iopub.status.idle":"2025-02-20T05:32:09.166865Z","shell.execute_reply.started":"2025-02-20T05:30:18.299152Z","shell.execute_reply":"2025-02-20T05:32:09.166040Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def translate_bbox(bbox):\n    img_size = 1024 # rsna defualt image size\n    \n    top_left_x = bbox[0]\n    top_left_y = bbox[1]\n    absolute_w = bbox[2]\n    absolute_h = bbox[3]\n\n    relative_w = absolute_w / img_size\n    relative_h = absolute_h / img_size\n    \n    relative_x = top_left_x / img_size + relative_w / 2\n    relative_y = top_left_y / img_size + relative_h / 2\n    \n    return relative_x, relative_y, relative_w, relative_h\n    \ndef revert_bbox(rx, ry, rw, rh):\n    img_size = 1024 # rsna defualt image size\n    \n    x = (rx-rw/2)*img_size\n    y = (ry-rh/2)*img_size\n    w = rw*img_size\n    h = rh*img_size\n    \n    return x, y, w, h\n    \n    \ndef save_label(label_dir, patient_id, bbox):\n    label_fp = os.path.join(label_dir, patient_id + '.txt')\n    \n    f = open(label_fp, \"a\")\n    if (bbox == 'nan').all():\n        f.close()\n        return\n    \n    x, y, w, h = translate_bbox(bbox)\n    \n    line = f\"0 {x} {y} {w} {h}\\n\"\n    \n    f.write(line)\n    f.close()","metadata":{"execution":{"iopub.status.busy":"2025-02-20T05:32:09.168860Z","iopub.execute_input":"2025-02-20T05:32:09.169116Z","iopub.status.idle":"2025-02-20T05:32:09.177896Z","shell.execute_reply.started":"2025-02-20T05:32:09.169096Z","shell.execute_reply":"2025-02-20T05:32:09.177080Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"LABELS_DIR = \"./labels_temp/\"\nos.mkdir(LABELS_DIR)\n\nfor row in annotations.values:\n    if pd.notna(row[1:5]).all():\n        save_label(LABELS_DIR, row[0], row[1:5])\n    \nfor patient_id in train_series:\n    if os.path.isfile(LABELS_DIR + patient_id + '.txt'):\n        shutil.copy(LABELS_DIR + patient_id + '.txt', DATASET_DIR + 'labels/train/')\n    \nfor patient_id in val_series:\n    if os.path.isfile(LABELS_DIR + patient_id + '.txt'):\n        shutil.copy(LABELS_DIR + patient_id + '.txt', DATASET_DIR + 'labels/val/')\n    \nshutil.rmtree(LABELS_DIR)","metadata":{"execution":{"iopub.status.busy":"2025-02-20T05:32:09.178942Z","iopub.execute_input":"2025-02-20T05:32:09.179186Z","iopub.status.idle":"2025-02-20T05:32:10.592513Z","shell.execute_reply.started":"2025-02-20T05:32:09.179155Z","shell.execute_reply":"2025-02-20T05:32:10.591830Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Display example from dataset","metadata":{}},{"cell_type":"code","source":"demo_patient_id = val_series.values[8]\ndemo_img_path = DATASET_DIR + 'images/val/' + demo_patient_id + '.jpg'\ndemo_label_path = DATASET_DIR + 'labels/val/' + demo_patient_id + '.txt'\n\nplt.imshow(cv2.imread(demo_img_path))\n\nwith open(demo_label_path, \"r\") as f:\n    for line in f:\n        print(line)\n        class_id, rx, ry, rw, rh = list(map(float, line.strip().split()))\n        \n        x, y, w, h = revert_bbox(rx, ry, rw, rh)\n        plt.plot([x, x, x+w, x+w, x], [y, y+h, y+h, y, y])","metadata":{"execution":{"iopub.status.busy":"2025-02-20T05:32:35.313551Z","iopub.execute_input":"2025-02-20T05:32:35.313883Z","iopub.status.idle":"2025-02-20T05:32:35.707030Z","shell.execute_reply.started":"2025-02-20T05:32:35.313861Z","shell.execute_reply":"2025-02-20T05:32:35.706150Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prepare configuration and run training","metadata":{}},{"cell_type":"code","source":"%%writefile config.yaml\n\npath: '/kaggle/working/dataset' # dataset root dir\ntrain: images/train  # train images (relative to 'path')\nval: images/val  # val images (relative to 'path')\n\n# Classes\nnames:\n  0: pneumonia","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-20T05:32:39.920915Z","iopub.execute_input":"2025-02-20T05:32:39.921230Z","iopub.status.idle":"2025-02-20T05:32:39.926897Z","shell.execute_reply.started":"2025-02-20T05:32:39.921208Z","shell.execute_reply":"2025-02-20T05:32:39.926078Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = YOLO('yolov8l.pt') # yaml","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-20T05:32:44.725898Z","iopub.execute_input":"2025-02-20T05:32:44.726207Z","iopub.status.idle":"2025-02-20T05:32:45.930020Z","shell.execute_reply.started":"2025-02-20T05:32:44.726183Z","shell.execute_reply":"2025-02-20T05:32:45.929320Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"wandb.login(key='da27b138f0f392e3f931cf71acdab08543ac649c')\nresults = model.train(data='config.yaml', epochs=10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-20T05:34:14.160292Z","iopub.execute_input":"2025-02-20T05:34:14.160645Z","iopub.status.idle":"2025-02-20T06:30:03.214051Z","shell.execute_reply.started":"2025-02-20T05:34:14.160591Z","shell.execute_reply":"2025-02-20T06:30:03.212955Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inspect model predictions","metadata":{}},{"cell_type":"code","source":"def plot_val_pred(demo_patient_id, verbose=True, split='val'):\n    demo_img_path = DATASET_DIR + f'images/{split}/' + demo_patient_id + '.jpg'\n    demo_label_path = DATASET_DIR + f'labels/{split}/' + demo_patient_id + '.txt'\n\n    res = model(demo_img_path, verbose=verbose)\n    if verbose:\n        print(res[0].probs)\n        print(res[0].boxes.xywh)\n\n    plt.imshow(cv2.imread(demo_img_path))\n\n    img_size = 1014\n    if os.path.isfile(demo_label_path):\n        with open(demo_label_path, \"r\") as f:\n            for line in f:\n                if verbose:\n                    print(line)\n                class_id, rx, ry, rw, rh = list(map(float, line.strip().split()))\n\n                x, y, w, h = revert_bbox(rx, ry, rw, rh)\n                plt.plot([x, x, x+w, x+w, x], [y, y+h, y+h, y, y], c='blue')\n\n                \n    for box in res[0].boxes.xywh.cpu():\n        px, py, pw, ph = box\n        plt.plot([px-pw/2, px-pw/2, px+pw/2, px+pw/2, px-pw/2], [py-ph/2, py+ph/2, py+ph/2, py-ph/2, py-ph/2], c='orange')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-20T06:38:15.742171Z","iopub.execute_input":"2025-02-20T06:38:15.742734Z","iopub.status.idle":"2025-02-20T06:38:15.750497Z","shell.execute_reply.started":"2025-02-20T06:38:15.742695Z","shell.execute_reply":"2025-02-20T06:38:15.749479Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def random_value(series):\n    return series.iloc[random.randrange(0, len(series))]\n\ndef plot_examples(series, rows = 5, cols = 2, split='val'):\n    plt.suptitle(split)\n    plt.figure(figsize=(10*cols,10*rows))\n    for h in range(rows):\n        for w in range(cols):\n            plt.subplot(rows, cols, h*2+w+1)\n            plot_val_pred(random_value(series), verbose=False, split=split)\n\nplot_examples(train_series, 2, 2, 'train')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-20T06:38:26.746931Z","iopub.execute_input":"2025-02-20T06:38:26.747577Z","iopub.status.idle":"2025-02-20T06:38:29.412632Z","shell.execute_reply.started":"2025-02-20T06:38:26.747550Z","shell.execute_reply":"2025-02-20T06:38:29.411418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_examples(val_series, 2, 2, 'val')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-20T06:38:51.686755Z","iopub.execute_input":"2025-02-20T06:38:51.687389Z","iopub.status.idle":"2025-02-20T06:38:53.730155Z","shell.execute_reply.started":"2025-02-20T06:38:51.687364Z","shell.execute_reply":"2025-02-20T06:38:53.729036Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prepare test dataset","metadata":{}},{"cell_type":"code","source":"for file in tqdm(os.listdir(TEST_SRC_DIR)):\n    src_path = TEST_SRC_DIR + file\n    dcm_data = pydicom.dcmread(src_path)\n    image_array = dcm_data.pixel_array\n    image = Image.fromarray(image_array)\n    image.save(TEST_IMG_DIR + os.path.splitext(file)[0] + '.jpg')","metadata":{"execution":{"iopub.status.busy":"2025-02-20T06:39:28.812464Z","iopub.execute_input":"2025-02-20T06:39:28.813359Z","iopub.status.idle":"2025-02-20T06:40:19.883992Z","shell.execute_reply.started":"2025-02-20T06:39:28.813331Z","shell.execute_reply":"2025-02-20T06:40:19.883174Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Generate submission.csv","metadata":{}},{"cell_type":"code","source":"# model = YOLO('../input/yolov8n-trained-on-rsna-pneumonia-detection/best.pt')","metadata":{"execution":{"iopub.status.busy":"2025-02-20T06:42:16.093511Z","iopub.execute_input":"2025-02-20T06:42:16.094187Z","iopub.status.idle":"2025-02-20T06:42:16.606799Z","shell.execute_reply.started":"2025-02-20T06:42:16.094159Z","shell.execute_reply":"2025-02-20T06:42:16.605590Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results = model(TEST_IMG_DIR, verbose=False, conf=0.28, stream=True) # conf=0.26 gives better score on private","metadata":{"execution":{"iopub.status.busy":"2025-02-20T06:42:50.471012Z","iopub.execute_input":"2025-02-20T06:42:50.471830Z","iopub.status.idle":"2025-02-20T06:42:50.476217Z","shell.execute_reply.started":"2025-02-20T06:42:50.471799Z","shell.execute_reply":"2025-02-20T06:42:50.475262Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results = model(TEST_IMG_DIR, verbose=False, conf=0.28, stream=True)\n\ndef get_id_from_path(path):\n    return os.path.splitext(os.path.basename(path))[0]\n\nfor result in results:  # Iterate over the generator\n    print(get_id_from_path(result.path))\n    print(result.boxes.xywh)\n    print(result.boxes.conf)\n    break  # Remove this if you want to process all images\n","metadata":{"execution":{"iopub.status.busy":"2025-02-20T06:44:16.608961Z","iopub.execute_input":"2025-02-20T06:44:16.609319Z","iopub.status.idle":"2025-02-20T06:44:16.705507Z","shell.execute_reply.started":"2025-02-20T06:44:16.609292Z","shell.execute_reply":"2025-02-20T06:44:16.704484Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open('submission.csv', 'w') as file:\n    file.write(\"patientId,PredictionString\\n\")\n\n    for result in tqdm(results):\n        line = get_id_from_path(result.path) + ','\n        \n        for conf, xywh in zip(result.boxes.conf, result.boxes.xywh):\n            x, y, w, h = xywh\n            line += f\"{conf:.2f} {x-w/2:.2f} {y-h/2:.2f} {w:.2f} {h:.2f} \"\n            \n        line = line.strip()\n        file.write(line+\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2025-02-20T06:44:24.198983Z","iopub.execute_input":"2025-02-20T06:44:24.199389Z","iopub.status.idle":"2025-02-20T06:47:02.146231Z","shell.execute_reply.started":"2025-02-20T06:44:24.199364Z","shell.execute_reply":"2025-02-20T06:47:02.145473Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Collapse overlapping boxes","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom PIL import Image","metadata":{"execution":{"iopub.status.busy":"2025-02-20T07:16:59.997635Z","iopub.execute_input":"2025-02-20T07:16:59.997962Z","iopub.status.idle":"2025-02-20T07:17:00.002585Z","shell.execute_reply.started":"2025-02-20T07:16:59.997936Z","shell.execute_reply":"2025-02-20T07:17:00.001670Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path = Path('submission.csv')\ndf = pd.read_csv(path)\nprint(path.name)\n\ndf['pred_count'] = df['PredictionString'].apply(lambda x: 0 if pd.isna(x) else int(len(x.split()) / 5))\ndf['pred_count'].value_counts().sort_index()","metadata":{"execution":{"iopub.status.busy":"2025-02-20T07:17:03.855310Z","iopub.execute_input":"2025-02-20T07:17:03.855989Z","iopub.status.idle":"2025-02-20T07:17:03.874316Z","shell.execute_reply.started":"2025-02-20T07:17:03.855961Z","shell.execute_reply":"2025-02-20T07:17:03.873330Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def str_to_boxes(s: str) -> list:  # return c,x,y,w,h\n    if pd.isna(s) or len(s) == 0:\n        return []\n\n    boxes = []\n    n = len(s.split()) // 5\n    for i in range(n):\n        box = s.split()[i * 5:i * 5 + 5]\n        boxes.append(list(map(float, box)))\n\n    return boxes\n\ndef remove_empty_boxes(s: str) -> str:\n    if pd.isna(s) or len(s) == 0:\n        return s\n\n    n = len(s.split()) // 5\n    data = s.split()\n    for i in range(n):\n        box = data[i * 5:i * 5 + 5]\n        if float(box[2]) * float(box[3]) == 0:\n            s = s.replace(' '.join(map(str, box)), '').strip()\n\n    return s","metadata":{"execution":{"iopub.status.busy":"2025-02-20T07:17:09.567053Z","iopub.execute_input":"2025-02-20T07:17:09.567740Z","iopub.status.idle":"2025-02-20T07:17:09.574210Z","shell.execute_reply.started":"2025-02-20T07:17:09.567713Z","shell.execute_reply":"2025-02-20T07:17:09.573201Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['PredictionString'] = df['PredictionString'].apply(remove_empty_boxes)","metadata":{"execution":{"iopub.status.busy":"2025-02-20T07:17:13.725787Z","iopub.execute_input":"2025-02-20T07:17:13.726687Z","iopub.status.idle":"2025-02-20T07:17:13.736027Z","shell.execute_reply.started":"2025-02-20T07:17:13.726648Z","shell.execute_reply":"2025-02-20T07:17:13.735148Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def box_areas_utils(box1, box2):  # corner coords\n    _, left_x1, left_y1, w1, h1 = box1\n    _, left_x2, left_y2, w2, h2 = box2\n\n    assert w1 * h1 * w2 * h2 > 0, 'w or h is 0'\n\n    right_x1, right_x2 = left_x1 + w1, left_x2 + w2\n    top_y1, top_y2 = left_y1 + h1, left_y2 + h2\n\n    area1, area2 = w1 * h1, w2 * h2\n    right_xi = min(right_x1, right_x2)\n    left_xi = max(left_x1, left_x2)\n    top_yi = min(top_y1, top_y2)\n    bottom_yi = max(left_y1, left_y2)\n\n    if right_xi <= left_xi or top_yi <= bottom_yi:\n        intersection = 0\n    else:\n        intersection = (right_xi - left_xi) * (top_yi - bottom_yi)\n\n    union = area1 + area2 - intersection\n    return area1, area2, intersection, union\n\n\ndef iou(box1, box2):\n    area1, area2, intersection, union = box_areas_utils(box1, box2)\n    return intersection / union\n\n\ndef two_boxes_overlap(box1, box2) -> bool:\n    return iou(box1, box2) > 0.3\n\n\ndef one_box_inside_another(box1, box2) -> bool:\n    area1, area2, intersection, union = box_areas_utils(box1, box2)\n    return intersection / area1 > 0.7 or intersection / area2 > 0.7\n\n\ndef merge_boxes(box1, box2) -> (float, float, float, float, float):  # c,x,y,w,h - bottom left corner (0,0)\n    c1, x1, y1, w1, h1 = box1\n    c2, x2, y2, w2, h2 = box2\n    min_x, min_y = min([x1, x2]), min([y1, y2])\n    max_x, max_y = max([x1 + w1, x2 + w2]), max([y1 + h1, y2 + h2])\n\n    w, h = max_x - min_x, max_y - min_y\n    # reduce w, h by 10%\n    dw, dh = w * 0.05, h * 0.05\n    w, h = w * 0.9, h * 0.9\n\n    return (c1 + c2) / 2, (x1 + x2) / 2, (y1 + y2) / 2, (w1+w2) / 2, (h1+h2) / 2\n\n\ndef detect_overlapping(s: str, type='both') -> bool:\n    if pd.isna(s):\n        return False\n\n    n = len(s.split()) // 5\n    for i in range(n):\n        box1 = list(map(float, s.split()[i * 5:i * 5 + 5]))\n        for j in range(n):\n            if i == j:\n                continue\n            box2 = list(map(float, s.split()[j * 5:j * 5 + 5]))\n\n            if type == 'both':\n                if two_boxes_overlap(box1, box2) or one_box_inside_another(box1, box2):\n                    return True\n            elif type == 'overlap':\n                if two_boxes_overlap(box1, box2) and not one_box_inside_another(box1, box2):\n                    return True\n            elif type == 'inside':\n                if one_box_inside_another(box1, box2):\n                    return True\n\n    return False\n\n\ndef merge_overlapping(s: str) -> str:\n    if pd.isna(s):\n        return s\n\n    boxes = str_to_boxes(s)\n\n    for i in range(len(boxes)):\n        for j in range(i + 1, len(boxes)):\n            if boxes[i] is None or boxes[j] is None:\n                continue\n            if two_boxes_overlap(boxes[i], boxes[j]) or \\\n                    one_box_inside_another(boxes[i], boxes[j]):\n                boxes[i] = merge_boxes(boxes[i], boxes[j])\n                boxes[j] = None\n\n    return ' '.join([' '.join(map(str, c)) for c in boxes if c is not None]).strip()\n","metadata":{"execution":{"iopub.status.busy":"2025-02-20T07:17:16.995850Z","iopub.execute_input":"2025-02-20T07:17:16.996167Z","iopub.status.idle":"2025-02-20T07:17:17.009933Z","shell.execute_reply.started":"2025-02-20T07:17:16.996143Z","shell.execute_reply":"2025-02-20T07:17:17.008992Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['overlaps'] = df['PredictionString'].apply(lambda s: detect_overlapping(s, type='overlap'));\ndf['inside_box'] = df['PredictionString'].apply(lambda s: detect_overlapping(s, type='inside'));\nprint(df.query('pred_count>0')['overlaps'].value_counts())\nprint(df.query('pred_count>0')['inside_box'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2025-02-20T07:17:22.327029Z","iopub.execute_input":"2025-02-20T07:17:22.327344Z","iopub.status.idle":"2025-02-20T07:17:22.348804Z","shell.execute_reply.started":"2025-02-20T07:17:22.327319Z","shell.execute_reply":"2025-02-20T07:17:22.347787Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[df['overlaps']]['pred_count'].value_counts().sort_index()","metadata":{"execution":{"iopub.status.busy":"2025-02-20T07:17:25.385128Z","iopub.execute_input":"2025-02-20T07:17:25.385792Z","iopub.status.idle":"2025-02-20T07:17:25.392772Z","shell.execute_reply.started":"2025-02-20T07:17:25.385764Z","shell.execute_reply":"2025-02-20T07:17:25.392072Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['PredictionString_'] = df['PredictionString']\n\ns = df.query('inside_box').sample(1).iloc[0]\n\nimg = Image.open(f'test_images/{s[\"patientId\"]}.jpg')\nimg = img.convert('RGB')\nax = plt.gca()\nboxes = str_to_boxes(s['PredictionString_'])\nfor b in boxes:\n    rect = patches.Rectangle((b[1], b[2]), b[3], b[4], linewidth=1, edgecolor='r', facecolor='none')\n    ax.add_patch(rect)\n\nfixed_s = merge_overlapping(s['PredictionString_'])\nboxes_fixed = str_to_boxes(fixed_s)\nfor b in boxes_fixed:\n    rect = patches.Rectangle((b[1], b[2]), b[3], b[4], linewidth=1, edgecolor='b', facecolor='none', linestyle=':')\n    ax.add_patch(rect)\nplt.imshow(img);","metadata":{"execution":{"iopub.status.busy":"2025-02-20T07:17:28.199437Z","iopub.execute_input":"2025-02-20T07:17:28.199787Z","iopub.status.idle":"2025-02-20T07:17:28.536327Z","shell.execute_reply.started":"2025-02-20T07:17:28.199759Z","shell.execute_reply":"2025-02-20T07:17:28.535522Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['PredictionString'] = df['PredictionString'].apply(merge_overlapping)","metadata":{"execution":{"iopub.status.busy":"2025-02-20T07:17:35.292188Z","iopub.execute_input":"2025-02-20T07:17:35.293012Z","iopub.status.idle":"2025-02-20T07:17:35.300788Z","shell.execute_reply.started":"2025-02-20T07:17:35.292983Z","shell.execute_reply":"2025-02-20T07:17:35.299889Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[['patientId', 'PredictionString']].to_csv(path.name, index=False)","metadata":{"execution":{"iopub.status.busy":"2025-02-20T07:17:38.044506Z","iopub.execute_input":"2025-02-20T07:17:38.045320Z","iopub.status.idle":"2025-02-20T07:17:38.057181Z","shell.execute_reply.started":"2025-02-20T07:17:38.045293Z","shell.execute_reply":"2025-02-20T07:17:38.056333Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}