{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"![](https://i.ibb.co/7pC1Y9q/lungs.jpg)\n\n<p style='text-align: right;'><span style=\"color: #000508; font-family: Segoe UI; font-size: 0.7em; font-weight: 300;\">Image Source: www.nhlbi.nih.gov</span></p>\n\n<p style='text-align: center;'><span style=\"color: #000508; font-family: Segoe UI; font-size: 2.5em; font-weight: 300;\">SIIM COVID-19 Resize & Process + Coco Dataset</span></p>","metadata":{}},{"cell_type":"markdown","source":"<span style=\"color: #0087e4; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">Overview</span>\n\n<p style='text-align: justify;'><span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\">This notebook covers the following:</span></p>\n\n- Read Dicom Images\n- Resize and Save to PNG\n- Generate Metadata for Train and Test Datasets\n- Process and Resize Bounding Boxes\n- Visualise Images\n- GroupKFold Train-Val Split\n- Generate GroupKFold COCO Dataset\n\n<br>\n\n<p style='text-align: justify;'><span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 400;\">Version Notes:</span></p>\n\n- V5: Added segmentations=[] to coco annotations. Removed images without bboxes in coco annotations\n- V3: Generated resized images, annotations and COCO GroupKFold dataset for sizes 256x256, 512x512 & 768x768\n- V2: Initial version, generated 512x512 resized images, annotations and COCO GroupKFold dataset for images with bboxes\n\n\n<br>\n\n<p style='text-align: justify;'><span style=\"color: #000508; font-family: Segoe UI; font-size: 1.2em; font-weight: 300;\">The resized images and the COCO Annotations in this notebook are registered as datasets:</span></p>\n\n**SIIM Covid19 Images & Metadata 256 512 768**\n\nhttps://www.kaggle.com/sreevishnudamodaran/siim-covid19-images-metadata-256-512-768\n\n**SIIM Covid-19 COCO 256 512 768 GroupKFold**\n\nhttps://www.kaggle.com/sreevishnudamodaran/siim-covid19-coco-256-512-768-groupkfold\n\n**SIIM covid19 512 images and metadata**\n\nhttps://www.kaggle.com/sreevishnudamodaran/siim-covid19-512-images-and-metadata\n\n**SIIM Covid-19 COCO 512x512 GroupKFold**\n\nhttps://www.kaggle.com/sreevishnudamodaran/siim-covid19-coco-512x512-groupkfold\n\nI will update these datasets if needed, when the issues in the discussions regarding the dataset are resolved by the organizers.\n\n\n### References\n\n- https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n- https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n- https://www.kaggle.com/sreevishnudamodaran/vinbigdata-fusing-bboxes-coco-dataset\n\n<br>\n\n<a href=\"https://www.kaggle.com/sreevishnudamodaran\"><center><img border=\"0\" alt=\"Ask Me Something\" src=\"https://img.shields.io/badge/Ask%20me-something-1abc9c.svg?style=flat-square&logo=kaggle\" width=\"130\" height=\"10\"></center></a>\n\n<center><img border=\"0\" alt=\"Ask Me Something\" src=\"https://img.shields.io/badge/Please-Upvote%20If%20you%20like%20this-07b3c8?style=for-the-badge&logo=kaggle\" width=\"260\" height=\"20\"></center>","metadata":{}},{"cell_type":"code","source":"!conda install gdcm -c conda-forge -y","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-25T08:35:24.712077Z","iopub.execute_input":"2021-07-25T08:35:24.712507Z","iopub.status.idle":"2021-07-25T08:36:38.184565Z","shell.execute_reply.started":"2021-07-25T08:35:24.712417Z","shell.execute_reply":"2021-07-25T08:36:38.183461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport ast\nimport numpy as np\nimport pandas as pd\nfrom path import Path\nimport datetime\nimport glob\nimport json\nimport shutil\n\nfrom PIL import Image\nfrom tqdm.auto import tqdm\nimport pydicom\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport cv2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-25T08:36:38.18635Z","iopub.execute_input":"2021-07-25T08:36:38.18668Z","iopub.status.idle":"2021-07-25T08:36:38.705115Z","shell.execute_reply.started":"2021-07-25T08:36:38.186642Z","shell.execute_reply":"2021-07-25T08:36:38.703896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #0087e4; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">Read Dicom Images and  Resize and Save to PNG</span>","metadata":{}},{"cell_type":"code","source":"from pydicom.pixel_data_handlers.util import apply_voi_lut\n\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)    \n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2021-07-25T08:36:38.706997Z","iopub.execute_input":"2021-07-25T08:36:38.707301Z","iopub.status.idle":"2021-07-25T08:36:38.71684Z","shell.execute_reply.started":"2021-07-25T08:36:38.707271Z","shell.execute_reply":"2021-07-25T08:36:38.715608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail(size, resample)\n    else:\n        im = im.resize(size, resample)\n    \n    return im","metadata":{"execution":{"iopub.status.busy":"2021-07-25T08:36:38.718502Z","iopub.execute_input":"2021-07-25T08:36:38.718849Z","iopub.status.idle":"2021-07-25T08:36:38.728025Z","shell.execute_reply.started":"2021-07-25T08:36:38.718818Z","shell.execute_reply":"2021-07-25T08:36:38.726946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_meta_folder = '/kaggle/working/images_metadata_256_512_768'\nos.makedirs(images_meta_folder, exist_ok=True)\n\n# Define sizes\nnew_sizes = [(256, 256), (512, 512), (768, 768)]","metadata":{"execution":{"iopub.status.busy":"2021-07-25T08:36:38.729388Z","iopub.execute_input":"2021-07-25T08:36:38.7297Z","iopub.status.idle":"2021-07-25T08:36:38.742261Z","shell.execute_reply.started":"2021-07-25T08:36:38.729668Z","shell.execute_reply":"2021-07-25T08:36:38.741019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for new_size in new_sizes:\n\n    for split in ['train', 'test']:\n    # for split in ['test']:\n        save_dir = f'/kaggle/tmp/{split}_{new_size[0]}x{new_size[1]}/'\n        dcm_paths = glob.glob(f'../input/siim-covid19-detection/{split}/*/*/*')\n        os.makedirs(save_dir, exist_ok=True)\n\n        image_ids = []\n        folder_ids = []\n        study_ids = []\n        widths = []\n        heights = []\n\n        for path in tqdm(dcm_paths):\n            # set keep_ratio=True to have original aspect ratio\n            xray = read_xray(path)\n            im = resize(xray, size=new_size)\n\n            path_split = path.split('/')\n            study_id = path_split[-3]\n            folder_id = path_split[-2]\n            image_name = path_split[-1].replace('.dcm', '_image')\n\n            im.save(os.path.join(save_dir, image_name+'.png'))\n\n            image_ids.append(image_name)\n            folder_ids.append(folder_id)\n            study_ids.append(study_id)\n            widths.append(xray.shape[1])\n            heights.append(xray.shape[0])\n            \n        df = pd.DataFrame.from_dict({'id': image_ids, 'folder_id': folder_ids,\n                                     'study_id': study_ids, 'width': widths,\n                                     'height': heights})\n        df.to_csv(f'{images_meta_folder}/{split}_meta_{new_size[0]}x{new_size[1]}.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-25T08:36:38.74369Z","iopub.execute_input":"2021-07-25T08:36:38.744038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #0087e4; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">Process and Resize Bounding Boxes</span>","metadata":{}},{"cell_type":"code","source":"for new_size in new_sizes:\n    print(f\"*****{new_size}*****\\n\")\n    df_train_meta = pd.read_csv(f'{images_meta_folder}/train_meta_{new_size[0]}x{new_size[1]}.csv')\n    df_train = pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")\n    df_train_meta = df_train.merge(df_train_meta, on='id')\n    \n    ## Drop all rows of images without annotations\n    df_train_meta = df_train_meta.dropna()\n    imagepaths = df_train_meta.id.unique()\n    print(\"Number of Images with Covid_Abnormality:\",len(imagepaths))\n    \n    display(df_train_meta.head(3))\n    print()\n    \n    df_idx=0\n\n    for idx, row in tqdm(df_train_meta.iterrows(), total=df_train_meta.shape[0]):\n        img = cv2.imread(os.path.join(f\"/kaggle/tmp/train_{new_size[0]}x{new_size[1]}/\",\n                                      row.id.replace(\"_image\", \".png\")))\n        bboxes = [list(bbox.values()) for bbox in ast.literal_eval(row.boxes)]\n        height_ratio, width_ratio = (new_size[0]/row.height, new_size[1]/row.width)\n\n        for box in bboxes:\n            box[2] = box[2]+box[0]\n            box[3] = box[3]+box[1]\n            box = (box[0]*width_ratio, box[1]*height_ratio,\n                   box[2]*width_ratio, box[3]*height_ratio)\n\n            row_df = pd.DataFrame({'id':row.id,\n                           'StudyInstanceUID':row.StudyInstanceUID,\n                           'folder_id':row.folder_id,\n                           'study_id':row.study_id,\n                           'width':row.width,\n                           'height':row.height,\n                           'xmin':round(box[0]),\n                           'ymin':round(box[1]),\n                           'xmax':round(box[2]),\n                           'ymax':round(box[3])}, index=[df_idx])\n\n            if df_idx==0:\n                df_train_processed = row_df\n            else:\n                df_train_processed = pd.concat([df_train_processed, row_df])\n\n            df_idx+=1\n\n    display(df_train_processed.head(3))\n    print()\n    df_train_processed.to_csv(f'{images_meta_folder}/df_train_processed_meta_{new_size[0]}x{new_size[1]}.csv',\n                              index=False)\n    df_train_processed.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #0087e4; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">Visualise Images</span>","metadata":{}},{"cell_type":"code","source":"def draw_bbox(image, box, label, color, scale=1.0):   \n    alpha = 0.1\n    alpha_box = 0.6\n    font_size = round(0.6*scale, 1)\n    overlay_bbox = image.copy()\n    overlay_text = image.copy()\n    output = image.copy()\n\n    text_width, text_height = cv2.getTextSize(label.upper(),\n                                              cv2.FONT_HERSHEY_SIMPLEX, font_size, 1)[0]\n    cv2.rectangle(overlay_bbox, (box[0], box[1]), (box[2], box[3]),\n                  color, -1)\n    cv2.addWeighted(overlay_bbox, alpha, output, 1 - alpha, 0, output)\n    \n    cv2.rectangle(overlay_text, (box[0], box[1]-7-text_height),\n                  (box[0]+text_width+2, box[1]), (0, 0, 0), -1)\n    \n    cv2.addWeighted(overlay_text, alpha_box, output, 1 - alpha_box, 0, output)\n    output = cv2.rectangle(output, (box[0], box[1]), (box[2], box[3]),\n                           color, 2)\n    cv2.putText(output, label.upper(), (box[0], box[1]-5),\n            cv2.FONT_HERSHEY_SIMPLEX, font_size, (255, 255, 255), 1, cv2.LINE_AA)\n    return output","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for new_size in new_sizes:\n    imgs_path = f\"/kaggle/tmp/train_{new_size[0]}x{new_size[1]}/\"\n    df_train_processed = pd.read_csv(f\"{images_meta_folder}/df_train_processed_meta_{new_size[0]}x{new_size[1]}.csv\")\n    \n    fig, axes = plt.subplots(1,3, figsize=(19,7))\n    fig.subplots_adjust(hspace=0.2, wspace=0.2)\n    axes = axes.ravel()\n    pos = 0\n\n    for idx, img_id in enumerate(imagepaths[:3]):\n        img = cv2.imread(os.path.join(imgs_path, img_id+\".png\"))\n        img_annotations = df_train_processed[df_train_processed.id==img_id]\n        bboxes = img_annotations[['xmin', 'ymin', 'xmax', 'ymax']].to_numpy().tolist()\n\n        for box in bboxes:\n            img = draw_bbox(img, list(np.int_(box)), \"Covid_Abnormality\",\n                            (255, 243, 0), scale=new_size[0]/512)\n\n        axes[idx].imshow(img, cmap='gray')\n        axes[idx].set_title(f'{img_id}_{new_size[0]}x{new_size[1]}', size=12, pad=10)\n        axes[idx].set_xticklabels([])\n        axes[idx].set_yticklabels([])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for new_size in new_sizes:\n    df_test = pd.read_csv(f\"{images_meta_folder}/test_meta_{new_size[0]}x{new_size[1]}.csv\")\n\n    fig, axes = plt.subplots(1, 3, figsize=(19,21))\n    fig.subplots_adjust(hspace=0.2, wspace=0.2)\n    axes = axes.ravel()\n    pos = 0\n\n    for idx in df_test.id[:3]:\n        img = cv2.imread(os.path.join(f\"/kaggle/tmp/test_{new_size[0]}x{new_size[1]}/\", idx+'.png'))\n        axes[pos].imshow(img, cmap='gray')\n        axes[pos].set_title(f'{img_id}_{new_size[0]}x{new_size[1]}', size=12, pad=10)\n        axes[pos].set_xticklabels([])\n        axes[pos].set_yticklabels([])\n        pos += 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !cp -r /kaggle/tmp/train* images_metadata_256_512_768/\n# !cp -r /kaggle/tmp/test* images_metadata_256_512_768/\n# !zip -rq images_metadata_256_512_768.zip images_metadata_256_512_768/*","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from kaggle_secrets import UserSecretsClient\n# from google.cloud import storage\n\n# def upload_blob(bucket_name, source_file_name, destination_blob_name):\n#     \"\"\"Uploads a file to the bucket. https://cloud.google.com/storage/docs/ \"\"\"\n#     bucket = storage_client.get_bucket(bucket_name)\n#     blob = bucket.blob(destination_blob_name)\n#     blob.upload_from_filename(source_file_name)\n#     print('File {} uploaded to {}.'.format(\n#         source_file_name,\n#         destination_blob_name))\n\n# user_secrets = UserSecretsClient()\n# gcp_project_id = user_secrets.get_secret(\"gcp_project_id\")\n# storage_client = storage.Client(project=gcp_project_id)\n# upload_blob('siim-files', 'images_metadata_256_512_768.zip', 'images_metadata_256_512_768_2.zip')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !cd ./images_metadata_256_512_768 && rm -r train_256x256 train_512x512 train_768x768 test_256x256 test_512x512 test_768x768\n# !rm ./images_metadata_256_512_768.zip","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #0087e4; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">Images and Metadata Datasets</span>\n\n\nThe resized images and the annotations generated above are in this dataset:\n\n**SIIM Covid19 Images & Metadata 256 512 768**\n\nhttps://www.kaggle.com/sreevishnudamodaran/siim-covid19-images-metadata-256-512-768\n\n\nThe resized images and the annotations generated in the previous version of this notebook are in this dataset:\n\n**SIIM covid19 512 images and metadata**\n\nhttps://www.kaggle.com/sreevishnudamodaran/siim-covid19-512-images-and-metadata","metadata":{}},{"cell_type":"markdown","source":"<span style=\"color: #0087e4; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">GroupKFold Train-Val Split</span>","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import GroupKFold, train_test_split\n\n# Remove images without bboxes\ndf_kfold = pd.DataFrame(pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\"))\nprint(\"Shape before removing images without bboxes:\", df_kfold.shape)\ndf_kfold = (df_kfold[df_kfold.label!='none 1 0 0 1 1']).reset_index(drop=True)\nprint(\"Shape after removing images without bboxes:\", df_kfold.shape)\n\nkfold = 5\ndf_kfold['fold'] = -1\ngroup_kfold  = GroupKFold(n_splits = kfold)\n\nfor fold, (train_index, val_index) in enumerate(group_kfold.split(df_kfold,\n                                                              groups=df_kfold.StudyInstanceUID.tolist())):\n    df_kfold.loc[val_index, 'fold'] = fold\n    \ndisplay(df_kfold.head(3))\ndf_kfold.to_csv(\"/kaggle/working/df_meta_kfold.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_fold = 0\ntrain_ids = df_kfold[df_kfold['fold'] != fold].id.unique()\nval_ids = df_kfold[df_kfold['fold'] == fold].id.unique()\n\nprint(\"Split Counts\\nTrain Images:\\t\\t{0}\\nVal Images:\\t\\t{1}\"\n      .format(len(train_ids), len(val_ids)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color: #0087e4; font-family: Segoe UI; font-size: 1.9em; font-weight: 300;\">Generate GroupKFold COCO Dataset</span>\n\n### COCO Format Overview","metadata":{}},{"cell_type":"markdown","source":"\n```python\n{\n  \"type\": \"instances\",\n  \"images\": [\n    {\n      \"file_name\": \"<image_name.png>\",\n      \"height\": \"<height>\",\n      \"width\": \"<width>\",\n      \"id\": \"<Used to reference each image and it should be unique for each image>\"\n    }\n#    .\n#    .\n  ],\n  \"categories\": [\n    {\n      \"supercategory\": \"none\",\n      \"name\": \"<Class One>\",\n      \"id\": 0\n    },\n#    .\n#    .\n  ],\n  \"annotations\": [\n    {\n      \"id\": 1,\n      \"bbox\": [\n        \"<xmin>\",\n        \"<ymin>\",\n        \"<bbox-width>\",\n        \"<bbox-height>\"\n      ],\n      \"image_id\": \"<id of the image from which the polygon annotation is from as defined in the 'images' block above>\",\n      \"segmentation\": [\n          \"<x1>\",\n          \"<y1>\",\n          \"<x2>\",\n          \"<y2>\"\n#          .\n#          .\n      ],\n      \"ignore\": 0,\n      \"area\": \"<Area of the Polygon represented by the points in the 'segmentation' block>\",\n      \"iscrowd\": 0,\n      \"category_id\": \"<Class category ID as an integer which will be defined below>\"\n    },\n  ],\n\"categories\": [\n    {\n        \"supercategory\": null,\n        \"id\": \"<Integer ID for the Class Label>\",\n        \"name\": \"<Class One Label as a String>\"\n    },\n#    .\n#    .\n]\n}\n```","metadata":{"execution":{"iopub.status.busy":"2021-05-24T08:03:59.576385Z","iopub.execute_input":"2021-05-24T08:03:59.576893Z","iopub.status.idle":"2021-05-24T08:03:59.585968Z","shell.execute_reply.started":"2021-05-24T08:03:59.576841Z","shell.execute_reply":"2021-05-24T08:03:59.585078Z"}}},{"cell_type":"code","source":"now = datetime.datetime.now()\n\ndata = dict(\n    info=dict(\n        description='SIIM Covid-19 GroupKfold',\n        url=None,\n        version=None,\n        year=now.year,\n        contributor=None,\n        date_created=now.strftime('%Y-%m-%d %H:%M:%S.%f'),\n    ),\n    licenses=[dict(\n        url=None,\n        id=0,\n        name=None,\n    )],\n    images=[\n        # license, url, file_name, height, width, date_captured, id\n    ],\n    type='instances',\n    annotations=[\n        # segmentation, area, iscrowd, image_id, bbox, category_id, id\n    ],\n    categories=[\n        # supercategory, id, name\n    ],\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_name_to_id = {}\nlabels =  [\"__ignore__\",\n            \"Covid_Abnormality\"]\n\nfor i, each_label in enumerate(labels):\n    class_id = i - 1  # starts with -1\n    class_name = each_label\n    if class_id == -1:\n        assert class_name == '__ignore__'\n        continue\n    class_name_to_id[class_name] = class_id\n    data['categories'].append(dict(\n        supercategory=None,\n        id=class_id,\n        name=class_name,\n    ))\ndata","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_meta_folder = '/kaggle/input/siim-covid19-images-metadata-256-512-768/images_metadata_256_512_768'\n\nfor new_size in new_sizes:\n    for fold in range(kfold):\n        train_ids = df_kfold[df_kfold['fold'] != fold].id.unique()\n        val_ids = df_kfold[df_kfold['fold'] == fold].id.unique()\n        print(f\"\\nFold: {fold}\\n Train images count: {len(train_ids)}, Val images count: {len(val_ids)}\")\n        \n        coco_dir = f'coco_{new_size[0]}x{new_size[1]}'\n        os.makedirs(coco_dir, exist_ok=True)\n\n        df_annotations = pd.read_csv(f\"{images_meta_folder}/df_train_processed_meta_{new_size[0]}x{new_size[1]}.csv\")\n\n        ## Setting the output annotation json file paths\n        train_out_file = f'{coco_dir}/train_annotations_fold{fold}_{new_size[0]}x{new_size[1]}.json'\n        val_out_file = f'{coco_dir}/val_annotations_fold{fold}_{new_size[0]}x{new_size[1]}.json'\n\n        data_train = data.copy()\n        data_train['images'] = []\n        data_train['annotations'] = []\n        data_val = data.copy()\n        data_val['images'] = []\n        data_val['annotations'] = []\n\n        for i, img_id in tqdm(enumerate(train_ids), total=len(train_ids)):\n\n            data_train['images'].append(dict(license=0,\n                                             url=None,\n                                             file_name=img_id+'.png',\n                                             height=new_size[0],\n                                             width=new_size[1],\n                                             date_captured=None,\n                                             id=i\n                                            ))\n\n            img_annotations = df_annotations[df_annotations.id==img_id]\n            boxes = img_annotations[['xmin', 'ymin', 'xmax', 'ymax']].to_numpy()\n            box_labels = np.zeros(img_annotations.shape[0])\n\n            for box, label in zip(boxes, box_labels):\n                x_min, y_min, x_max, y_max = (box[0], box[1], box[2], box[3])\n                area = round((x_max-x_min)*(y_max-y_min),1)\n                bbox =[\n                        int(x_min),\n                        int(y_min),\n                        int(x_max-x_min),\n                        int(y_max-y_min)\n                        ]\n\n                data_train['annotations'].append(dict(id=len(data_train['annotations']),\n                                                      image_id=i,\n                                                      category_id=int(label),\n                                                      area=int(area),\n                                                      bbox=bbox,\n                                                      segmentation=[],\n                                                      iscrowd=0)) \n        with open(train_out_file, 'w') as f:\n            json.dump(data_train, f, indent=4)\n\n        for i, img_id in tqdm(enumerate(val_ids), total=len(val_ids)):\n            data_val['images'].append(dict(license=0,\n                                             url=None,\n                                             file_name=img_id+'.png',\n                                             height=new_size[0],\n                                             width=new_size[1],\n                                             date_captured=None,\n                                             id=i\n                                            ))\n\n            img_annotations = df_annotations[df_annotations.id==img_id]\n            boxes = img_annotations[['xmin', 'ymin', 'xmax', 'ymax']].to_numpy()\n            box_labels = np.zeros(img_annotations.shape[0])\n\n            for box, label in zip(boxes, box_labels):\n                x_min, y_min, x_max, y_max = (box[0], box[1], box[2], box[3])\n                area = round((x_max-x_min)*(y_max-y_min),1)\n                bbox =[\n                        int(x_min),\n                        int(y_min),\n                        int(x_max-x_min),\n                        int(y_max-y_min)\n                        ]\n\n                data_val['annotations'].append(dict(id=len(data_val['annotations']),\n                                                    image_id=i,\n                                                    category_id=int(label),\n                                                    area=int(area),\n                                                    bbox=bbox,\n                                                    segmentation=[],\n                                                    iscrowd=0))             \n        with open(val_out_file, 'w') as f:\n            json.dump(data_val, f, indent=4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir ./metadata_256_512_768\n!cp ../input/siim-covid19-images-metadata-256-512-768/images_metadata_256_512_768/*.csv ./metadata_256_512_768\n!mv /kaggle/working/df_meta_kfold.csv ./metadata_256_512_768","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<p style='text-align: center;'><span style=\"color: #000508; font-family: Segoe UI; font-size: 1.4em; font-weight: 300;\">Let me know if you have any suggestions!</span></p>\n\n<p style='text-align: center;'><span style=\"color: #000508; font-family: Segoe UI; font-size: 2.0em; font-weight: 300;\">THANKS!</span></p>","metadata":{}}]}