{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Objective of this EDA notebook.\n\n1. Basic train_study inspections\n2. Checking missing values, outliers (if any), and train_study distribution.\n3. Identifying the preprocessing step.\n\nAlso some of the codes are copied from these notebooks. I recommend reading those people's EDA work:\n\n* https://www.kaggle.com/code/sadidul012/rsna24-analysis-and-kfold-with-study-id?scriptVersionId=187067156\n* https://www.kaggle.com/code/gunesevitan/rsna-2024-lsdc-eda\n\nHope you enjoy:)","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport random\nimport pandas as pd\nfrom glob import glob\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nimport polars as pl\npl.Config.set_tbl_rows(40)\npl.Config.set_fmt_str_lengths(n=40)\nimport seaborn as sns\nimport pydicom\nimport math\nimport missingno as msno\nfrom skimage.draw import disk\nfrom skimage.io import imsave\nfrom zipfile import ZipFile\n\ntqdm.pandas()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:20:27.509168Z","iopub.execute_input":"2024-07-13T21:20:27.509534Z","iopub.status.idle":"2024-07-13T21:20:30.780856Z","shell.execute_reply.started":"2024-07-13T21:20:27.509504Z","shell.execute_reply":"2024-07-13T21:20:30.778924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rd = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification'\ntrain = pd.read_csv(os.path.join(rd, \"train.csv\"))\ntrain_labels = pd.read_csv(os.path.join(rd, \"train_label_coordinates.csv\"))\ntrain_des = pd.read_csv(os.path.join(rd, \"train_series_descriptions.csv\"))\ntest_des = pd.read_csv(os.path.join(rd, \"test_series_descriptions.csv\"))\nsample_sub = pd.read_csv(os.path.join(rd, \"sample_submission.csv\"))","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:23:29.973342Z","iopub.execute_input":"2024-07-13T21:23:29.973883Z","iopub.status.idle":"2024-07-13T21:23:30.143911Z","shell.execute_reply.started":"2024-07-13T21:23:29.973854Z","shell.execute_reply":"2024-07-13T21:23:30.143133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Basic training data inspections","metadata":{}},{"cell_type":"code","source":"train.shape, train_des.shape, train_labels.shape","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:23:31.637650Z","iopub.execute_input":"2024-07-13T21:23:31.638238Z","iopub.status.idle":"2024-07-13T21:23:31.644966Z","shell.execute_reply.started":"2024-07-13T21:23:31.638209Z","shell.execute_reply":"2024-07-13T21:23:31.643984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:23:31.861705Z","iopub.execute_input":"2024-07-13T21:23:31.862432Z","iopub.status.idle":"2024-07-13T21:23:31.882445Z","shell.execute_reply.started":"2024-07-13T21:23:31.862396Z","shell.execute_reply":"2024-07-13T21:23:31.881362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"msno.dendrogram(train)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:23:32.000822Z","iopub.execute_input":"2024-07-13T21:23:32.001420Z","iopub.status.idle":"2024-07-13T21:23:32.841663Z","shell.execute_reply.started":"2024-07-13T21:23:32.001387Z","shell.execute_reply":"2024-07-13T21:23:32.840695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.isnull().sum(),train_des.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:23:32.843664Z","iopub.execute_input":"2024-07-13T21:23:32.844090Z","iopub.status.idle":"2024-07-13T21:23:32.863524Z","shell.execute_reply.started":"2024-07-13T21:23:32.844058Z","shell.execute_reply":"2024-07-13T21:23:32.862571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:23:32.864670Z","iopub.execute_input":"2024-07-13T21:23:32.864978Z","iopub.status.idle":"2024-07-13T21:23:32.899778Z","shell.execute_reply.started":"2024-07-13T21:23:32.864937Z","shell.execute_reply":"2024-07-13T21:23:32.898911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_des","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:23:32.901363Z","iopub.execute_input":"2024-07-13T21:23:32.901631Z","iopub.status.idle":"2024-07-13T21:23:32.912105Z","shell.execute_reply.started":"2024-07-13T21:23:32.901608Z","shell.execute_reply":"2024-07-13T21:23:32.911172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:23:32.916819Z","iopub.execute_input":"2024-07-13T21:23:32.917176Z","iopub.status.idle":"2024-07-13T21:23:32.931091Z","shell.execute_reply.started":"2024-07-13T21:23:32.917144Z","shell.execute_reply":"2024-07-13T21:23:32.930199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Checking train_label distribtions","metadata":{}},{"cell_type":"code","source":"# Combining train_labels and train_desc\n\ntrain_labels[\"series_description\"] = train_labels.progress_apply(\nlambda row: train_des.loc[(train_des.study_id == row.study_id) & (train_des.series_id == row.series_id)]\n[\"series_description\"].tolist()[0], axis=1)\ntrain_labels","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:23:33.269099Z","iopub.execute_input":"2024-07-13T21:23:33.269768Z","iopub.status.idle":"2024-07-13T21:24:01.053958Z","shell.execute_reply.started":"2024-07-13T21:23:33.269735Z","shell.execute_reply":"2024-07-13T21:24:01.053054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_labels[[\"series_description\", \"instance_number\"]].groupby([\"series_description\"]).count()\ntrain_labels[\"series_description\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:01.056028Z","iopub.execute_input":"2024-07-13T21:24:01.056705Z","iopub.status.idle":"2024-07-13T21:24:01.075719Z","shell.execute_reply.started":"2024-07-13T21:24:01.056667Z","shell.execute_reply":"2024-07-13T21:24:01.074740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels[[\"series_description\", \"condition\", \"instance_number\"]].groupby([\"series_description\", \"condition\"]).count()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:01.076799Z","iopub.execute_input":"2024-07-13T21:24:01.077118Z","iopub.status.idle":"2024-07-13T21:24:01.103495Z","shell.execute_reply.started":"2024-07-13T21:24:01.077092Z","shell.execute_reply":"2024-07-13T21:24:01.102564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_condition_counts(df, title, path=None):\n    \n    \"\"\"\n    Visualize condition counts on training set\n    \n    Parameters\n    ----------\n    df: pandas.train_studyFrame\n        Counts and percentages of conditions\n        \n    title: str\n        Title of the plot\n        \n    path: str, pathlib.Path or None\n        Path of the output file (if path is None, plot is displayed with selected backend)\n    \"\"\"\n    \n    fig, ax = plt.subplots(figsize=(24, 16))\n\n    ax.barh(\n        y=np.arange(df.shape[0] // 3) - 0.2,\n        width=df['count'].values[0::3],\n        height=0.2,\n        align='center',\n        label='Normal/Mild'\n    )\n    ax.barh(\n        y=np.arange(df.shape[0] // 3),\n        width=df['count'].values[1::3],\n        height=0.2,\n        align='center',\n        label='Moderate'\n    )\n    ax.barh(\n        y=np.arange(df.shape[0] // 3) + 0.2,\n        width=df['count'].values[2::3],\n        height=0.2,\n        align='center',\n        label='Severe'\n    )\n\n    ax.set_yticks(np.arange(df.shape[0] // 3))\n    ax.set_yticklabels([\n        f'{level}\\nNormal Count: {normal_count} ({normal_percentage:.2f}%)\\nModerate Count: {moderate_count} ({moderate_percentage:.2f}%)\\nSevere Count: {severe_count} ({severe_percentage:.2f}%)' for level, normal_count, normal_percentage, moderate_count, moderate_percentage, severe_count, severe_percentage, in zip(\n            df['level'].values[0::3],\n            df['count'].values[0::3],\n            df['percentage'].values[0::3],\n            df['count'].values[1::3],\n            df['percentage'].values[1::3],\n            df['count'].values[2::3],\n            df['percentage'].values[2::3],\n        )\n    ])\n    ax.set_xlabel('')\n    ax.tick_params(axis='x', labelsize=17.5, pad=10)\n    ax.tick_params(axis='y', labelsize=17.5, pad=10)\n    ax.set_title(title, size=20, pad=15)\n    ax.legend(loc='best', prop={'size': 18})\n    plt.gca().invert_yaxis()\n\n    plt.show()\n\n    if path is None:\n        plt.show()\n    else:\n        plt.savefig(path)\n        plt.close(fig)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:01.106645Z","iopub.execute_input":"2024-07-13T21:24:01.107056Z","iopub.status.idle":"2024-07-13T21:24:01.119785Z","shell.execute_reply.started":"2024-07-13T21:24:01.107019Z","shell.execute_reply":"2024-07-13T21:24:01.118785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spinal_canal_stenosis_columns = [column for column in train.columns if column.startswith('spinal_canal_stenosis')]\ntrain_spinal_canal_stenosis = []\n\nfor column in spinal_canal_stenosis_columns:\n    df = train[[column]].copy(deep=True)\n    df['level'] = '_'.join(column.split('_')[-2:])\n    df = df.rename(columns={column: 'spinal_canal_stenosis'})\n    train_spinal_canal_stenosis.append(df)\n    \ntrain_spinal_canal_stenosis = pd.concat(train_spinal_canal_stenosis, axis=0).reset_index(drop=True)\n\ntrain_spinal_canal_stenosis_counts = train_spinal_canal_stenosis.groupby('level').value_counts().reset_index()\ntrain_spinal_canal_stenosis_counts['severity'] = train_spinal_canal_stenosis_counts['spinal_canal_stenosis'].map({\n    'Normal/Mild': 0,\n    'Moderate': 1,\n    'Severe': 2\n})\ntrain_spinal_canal_stenosis_counts = train_spinal_canal_stenosis_counts.sort_values(by=['level', 'severity'], ascending=True)\ntrain_spinal_canal_stenosis_counts['percentage'] = train_spinal_canal_stenosis_counts['count'] / train_spinal_canal_stenosis_counts.groupby('level')['count'].transform('sum') * 100\n\nvisualize_condition_counts(\n    df=train_spinal_canal_stenosis_counts,\n    title='Spinal Canal Stenosis Counts On Levels'\n)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:01.121013Z","iopub.execute_input":"2024-07-13T21:24:01.121334Z","iopub.status.idle":"2024-07-13T21:24:01.703654Z","shell.execute_reply.started":"2024-07-13T21:24:01.121291Z","shell.execute_reply":"2024-07-13T21:24:01.702697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Identifying annotations","metadata":{}},{"cell_type":"code","source":"study_id = 901299313\ntrain.loc[train[\"study_id\"] == study_id]","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:02.741461Z","iopub.execute_input":"2024-07-13T21:24:02.741718Z","iopub.status.idle":"2024-07-13T21:24:02.761658Z","shell.execute_reply.started":"2024-07-13T21:24:02.741696Z","shell.execute_reply":"2024-07-13T21:24:02.760627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study = train_labels.loc[train_labels[\"study_id\"] == study_id]\ntrain_study.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:02.763030Z","iopub.execute_input":"2024-07-13T21:24:02.763411Z","iopub.status.idle":"2024-07-13T21:24:02.779067Z","shell.execute_reply.started":"2024-07-13T21:24:02.763374Z","shell.execute_reply":"2024-07-13T21:24:02.777877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc_study = train_des.loc[train_des[\"study_id\"] == study_id]\ntrain_desc_study","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:02.780402Z","iopub.execute_input":"2024-07-13T21:24:02.780767Z","iopub.status.idle":"2024-07-13T21:24:02.792023Z","shell.execute_reply.started":"2024-07-13T21:24:02.780731Z","shell.execute_reply":"2024-07-13T21:24:02.791046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, row in train_desc_study.iterrows():\n    train_study.loc[train_study.series_id == row.series_id, \"series_description\"]  = row.series_description\n\ntrain_study.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:02.793069Z","iopub.execute_input":"2024-07-13T21:24:02.793332Z","iopub.status.idle":"2024-07-13T21:24:02.812494Z","shell.execute_reply.started":"2024-07-13T21:24:02.793309Z","shell.execute_reply":"2024-07-13T21:24:02.811441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def image_position(ints):\n    sid = ints.study_id\n    series_id = ints.series_id\n    instance_id = ints.instance_number\n\n    dicom = pydicom.read_file(rd + f\"/train_images/{sid}/{series_id}/{instance_id}.dcm\")\n    return dicom.ImagePositionPatient\n\ntrain_study.loc[:, [\"px\", \"py\", \"pz\"]] = train_study.apply(lambda x: image_position(x),axis=1).to_list()\ntrain_study.sort_values(by=[\"series_description\", \"instance_number\"]).head()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:02.814206Z","iopub.execute_input":"2024-07-13T21:24:02.814507Z","iopub.status.idle":"2024-07-13T21:24:02.877229Z","shell.execute_reply.started":"2024-07-13T21:24:02.814482Z","shell.execute_reply":"2024-07-13T21:24:02.876289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom(path):\n    dicom = pydicom.read_file(path)\n    train_study = dicom.pixel_array\n    if np.max(train_study - np.min(train_study)) != 0:\n        train_study = (train_study - np.min(train_study)) / np.max(train_study)\n    train_study = (train_study * 255).astype(np.uint8)\n    return train_study\n\n\ndef plot_image(series_description, columns = 3, size=[16, 6]):\n    ints = train_study.loc[train_study[\"series_description\"] == series_description]\n    instance_number = ints.instance_number.unique()\n    \n    rows = math.floor(len(instance_number) / columns) + 1\n    size[1] = size[1] * rows\n    fig = plt.figure(figsize=size)\n    \n    for i, i_n in enumerate(instance_number):\n        ints_df = ints.loc[ints.instance_number == i_n]\n        sid = ints_df.iloc[0].study_id\n        series_id = ints_df.iloc[0].series_id\n        instance_id = ints_df.iloc[0].instance_number\n        \n        try:\n            image_path = os.path.join(rd, 'train_images', str(sid), str(series_id), f\"{instance_id}.dcm\")\n            image = load_dicom(image_path)\n            ax = fig.add_subplot(rows, columns, i+1)\n            plt.imshow(image, cmap=\"gray\")\n\n            for i, row in ints_df.iterrows():\n                circle = plt.Circle((row[\"x\"], row[\"y\"]), 10, color='r', fill=False)\n                ax.add_patch(circle)\n        except:\n            continue\n            \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:02.878338Z","iopub.execute_input":"2024-07-13T21:24:02.878602Z","iopub.status.idle":"2024-07-13T21:24:02.889876Z","shell.execute_reply.started":"2024-07-13T21:24:02.878580Z","shell.execute_reply":"2024-07-13T21:24:02.888884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_image(\"Sagittal T1\", size=[16, 3])","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:02.891038Z","iopub.execute_input":"2024-07-13T21:24:02.891369Z","iopub.status.idle":"2024-07-13T21:24:03.579369Z","shell.execute_reply.started":"2024-07-13T21:24:02.891337Z","shell.execute_reply":"2024-07-13T21:24:03.578391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_image(\"Sagittal T2/STIR\", size=[16, 3])","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:03.580983Z","iopub.execute_input":"2024-07-13T21:24:03.581368Z","iopub.status.idle":"2024-07-13T21:24:04.000215Z","shell.execute_reply.started":"2024-07-13T21:24:03.581335Z","shell.execute_reply":"2024-07-13T21:24:03.999299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_image(\"Axial T2\", size=[16, 7])","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:04.001438Z","iopub.execute_input":"2024-07-13T21:24:04.001802Z","iopub.status.idle":"2024-07-13T21:24:05.682292Z","shell.execute_reply.started":"2024-07-13T21:24:04.001772Z","shell.execute_reply":"2024-07-13T21:24:05.681387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Checking the image size distribution\n\nSince there are numerious training images, I randomly sample 500 images and check the image size distribution.\nThis step is crucial in image masking/ segmentaiton where we need to compare each image to another.","metadata":{}},{"cell_type":"code","source":"def checking_size_dist(series_description, sample_num):\n    random_samples = train_labels.sample(n=sample_num)\n    image_shapes = []\n    for _, row in tqdm(random_samples.iterrows()):\n        try:\n            sid = row['study_id']\n            series_id = row['series_id']\n            i_n = row['instance_number']\n            image = load_dicom(rd + f\"/train_images/{sid}/{series_id}/{i_n}.dcm\")\n            image_shapes.append(image.shape)\n        except Exception as e:\n            print(f\"Could not load image {image_path}: {e}\")\n            continue\n    \n    shapes_df = pd.DataFrame(image_shapes, columns=['Height', 'Width'])\n    \n    print(f'Number of samples:{sample_num}')\n    plt.figure(figsize=(12, 6))\n    plt.subplot(1, 2, 1)\n    plt.hist(shapes_df['Height'], bins=30, color='skyblue', edgecolor='black')\n    plt.title(f'Height Distribution for {series_description}')\n    plt.xlabel('Height')\n    plt.ylabel('Frequency')\n    \n    plt.subplot(1, 2, 2)\n    plt.hist(shapes_df['Width'], bins=30, color='skyblue', edgecolor='black')\n    plt.title(f'Width Distribution for {series_description}')\n    plt.xlabel('Width')\n    plt.ylabel('Frequency')\n    \n    plt.tight_layout()\n    plt.show()\n\n    return shapes_df","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:05.683453Z","iopub.execute_input":"2024-07-13T21:24:05.683727Z","iopub.status.idle":"2024-07-13T21:24:05.694729Z","shell.execute_reply.started":"2024-07-13T21:24:05.683702Z","shell.execute_reply":"2024-07-13T21:24:05.693763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sagittal_t1_shapes = checking_size_dist('Sagittal T1', 500)\naxial_t2_shapes = checking_size_dist('Axial T2', 500)\nsagittal_t2_stir_shapes = checking_size_dist('Sagittal T2/STIR', 500)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:05.695863Z","iopub.execute_input":"2024-07-13T21:24:05.696205Z","iopub.status.idle":"2024-07-13T21:24:46.753051Z","shell.execute_reply.started":"2024-07-13T21:24:05.696174Z","shell.execute_reply":"2024-07-13T21:24:46.752140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Based on this, it may be a better idea to keep all the mask images into 500 times 500.","metadata":{}},{"cell_type":"markdown","source":"# Possible preprocessing steps\n1. KFold\n- Separating the training dataset into the training data and validation data, and conduct cv on them.\n\n2. Mask creation for each axis plane\n- Combining all masks of each image (Do this on GPU)\n- The images are saved as the png format.\n","metadata":{}},{"cell_type":"markdown","source":"# KFold for the hold-out method","metadata":{}},{"cell_type":"code","source":"study_ids = np.array(train_labels.study_id.unique())\nlen(study_ids)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:46.754503Z","iopub.execute_input":"2024-07-13T21:24:46.754868Z","iopub.status.idle":"2024-07-13T21:24:46.763010Z","shell.execute_reply.started":"2024-07-13T21:24:46.754834Z","shell.execute_reply":"2024-07-13T21:24:46.762062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\n\nfor fold, (trn_idx, val_idx) in enumerate(kf.split(range(len(study_ids)))):\n    print(f\"Fold {fold}\")\n    print(\"Train studies:\", len(trn_idx))\n    trn_study_id = study_ids[trn_idx]\n    fold_train_labels = train_labels.loc[train_labels.study_id.isin(trn_study_id)].reset_index(drop=True)\n    fold_train_des = train_des.loc[train_des.study_id.isin(trn_study_id)].reset_index(drop=True)\n    print(\"Fold train labels shape:\", fold_train_labels.shape, \"\\nFold train des shape:\", fold_train_des.shape)\n\n    print(\"Valid studies:\", len(val_idx))\n    val_study_id = study_ids[val_idx]\n    fold_valid_labels = train_labels.loc[train_labels.study_id.isin(val_study_id)].reset_index(drop=True)\n    fold_valid_des = train_des.loc[train_des.study_id.isin(val_study_id)].reset_index(drop=True)\n    print(\"Fold valid labels shape:\", fold_valid_labels.shape, \"\\nFold valid des shape:\", fold_valid_des.shape)\n    print(\"*\" * 40)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:46.764083Z","iopub.execute_input":"2024-07-13T21:24:46.764376Z","iopub.status.idle":"2024-07-13T21:24:46.825661Z","shell.execute_reply.started":"2024-07-13T21:24:46.764351Z","shell.execute_reply":"2024-07-13T21:24:46.824626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating a mask of coordinates","metadata":{}},{"cell_type":"code","source":"def create_mask(image_shape, coordinates, radius=20):\n    \n    mask = np.zeros(image_shape, dtype=np.uint8)\n    for (x, y) in coordinates:\n        rr, cc = disk((y,x), radius, shape=image_shape)\n        mask[rr, cc]= 1\n    return mask","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:46.827056Z","iopub.execute_input":"2024-07-13T21:24:46.827465Z","iopub.status.idle":"2024-07-13T21:24:46.833749Z","shell.execute_reply.started":"2024-07-13T21:24:46.827428Z","shell.execute_reply":"2024-07-13T21:24:46.832767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_count = 0\nfor (series_description, study_id, series_id), group in train_labels.groupby(['series_description', 'study_id', 'series_id']):\n\n    coordinates = list(zip(group['x'], group['y']))\n\n    example_instance_number = group['instance_number'].iloc[0]\n    image_path = os.path.join(rd, f'train_images/{study_id}/{series_id}/{example_instance_number}.dcm')\n    dicom_image = pydicom.dcmread(image_path).pixel_array\n    \n    mask = create_mask(dicom_image.shape, coordinates)\n    \n    plt.figure(figsize=(12, 6))\n    plt.subplot(1, 2, 1)\n    plt.title('Original Image')\n    plt.imshow(dicom_image, cmap='gray')\n    plt.subplot(1, 2, 2)\n    plt.title('Mask')\n    plt.imshow(mask, cmap='gray')\n    plt.show()\n    \n    display_count += 1\n    \n    if display_count >= 5:\n        break","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2024-07-13T21:24:46.835112Z","iopub.execute_input":"2024-07-13T21:24:46.835429Z","iopub.status.idle":"2024-07-13T21:24:49.565696Z","shell.execute_reply.started":"2024-07-13T21:24:46.835400Z","shell.execute_reply":"2024-07-13T21:24:49.564658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARGET_SHAPE = (500, 500)\ndef pad_or_trim_image(image, target_shape):\n    \"\"\"\n    Pad or trim the image to the target shape.\n    \"\"\"\n    original_shape = image.shape\n    padded_image = np.zeros(target_shape, dtype=image.dtype)\n    \n    y_offset = (target_shape[0] - original_shape[0]) // 2\n    x_offset = (target_shape[1] - original_shape[1]) // 2\n    \n    y_start = max(y_offset, 0)\n    y_end = y_start + min(original_shape[0], target_shape[0])\n    x_start = max(x_offset, 0)\n    x_end = x_start + min(original_shape[1], target_shape[1])\n    \n    padded_image[y_start:y_end, x_start:x_end] = image[:target_shape[0], :target_shape[1]]\n    \n    return padded_image","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:49.570261Z","iopub.execute_input":"2024-07-13T21:24:49.570580Z","iopub.status.idle":"2024-07-13T21:24:49.577706Z","shell.execute_reply.started":"2024-07-13T21:24:49.570552Z","shell.execute_reply":"2024-07-13T21:24:49.576757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_masks={\n    'Sagittal T1': None,\n    'Axial T2': None,\n    'Sagittal T2/STIR': None\n}\n\n\nfor (series_description, study_id, series_id), group in tqdm(train_labels.groupby(['series_description', 'study_id', 'series_id'])):\n\n    coordinates = list(zip(group['x'], group['y']))\n    \n\n    example_instance_number = group['instance_number'].iloc[0]\n    image_path = os.path.join(rd, f'train_images/{study_id}/{series_id}/{example_instance_number}.dcm')\n    dicom_image = pydicom.dcmread(image_path).pixel_array\n    \n    mask = create_mask(dicom_image.shape, coordinates)\n    padded_mask = pad_or_trim_image(mask, TARGET_SHAPE)\n    \n    if combined_masks[series_description] is None:\n        combined_masks[series_description] = padded_mask\n    else:\n        combined_masks[series_description] = np.maximum(combined_masks[series_description], padded_mask)","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:24:49.578818Z","iopub.execute_input":"2024-07-13T21:24:49.579118Z","iopub.status.idle":"2024-07-13T21:27:29.560342Z","shell.execute_reply.started":"2024-07-13T21:24:49.579092Z","shell.execute_reply":"2024-07-13T21:27:29.559384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"combined_masks","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:27:29.561456Z","iopub.execute_input":"2024-07-13T21:27:29.561706Z","iopub.status.idle":"2024-07-13T21:27:29.569280Z","shell.execute_reply.started":"2024-07-13T21:27:29.561683Z","shell.execute_reply":"2024-07-13T21:27:29.568275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for series_description, combined_mask in combined_masks.items():\n#     mask_path = os.path.join(rd, f'combined_mask_{series_description.replace(\"/\", \"_\")}.npy')\n#     np.save(mask_path, combined_mask)\n    \n    plt.figure(figsize=(6, 6))\n    plt.title(f'Combined Mask for {series_description}')\n    plt.imshow(combined_mask, cmap='gray')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:27:29.570420Z","iopub.execute_input":"2024-07-13T21:27:29.570726Z","iopub.status.idle":"2024-07-13T21:27:30.399049Z","shell.execute_reply.started":"2024-07-13T21:27:29.570699Z","shell.execute_reply":"2024-07-13T21:27:30.398036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the combined mask image\n\nfrom zipfile import ZipFile\nimport os\n\nsave_dir = 'train_masks'\nos.makedirs(save_dir, exist_ok = True)\n\nfor series_description, combined_mask in combined_masks.items():\n\n    mask_path_png = os.path.join(save_dir, f'combined_mask_{series_description.replace(\"/\", \"_\")}.png')\n    imsave(mask_path_png, combined_mask)\n\nzip_path = 'Axes_mask.zip'\nwith ZipFile(zip_path, 'w') as zipObj:\n    for filename in os.listdir(save_dir):\n        if filename.endswith(\".png\"):\n            file_path = os.path.join(save_dir, filename)\n            zipObj.write(file_path, filename)\n\n# Verify the contents of the zip file\nwith ZipFile(zip_path, 'r') as zipObj:\n    print(\"Contents of the zip file:\")\n    print(zipObj.namelist())","metadata":{"execution":{"iopub.status.busy":"2024-07-13T21:27:30.400386Z","iopub.execute_input":"2024-07-13T21:27:30.401051Z","iopub.status.idle":"2024-07-13T21:27:30.499860Z","shell.execute_reply.started":"2024-07-13T21:27:30.401014Z","shell.execute_reply":"2024-07-13T21:27:30.498953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You can ignore the last warning because we are saving hte binary masks.","metadata":{}}]}