{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":61446,"databundleVersionId":6962461,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# SenNet + HOA","metadata":{}},{"cell_type":"markdown","source":"## Importing Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nfrom tqdm import tqdm\nfrom pathlib import Path\nimport seaborn as sns\nfrom PIL import Image\nimport numpy as np\n\nfrom matplotlib import animation, rc\nrc('animation', html='jshtml')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-25T09:00:14.122590Z","iopub.execute_input":"2023-11-25T09:00:14.123049Z","iopub.status.idle":"2023-11-25T09:00:16.376010Z","shell.execute_reply.started":"2023-11-25T09:00:14.123001Z","shell.execute_reply":"2023-11-25T09:00:16.374676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/blood-vessel-segmentation/train_rles.csv')\ndf['subset']=df['id'].map(lambda x:'_'.join(x.split('_')[:-1]))\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:00:16.378569Z","iopub.execute_input":"2023-11-25T09:00:16.379456Z","iopub.status.idle":"2023-11-25T09:00:17.573653Z","shell.execute_reply.started":"2023-11-25T09:00:16.379419Z","shell.execute_reply":"2023-11-25T09:00:17.572497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of unique slices:\", df['id'].nunique())\nprint(\"Number of unique datasets:\", df['id'].str.split('_').str[0].nunique())\n","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:00:17.575460Z","iopub.execute_input":"2023-11-25T09:00:17.576294Z","iopub.status.idle":"2023-11-25T09:00:17.606907Z","shell.execute_reply.started":"2023-11-25T09:00:17.576249Z","shell.execute_reply":"2023-11-25T09:00:17.605784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Distribution of Segmentation Mask Lengths","metadata":{}},{"cell_type":"code","source":"df['mask_length'] = df['rle'].apply(lambda x: len(str(x).split()))\n\nfig = px.histogram(df, x='mask_length', nbins=20,\n                   labels={'mask_length': 'Number of Pixels in Mask', 'count': 'Frequency'},\n                   title='Distribution of Segmentation Mask Lengths',\n                   marginal=\"box\",  # or violin, rug\n                   color_discrete_sequence=['skyblue'])\n\nfig.update_layout(bargap=0.1)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:00:17.608562Z","iopub.execute_input":"2023-11-25T09:00:17.609342Z","iopub.status.idle":"2023-11-25T09:00:20.336480Z","shell.execute_reply.started":"2023-11-25T09:00:17.609295Z","shell.execute_reply":"2023-11-25T09:00:20.335255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['scan'] = df['id'].apply(lambda x: x[:-5])\ndf['mask_is_empty'] = df['id'].apply(lambda x: x[:-5])\n\ndf['scan'] = df['id'].apply(lambda x: x[:-5])\ndf['mask_is_empty'] = df['rle']=='1 0'","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:00:20.339185Z","iopub.execute_input":"2023-11-25T09:00:20.339537Z","iopub.status.idle":"2023-11-25T09:00:20.366259Z","shell.execute_reply.started":"2023-11-25T09:00:20.339507Z","shell.execute_reply":"2023-11-25T09:00:20.364738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Number of Images per Scan","metadata":{}},{"cell_type":"code","source":"\n\nfig = px.histogram(df, y='scan', title='Number of Images per Scan', labels={'scan': 'Scan'}, color_discrete_sequence=['skyblue'])\nfig.update_layout(yaxis_title='Scan', xaxis_title='Number of Images')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:00:20.367600Z","iopub.execute_input":"2023-11-25T09:00:20.368473Z","iopub.status.idle":"2023-11-25T09:00:20.477414Z","shell.execute_reply.started":"2023-11-25T09:00:20.368434Z","shell.execute_reply":"2023-11-25T09:00:20.476134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Mask is Empty Percentage","metadata":{}},{"cell_type":"code","source":"x = 'scan'\ny = 'mask_is_empty'\n\ngb = df.groupby(x)[y].value_counts(normalize=True)\ngb = gb.round(3)*100\ngb = gb.rename('percent').reset_index()\n\nfig = px.bar(gb, \n             x='percent',\n             y='scan',\n             color='mask_is_empty',\n             orientation='h',\n             title='Mask is Empty Percentage',\n             labels={'scan': 'Scan', 'percent': 'Percentage'},\n             category_orders={'scan': sorted(df['scan'].unique())},  \n             color_discrete_sequence=px.colors.qualitative.Set1 \n            )\n\nfor trace in fig.data:\n    trace['text'] = [f\"{round(val, 1)}%\" for val in trace['x']]\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:00:20.478902Z","iopub.execute_input":"2023-11-25T09:00:20.479794Z","iopub.status.idle":"2023-11-25T09:00:20.621732Z","shell.execute_reply.started":"2023-11-25T09:00:20.479760Z","shell.execute_reply":"2023-11-25T09:00:20.620490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_image(file_path):\n    with Image.open(file_path) as img:\n        return np.array(img)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:00:20.623475Z","iopub.execute_input":"2023-11-25T09:00:20.624007Z","iopub.status.idle":"2023-11-25T09:00:20.630042Z","shell.execute_reply.started":"2023-11-25T09:00:20.623976Z","shell.execute_reply":"2023-11-25T09:00:20.628826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize_intensity(image):\n    return image / 255.0","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:00:20.631675Z","iopub.execute_input":"2023-11-25T09:00:20.632036Z","iopub.status.idle":"2023-11-25T09:00:20.639437Z","shell.execute_reply.started":"2023-11-25T09:00:20.631994Z","shell.execute_reply":"2023-11-25T09:00:20.638559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show(sample_df, idx):\n    sample = sample_df[sample_df['slice_id'] == str.zfill(f'{idx}', 4)]\n\n    image = load_image(sample['image'].values[0])\n    label = load_image(sample['label'].values[0])\n\n    image = normalize_intensity(image)\n\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4))\n    ax1.imshow(image, cmap='gray')\n    ax2.imshow(label, cmap='gray')\n    ax1.axis('off')\n    ax2.axis('off')\n    plt.subplots_adjust(wspace=0.05)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:00:20.640342Z","iopub.execute_input":"2023-11-25T09:00:20.640631Z","iopub.status.idle":"2023-11-25T09:00:20.650985Z","shell.execute_reply.started":"2023-11-25T09:00:20.640605Z","shell.execute_reply":"2023-11-25T09:00:20.650025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def animate(sample_df, id_range):\n    fig, [ax1, ax2] = plt.subplots(1, 2)\n    ax1.axis('off')\n    ax2.axis('off')\n    images = []\n\n    for i in tqdm(id_range):\n        sample = sample_df[sample_df['slice_id'] == str.zfill(f'{i}', 4)]\n\n        image = load_image(sample['image'].values[0])\n        label = load_image(sample['label'].values[0])\n\n        image = normalize_intensity(image)\n    \n\n        im1 = ax1.imshow(image, animated=True, cmap='gray')\n        im2 = ax2.imshow(label, animated=True, cmap='gray')\n        \n        if i == id_range[0]:\n            ax1.imshow(image, cmap='gray')\n            ax2.imshow(label, cmap='gray')\n        \n        images.append([im1, im2])\n\n    ani = animation.ArtistAnimation(fig, images, interval=50, blit=True, repeat_delay=1000)\n    plt.close()\n    return ani","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:00:20.652082Z","iopub.execute_input":"2023-11-25T09:00:20.653319Z","iopub.status.idle":"2023-11-25T09:00:20.663693Z","shell.execute_reply.started":"2023-11-25T09:00:20.653286Z","shell.execute_reply":"2023-11-25T09:00:20.662606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_df(im_dir='kidney_1_dense',lb_dir='kidney_1_dense'):\n    df = pd.read_csv('/kaggle/input/blood-vessel-segmentation/train_rles.csv')\n    base_dir = Path('/kaggle/input/blood-vessel-segmentation/train')\n    subset_df = df[df.id.str.startswith(lb_dir)].reset_index(drop=True)\n    subset_df['slice_id'] = subset_df['id'].map(lambda x:x.split('_')[-1]) \n    subset_df['image'] = subset_df['slice_id'].map(lambda x: base_dir / im_dir / 'images' / f'{x}.tif')\n    subset_df['label'] = subset_df['slice_id'].map(lambda x: base_dir / lb_dir / 'labels' / f'{x}.tif')\n    return subset_df","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:00:20.665119Z","iopub.execute_input":"2023-11-25T09:00:20.666429Z","iopub.status.idle":"2023-11-25T09:00:20.682915Z","shell.execute_reply.started":"2023-11-25T09:00:20.666382Z","shell.execute_reply":"2023-11-25T09:00:20.681414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## kidney_1_dense","metadata":{}},{"cell_type":"code","source":"dense_1_df = prepare_df(im_dir='kidney_1_dense',lb_dir='kidney_1_dense')\nshow(dense_1_df,1234)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:00:20.685284Z","iopub.execute_input":"2023-11-25T09:00:20.685809Z","iopub.status.idle":"2023-11-25T09:00:21.996956Z","shell.execute_reply.started":"2023-11-25T09:00:20.685763Z","shell.execute_reply":"2023-11-25T09:00:21.995694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"animate(dense_1_df,id_range=range(1200,1300))","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:00:22.001016Z","iopub.execute_input":"2023-11-25T09:00:22.001448Z","iopub.status.idle":"2023-11-25T09:01:17.061954Z","shell.execute_reply.started":"2023-11-25T09:00:22.001410Z","shell.execute_reply":"2023-11-25T09:01:17.060700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## kidney_1_voi","metadata":{}},{"cell_type":"code","source":"kidney_1_voi_df = prepare_df(im_dir='kidney_1_voi',lb_dir='kidney_1_voi')\nshow(kidney_1_voi_df,454)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:01:17.063497Z","iopub.execute_input":"2023-11-25T09:01:17.063885Z","iopub.status.idle":"2023-11-25T09:01:18.797560Z","shell.execute_reply.started":"2023-11-25T09:01:17.063851Z","shell.execute_reply":"2023-11-25T09:01:18.796243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"animate(kidney_1_voi_df,id_range=range(500,600))","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:01:18.799037Z","iopub.execute_input":"2023-11-25T09:01:18.799518Z","iopub.status.idle":"2023-11-25T09:03:39.294674Z","shell.execute_reply.started":"2023-11-25T09:01:18.799473Z","shell.execute_reply":"2023-11-25T09:03:39.292591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## kidney_2","metadata":{}},{"cell_type":"code","source":"kidney_2_df = prepare_df(im_dir='kidney_2',lb_dir='kidney_2')\nshow(kidney_2_df,1234)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:03:39.296844Z","iopub.execute_input":"2023-11-25T09:03:39.297464Z","iopub.status.idle":"2023-11-25T09:03:40.540961Z","shell.execute_reply.started":"2023-11-25T09:03:39.297402Z","shell.execute_reply":"2023-11-25T09:03:40.539708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"animate(kidney_2_df,id_range=range(800,900))","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:03:40.542437Z","iopub.execute_input":"2023-11-25T09:03:40.542805Z","iopub.status.idle":"2023-11-25T09:04:46.409036Z","shell.execute_reply.started":"2023-11-25T09:03:40.542766Z","shell.execute_reply":"2023-11-25T09:04:46.407376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## kidney_3_dense","metadata":{}},{"cell_type":"code","source":"kidney_3_dense_df = prepare_df(im_dir='kidney_3_sparse',lb_dir='kidney_3_dense')\nshow(kidney_3_dense_df,533)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:04:46.410884Z","iopub.execute_input":"2023-11-25T09:04:46.411530Z","iopub.status.idle":"2023-11-25T09:04:47.769411Z","shell.execute_reply.started":"2023-11-25T09:04:46.411466Z","shell.execute_reply":"2023-11-25T09:04:47.768068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"animate(kidney_3_dense_df,id_range=range(500,600))","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:04:47.770968Z","iopub.execute_input":"2023-11-25T09:04:47.771476Z","iopub.status.idle":"2023-11-25T09:06:28.863731Z","shell.execute_reply.started":"2023-11-25T09:04:47.771439Z","shell.execute_reply":"2023-11-25T09:06:28.861251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## kidney_3_sparse","metadata":{}},{"cell_type":"code","source":"kidney_3_sparse_df = prepare_df(im_dir='kidney_3_sparse',lb_dir='kidney_3_sparse')\nshow(kidney_3_sparse_df,343)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:06:28.865143Z","iopub.execute_input":"2023-11-25T09:06:28.865560Z","iopub.status.idle":"2023-11-25T09:06:30.204101Z","shell.execute_reply.started":"2023-11-25T09:06:28.865527Z","shell.execute_reply":"2023-11-25T09:06:30.203193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"animate(kidney_3_sparse_df,id_range=range(300,400))","metadata":{"execution":{"iopub.status.busy":"2023-11-25T09:06:30.205233Z","iopub.execute_input":"2023-11-25T09:06:30.206379Z","iopub.status.idle":"2023-11-25T09:08:13.598792Z","shell.execute_reply.started":"2023-11-25T09:06:30.206340Z","shell.execute_reply":"2023-11-25T09:08:13.596433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Continue...**","metadata":{}}]}