{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom skimage import io\nimport seaborn as sns\nimport math\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-27T03:15:21.798810Z","iopub.execute_input":"2022-05-27T03:15:21.799179Z","iopub.status.idle":"2022-05-27T03:15:24.134655Z","shell.execute_reply.started":"2022-05-27T03:15:21.799084Z","shell.execute_reply":"2022-05-27T03:15:24.133799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sartorius_df = pd.read_csv('../input/sartorius-cell-instance-segmentation/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:15:24.136339Z","iopub.execute_input":"2022-05-27T03:15:24.137138Z","iopub.status.idle":"2022-05-27T03:15:24.813752Z","shell.execute_reply.started":"2022-05-27T03:15:24.137096Z","shell.execute_reply":"2022-05-27T03:15:24.812708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Helper functions\ndef make_mask(mask_files, image_shape=(520, 704), color=False):\n    mask = np.zeros(image_shape).ravel()\n    for i, mask_file in enumerate(mask_files):\n        couples = np.array(mask_file.split()).reshape(-1, 2).astype(int)\n        couples[:, 1] = couples[:, 0] + couples[:, 1]\n        for couple in couples:\n            if color:\n                mask[couple[0]:couple[1]] = i\n            else:\n                mask[couple[0]:couple[1]] = 1\n    mask = mask.reshape(520, 704)\n    return mask\n\ndef plot_image(image_id='0030fd0e6378'):\n    fig, ax = plt.subplots(1, 2, figsize=(14, 5))\n    cell_type = sartorius_df.loc[sartorius_df['id'] == image_id, 'cell_type'][0:1].values\n    \n    file_name = os.path.join(\n        '../input/sartorius-cell-instance-segmentation',\n        'train', image_id + '.png')\n    image = plt.imread(file_name)\n    mask_files = sartorius_df.loc[sartorius_df['id'] == image_id, 'annotation']\n    mask = make_mask(mask_files)\n    \n    ax[0].imshow(image, cmap='gray')\n    ax[0].axis('off')\n    ax[1].imshow(mask, alpha=1, cmap=plt.get_cmap('seismic'))\n    ax[1].set_title(f'Source [{image_id}] + Mask {cell_type}')\n    ax[1].axis('off')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:15:24.815130Z","iopub.execute_input":"2022-05-27T03:15:24.815419Z","iopub.status.idle":"2022-05-27T03:15:24.827374Z","shell.execute_reply.started":"2022-05-27T03:15:24.815379Z","shell.execute_reply":"2022-05-27T03:15:24.825859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <font color='green'/> **Unique cell type in dataset**","metadata":{}},{"cell_type":"code","source":"cell_labels = np.sort(sartorius_df['cell_type'].unique())\nprint('Label in sartorius dataset: ', cell_labels)","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:15:24.830234Z","iopub.execute_input":"2022-05-27T03:15:24.830517Z","iopub.status.idle":"2022-05-27T03:15:24.855395Z","shell.execute_reply.started":"2022-05-27T03:15:24.830485Z","shell.execute_reply":"2022-05-27T03:15:24.853995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The dataset contains three cell types: `astro`, `cort` and `shsy5y`","metadata":{}},{"cell_type":"code","source":"HEIGHT = sartorius_df['height'].unique()[0]\nWIDTH = sartorius_df['width'].unique()[0]\nSHAPE = (HEIGHT, WIDTH)\nprint('Image shape: ', SHAPE)\n\nassert sartorius_df['height'].nunique() == 1\nassert sartorius_df['width'].nunique() == 1","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:15:24.857071Z","iopub.execute_input":"2022-05-27T03:15:24.858047Z","iopub.status.idle":"2022-05-27T03:15:24.871180Z","shell.execute_reply.started":"2022-05-27T03:15:24.858003Z","shell.execute_reply":"2022-05-27T03:15:24.870278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"All the image have the same size, shape of image is: `704 x 520`","metadata":{}},{"cell_type":"code","source":"print('Number of unique images: ', len(sartorius_df.groupby('id')))\nprint('Number of unique image and cell type combinations: ', len(sartorius_df.groupby(['id', 'cell_type'])))","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:15:24.873120Z","iopub.execute_input":"2022-05-27T03:15:24.873427Z","iopub.status.idle":"2022-05-27T03:15:24.982065Z","shell.execute_reply.started":"2022-05-27T03:15:24.873366Z","shell.execute_reply":"2022-05-27T03:15:24.980351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Number of unique image and cell type combinations is equal to the number of unique images. This means each image is associated with a unique cell type. So that we can calculate the distribution of image type.","metadata":{}},{"cell_type":"markdown","source":"### <font color='green'/> **Data distribution**","metadata":{}},{"cell_type":"code","source":"ins_dist = sartorius_df['cell_type'].value_counts(normalize=True) * 100\nimg_dist = sartorius_df.groupby('id').first()['cell_type'].value_counts(normalize=True) * 100","metadata":{"execution":{"iopub.status.busy":"2022-05-27T06:17:43.690993Z","iopub.execute_input":"2022-05-27T06:17:43.691334Z","iopub.status.idle":"2022-05-27T06:17:43.740336Z","shell.execute_reply.started":"2022-05-27T06:17:43.691297Z","shell.execute_reply":"2022-05-27T06:17:43.739471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ins_dist=pd.DataFrame({'cell_type': ins_dist.index.to_list(),'percentile': ins_dist.values})\nimg_dist=pd.DataFrame({'cell_type': img_dist.index.to_list(),'percentile': img_dist.values})\nins_dist['hue'] = 'Ins dist'\nimg_dist['hue'] = 'Img dis'\nres=pd.concat([ins_dist, img_dist])","metadata":{"execution":{"iopub.status.busy":"2022-05-27T06:17:44.274991Z","iopub.execute_input":"2022-05-27T06:17:44.275913Z","iopub.status.idle":"2022-05-27T06:17:44.283594Z","shell.execute_reply.started":"2022-05-27T06:17:44.275868Z","shell.execute_reply":"2022-05-27T06:17:44.283004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res","metadata":{"execution":{"iopub.status.busy":"2022-05-27T06:17:46.309749Z","iopub.execute_input":"2022-05-27T06:17:46.310669Z","iopub.status.idle":"2022-05-27T06:17:46.322990Z","shell.execute_reply.started":"2022-05-27T06:17:46.310627Z","shell.execute_reply":"2022-05-27T06:17:46.321857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(8, 6))\naxs = plt.subplot(1, 1, 1)\n\nsns.despine()\nsns.set_theme(palette='Set2')\nsns.set_style('ticks')\n\nsns.barplot(data=res, x='cell_type', y='percentile', hue='hue')\naxs.set_xlabel('Cell type', fontdict={'family': 'serif', 'size': 14})\naxs.set_ylabel('Percentile', fontdict={'family': 'serif', 'size': 14})\naxs.set_title('Image and instance distribution (%)',\n              fontdict={'family': 'serif', 'size': 14, 'weight': 'bold'})\nleg = axs.get_legend()\nleg.set_title('')\nleg.set_frame_on(False)","metadata":{"execution":{"iopub.status.busy":"2022-05-27T06:18:43.912590Z","iopub.execute_input":"2022-05-27T06:18:43.912965Z","iopub.status.idle":"2022-05-27T06:18:44.179085Z","shell.execute_reply.started":"2022-05-27T06:18:43.912932Z","shell.execute_reply":"2022-05-27T06:18:44.177326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Observation**\n* The `cort` image takes ~ 52% of entire images, the `astro` and `shsy5y` almost have the same distribution\n* Each image has many cells, and over 70% cells of the dataset are `shsy5y`, `cort` and `astro` cell have the same distribution","metadata":{}},{"cell_type":"markdown","source":"### <font color='green'/> **Number instances per image**","metadata":{}},{"cell_type":"code","source":"group = sartorius_df.groupby(['id', 'cell_type']).size()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:15:25.231554Z","iopub.execute_input":"2022-05-27T03:15:25.231910Z","iopub.status.idle":"2022-05-27T03:15:25.252936Z","shell.execute_reply.started":"2022-05-27T03:15:25.231874Z","shell.execute_reply":"2022-05-27T03:15:25.251691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group","metadata":{"execution":{"iopub.status.busy":"2022-05-27T06:12:45.854671Z","iopub.execute_input":"2022-05-27T06:12:45.855660Z","iopub.status.idle":"2022-05-27T06:12:45.867733Z","shell.execute_reply.started":"2022-05-27T06:12:45.855614Z","shell.execute_reply":"2022-05-27T06:12:45.866440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(10, 7))\naxs = plt.subplot(1, 1, 1)\n\nsns.despine()\nsns.set_theme()\nsns.set_style('ticks')\n\nsns.histplot(x=group.values, data=group, hue='cell_type', element=\"step\")\naxs.set_xlabel('Instance count', fontdict={'family': 'serif', 'size': 14})\naxs.set_ylabel('Freq', fontdict={'family': 'serif', 'size': 14})\naxs.set_title('Instance count per image',\n              fontdict={'family': 'serif', 'size': 14, 'weight': 'bold'})\nleg = axs.get_legend()\nleg.set_title('Cell type', prop={'family': 'serif'})\nleg.set_frame_on(False)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T06:19:58.557983Z","iopub.execute_input":"2022-05-27T06:19:58.558381Z","iopub.status.idle":"2022-05-27T06:19:58.819837Z","shell.execute_reply.started":"2022-05-27T06:19:58.558344Z","shell.execute_reply":"2022-05-27T06:19:58.818897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Total cell type\ninstances_per_img = sartorius_df.groupby('id').size().sort_values()\n\n# Astro type\nastro_df = sartorius_df[sartorius_df['cell_type'] == 'astro']\nastro_instances_per_img = astro_df.groupby('id').size()\n\n# Cort type\ncort_df = sartorius_df[sartorius_df['cell_type'] == 'cort']\ncort_instances_per_img = cort_df.groupby('id').size()\n\n# Shsy5y type\nshsy5y_df = sartorius_df[sartorius_df['cell_type'] == 'shsy5y']\nshsy5y_instances_per_img = shsy5y_df.groupby('id').size()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T04:31:50.989675Z","iopub.execute_input":"2022-05-27T04:31:50.989988Z","iopub.status.idle":"2022-05-27T04:31:51.040735Z","shell.execute_reply.started":"2022-05-27T04:31:50.989957Z","shell.execute_reply":"2022-05-27T04:31:51.039523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('---------- Total cell type statistic ----------')\nprint(instances_per_img.describe())\nprint('---------- Astro type statistic ----------')\nprint(astro_instances_per_img.describe())\nprint('---------- Cort type statistic ----------')\nprint(cort_instances_per_img.describe())\nprint('---------- Shsy5y type statistic ----------')\nprint(shsy5y_instances_per_img.describe())","metadata":{"execution":{"iopub.status.busy":"2022-05-27T04:31:53.796457Z","iopub.execute_input":"2022-05-27T04:31:53.796743Z","iopub.status.idle":"2022-05-27T04:31:53.817672Z","shell.execute_reply.started":"2022-05-27T04:31:53.796715Z","shell.execute_reply":"2022-05-27T04:31:53.816849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Observation**\n* **Fig 1:** Almost images have less than 200 instances.\n* **Fig 2:** We have total 131 `astro` images, just 2 images have more than 200 instances, and the remaining images have less than 200 instances.\n* **Fig 3:** We have total 320 `cort` images, just 1 images has around 105-109 instances, and almost images have between 15 and 50 instances.\n* **Fig 4:** We have total 155 `shsy5y` images, number of instances per this image type is largest, almost images have more than 200 instances in entire image is `shsy5y`, 80% `shsy5y` images have between 50 and 450 instances.","metadata":{}},{"cell_type":"markdown","source":"**Plot images have max and min instances**","metadata":{}},{"cell_type":"code","source":"max_num_ins_id = sartorius_df.groupby('id').size().sort_values().index[-1]\nplot_image(max_num_ins_id)","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:15:25.847864Z","iopub.status.idle":"2022-05-27T03:15:25.848158Z","shell.execute_reply.started":"2022-05-27T03:15:25.848008Z","shell.execute_reply":"2022-05-27T03:15:25.848024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_num_ins_id = sartorius_df.groupby('id').size().sort_values().index[0]\nplot_image(min_num_ins_id)","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:15:25.849691Z","iopub.status.idle":"2022-05-27T03:15:25.849992Z","shell.execute_reply.started":"2022-05-27T03:15:25.849841Z","shell.execute_reply":"2022-05-27T03:15:25.849856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <font color='green'/> **Calculating bounding box size**","metadata":{}},{"cell_type":"code","source":"def rle2mask(rle):\n    array = np.fromiter(rle.split(), dtype=np.uint32)\n    array = array.reshape(-1, 2)\n    array[:, 0] = array[:, 0] - 1\n    \n    mask_decompressed = np.concatenate([np.arange(i[0], i[0] + i[1], \n                                                 dtype=np.uint32) for i in array])\n    \n    mask = np.zeros(SHAPE[0] * SHAPE[1], dtype=np.uint32)\n    mask[mask_decompressed] = 1\n    mask = mask.reshape(SHAPE)\n    \n    return mask\n\ndef get_box_height_width(rle):\n    mask = rle2mask(rle)\n    nonzero_coords = np.transpose(np.nonzero(mask))\n    box_height = nonzero_coords[:, 0].max() - nonzero_coords[:, 0].min()\n    box_width = nonzero_coords[:, 1].max() - nonzero_coords[:, 1].min()\n    \n    return box_height, box_width\n\nsartorius_df[['box_height', 'box_width']] = sartorius_df['annotation'].apply(get_box_height_width).to_list()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:15:25.851605Z","iopub.status.idle":"2022-05-27T03:15:25.852378Z","shell.execute_reply.started":"2022-05-27T03:15:25.852166Z","shell.execute_reply":"2022-05-27T03:15:25.852189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('---------- Total bounding box statistic ----------')\nprint(sartorius_df[['box_height', 'box_width']].describe())\nprint('---------- Bounding box of astro type statistic ----------')\nprint(sartorius_df[sartorius_df['cell_type'] == 'astro'][['box_height', 'box_width']].describe())\nprint('---------- Bounding box of cort type statistic ----------')\nprint(sartorius_df[sartorius_df['cell_type'] == 'cort'][['box_height', 'box_width']].describe())\nprint('---------- Bounding box of shsy5y type statistic ----------')\nprint(sartorius_df[sartorius_df['cell_type'] == 'shsy5y'][['box_height', 'box_width']].describe())","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:15:25.853434Z","iopub.status.idle":"2022-05-27T03:15:25.854238Z","shell.execute_reply.started":"2022-05-27T03:15:25.854009Z","shell.execute_reply":"2022-05-27T03:15:25.854030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter(sartorius_df, x='box_width', y='box_height', color='cell_type',\n                labels={\n                    'box_width': 'Box Width (px)',\n                    'box_height': 'Box Height (px)',\n                    'cell_type': 'Cell type'\n                }, title='Bounding box size for each instance')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:15:25.855331Z","iopub.status.idle":"2022-05-27T03:15:25.856111Z","shell.execute_reply.started":"2022-05-27T03:15:25.855904Z","shell.execute_reply":"2022-05-27T03:15:25.855927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Observation**\n* `Cort` type is the smallest cell type in dataset, the average size of box is (17, 17)\n* Almost bounding box' shape cover `cort` type is smaller than (70, 70)\n* `Shsy5y` is bit bigger than `cort` type, almost bounding box is smaller than (80, 100)\n* Average size of `Shsy5y` box is (18, 18)\n* `Astro` is the largest cell type in dataset\n* Shape of `astro` type is quite unbalanced, this type has smallest and biggest shape over entire instances\n* Shape of `astro` goes from 2 pixel to over 300 pixel, but box's size tend to centralize in range [0, 150]","metadata":{}},{"cell_type":"markdown","source":"**Plot max width and height instances**","metadata":{}},{"cell_type":"code","source":"max_width_rle = sartorius_df.loc[sartorius_df['box_width'] == sartorius_df['box_width'].max(), \n                                 'annotation'].values[0]\nmax_height_rle = sartorius_df.loc[sartorius_df['box_height'] == sartorius_df['box_height'].max(),\n                                 'annotation'].values[0]\nmax_width_mask = rle2mask(max_width_rle)\nmax_height_mask = rle2mask(max_height_rle)\n\nfig, axs = plt.subplots(nrows=1, ncols= 2, figsize=(10, 4))\n\naxs[0].set_title('Max height instance')\naxs[0].imshow(max_height_mask, alpha=1, cmap=plt.get_cmap('seismic'))\naxs[0].axis('off')\n\naxs[1].set_title('Max width instance')\naxs[1].imshow(max_width_mask, alpha=1, cmap=plt.get_cmap('seismic'))\naxs[1].axis('off')\n\n\nfig.tight_layout()\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-27T03:15:25.857165Z","iopub.status.idle":"2022-05-27T03:15:25.857893Z","shell.execute_reply.started":"2022-05-27T03:15:25.857713Z","shell.execute_reply":"2022-05-27T03:15:25.857732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### <font color='green'/> **References**\n\nhttps://www.kaggle.com/code/tolgadincer/sartorius-eda-general-overview-and-outliers/notebook","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}